tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

executors: server executor

Browse Source
This commit is contained in:
Anton Martynov 2015-08-11 19:47:54 +03:00
parent 7a69d2b83c
commit fb272395df
8 changed files with 351 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
devops-service/app/api2/handlers/deploy.rb
View File

@ -1,4 +1,4 @@
require "commands/deploy"
require "executors/server_executor"
require "commands/status"
require "workers/deploy_worker"
require "exceptions/deploy_info_error"
@ -9,7 +9,7 @@ module Devops
module API2_0
module Handler
class Deploy < RequestHandler
extend DeployCommands
# extend DeployCommands
extend StatusCommands
set_parser Devops::API2_0::Parser::DeployParser
@ -81,7 +81,7 @@ module Devops
end
begin
deploy_info = create_deploy_info(s, project, body["build_number"])
res = deploy_server_proc.call(out, s, tags, deploy_info)
res = Devops::Executor::ServerExecutor.new(s, out).deploy_server_with_tags(tags, deploy_info)
status.push(res)
rescue DeployInfoError => e
msg = "Can not get deploy info: " + e.message

15
devops-service/app/api2/handlers/server.rb
View File

@ -4,7 +4,6 @@ require "uri"
require "commands/status"
require "commands/server"
require "commands/bootstrap_templates"
require "commands/knife_commands"
require "providers/provider_factory"
@ -34,7 +33,7 @@ module Devops
end
def chef_servers
KnifeCommands.chef_node_list
KnifeFactory.instance.chef_node_list
end
def provider_servers provider
@ -148,8 +147,12 @@ module Devops
cert = Devops::Db.connector.key s.key
DevopsLogger.logger.debug "Bootstrap certificate path: #{cert.path}"
#bootstrap s, out, cert.path, logger
provider = ::Provider::ProviderFactory.get(s.provider)
r = two_phase_bootstrap s, provider.run_list, bt, cert.path, out
options = {
:bootstrap_template => bt,
:cert_path => cert.path,
:run_list => rl
}
r = two_phase_bootstrap s, options, out
str = nil
r = if check_server(s)
Devops::Db.connector.server_set_chef_node_name s
@ -270,9 +273,9 @@ module Devops
# server not found - OK
s = provider.servers.detect {|s| s["name"] == name}
halt(400, "#{provider.name} node with name '#{name}' already exist") unless s.nil?
s = KnifeCommands.chef_node_list.detect {|n| n == name}
s = KnifeFactory.instance.chef_node_list.detect {|n| n == name}
halt(400, "Chef node with name '#{name}' already exist") unless s.nil?
s = KnifeCommands.chef_client_list.detect {|c| c == name}
s = KnifeFactory.instance.chef_client_list.detect {|c| c == name}
halt(400, "Chef client with name '#{name}' already exist") unless s.nil?
end

49
devops-service/commands/knife_commands.rb
View File

@ -2,37 +2,43 @@ require "json"
class KnifeCommands
def self.chef_node_list
attr_accessor :config
def initialize config
self.config = config
end
def chef_node_list
knife("node list")[0].split.map{|c| c.strip}
end
def self.chef_client_list
def chef_client_list
knife("client list")[0].split.map{|c| c.strip}
end
def self.chef_node_delete name
def chef_node_delete name
o = knife("node delete #{name} -y")[0]
(o.nil? ? o : o.strip)
end
def self.chef_client_delete name
def chef_client_delete name
o = knife("client delete #{name} -y")[0]
(o.nil? ? o : o.strip)
end
def self.tags_list name
def tags_list name
knife("tag list #{name}")[0].split.map{|c| c.strip}
end
def self.tags_create name, tagsStr
def tags_create name, tagsStr
knife("tag create #{name} #{tagsStr}")
end
def self.tags_delete name, tagsStr
def tags_delete name, tagsStr
knife("tag delete #{name} #{tagsStr}")
end
def self.create_role role_name, project, env
def create_role role_name, project, env
file = "/tmp/new_role.json"
File.open(file, "w") do |f|
f.puts <<-EOH
@ -51,31 +57,31 @@ class KnifeCommands
}
EOH
end
out = `knife role from file #{file}`
raise "Cannot create role '#{role_name}': #{out}" unless $?.success?
out, res = knife("role from file #{file}")
raise "Cannot create role '#{role_name}': #{out}" unless res
true
end
def self.roles(chef_env=nil)
def roles(chef_env=nil)
o, s = knife("role list --format json")
return (s ? JSON.parse(o) : nil)
end
def self.role_name project_name, deploy_env
def role_name project_name, deploy_env
project_name + (DevopsConfig.config[:role_separator] || "_") + deploy_env
end
def self.knife cmd
o = `knife #{cmd} 2>&1`
def knife cmd
o = `knife #{cmd} -c #{self.config} 2>&1`
return o, $?.success?
end
def self.ssh_options cmd, host, user, cert
def ssh_options cmd, host, user, cert
["-m", "-x", user, "-i", cert, "--no-host-key-verify", host, "'#{(user == "root" ? cmd : "sudo #{cmd}")}'"]
end
def self.ssh_stream out, cmd, host, user, cert
knife_cmd = "knife ssh -c #{get_config()} #{ssh_options(cmd, host, user, cert).join(" ")}"
def ssh_stream out, cmd, host, user, cert
knife_cmd = "knife ssh -c #{self.config} #{ssh_options(cmd, host, user, cert).join(" ")}"
out << "\nExecuting '#{knife_cmd}' \n\n"
out.flush if out.respond_to?(:flush)
status = 2
@ -91,13 +97,12 @@ EOH
return lline
end
def self.knife_bootstrap out, ip, options
options << "-c ~/.chef/knife.rb"
def knife_bootstrap out, ip, options
knife_stream(out, "bootstrap", options + [ ip ])
end
def self.knife_stream out, cmd, options=[]
knife_cmd = "knife #{cmd} #{options.join(" ")}"
def knife_stream out, cmd, options=[]
knife_cmd = "knife #{cmd} #{options.join(" ")} -c #{self.config}"
out << "\nExecuting '#{knife_cmd}' \n\n"
out.flush if out.respond_to?(:flush)
status = nil
@ -112,7 +117,7 @@ EOH
return status
end
def self.set_run_list node, list
def set_run_list node, list
knife("node run_list set #{node} '#{list.join("','")}'")
end

170
devops-service/commands/server.rb
View File

@ -6,30 +6,6 @@ require "exceptions/record_not_found"
module ServerCommands
include DeployCommands
=begin
def create_server_proc
lambda do |out, s, provider|
mongo = ::Devops::Db.connector
begin
out << "Create server...\n"
out.flush if out.respond_to?(:flush)
unless provider.create_server(s, out)
return 3
end
s.create
out.flush if out.respond_to?(:flush)
DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
key = mongo.key(s.key)
return two_phase_bootstrap(s, out, provider, key.path)
rescue => e
DevopsLogger.logger.error e.message
DevopsLogger.logger.warn roll_back(s, provider)
mongo.server_delete s.id
return 5
end
end
end
=end
def create_server project, env, params, user, out
provider = ::Provider::ProviderFactory.get(env.provider)
@ -57,7 +33,11 @@ module ServerCommands
s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
key = mongo.key(s.key)
s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
return two_phase_bootstrap(s, provider.run_list, i.bootstrap_template, key.path, out)
options = {
bootstrap_template: i.bootstrap_template,
cert_path: key.path
}
return two_phase_bootstrap(s, options, out)
else
return 0
end
@ -69,12 +49,13 @@ module ServerCommands
end
end
def two_phase_bootstrap s, provider_run_list, bootstrap_template, cert_path, out
def two_phase_bootstrap s, options, out
provider = ::Provider::ProviderFactory.get(s.provider)
mongo = ::Devops::Db.connector
out << "\n\nBootstrap...\n"
out.flush if out.respond_to?(:flush)
status = bootstrap(s, out, bootstrap_template, provider_run_list, cert_path)
options[:run_list] = provider.run_list
status = bootstrap(s, options, out)
out.flush if out.respond_to?(:flush)
if status == 0
DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
@ -88,7 +69,7 @@ module ServerCommands
out << "\n"
out.flush if out.respond_to?(:flush)
run_list = s.run_list + provider_run_list
run_list = s.run_list + provider.run_list
out << "\nRun list: #{run_list.inspect}"
# s.options[:run_list] += run_list
KnifeCommands.set_run_list(s.chef_node_name, run_list)
@ -108,139 +89,6 @@ module ServerCommands
return status
end
def delete_from_chef_server node_name
{
:chef_node => KnifeCommands.chef_node_delete(node_name),
:chef_client => KnifeCommands.chef_client_delete(node_name)
}
end
def self.delete_etc_chef s, cert_path
cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
cmd += " sudo " unless s.remote_user == "root"
cmd += "rm -Rf /etc/chef"
r = `#{cmd}`
raise(r) unless $?.success?
end
def check_server s
KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
end
def bootstrap s, out, bootstrap_template, run_list, cert_path
out << "Before bootstrap hooks...\n"
res = s.run_hook(:before_bootstrap, out)
out << "Done\n"
if s.private_ip.nil?
out << "Error: Private IP is null"
return false
end
out << "\nBootstrap with run list: #{run_list.inspect}\n"
ja = {
:provider => s.provider,
:devops_host => `hostname`.strip
}
bootstrap_options = [
"-x #{s.remote_user}",
"-i #{cert_path}",
"--json-attributes '#{ja.to_json}'",
"-N #{s.chef_node_name}"
]
bootstrap_options.push "--sudo" unless s.remote_user == "root"
bootstrap_options.push "-d #{bootstrap_template}" if bootstrap_template
bootstrap_options.push "-r #{run_list.join(",")}" unless run_list.empty?
ip = s.private_ip
unless s.public_ip.nil? || s.public_ip.strip.empty?
ip = s.public_ip
out << "\nPublic IP is present\n"
end
out << "\nWaiting for SSH..."
out.flush if out.respond_to?(:flush)
i = 0
cmd = "ssh -i #{cert_path} -q #{s.remote_user}@#{ip} 'exit' 2>&1"
begin
sleep(5)
res = `#{cmd}`
i += 1
if i == 120
out << "\nCan not connect to #{s.remote_user}@#{ip}"
out << "\n" + res
DevopsLogger.logger.error "Can not connect with command 'ssh -i #{cert_path} #{s.remote_user}@#{ip}':\n#{res}"
return false
end
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
rescue ArgumentError => e
retry
end
r = KnifeCommands.knife_bootstrap(out, ip, bootstrap_options)
if r == 0
out << "Chef node name: #{s.chef_node_name}\n"
::Devops::Db.connector.server_set_chef_node_name s
out << "Chef node name has been updated\n"
out << "After bootstrap hooks...\n"
res = s.run_hook(:after_bootstrap, out)
out << "Done\n"
else
end
r
end
def self.unbootstrap s, cert_path
i = 0
begin
r = `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef`
raise(r) unless $?.success?
rescue => e
DevopsLogger.logger.error "Unbootstrap error: " + e.message
i += 1
sleep(1)
retry unless i == 5
return e.message
end
nil
end
def delete_server s
mongo = ::Devops::Db.connector
if s.static?
if !s.chef_node_name.nil?
cert = mongo.key s.key
ServerCommands.unbootstrap(s, cert.path)
end
mongo.server_delete s.id
msg = "Static server '#{s.id}' is removed"
DevopsLogger.logger.info msg
return msg, nil
end
r = delete_from_chef_server(s.chef_node_name)
provider = ::Provider::ProviderFactory.get(s.provider)
begin
r[:server] = provider.delete_server s
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
DevopsLogger.logger.warn r[:server]
end
mongo.server_delete s.id
info = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
DevopsLogger.logger.info info
r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
return info, r
end
def roll_back s, provider
str = ""
unless s.id.nil?
str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n"
str << delete_from_chef_server(s.chef_node_name).values.join("\n")
begin
str << provider.delete_server(s)
rescue => e
str << e.message
end
str << "\nRolled back\n"
end
return str
end
end

3
devops-service/core/devops-service.rb
View File

@ -19,6 +19,8 @@ require_relative "devops-db"
require_relative "devops-logger"
require_relative "devops-application"
require "knife/knife_factory"
require_relative "../sinatra/methods_with_headers"
require_relative "../applications"
@ -46,6 +48,7 @@ class DevopsService
# 6. init all routes classes
# 7. register routes for all classes
def init
KnifeFactory.init
# init database
Devops::Db.init
DevopsLogger.logger = DevopsLogger.create(STDOUT)

281
devops-service/lib/executors/server_executor.rb Normal file
View File

@ -0,0 +1,281 @@
require "knife/knife_factory"
module Devops
module Executor
class ServerExecutor
def initialize server, out
@project = Devops::Db.connector.project(server.project)
@deploy_env = @project.deploy_env(server.deploy_env)
@knife_instance = KnifeFactory.instance
@server = server
@out = out
end
def create_server options
provider = @server.provider_instance
mongo = ::Devops::Db.connector
begin
@out << "Create server...\n"
@out.flush if @out.respond_to?(:flush)
=begin
s = Devops::Model::Server.new
s.provider = provider.name
s.project = self.id
s.deploy_env = env.identifier
s.run_list = options["run_list"] || []
s.chef_node_name = options["name"]
s.key = options["key"] || provider.ssh_key
s.created_by = user
=end
i = mongo.image(@deploy_env.image)
@server.remote_user = i.remote_user
res = {}
res[:before] = self.run_hook :before_create
return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
mongo.server_insert @server
res[:after] = self.run_hook :after_create
res
# return 3 unless @server.create(provider, env.image, env.flavor, env.subnets, env.groups, @out)
@out.flush if @out.respond_to?(:flush)
DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
unless options["without_bootstrap"]
@server.run_list = Set.new.merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
bootstrap_options = {
bootstrap_template: i.bootstrap_template
}
return bootstrap(bootstrap_options)
else
return 0
end
rescue => e
DevopsLogger.logger.error e.message
roll_back
mongo.server_delete @server.id
return 5
end
end
def bootstrap options
k = Devops::Db.connector.key(@server.key)
cert_path = k.path
@out << "Before bootstrap hooks...\n"
res = self.run_hook(:before_bootstrap, @out)
@out << "Done\n"
if @server.private_ip.nil?
@out << "Error: Private IP is null"
return false
end
ja = {
:provider => @server.provider,
:devops_host => `hostname`.strip
}
ip = @server.private_ip
unless @server.public_ip.nil?
ip = @server.public_ip
@out << "\nPublic IP is present\n"
end
@out << "\nWaiting for SSH..."
@out.flush if @out.respond_to?(:flush)
i = 0
cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{ip} 'exit' 2>&1"
begin
sleep(5)
res = `#{cmd}`
i += 1
if i == 120
@out << "\nCan not connect to #{@server.remote_user}@#{ip}"
@out << "\n" + res
DevopsLogger.logger.error "Can not connect with command 'ssh -i #{cert_path} #{@server.remote_user}@#{ip}':\n#{res}"
return false
end
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
rescue ArgumentError => e
retry
end
r = @knife_instance.knife_bootstrap(out, ip, self.bootstrap_options(ja, options))
if r == 0
@out << "Chef node name: #{@server.chef_node_name}\n"
::Devops::Db.connector.server_set_chef_node_name @server
@out << "Chef node name has been updated\n"
@out << "After bootstrap hooks...\n"
res = self.run_hook(:after_bootstrap, @out)
@out << "Done\n"
else
end
r
end
def bootstrap_options attributes, options
bootstrap_options = [
"-x #{@server.remote_user}",
"-i #{options[:cert_path]}",
"--json-attributes '#{attributes.to_json}'",
"-N #{@server.chef_node_name}"
]
bootstrap_options.push "--sudo" unless @server.remote_user == "root"
bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
rl = options[:run_list]
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil?# rl.empty?
bootstrap_options.push "-c #{options[:config]}" if options[:config]
bootstrap_options
end
def check_server
@knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
end
def unbootstrap
k = Devops::Db.connector.key(@server.key)
cert_path = k.path
i = 0
begin
r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
raise(r) unless $?.success?
rescue => e
DevopsLogger.logger.error "Unbootstrap error: " + e.message
i += 1
sleep(1)
retry unless i == 5
return e.message
end
nil
end
def deploy_server_with_tags tags, deploy_info
old_tags_str = nil
new_tags_str = nil
unless tags.empty?
old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
@out << "Server tags: #{old_tags_str}\n"
@knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
new_tags_str = tags.join(" ")
@out << "Server new tags: #{new_tags_str}\n"
cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
unless cmd[1]
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
DevopsLogger.logger.error(m)
@out << m + "\n"
return 3
end
DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
end
deploy_server deploy_info
unless tags.empty?
@out << "Restore tags\n"
cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
end
return r
end
def deploy_server deploy_info
@out << "Before deploy hooks...\n"
res = self.run_hook(:before_deploy, @out, deploy_info)
@out << "Done\n"
@out << "\nRun chef-client on '#{@server.chef_node_name}'\n"
cmd = "chef-client --no-color"
if deploy_info["use_json_file"]
deploy_info.delete["use_json_file"]
@out << "Build information:\n"
json = JSON.pretty_generate(deploy_info)
@out << json
@out << "\n"
file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
dir = DevopsConfig.config[:project_info_dir]
File.open(File.join(dir, file), "w") do |f|
f.write json
end
@out.flush if @out.respond_to?(:flush)
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
end
ip = if @server.public_ip.nil?
@server.private_ip
else
@out << "Public IP detected\n"
@server.public_ip
end
@out.flush if @out.respond_to?(:flush)
k = Devops::Db.connector.key(@server.key)
lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
r = /Chef\sClient\sfinished/i
if lline[r].nil?
1
else
@out << "After deploy hooks...\n"
res = server.run_hook(:after_deploy, @out, deploy_info)
@out << "Done\n"
0
end
end
def delete_from_chef_server node_name
{
:chef_node => @knife_instance.chef_node_delete(node_name),
:chef_client => @knife_instance.chef_client_delete(node_name)
}
end
=begin
def delete_etc_chef s, cert_path
cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
cmd += " sudo " unless s.remote_user == "root"
cmd += "rm -Rf /etc/chef"
r = `#{cmd}`
raise(r) unless $?.success?
end
=end
def delete_server
mongo = ::Devops::Db.connector
if @server.static?
if !@server.chef_node_name.nil?
unbootstrap
end
mongo.server_delete @server.id
msg = "Static server '#{@server.id}' is removed"
DevopsLogger.logger.info msg
return msg, nil
end
r = delete_from_chef_server(@server.chef_node_name)
provider = @server.provider_instance
begin
r[:server] = provider.delete_server @server
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
r[:server] = "Server with id '#{@server.id}' not found in '#{provider.name}' servers"
DevopsLogger.logger.warn r[:server]
end
mongo.server_delete @server.id
info = "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed"
DevopsLogger.logger.info info
r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
return info, r
end
def roll_back
unless @server.id.nil?
@out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n"
@out << delete_from_chef_server(@server.chef_node_name).values.join("\n")
begin
@out << @server.provider_instance.delete_server(@server)
rescue => e
@out << e.message
end
@out << "\nRolled back\n"
end
end
end
end
end

17
devops-service/lib/knife/knife_factory.rb Normal file
View File

@ -0,0 +1,17 @@
require "commands/knife_commands"
class KnifeFactory
class << self
def init
c = DevopsConfig.config[:knife_config]
raise "Option ':knife_config' is undefined, please check config.rb file" unless c
@instance = KnifeCommands.new(c)
end
def instance
@instance
end
end
end

5
devops-service/workers/deploy_worker.rb
View File

@ -1,11 +1,10 @@
require File.join(File.dirname(__FILE__), "worker")
require "commands/deploy"
require "lib/executors/server_executor"
require "db/mongo/models/server"
require "db/mongo/models/report"
class DeployWorker < Worker
include DeployCommands
def perform(dir, server, tags, owner, conf, deploy_info)
call(conf, nil, dir) do |provider, out, file|
@ -24,7 +23,7 @@ class DeployWorker < Worker
}
mongo.save_report(Report.new(o))
status = deploy_server_proc.call(out, s, tags, deploy_info)
status = Devops::Executor::ServerExecutor.new(s, out).deploy_server_with_tags(tags, deploy_info)
status
end
end