509 lines
18 KiB
Ruby
509 lines
18 KiB
Ruby
require "lib/knife/knife_factory"
|
|
require "workers/worker"
|
|
require "workers/delete_server_worker"
|
|
require "hooks"
|
|
|
|
module Devops
|
|
module Executor
|
|
class ServerExecutor
|
|
include Hooks
|
|
|
|
RESULT_CODES = {
|
|
server_bootstrap_fail: 2,
|
|
server_not_in_chef_nodes: 5,
|
|
server_bootstrap_unknown_error: 7,
|
|
deploy_unknown_error: 6,
|
|
deploy_failed: 8
|
|
}
|
|
|
|
# waiting for 5*60 seconds (5 min)
|
|
MAX_SSH_RETRIES_AMOUNT = 60
|
|
|
|
#params:
|
|
# out - container for output data
|
|
# deploy_info - hash with deploy data
|
|
define_hook :before_deploy
|
|
define_hook :after_deploy
|
|
|
|
define_hook :before_create
|
|
define_hook :after_create
|
|
|
|
#params:
|
|
# out - container for output data
|
|
define_hook :before_bootstrap
|
|
define_hook :after_bootstrap
|
|
|
|
before_deploy :create_run_list
|
|
|
|
def initialize server, out
|
|
if server
|
|
@project = Devops::Db.connector.project(server.project)
|
|
@deploy_env = @project.deploy_env(server.deploy_env)
|
|
end
|
|
@knife_instance = KnifeFactory.instance
|
|
@server = server
|
|
@out = out
|
|
@out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
|
|
end
|
|
|
|
def self.result_code(symbolic_code)
|
|
RESULT_CODES.fetch(symbolic_code)
|
|
end
|
|
|
|
def self.symbolic_result_code(integer_code)
|
|
RESULT_CODES.key(integer_code) || :unknown_error
|
|
end
|
|
|
|
def result_code(symbolic_code)
|
|
self.class.result_code(symbolic_code)
|
|
end
|
|
|
|
def report= r
|
|
@report = r
|
|
end
|
|
|
|
def project= p
|
|
@project = p
|
|
end
|
|
|
|
def deploy_env= e
|
|
@deploy_env = e
|
|
end
|
|
|
|
def server
|
|
@server
|
|
end
|
|
|
|
def create_server options
|
|
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider, "provider_account" => @deploy_env.provider_account})
|
|
provider = @deploy_env.provider_instance
|
|
mongo = ::Devops::Db.connector
|
|
begin
|
|
@out.puts "Using '#{@deploy_env.provider}' account '#{@deploy_env.provider_account}'\n" if @deploy_env.provider_account
|
|
@out.puts "Create server..."
|
|
@out.flush
|
|
|
|
@server.run_list = options["run_list"] || []
|
|
@server.chef_node_name = options["name"]
|
|
@server.key = options["key"] || provider.ssh_key
|
|
|
|
i = mongo.image(@deploy_env.image)
|
|
@server.remote_user = i.remote_user
|
|
|
|
res = {}
|
|
@out << "\nBefore create hooks...\n"
|
|
res[:before] = self.run_hook :before_create
|
|
@out << "Done\n"
|
|
|
|
return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
|
|
mongo.server_insert @server
|
|
|
|
@out << "\nAfter create hooks...\n"
|
|
res[:after] = self.run_hook :after_create
|
|
@out << "Done\n"
|
|
@out.flush
|
|
DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
|
|
|
|
schedule_expiration(@server)
|
|
|
|
unless options["without_bootstrap"]
|
|
bootstrap_options = {
|
|
bootstrap_template: i.bootstrap_template
|
|
}
|
|
return two_phase_bootstrap(bootstrap_options)
|
|
else
|
|
return 0
|
|
end
|
|
rescue => e
|
|
@out << e.message
|
|
DevopsLogger.logger.error e.message
|
|
roll_back
|
|
mongo.server_delete @server.id
|
|
# return 5
|
|
return result_code(:server_not_in_chef_nodes)
|
|
end
|
|
end
|
|
|
|
def bootstrap options
|
|
@out << "\n\nBootstrap...\n"
|
|
@out.flush
|
|
k = Devops::Db.connector.key(@server.key)
|
|
cert_path = k.path
|
|
options[:cert_path] = k.path
|
|
@out << "\nBefore bootstrap hooks...\n"
|
|
res = self.run_hook(:before_bootstrap, @out)
|
|
@out << "Done\n"
|
|
if @server.private_ip.nil?
|
|
@out << "Error: Private IP is null"
|
|
return false
|
|
end
|
|
ja = {
|
|
:provider => @server.provider,
|
|
:provider_account => @server.provider_account,
|
|
:devops_host => `hostname`.strip
|
|
}
|
|
ip = @server.private_ip
|
|
unless @server.public_ip.nil?
|
|
ip = @server.public_ip
|
|
@out << "\nPublic IP is present\n"
|
|
end
|
|
|
|
address = "#{@server.remote_user}@#{ip}"
|
|
|
|
cmd = 'ssh '
|
|
cmd << "-i #{cert_path} "
|
|
cmd << '-q '
|
|
cmd << '-o StrictHostKeyChecking=no '
|
|
cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
|
|
cmd << "#{address} 'exit'"
|
|
cmd << " 2>&1"
|
|
|
|
@out << "\nWaiting for SSH..."
|
|
@out << "\nTest command: '#{cmd}'\n"
|
|
@out.flush
|
|
|
|
retries_amount = 0
|
|
begin
|
|
sleep(5)
|
|
res = `#{cmd}`
|
|
retries_amount += 1
|
|
if retries_amount > MAX_SSH_RETRIES_AMOUNT
|
|
@out.puts "Can not connect to #{address}"
|
|
@out.puts res
|
|
@out.flush
|
|
DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
|
|
return result_code(:server_bootstrap_fail)
|
|
end
|
|
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
|
|
rescue ArgumentError => e
|
|
@out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
|
|
@out.flush
|
|
retry
|
|
end
|
|
|
|
provider = @server.provider_instance
|
|
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
|
|
|
|
r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
|
|
|
|
if r == 0
|
|
@out << "Chef node name: #{@server.chef_node_name}\n"
|
|
::Devops::Db.connector.server_set_chef_node_name @server
|
|
@out << "Chef node name has been updated\n"
|
|
@out << "After bootstrap hooks...\n"
|
|
res = self.run_hook(:after_bootstrap, @out)
|
|
@out << "Done\n"
|
|
@out.flush
|
|
DevopsLogger.logger.info "Server with id '#{@server.id}' is bootstraped"
|
|
r
|
|
else
|
|
@out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
|
|
@out.flush
|
|
result_code(:server_bootstrap_fail)
|
|
end
|
|
end
|
|
|
|
def bootstrap_options attributes, options
|
|
bootstrap_options = [
|
|
"-x #{@server.remote_user}",
|
|
"-i #{options[:cert_path]}",
|
|
"--json-attributes '#{attributes.to_json}'"
|
|
]
|
|
bootstrap_options.push "-N #{@server.chef_node_name}" if @server.chef_node_name
|
|
bootstrap_options.push "--sudo" unless @server.remote_user == "root"
|
|
bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
|
|
rl = options[:run_list]
|
|
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil? or rl.empty?
|
|
bootstrap_options.push "-c #{options[:config]}" if options[:config]
|
|
bootstrap_options
|
|
end
|
|
|
|
def two_phase_bootstrap options
|
|
# bootstrap phase
|
|
begin
|
|
provider = @server.provider_instance
|
|
mongo = ::Devops::Db.connector
|
|
options[:run_list] = provider.run_list
|
|
bootstrap_status = bootstrap(options)
|
|
|
|
if bootstrap_status == 0
|
|
if check_server
|
|
@out << "Server #{@server.chef_node_name} is created"
|
|
else
|
|
@out << roll_back
|
|
@out.flush
|
|
mongo.server_delete @server.id
|
|
return result_code(:server_not_in_chef_nodes)
|
|
end
|
|
else
|
|
# @out << roll_back
|
|
# mongo.server_delete @server.id
|
|
msg = "Failed while bootstraping server with id '#{@server.id}'\n"
|
|
msg << "Bootstraping operation result was #{bootstrap_status}"
|
|
DevopsLogger.logger.error msg
|
|
@out.puts msg
|
|
@out.flush
|
|
return result_code(:server_bootstrap_fail)
|
|
end
|
|
rescue => e
|
|
@out << "\nError: #{e.message}\n"
|
|
@out.flush
|
|
return result_code(:server_bootstrap_unknown_error)
|
|
end
|
|
|
|
# deploy phase. Assume that all servers are bootstraped successfully here.
|
|
begin
|
|
#raise "hello"
|
|
@out << "\n"
|
|
run_list = compute_run_list
|
|
@out << "\nComputed run list: #{run_list.join(", ")}"
|
|
@out.flush
|
|
@knife_instance.set_run_list(@server.chef_node_name, run_list)
|
|
deploy_info = @project.deploy_info(@deploy_env)
|
|
deploy_status = deploy_server(deploy_info)
|
|
if deploy_status == 0
|
|
0
|
|
else
|
|
msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
|
|
msg << "\nDeploing server operation status was #{deploy_status}"
|
|
DevopsLogger.logger.error msg
|
|
@out << "\n" + msg + "\n"
|
|
result_code(:deploy_failed)
|
|
end
|
|
rescue => e
|
|
@out << "\nError: #{e.message}\n"
|
|
DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
|
|
result_code(:deploy_unknown_error)
|
|
end
|
|
end
|
|
|
|
def check_server
|
|
@knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
|
|
end
|
|
|
|
def unbootstrap
|
|
k = Devops::Db.connector.key(@server.key)
|
|
cert_path = k.path
|
|
i = 0
|
|
res = delete_from_chef_server(@server.chef_node_name)
|
|
begin
|
|
new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}"
|
|
# r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
|
|
cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} \"/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name}; else echo not found; fi'\""
|
|
DevopsLogger.logger.info("Trying to run command '#{cmd}'")
|
|
r = `#{cmd}`.strip
|
|
if r == 'not found'
|
|
res[:server] = "Directory '/etc/chef' does not exists"
|
|
else
|
|
raise(r) unless $?.success?
|
|
res[:server] = "'/etc/chef' renamed to '#{new_name}'"
|
|
end
|
|
rescue => e
|
|
DevopsLogger.logger.error "Unbootstrap error: " + e.message
|
|
i += 1
|
|
sleep(1)
|
|
retry unless i == 5
|
|
return {error: e.message}
|
|
end
|
|
res
|
|
end
|
|
|
|
def deploy_server_with_tags tags, deploy_info
|
|
old_tags_str = nil
|
|
new_tags_str = nil
|
|
unless tags.empty?
|
|
old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
|
|
@out << "Server tags: #{old_tags_str}\n"
|
|
@knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
|
|
|
|
new_tags_str = tags.join(" ")
|
|
@out << "Server new tags: #{new_tags_str}\n"
|
|
cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
|
|
unless cmd[1]
|
|
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
|
|
DevopsLogger.logger.error(m)
|
|
@out << m + "\n"
|
|
return 3
|
|
end
|
|
DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
|
|
end
|
|
|
|
r = deploy_server deploy_info
|
|
|
|
unless tags.empty?
|
|
@out << "Restore tags\n"
|
|
cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
|
|
DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
|
|
cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
|
|
DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
|
|
end
|
|
return r
|
|
end
|
|
|
|
def deploy_server deploy_info
|
|
@out << "\nBefore deploy hooks...\n"
|
|
res = self.run_hook(:before_deploy, @out, deploy_info)
|
|
@out << "Done\n"
|
|
@out << "\nRun chef-client on '#{@server.chef_node_name}'\n"
|
|
cmd = "chef-client --no-color"
|
|
if deploy_info["use_json_file"]
|
|
deploy_info.delete("use_json_file")
|
|
@out << "Deploy Input Parameters:\n"
|
|
json = JSON.pretty_generate(deploy_info)
|
|
@out << json
|
|
@out << "\n"
|
|
file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
|
|
dir = DevopsConfig.config[:project_info_dir]
|
|
File.open(File.join(dir, file), "w") do |f|
|
|
f.write json
|
|
end
|
|
@out.flush
|
|
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
|
|
else
|
|
cmd << " -r #{deploy_info["run_list"].join(",")}" unless @server.stack.nil?
|
|
end
|
|
ip = if @server.public_ip.nil?
|
|
@server.private_ip
|
|
else
|
|
@out << "Public IP detected\n"
|
|
@server.public_ip
|
|
end
|
|
@out.flush
|
|
k = Devops::Db.connector.key(@server.key)
|
|
lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
|
|
r = /Chef\sClient\sfinished/i
|
|
if lline[r].nil?
|
|
1
|
|
else
|
|
@out << "\nAfter deploy hooks...\n"
|
|
res = self.run_hook(:after_deploy, @out, deploy_info)
|
|
@out << "Done\n"
|
|
0
|
|
end
|
|
end
|
|
|
|
def delete_from_chef_server node_name
|
|
{
|
|
:chef_node => @knife_instance.chef_node_delete(node_name),
|
|
:chef_client => @knife_instance.chef_client_delete(node_name)
|
|
}
|
|
end
|
|
|
|
=begin
|
|
def delete_etc_chef s, cert_path
|
|
cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
|
|
cmd += " sudo " unless s.remote_user == "root"
|
|
cmd += "rm -Rf /etc/chef"
|
|
r = `#{cmd}`
|
|
raise(r) unless $?.success?
|
|
end
|
|
=end
|
|
|
|
def delete_server
|
|
mongo = ::Devops::Db.connector
|
|
if @server.static?
|
|
if !@server.chef_node_name.nil?
|
|
unbootstrap
|
|
end
|
|
mongo.server_delete @server.id
|
|
msg = "Static server '#{@server.id}' is removed"
|
|
DevopsLogger.logger.info msg
|
|
return msg, nil
|
|
end
|
|
r = delete_from_chef_server(@server.chef_node_name)
|
|
provider = @server.provider_instance
|
|
begin
|
|
r[:server] = provider.delete_server @server
|
|
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
|
|
r[:server] = "Server with id '#{@server.id}' not found in '#{provider.name}' servers"
|
|
DevopsLogger.logger.warn r[:server]
|
|
end
|
|
mongo.server_delete @server.id
|
|
info = "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed"
|
|
DevopsLogger.logger.info info
|
|
r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
|
|
return info, r
|
|
end
|
|
|
|
def roll_back
|
|
unless @server.id.nil?
|
|
@out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n"
|
|
@out << delete_from_chef_server(@server.chef_node_name).values.join("\n")
|
|
begin
|
|
@out << @server.provider_instance.delete_server(@server)
|
|
rescue => e
|
|
@out << e.message
|
|
end
|
|
@out << "\nRolled back\n"
|
|
end
|
|
end
|
|
|
|
def create_run_list out, deploy_info
|
|
out << "\nGenerate run list hook...\n"
|
|
if deploy_info["run_list"]
|
|
out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
|
|
return
|
|
end
|
|
out << "Project run list: #{@project.run_list.join(", ")}\n"
|
|
out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
|
|
out << "Server run list: #{@server.run_list.join(", ")}\n"
|
|
=begin
|
|
rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
|
|
if @server.stack
|
|
stack = Devops::Db.connector.stack(@server.stack)
|
|
out << "Stack run list: #{stack.run_list.join(", ")}\n"
|
|
rlist.merge(stack.run_list)
|
|
end
|
|
=end
|
|
deploy_info["run_list"] = compute_run_list
|
|
out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
|
|
end
|
|
|
|
def compute_run_list
|
|
rlist = []
|
|
[@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
|
|
rlist += sub_run_list if sub_run_list.is_a?(Array)
|
|
end
|
|
rlist = Set.new(rlist)
|
|
if @server.stack
|
|
stack = Devops::Db.connector.stack(@server.stack)
|
|
# out << "Stack run list: #{stack.run_list.join(", ")}\n"
|
|
srl = stack.run_list
|
|
rlist.merge(srl) if srl.is_a?(Array)
|
|
end
|
|
rlist.to_a
|
|
end
|
|
|
|
private
|
|
|
|
def schedule_expiration(server)
|
|
expires = @deploy_env.expires
|
|
return unless expires
|
|
interval = interval_in_seconds(expires)
|
|
@out << "Planning expiration in #{expires}"
|
|
DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name)
|
|
end
|
|
|
|
def interval_in_seconds(interval_as_string)
|
|
interval = interval_as_string.to_i
|
|
measure_unit = interval_as_string.chars.last
|
|
case measure_unit
|
|
when 's'
|
|
interval
|
|
when 'm'
|
|
interval * 60
|
|
when 'h'
|
|
interval * 60 * 60
|
|
when 'd'
|
|
interval * 60 * 60 * 24
|
|
when 'w'
|
|
interval * 60 * 60 * 24 * 7
|
|
else
|
|
raise 'Wrong interval format'
|
|
end
|
|
end
|
|
|
|
end
|
|
end
|
|
end
|