556 lines
20 KiB
Ruby
556 lines
20 KiB
Ruby
require "lib/knife/knife_factory"
|
||
require "lib/puts_and_flush"
|
||
require "hooks"
|
||
require 'net/ssh'
|
||
require_relative "server_executor/expiration_scheduler"
|
||
require_relative 'server_executor/server_operation_result'
|
||
|
||
module Devops
|
||
module Executor
|
||
class ServerExecutor
|
||
include Hooks
|
||
include PutsAndFlush
|
||
|
||
# waiting for 5*60 seconds (5 min)
|
||
MAX_SSH_RETRIES_AMOUNT = 60
|
||
|
||
#params:
|
||
# out - container for output data
|
||
# deploy_info - hash with deploy data
|
||
define_hook :before_deploy
|
||
define_hook :after_deploy
|
||
|
||
define_hook :before_create
|
||
define_hook :after_create
|
||
|
||
#params:
|
||
# out - container for output data
|
||
define_hook :before_bootstrap
|
||
define_hook :after_bootstrap
|
||
|
||
before_deploy :add_run_list_to_deploy_info
|
||
|
||
|
||
attr_accessor :server, :deploy_env, :report, :project
|
||
attr_reader :out
|
||
|
||
def initialize server, out, options={}
|
||
if server
|
||
@project = Devops::Db.connector.project(server.project)
|
||
@deploy_env = @project.deploy_env(server.deploy_env)
|
||
end
|
||
@server = server
|
||
@out = out
|
||
@out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
|
||
@current_user = options[:current_user]
|
||
end
|
||
|
||
def error_code(reason)
|
||
ServerOperationResult.code_of_reason(reason)
|
||
end
|
||
|
||
def create_server_object options
|
||
Devops::Model::Server.new({
|
||
"project" => @project.id,
|
||
"deploy_env" => @deploy_env.identifier,
|
||
"created_by" => options["created_by"],
|
||
"provider" => @deploy_env.provider,
|
||
"provider_account" => @deploy_env.provider_account,
|
||
"private_ip" => options["private_ip"]
|
||
})
|
||
end
|
||
|
||
def create_server options
|
||
@server = create_server_object(options)
|
||
@server.set_last_operation(Devops::Model::Server::OperationType::CREATION, options["created_by"])
|
||
provider = @deploy_env.provider_instance
|
||
mongo = ::Devops::Db.connector
|
||
begin
|
||
@out.puts "Using '#{@deploy_env.provider}' account '#{@deploy_env.provider_account}'\n" if @deploy_env.provider_account
|
||
@out.puts "Create server..."
|
||
@out.flush
|
||
|
||
@server.run_list = options["run_list"] || []
|
||
@server.chef_node_name = options["name"]
|
||
@server.key = options["key"] || provider.ssh_key
|
||
|
||
i = mongo.image(@deploy_env.image)
|
||
@server.remote_user = i.remote_user
|
||
|
||
res = {}
|
||
@out << "\nBefore create hooks...\n"
|
||
res[:before] = self.run_hook :before_create
|
||
@out << "Done\n"
|
||
|
||
unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
|
||
return error_code(:creating_server_in_cloud_failed)
|
||
end
|
||
mongo.server_insert @server
|
||
|
||
@out << "\nAfter create hooks...\n"
|
||
res[:after] = self.run_hook :after_create
|
||
@out << "Done\n"
|
||
@out.flush
|
||
DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
|
||
|
||
schedule_expiration()
|
||
|
||
unless options["without_bootstrap"]
|
||
bootstrap_options = { bootstrap_template: i.bootstrap_template }
|
||
if @project.is_sandbox?
|
||
bootstrap_options[:deployers] = [options['created_by']]
|
||
bootstrap_options[:deployers] += (options['project_info']['deployers'] || []) if options['project_info']
|
||
end
|
||
return two_phase_bootstrap(bootstrap_options)
|
||
else
|
||
return 0
|
||
end
|
||
rescue => e
|
||
@out.puts e.message
|
||
@out.puts e.backtrace.join("\n")
|
||
|
||
DevopsLogger.logger.error e.message
|
||
roll_back
|
||
mongo.server_delete @server.id
|
||
error_code(:creating_server_unknown_error)
|
||
end
|
||
end
|
||
|
||
# options:
|
||
# :run_list (optional)
|
||
# :bootstrap_template (optional)
|
||
# :chef_environment (optional)
|
||
# :config (optional)
|
||
def bootstrap options
|
||
@out << "\n\nBootstrap...\n"
|
||
@out.flush
|
||
k = Devops::Db.connector.key(@server.key)
|
||
cert_path = k.path
|
||
options[:cert_path] = k.path
|
||
@out << "\nBefore bootstrap hooks...\n"
|
||
res = self.run_hook(:before_bootstrap, @out)
|
||
@out << "Done\n"
|
||
if @server.private_ip.nil?
|
||
@out.puts "Error: Private IP is null"
|
||
return error_code(:server_bootstrap_private_ip_unset)
|
||
end
|
||
ja = {
|
||
:provider => @server.provider,
|
||
:provider_account => @server.provider_account,
|
||
:devops_host => `hostname`.strip
|
||
}
|
||
ip = @server.private_ip.strip
|
||
unless @server.public_ip.nil?
|
||
ip = @server.public_ip.strip
|
||
@out << "\nPublic IP is present\n"
|
||
end
|
||
|
||
address = "#{@server.remote_user}@#{ip}"
|
||
|
||
cmd = check_ssh_command(cert_path, address)
|
||
|
||
@out << "\nWaiting for SSH..."
|
||
@out << "\nTest command: '#{cmd}'\n"
|
||
@out.flush
|
||
|
||
retries_amount = 0
|
||
begin
|
||
sleep(5)
|
||
res = execute_system_command(cmd)
|
||
retries_amount += 1
|
||
if retries_amount >= MAX_SSH_RETRIES_AMOUNT
|
||
@out.puts "Can not connect to #{address}"
|
||
@out.puts res
|
||
@out.flush
|
||
DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
|
||
return error_code(:server_bootstrap_fail)
|
||
end
|
||
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless last_command_successful?
|
||
rescue ArgumentError => e
|
||
@out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
|
||
@out.flush
|
||
retry
|
||
end
|
||
|
||
provider = @server.provider_instance
|
||
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
|
||
|
||
r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
|
||
|
||
if r == 0
|
||
@out << "Chef node name: #{@server.chef_node_name}\n"
|
||
::Devops::Db.connector.server_set_chef_node_name @server
|
||
@out << "Chef node name has been updated\n"
|
||
@out << "After bootstrap hooks...\n"
|
||
res = self.run_hook(:after_bootstrap, @out)
|
||
@out << "Done\n"
|
||
msg = "Server with id '#{@server.id}' is bootstraped"
|
||
@out.puts msg
|
||
@out.flush
|
||
DevopsLogger.logger.info msg
|
||
r
|
||
else
|
||
@out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
|
||
@out.flush
|
||
error_code(:server_bootstrap_fail)
|
||
end
|
||
end
|
||
|
||
# options:
|
||
# :cert_path (required)
|
||
# :run_list (optional)
|
||
# :bootstrap_template (optional)
|
||
# :chef_environment (optional)
|
||
# :config (optional)
|
||
def bootstrap_options attributes, options
|
||
bootstrap_options = [
|
||
"-x #{@server.remote_user}",
|
||
"-i #{options[:cert_path]}",
|
||
"--json-attributes '#{attributes.to_json}'"
|
||
]
|
||
bootstrap_options.push "-N #{@server.chef_node_name}" if @server.chef_node_name
|
||
bootstrap_options.push "--sudo" unless @server.remote_user == "root"
|
||
bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
|
||
bootstrap_options.push "-E #{options[:chef_environment]}" if options[:chef_environment]
|
||
rl = options[:run_list]
|
||
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil? or rl.empty?
|
||
bootstrap_options.push "-c #{options[:config]}" if options[:config]
|
||
bootstrap_options
|
||
end
|
||
|
||
def prepare_two_phase_bootstrap options
|
||
@out << "Prepare bootstrap...\n"
|
||
@out << "Done\n"
|
||
end
|
||
|
||
# essentially, it just bootstrap and then deploy
|
||
def two_phase_bootstrap options
|
||
prepare_two_phase_bootstrap(options)
|
||
# bootstrap phase
|
||
begin
|
||
provider = @server.provider_instance
|
||
mongo = ::Devops::Db.connector
|
||
options[:run_list] = provider.run_list
|
||
bootstrap_status = bootstrap(options)
|
||
|
||
if bootstrap_status == 0
|
||
if check_server_on_chef_server
|
||
@out << "Server #{@server.chef_node_name} is created"
|
||
else
|
||
@out.puts "Can not find client or node on chef-server"
|
||
@out.puts "Skip rollback because :skip_rollback option is set"
|
||
unless options[:skip_rollback]
|
||
roll_back
|
||
mongo.server_delete @server.id
|
||
end
|
||
@out.flush
|
||
return error_code(:server_not_in_chef_nodes)
|
||
end
|
||
else
|
||
@out.puts "Skip rollback because :skip_rollback option is set"
|
||
unless options[:skip_rollback]
|
||
roll_back
|
||
mongo.server_delete @server.id
|
||
end
|
||
msg = "Failed while bootstraping server with id '#{@server.id}'\n"
|
||
msg << "Bootstraping operation result was #{bootstrap_status}"
|
||
DevopsLogger.logger.error msg
|
||
@out.puts msg
|
||
@out.flush
|
||
return error_code(:server_bootstrap_fail)
|
||
end
|
||
rescue => e
|
||
@out << "\nError: #{e.message}\n"
|
||
@out.flush
|
||
return error_code(:server_bootstrap_unknown_error)
|
||
end
|
||
|
||
# deploy phase. Assume that all servers are bootstraped successfully here.
|
||
begin
|
||
run_list = compute_run_list
|
||
@out << "\n\nComputed run list: #{run_list.join(", ")}"
|
||
@out.flush
|
||
knife_instance.set_run_list(@server.chef_node_name, run_list)
|
||
deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
|
||
deploy_info['project_info']['deployers'] = options[:deployers] if options[:deployers]
|
||
deploy_status = deploy_server(deploy_info)
|
||
if deploy_status == 0
|
||
0
|
||
else
|
||
msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
|
||
msg << "\nDeploing server operation status was #{deploy_status}"
|
||
DevopsLogger.logger.error msg
|
||
@out << "\n" + msg + "\n"
|
||
error_code(:deploy_failed)
|
||
end
|
||
rescue => e
|
||
@out << "\nError: #{e.message}\n"
|
||
DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
|
||
error_code(:deploy_unknown_error)
|
||
end
|
||
end
|
||
|
||
def check_server_on_chef_server
|
||
knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
|
||
end
|
||
|
||
# returns a hash with :chef_node, :chef_client and :server keys
|
||
def unbootstrap
|
||
k = Devops::Db.connector.key(@server.key)
|
||
cert_path = k.path
|
||
i = 0
|
||
res = delete_from_chef_server(@server.chef_node_name)
|
||
begin
|
||
new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}"
|
||
cmd = (@server.remote_user == 'root' ? "" : "sudo ")
|
||
cmd = cmd + "/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name} && echo ok; else echo not found; fi'"
|
||
DevopsLogger.logger.info("SSH: trying to run command '#{cmd}'")
|
||
Net::SSH.start(@server.private_ip, @server.remote_user, :keys => [cert_path]) do |session|
|
||
session.open_channel do |channel|
|
||
channel.request_pty(:modes => { Net::SSH::Connection::Term::ECHO => 0 }) do |c, success|
|
||
raise "could not request pty" unless success
|
||
channel.exec cmd
|
||
channel.on_data do |c_, data|
|
||
if data == 'not found'
|
||
res[:server] = "Directory '/etc/chef' does not exists"
|
||
elsif data == 'ok'
|
||
res[:server] = "'/etc/chef' renamed to '#{new_name}'"
|
||
else
|
||
DevopsLogger.logger.error "Unexpected error: " + data
|
||
raise(data)
|
||
end
|
||
end
|
||
end
|
||
end
|
||
end
|
||
rescue => e
|
||
DevopsLogger.logger.error "Unbootstrap error: " + e.message
|
||
i += 1
|
||
sleep(1)
|
||
retry unless i == 5
|
||
return {error: e.message}
|
||
end
|
||
res
|
||
end
|
||
|
||
def deploy_server_with_tags tags, deploy_info
|
||
return deploy_server(deploy_info) if tags.empty?
|
||
|
||
old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
|
||
new_tags_str = tags.join(" ")
|
||
|
||
@out.puts "Temporarily changing tags (#{old_tags_str}) to (#{new_tags_str})"
|
||
unless knife_instance.swap_tags(@server.chef_node_name, old_tags_str, new_tags_str)
|
||
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
|
||
DevopsLogger.logger.error(m)
|
||
@out.puts m
|
||
return error_code(:server_cannot_update_tags)
|
||
end
|
||
DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
|
||
|
||
begin
|
||
deploy_result = deploy_server deploy_info
|
||
ensure
|
||
@out.puts "Restoring tags"
|
||
knife_instance.swap_tags(@server.chef_node_name, new_tags_str, old_tags_str)
|
||
DevopsLogger.logger.info("Restoring tags for #{@server.chef_node_name}: from #{new_tags_str} back to (#{old_tags_str})")
|
||
end
|
||
deploy_result
|
||
end
|
||
|
||
def deploy_server deploy_info
|
||
sync_project_metadata_with_chef!(deploy_info)
|
||
@out << "\nBefore deploy hooks...\n"
|
||
res = self.run_hook(:before_deploy, @out, deploy_info)
|
||
@out << "Done\n"
|
||
@out << "\nRun chef-client on '#{@server.chef_node_name}'\n"
|
||
cmd = "chef-client --no-color"
|
||
if deploy_info["use_json_file"]
|
||
deploy_info.delete("use_json_file")
|
||
json = nil
|
||
dir = DevopsConfig.config[:project_info_dir]
|
||
file = deploy_info.delete("json_file") || "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
|
||
path = File.join(dir, file)
|
||
if File.exists?(path)
|
||
json = File.read(path)
|
||
else
|
||
json = JSON.pretty_generate(deploy_info)
|
||
File.open(File.join(dir, file), "w") do |f|
|
||
f.write json
|
||
end
|
||
end
|
||
@out.puts "Deploy Input Parameters:"
|
||
@out.puts json
|
||
@out.flush
|
||
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
|
||
else
|
||
if deploy_info['named_task'].present?
|
||
named_task = @project.named_tasks.detect {|task| task['name'] == deploy_info['named_task']}
|
||
raise "Named task #{deploy_info['named_task']} doesn't exist." unless named_task
|
||
puts_and_flush "Using named task #{deploy_info['named_task']}."
|
||
cmd << " -o #{named_task['run_list'].join(',')}"
|
||
else
|
||
cmd << " -r #{deploy_info["run_list"].join(",")}" unless @server.stack.nil?
|
||
end
|
||
end
|
||
ip = if @server.public_ip.nil?
|
||
@server.private_ip
|
||
else
|
||
@out << "Public IP detected\n"
|
||
@server.public_ip
|
||
end
|
||
@out.flush
|
||
k = Devops::Db.connector.key(@server.key)
|
||
lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
|
||
r = /Chef\sClient\sfinished/i
|
||
|
||
if lline && lline[r]
|
||
@out << "\nAfter deploy hooks...\n"
|
||
res = self.run_hook(:after_deploy, @out, deploy_info)
|
||
@out << "Done\n"
|
||
|
||
@server.set_last_operation(Devops::Model::Server::OperationType::DEPLOY, @current_user)
|
||
Devops::Db.connector.server_update(@server)
|
||
|
||
0
|
||
else
|
||
@out << "An error occured during knife command executing"
|
||
1
|
||
end
|
||
end
|
||
|
||
def delete_from_chef_server node_name
|
||
return {chef_node: nil, chef_client: nil} if node_name.blank?
|
||
{
|
||
:chef_node => knife_instance.chef_node_delete(node_name),
|
||
:chef_client => knife_instance.chef_client_delete(node_name)
|
||
}
|
||
end
|
||
|
||
def delete_server
|
||
mongo = ::Devops::Db.connector
|
||
if @server.static?
|
||
if !@server.chef_node_name.nil?
|
||
unbootstrap
|
||
end
|
||
mongo.server_delete @server.id
|
||
puts_and_flush "Static server '#{@server.id}' is removed"
|
||
return 0
|
||
end
|
||
|
||
if @server.chef_node_name.present?
|
||
puts_and_flush "Deleting from chef server:"
|
||
delete_from_chef_server(@server.chef_node_name).each do |key, result|
|
||
@out.puts "#{key} - #{result}"
|
||
end
|
||
else
|
||
puts_and_flush "Chef node name is empty, skipping deleting from chef server."
|
||
end
|
||
|
||
puts_and_flush "Deleting from cloud:"
|
||
provider = @server.provider_instance
|
||
begin
|
||
puts_and_flush provider.delete_server @server
|
||
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
|
||
puts_and_flush "Server with id '#{@server.id}' not found among '#{provider.name}' servers"
|
||
end
|
||
mongo.server_delete @server.id
|
||
puts_and_flush "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed."
|
||
0
|
||
end
|
||
|
||
def roll_back
|
||
@out.puts "Trying to roll back..."
|
||
unless @server.id.nil?
|
||
@out.puts "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created"
|
||
@out.puts delete_from_chef_server(@server.chef_node_name).values.join("\n")
|
||
begin
|
||
@out.puts @server.provider_instance.delete_server(@server)
|
||
rescue => e
|
||
@out.puts e.message
|
||
end
|
||
@out << "\nRolled back\n"
|
||
end
|
||
end
|
||
|
||
def add_run_list_to_deploy_info out, deploy_info
|
||
out << "\nGenerate run list hook...\n"
|
||
if deploy_info["run_list"]
|
||
out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
|
||
return
|
||
end
|
||
out << "Project run list: #{@project.run_list.join(", ")}\n"
|
||
out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
|
||
out << "Server run list: #{@server.run_list.join(", ")}\n"
|
||
deploy_info["run_list"] = compute_run_list
|
||
out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
|
||
end
|
||
|
||
def compute_run_list
|
||
rlist = []
|
||
[@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
|
||
rlist += sub_run_list if sub_run_list
|
||
end
|
||
if @server.stack
|
||
stack = Devops::Db.connector.stack(@server.stack)
|
||
srl = stack.run_list
|
||
rlist += srl if srl
|
||
end
|
||
rlist.uniq
|
||
end
|
||
|
||
def sync_project_metadata_with_chef!(deploy_info)
|
||
version = deploy_info['project_info']['version']
|
||
deployers = deploy_info['project_info']['deployers']
|
||
api_cmd = %Q{
|
||
nodes.find(name: '#{@server.chef_node_name}') do |node|
|
||
node.set['project_info'] = {
|
||
'pinpromote' => { 'version'=> '#{version}' },
|
||
'deployers' => #{deployers}
|
||
}
|
||
node.save
|
||
end
|
||
}
|
||
knife_cmd = %Q{exec -E "#{api_cmd}"}
|
||
puts_and_flush "Going to execute knife command:\n#{knife_cmd}"
|
||
output, succeeded = knife_instance.knife(knife_cmd)
|
||
puts_and_flush "Command output: #{output}"
|
||
raise 'Failed to sync project metadata with chef' unless succeeded
|
||
end
|
||
|
||
private
|
||
|
||
def schedule_expiration
|
||
if @deploy_env.expires
|
||
job_id = ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration!
|
||
puts_and_flush "Planning expiration in #{@deploy_env.expires}, job_id: #{job_id}"
|
||
end
|
||
end
|
||
|
||
def check_ssh_command(cert_path, address)
|
||
cmd = 'ssh '
|
||
cmd << "-i #{cert_path} "
|
||
cmd << '-q '
|
||
cmd << '-o StrictHostKeyChecking=no '
|
||
cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
|
||
cmd << "#{address} 'exit'"
|
||
cmd << " 2>&1"
|
||
cmd
|
||
end
|
||
|
||
def execute_system_command(cmd)
|
||
`#{cmd}`
|
||
end
|
||
|
||
def last_command_successful?
|
||
$?.success?
|
||
end
|
||
|
||
def knife_instance
|
||
@knife_instance ||= KnifeFactory.instance
|
||
end
|
||
|
||
end
|
||
end
|
||
end
|