#730: separate deploy and bootstrap errors handling
This commit is contained in:
Anton Chuchkalov
parent
d354edd84f
commit
b6c49161e0
@ -10,9 +10,14 @@ module Devops
|
||||
|
||||
RESULT_CODES = {
|
||||
server_bootstrap_fail: 2,
|
||||
server_not_in_chef_nodes: 5
|
||||
server_not_in_chef_nodes: 5,
|
||||
server_bootstrap_unknown_error: 7,
|
||||
deploy_unknown_error: 6,
|
||||
deploy_failed: 8
|
||||
}
|
||||
|
||||
MAX_SSH_RETRIES_AMOUNT = 20
|
||||
|
||||
#params:
|
||||
# out - container for output data
|
||||
# deploy_info - hash with deploy data
|
||||
@ -45,7 +50,7 @@ module Devops
|
||||
end
|
||||
|
||||
def self.symbolic_result_code(integer_code)
|
||||
RESULT_CODES.key(integer_code) || :deploy_error
|
||||
RESULT_CODES.key(integer_code) || :unknown_error
|
||||
end
|
||||
|
||||
def result_code(symbolic_code)
|
||||
@ -159,7 +164,7 @@ module Devops
|
||||
sleep(5)
|
||||
res = `#{cmd}`
|
||||
retries_amount += 1
|
||||
if retries_amount > max_retries_amount
|
||||
if retries_amount > MAX_SSH_RETRIES_AMOUNT
|
||||
@out.puts "Can not connect to #{address}"
|
||||
@out.puts res
|
||||
@out.flush
|
||||
@ -168,13 +173,14 @@ module Devops
|
||||
end
|
||||
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
|
||||
rescue ArgumentError => e
|
||||
@out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})"
|
||||
@out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
|
||||
@out.flush
|
||||
retry
|
||||
end
|
||||
|
||||
provider = @server.provider_instance
|
||||
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
|
||||
|
||||
r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
|
||||
|
||||
if r == 0
|
||||
@ -210,43 +216,61 @@ module Devops
|
||||
end
|
||||
|
||||
def two_phase_bootstrap options
|
||||
# bootstrap phase
|
||||
begin
|
||||
provider = @server.provider_instance
|
||||
mongo = ::Devops::Db.connector
|
||||
options[:run_list] = provider.run_list
|
||||
status = bootstrap(options)
|
||||
if status == 0
|
||||
bootstrap_status = bootstrap(options)
|
||||
|
||||
if bootstrap_status == 0
|
||||
if check_server
|
||||
@out << "Server #{@server.chef_node_name} is created"
|
||||
else
|
||||
@out << roll_back
|
||||
@out.flush
|
||||
mongo.server_delete @server.id
|
||||
# return 5
|
||||
return result_code(:server_not_in_chef_nodes)
|
||||
end
|
||||
else
|
||||
# @out << roll_back
|
||||
# mongo.server_delete @server.id
|
||||
msg = "Failed while bootstraping server with id '#{@server.id}'\n"
|
||||
msg << "Bootstraping operation result was #{bootstrap_status}"
|
||||
DevopsLogger.logger.error msg
|
||||
@out.puts msg
|
||||
@out.flush
|
||||
return result_code(:server_bootstrap_fail)
|
||||
end
|
||||
rescue => e
|
||||
@out << "\nError: #{e.message}\n"
|
||||
@out.flush
|
||||
return result_code(:server_bootstrap_unknown_error)
|
||||
end
|
||||
|
||||
# deploy phase. Assume that all servers are bootstraped successfully here.
|
||||
begin
|
||||
raise "hello"
|
||||
@out << "\n"
|
||||
@out.flush
|
||||
|
||||
# run_list = @server.run_list + provider.run_list
|
||||
# @out << "\nRun list: #{run_list.inspect}"
|
||||
# s.options[:run_list] += run_list
|
||||
# @knife_instance.set_run_list(@server.chef_node_name, run_list)
|
||||
status = deploy_server(@project.deploy_info(@deploy_env))
|
||||
if status != 0
|
||||
msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
|
||||
DevopsLogger.logger.error msg
|
||||
@out << "\n" + msg + "\n"
|
||||
end
|
||||
deploy_status = deploy_server(@project.deploy_info(@deploy_env))
|
||||
if deploy_status == 0
|
||||
0
|
||||
else
|
||||
msg = "Failed while bootstraping server with id '#{@server.id}'"
|
||||
msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
|
||||
msg << "\nDeploing server operation status was #{deploy_status}"
|
||||
DevopsLogger.logger.error msg
|
||||
@out << "\n" + msg + "\n"
|
||||
# @out << roll_back
|
||||
# mongo.server_delete @server.id
|
||||
result_code(:deploy_failed)
|
||||
end
|
||||
status
|
||||
rescue => e
|
||||
@out << "\nError: #{e.message}\n"
|
||||
return 6
|
||||
result_code(:deploy_unknown_error)
|
||||
end
|
||||
end
|
||||
|
||||
def check_server
|
||||
@ -419,10 +443,6 @@ module Devops
|
||||
|
||||
private
|
||||
|
||||
def max_retries_amount
|
||||
20
|
||||
end
|
||||
|
||||
def schedule_expiration(server)
|
||||
expires = @deploy_env.expires
|
||||
return unless expires
|
||||
|
||||
@ -81,6 +81,7 @@ class StackBootstrapWorker < Worker
|
||||
executor = Devops::Executor::ServerExecutor.new(server, @out)
|
||||
executor.report = report
|
||||
bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({})
|
||||
@out.flush
|
||||
end
|
||||
check_bootstrap_results!(bootstraping_results)
|
||||
end
|
||||
@ -99,17 +100,20 @@ class StackBootstrapWorker < Worker
|
||||
@out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
|
||||
end
|
||||
|
||||
if errors_in_bootstrapping_present?(results)
|
||||
if errors_in_bootstrapping_present?(results.values)
|
||||
raise BootstrapingStackServerError # will cause rollback of a stack
|
||||
else
|
||||
raise DeployingStackServerError #will not cause rollback of a stack
|
||||
end
|
||||
end
|
||||
|
||||
def errors_in_bootstrapping_present?(results)
|
||||
results.values.any? do |result|
|
||||
result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail)
|
||||
def errors_in_bootstrapping_present?(result_codes)
|
||||
bootstrap_error_codes = []
|
||||
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
|
||||
bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
|
||||
end
|
||||
|
||||
(bootstrap_error_codes & result_codes).size > 0
|
||||
end
|
||||
|
||||
def save_report(file, stack_attrs)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user