From b6c49161e0c35c2b2fe951f38dead09a08262210 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Fri, 25 Sep 2015 17:31:10 +0300 Subject: [PATCH] #730: separate deploy and bootstrap errors handling --- .../lib/executors/server_executor.rb | 92 +++++++++++-------- .../workers/stack_bootstrap_worker.rb | 12 ++- 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 5f43993..3d3e809 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -10,9 +10,14 @@ module Devops RESULT_CODES = { server_bootstrap_fail: 2, - server_not_in_chef_nodes: 5 + server_not_in_chef_nodes: 5, + server_bootstrap_unknown_error: 7, + deploy_unknown_error: 6, + deploy_failed: 8 } + MAX_SSH_RETRIES_AMOUNT = 20 + #params: # out - container for output data # deploy_info - hash with deploy data @@ -45,7 +50,7 @@ module Devops end def self.symbolic_result_code(integer_code) - RESULT_CODES.key(integer_code) || :deploy_error + RESULT_CODES.key(integer_code) || :unknown_error end def result_code(symbolic_code) @@ -159,7 +164,7 @@ module Devops sleep(5) res = `#{cmd}` retries_amount += 1 - if retries_amount > max_retries_amount + if retries_amount > MAX_SSH_RETRIES_AMOUNT @out.puts "Can not connect to #{address}" @out.puts res @out.flush @@ -168,13 +173,14 @@ module Devops end raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success? rescue ArgumentError => e - @out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})" + @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})" @out.flush retry end provider = @server.provider_instance @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil? + r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options)) if r == 0 @@ -210,43 +216,61 @@ module Devops end def two_phase_bootstrap options - provider = @server.provider_instance - mongo = ::Devops::Db.connector - options[:run_list] = provider.run_list - status = bootstrap(options) - if status == 0 - if check_server - @out << "Server #{@server.chef_node_name} is created" + # bootstrap phase + begin + provider = @server.provider_instance + mongo = ::Devops::Db.connector + options[:run_list] = provider.run_list + bootstrap_status = bootstrap(options) + + if bootstrap_status == 0 + if check_server + @out << "Server #{@server.chef_node_name} is created" + else + @out << roll_back + @out.flush + mongo.server_delete @server.id + return result_code(:server_not_in_chef_nodes) + end else - @out << roll_back - mongo.server_delete @server.id - # return 5 - return result_code(:server_not_in_chef_nodes) + # @out << roll_back + # mongo.server_delete @server.id + msg = "Failed while bootstraping server with id '#{@server.id}'\n" + msg << "Bootstraping operation result was #{bootstrap_status}" + DevopsLogger.logger.error msg + @out.puts msg + @out.flush + return result_code(:server_bootstrap_fail) end + rescue => e + @out << "\nError: #{e.message}\n" + @out.flush + return result_code(:server_bootstrap_unknown_error) + end + + # deploy phase. Assume that all servers are bootstraped successfully here. + begin + raise "hello" @out << "\n" @out.flush - -# run_list = @server.run_list + provider.run_list -# @out << "\nRun list: #{run_list.inspect}" -# s.options[:run_list] += run_list -# @knife_instance.set_run_list(@server.chef_node_name, run_list) - status = deploy_server(@project.deploy_info(@deploy_env)) - if status != 0 + # run_list = @server.run_list + provider.run_list + # @out << "\nRun list: #{run_list.inspect}" + # s.options[:run_list] += run_list + # @knife_instance.set_run_list(@server.chef_node_name, run_list) + deploy_status = deploy_server(@project.deploy_info(@deploy_env)) + if deploy_status == 0 + 0 + else msg = "Failed on chef-client with run list, server with id '#{@server.id}'" + msg << "\nDeploing server operation status was #{deploy_status}" DevopsLogger.logger.error msg @out << "\n" + msg + "\n" + result_code(:deploy_failed) end - else - msg = "Failed while bootstraping server with id '#{@server.id}'" - DevopsLogger.logger.error msg - @out << "\n" + msg + "\n" -# @out << roll_back -# mongo.server_delete @server.id + rescue => e + @out << "\nError: #{e.message}\n" + result_code(:deploy_unknown_error) end - status - rescue => e - @out << "\nError: #{e.message}\n" - return 6 end def check_server @@ -419,10 +443,6 @@ module Devops private - def max_retries_amount - 20 - end - def schedule_expiration(server) expires = @deploy_env.expires return unless expires diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 5375735..253ba3a 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -81,6 +81,7 @@ class StackBootstrapWorker < Worker executor = Devops::Executor::ServerExecutor.new(server, @out) executor.report = report bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({}) + @out.flush end check_bootstrap_results!(bootstraping_results) end @@ -99,17 +100,20 @@ class StackBootstrapWorker < Worker @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}" end - if errors_in_bootstrapping_present?(results) + if errors_in_bootstrapping_present?(results.values) raise BootstrapingStackServerError # will cause rollback of a stack else raise DeployingStackServerError #will not cause rollback of a stack end end - def errors_in_bootstrapping_present?(results) - results.values.any? do |result| - result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail) + def errors_in_bootstrapping_present?(result_codes) + bootstrap_error_codes = [] + [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code| + bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code) end + + (bootstrap_error_codes & result_codes).size > 0 end def save_report(file, stack_attrs)