tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'devops_3' of http://git.stu.neva.ru/cloudtechlab/devops-service into devops_3

Browse Source
This commit is contained in:
amartynov 2015-09-28 14:10:28 +03:00
commit a0cfd3f3ad
2 changed files with 64 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
devops-service/lib/executors/server_executor.rb
View File

@ -10,9 +10,14 @@ module Devops
RESULT_CODES = {
server_bootstrap_fail: 2,
server_not_in_chef_nodes: 5
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8
}
MAX_SSH_RETRIES_AMOUNT = 20
#params:
# out - container for output data
# deploy_info - hash with deploy data
@ -45,7 +50,7 @@ module Devops
end
def self.symbolic_result_code(integer_code)
RESULT_CODES.key(integer_code) || :deploy_error
RESULT_CODES.key(integer_code) || :unknown_error
end
def result_code(symbolic_code)
@ -159,7 +164,7 @@ module Devops
sleep(5)
res = `#{cmd}`
retries_amount += 1
if retries_amount > max_retries_amount
if retries_amount > MAX_SSH_RETRIES_AMOUNT
@out.puts "Can not connect to #{address}"
@out.puts res
@out.flush
@ -168,13 +173,14 @@ module Devops
end
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
rescue ArgumentError => e
@out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})"
@out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
@out.flush
retry
end
provider = @server.provider_instance
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
if r == 0
@ -210,43 +216,61 @@ module Devops
end
def two_phase_bootstrap options
provider = @server.provider_instance
mongo = ::Devops::Db.connector
options[:run_list] = provider.run_list
status = bootstrap(options)
if status == 0
if check_server
@out << "Server #{@server.chef_node_name} is created"
# bootstrap phase
begin
provider = @server.provider_instance
mongo = ::Devops::Db.connector
options[:run_list] = provider.run_list
bootstrap_status = bootstrap(options)
if bootstrap_status == 0
if check_server
@out << "Server #{@server.chef_node_name} is created"
else
@out << roll_back
@out.flush
mongo.server_delete @server.id
return result_code(:server_not_in_chef_nodes)
end
else
@out << roll_back
mongo.server_delete @server.id
# return 5
return result_code(:server_not_in_chef_nodes)
# @out << roll_back
# mongo.server_delete @server.id
msg = "Failed while bootstraping server with id '#{@server.id}'\n"
msg << "Bootstraping operation result was #{bootstrap_status}"
DevopsLogger.logger.error msg
@out.puts msg
@out.flush
return result_code(:server_bootstrap_fail)
end
rescue => e
@out << "\nError: #{e.message}\n"
@out.flush
return result_code(:server_bootstrap_unknown_error)
end
# deploy phase. Assume that all servers are bootstraped successfully here.
begin
raise "hello"
@out << "\n"
@out.flush
# run_list = @server.run_list + provider.run_list
# @out << "\nRun list: #{run_list.inspect}"
# s.options[:run_list] += run_list
# @knife_instance.set_run_list(@server.chef_node_name, run_list)
status = deploy_server(@project.deploy_info(@deploy_env))
if status != 0
# run_list = @server.run_list + provider.run_list
# @out << "\nRun list: #{run_list.inspect}"
# s.options[:run_list] += run_list
# @knife_instance.set_run_list(@server.chef_node_name, run_list)
deploy_status = deploy_server(@project.deploy_info(@deploy_env))
if deploy_status == 0
0
else
msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
msg << "\nDeploing server operation status was #{deploy_status}"
DevopsLogger.logger.error msg
@out << "\n" + msg + "\n"
result_code(:deploy_failed)
end
else
msg = "Failed while bootstraping server with id '#{@server.id}'"
DevopsLogger.logger.error msg
@out << "\n" + msg + "\n"
# @out << roll_back
# mongo.server_delete @server.id
rescue => e
@out << "\nError: #{e.message}\n"
result_code(:deploy_unknown_error)
end
status
rescue => e
@out << "\nError: #{e.message}\n"
return 6
end
def check_server
@ -419,10 +443,6 @@ module Devops
private
def max_retries_amount
20
end
def schedule_expiration(server)
expires = @deploy_env.expires
return unless expires

12
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -81,6 +81,7 @@ class StackBootstrapWorker < Worker
executor = Devops::Executor::ServerExecutor.new(server, @out)
executor.report = report
bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({})
@out.flush
end
check_bootstrap_results!(bootstraping_results)
end
@ -99,17 +100,20 @@ class StackBootstrapWorker < Worker
@out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
end
if errors_in_bootstrapping_present?(results)
if errors_in_bootstrapping_present?(results.values)
raise BootstrapingStackServerError # will cause rollback of a stack
else
raise DeployingStackServerError #will not cause rollback of a stack
end
end
def errors_in_bootstrapping_present?(results)
results.values.any? do |result|
result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail)
def errors_in_bootstrapping_present?(result_codes)
bootstrap_error_codes = []
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
end
(bootstrap_error_codes & result_codes).size > 0
end
def save_report(file, stack_attrs)