217 lines
7.0 KiB
Ruby
217 lines
7.0 KiB
Ruby
require "commands/stack"
|
|
require "db/mongo/models/stack/stack_factory"
|
|
require "db/mongo/models/project"
|
|
require "db/mongo/models/report"
|
|
|
|
class StackCreatingError < StandardError; end
|
|
class BootstrapingStackServerError < StandardError; end
|
|
class DeployingStackServerError < StandardError; end
|
|
|
|
class StackBootstrapWorker < Worker
|
|
include StackCommands
|
|
|
|
def perform(options)
|
|
stack_attrs = options.fetch('stack_attributes')
|
|
|
|
call() do |out, file|
|
|
@out = out
|
|
without_bootstrap = stack_attrs.delete('without_bootstrap')
|
|
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
|
|
|
|
report = save_report(file, stack_attrs)
|
|
|
|
begin
|
|
stack = create_stack(stack_attrs)
|
|
|
|
#TODO: errors
|
|
begin
|
|
servers_with_priority = persist_stack_servers!(stack)
|
|
unless without_bootstrap
|
|
sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
|
|
sorted_keys.each do |key|
|
|
@out.puts "Servers with priority '#{key}':"
|
|
bootstrap_servers!(servers_with_priority[key], report)
|
|
end
|
|
end
|
|
@out.puts "Done."
|
|
0
|
|
rescue BootstrapingStackServerError
|
|
@out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
|
|
rollback_stack!(stack)
|
|
2
|
|
rescue DeployingStackServerError => e
|
|
@out.puts "\nStack was launched, but an error occured during deploying stack servers."
|
|
@out.puts "You can redeploy stack after fixing the error."
|
|
3
|
|
rescue StandardError => e
|
|
@out.puts "\nAn error occured. Initiating stack rollback."
|
|
rollback_stack!(stack)
|
|
raise e
|
|
end
|
|
rescue StackCreatingError
|
|
@out.puts "Stack creating error"
|
|
1
|
|
end
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def rollback_stack!(stack)
|
|
@out.puts "\nStart rollback of a stack"
|
|
stack.delete_stack_in_cloud!
|
|
Devops::Db.connector.stack_servers_delete(stack.name)
|
|
Devops::Db.connector.stack_delete(stack.id)
|
|
@out.puts "Rollback has been completed"
|
|
end
|
|
|
|
def create_stack(stack_attrs)
|
|
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
|
|
mongo.stack_insert(stack)
|
|
operation_result = sync_stack_proc.call(@out, stack, mongo)
|
|
|
|
if operation_result == 0
|
|
@out.puts "\nStack '#{stack.name}' has been created"
|
|
@out.flush
|
|
stack
|
|
else
|
|
human_readable_code = StackCommands.result_codes.key(operation_result)
|
|
@out.puts "An error ocurred during stack creating"
|
|
@out.puts "Stack creating operation result was #{human_readable_code}"
|
|
raise StackCreatingError
|
|
end
|
|
end
|
|
|
|
def bootstrap_servers!(servers, report)
|
|
@out << "\nStart bootstraping stack servers\n"
|
|
|
|
subreports = []
|
|
data = {}
|
|
servers.each do |server|
|
|
sjid = Worker.start_async(BootstrapWorker,
|
|
server_attrs: server.to_mongo_hash,
|
|
bootstrap_template: 'omnibus',
|
|
owner: server.created_by
|
|
)
|
|
subreports << sjid
|
|
@out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
|
|
data[server.id] = sjid
|
|
end
|
|
@out.puts
|
|
@out.flush
|
|
mongo.add_report_subreports(jid, subreports)
|
|
results = []
|
|
data.each do |server_id, subreport_id|
|
|
begin
|
|
sleep(5)
|
|
subreport = mongo.report(subreport_id)
|
|
status = subreport.status
|
|
if status == Worker::STATUS::COMPLETED
|
|
@out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
|
|
break
|
|
elsif status == Worker::STATUS::FAILED
|
|
results << subreport.job_result_code
|
|
@out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
|
|
break
|
|
end
|
|
end while(true)
|
|
end
|
|
@out.flush
|
|
results.empty? ? 0 : -5
|
|
end
|
|
|
|
def check_bootstrap_results!(results)
|
|
if results.values.all?(&:zero?)
|
|
# everything is OK
|
|
@out.puts "Stack servers have been bootstraped"
|
|
@out.flush
|
|
return 0
|
|
end
|
|
|
|
@out.puts
|
|
results.each do |chef_node_name, code|
|
|
human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code)
|
|
@out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
|
|
end
|
|
|
|
if errors_in_bootstrapping_present?(results.values)
|
|
raise BootstrapingStackServerError # will cause rollback of a stack
|
|
else
|
|
raise DeployingStackServerError #will not cause rollback of a stack
|
|
end
|
|
end
|
|
|
|
def errors_in_bootstrapping_present?(result_codes)
|
|
bootstrap_error_codes = []
|
|
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
|
|
bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
|
|
end
|
|
|
|
(bootstrap_error_codes & result_codes).size > 0
|
|
end
|
|
|
|
def save_report(file, stack_attrs)
|
|
report = ::Devops::Model::Report.new(
|
|
"file" => file,
|
|
"_id" => jid,
|
|
"created_by" => stack_attrs['owner'],
|
|
"project" => stack_attrs["project"],
|
|
"deploy_env" => stack_attrs["deploy_env"],
|
|
"type" => ::Devops::Model::Report::STACK_TYPE,
|
|
"subreports" => [],
|
|
"stack" => stack_attrs['name']
|
|
)
|
|
mongo.save_report(report)
|
|
report
|
|
end
|
|
|
|
# returns
|
|
# {
|
|
# "priority" => [Servers]
|
|
# }
|
|
def persist_stack_servers!(stack)
|
|
@out.puts "Start syncing stack servers with CID"
|
|
@out.flush
|
|
project = mongo.project(stack.project)
|
|
deploy_env = project.deploy_env(stack.deploy_env)
|
|
provider = stack.provider_instance
|
|
|
|
stack_servers = provider.stack_servers(stack)
|
|
stack_servers.each do |info|
|
|
info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
|
|
end
|
|
stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
|
|
stack_servers_with_priority = {}
|
|
stack_servers_info.each do |priority, info_array|
|
|
stack_servers_with_priority[priority] = info_array.map do |extended_info|
|
|
server_attrs = {
|
|
'provider' => provider.name,
|
|
'project' => project.id,
|
|
'deploy_env' => deploy_env.identifier,
|
|
'remote_user' => mongo.image(deploy_env.image).remote_user,
|
|
'key' => extended_info["key_name"] || provider.ssh_key,
|
|
'_id' => extended_info["id"],
|
|
'chef_node_name' => extended_info["name"],
|
|
'private_ip' => extended_info["private_ip"],
|
|
'public_ip' => extended_info["public_ip"],
|
|
'created_by' => stack.owner,
|
|
'run_list' => stack.run_list || [],
|
|
'stack' => stack.name
|
|
}
|
|
|
|
server = ::Devops::Model::Server.new(server_attrs)
|
|
mongo.server_insert(server)
|
|
# server.chef_node_name = provider.create_default_chef_node_name(server)
|
|
server
|
|
end
|
|
end
|
|
@out.puts "Stack servers have been synced with CID"
|
|
stack_servers_with_priority.each do |priority, servers|
|
|
@out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
|
|
end
|
|
@out.flush
|
|
stack_servers_with_priority
|
|
end
|
|
|
|
end
|