require "commands/stack" require "db/mongo/models/stack/stack_factory" require "db/mongo/models/project" require "db/mongo/models/report" class StackCreatingError < StandardError; end class BootstrapingStackServerError < StandardError; end class DeployingStackServerError < StandardError; end class StackBootstrapWorker < Worker include StackCommands def perform(options) stack_attrs = options.fetch('stack_attributes') call() do |out, file| @out = out without_bootstrap = stack_attrs.delete('without_bootstrap') @out.puts "Received 'without_bootstrap' option" if without_bootstrap report = save_report(file, stack_attrs) begin stack = create_stack(stack_attrs) #TODO: errors begin servers_with_priority = persist_stack_servers!(stack) unless without_bootstrap sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x} sorted_keys.each do |key| @out.puts "Servers with priority '#{key}':" bootstrap_servers!(servers_with_priority[key], report) end end @out.puts "Done." 0 rescue BootstrapingStackServerError @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback." rollback_stack!(stack) 2 rescue DeployingStackServerError => e @out.puts "\nStack was launched, but an error occured during deploying stack servers." @out.puts "You can redeploy stack after fixing the error." 3 rescue StandardError => e @out.puts "\nAn error occured. Initiating stack rollback." rollback_stack!(stack) raise e end rescue StackCreatingError @out.puts "Stack creating error" 1 end end end private def rollback_stack!(stack) @out.puts "\nStart rollback of a stack" stack.delete_stack_in_cloud! Devops::Db.connector.stack_servers_delete(stack.name) Devops::Db.connector.stack_delete(stack.id) @out.puts "Rollback has been completed" end def create_stack(stack_attrs) stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) mongo.stack_insert(stack) operation_result = sync_stack_proc.call(@out, stack, mongo) if operation_result == 0 @out.puts "\nStack '#{stack.name}' has been created" @out.flush stack else human_readable_code = StackCommands.result_codes.key(operation_result) @out.puts "An error ocurred during stack creating" @out.puts "Stack creating operation result was #{human_readable_code}" raise StackCreatingError end end def bootstrap_servers!(servers, report) @out << "\nStart bootstraping stack servers\n" subreports = [] data = {} servers.each do |server| sjid = Worker.start_async(BootstrapWorker, server_attrs: server.to_mongo_hash, bootstrap_template: 'omnibus', owner: server.created_by ) subreports << sjid @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}" data[server.id] = sjid end @out.puts @out.flush mongo.add_report_subreports(jid, subreports) results = [] data.each do |server_id, subreport_id| begin sleep(5) subreport = mongo.report(subreport_id) status = subreport.status if status == Worker::STATUS::COMPLETED @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}" break elsif status == Worker::STATUS::FAILED results << subreport.job_result_code @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'" break end end while(true) end @out.flush results.empty? ? 0 : -5 end def check_bootstrap_results!(results) if results.values.all?(&:zero?) # everything is OK @out.puts "Stack servers have been bootstraped" @out.flush return 0 end @out.puts results.each do |chef_node_name, code| human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code) @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}" end if errors_in_bootstrapping_present?(results.values) raise BootstrapingStackServerError # will cause rollback of a stack else raise DeployingStackServerError #will not cause rollback of a stack end end def errors_in_bootstrapping_present?(result_codes) bootstrap_error_codes = [] [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code| bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code) end (bootstrap_error_codes & result_codes).size > 0 end def save_report(file, stack_attrs) report = ::Devops::Model::Report.new( "file" => file, "_id" => jid, "created_by" => stack_attrs['owner'], "project" => stack_attrs["project"], "deploy_env" => stack_attrs["deploy_env"], "type" => ::Devops::Model::Report::STACK_TYPE, "subreports" => [], "stack" => stack_attrs['name'] ) mongo.save_report(report) report end # returns # { # "priority" => [Servers] # } def persist_stack_servers!(stack) @out.puts "Start syncing stack servers with CID" @out.flush project = mongo.project(stack.project) deploy_env = project.deploy_env(stack.deploy_env) provider = stack.provider_instance stack_servers = provider.stack_servers(stack) stack_servers.each do |info| info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i end stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]} stack_servers_with_priority = {} stack_servers_info.each do |priority, info_array| stack_servers_with_priority[priority] = info_array.map do |extended_info| @out.puts "Instance '#{extended_info["id"]}' has been launched with stack." server_attrs = { 'provider' => provider.name, 'project' => project.id, 'deploy_env' => deploy_env.identifier, 'remote_user' => mongo.image(deploy_env.image).remote_user, 'key' => extended_info["key_name"] || provider.ssh_key, '_id' => extended_info["id"], 'chef_node_name' => extended_info["name"], 'private_ip' => extended_info["private_ip"], 'public_ip' => extended_info["public_ip"], 'created_by' => stack.owner, 'run_list' => stack.run_list || [], 'stack' => stack.name } server = ::Devops::Model::Server.new(server_attrs) mongo.server_insert(server) # server.chef_node_name = provider.create_default_chef_node_name(server) server end end @out.puts "Stack servers have been synced with CID" stack_servers_with_priority.each do |priority, servers| @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" end @out.flush stack_servers_with_priority end end