diff --git a/devops-service/app/api2/handlers/deploy.rb b/devops-service/app/api2/handlers/deploy.rb index 2e42e2f..acaf5b1 100644 --- a/devops-service/app/api2/handlers/deploy.rb +++ b/devops-service/app/api2/handlers/deploy.rb @@ -19,10 +19,8 @@ module Devops names = body["names"] tags = body["tags"] || [] run_list = body["run_list"] - dir = DevopsConfig.config[:report_dir_v2] files = [] jid = nil - uri = URI.parse(parser.request.url) owner = parser.current_user @deploy_info_buf = {} servers(names).each do |s| @@ -36,7 +34,7 @@ module Devops deploy_info = create_deploy_info(s, project, body["build_number"]) deploy_info["run_list"] = run_list if run_list - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: owner, tags: tags, @@ -63,7 +61,7 @@ module Devops } Devops::Db.connector.save_report(Report.new(o)) end - files.push(uri) + files.push(jid) end files end diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index e82616c..68d0b39 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -251,13 +251,13 @@ module Devops deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil) end - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: parser.current_user, tags: [], deploy_info: deploy_info ) - files.push uri + files.push jid end files end @@ -288,14 +288,14 @@ module Devops raise InvalidRecord.new(msg) end - uri = Worker.start_async(ProjectTestWorker, @request, + jid = Worker.start_async(ProjectTestWorker, project: project.id, deploy_env: env.identifier, user: @request.env['REMOTE_USER'] ) sleep 1 - return [uri] + return [jid] end end diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb index f6d626d..fb4df48 100644 --- a/devops-service/app/api2/handlers/server.rb +++ b/devops-service/app/api2/handlers/server.rb @@ -71,11 +71,11 @@ module Devops user = parser.current_user check_if_server_attrs_are_valid(body, user) - uri = Worker.start_async(CreateServerWorker, @request, + jid = Worker.start_async(CreateServerWorker, server_attrs: body, owner: user ) - [uri] + [jid] end def deploy node_name @@ -185,19 +185,16 @@ module Devops end def bootstrap_server - options = prepare_bootstrap_server - s = options.delete(:server) - options[:provider_name] = s.provider - options[:server_attrs] = s.to_mongo_hash - options[:owner] = parser.current_user -# dir = DevopsConfig[:report_dir_v2] -# files = [] -# uri = URI.parse(@request.url) + server, rl, bootstrap_template = prepare_bootstrap_server - uri = Worker.start_async(BootstrapWorker, @request, options) + jid = Worker.start_async(BootstrapWorker, + server_attrs: server.to_mongo_hash, + bootstrap_template: bootstrap_template, + owner: parser.current_user + ) sleep 1 - [uri] + [jid] end def prepare_bootstrap_server diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 664df53..ffc1fe0 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -28,13 +28,10 @@ module Devops object["provider"] = env.provider object["provider_account"] = env.provider_account - # TODO: without provider_name - uri = Worker.start_async(StackBootstrapWorker, @request, - provider_name: env.provider, + jid = Worker.start_async(StackBootstrapWorker, stack_attributes: object ) - puts "Syncing report is located here: #{uri}" - [uri] + [jid] end def stack id @@ -102,7 +99,6 @@ module Devops def deploy id stack = self.stack(id) owner = parser.current_user - status = [] project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner) deploy_env_model = project.deploy_env(stack.deploy_env) body = parser.deploy @@ -114,7 +110,7 @@ module Devops servers.each do |s| begin deploy_info = project.deploy_info(deploy_env_model, nil) - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: owner, tags: tags, @@ -140,7 +136,7 @@ module Devops } Devops::Db.connector.save_report(Report.new(o)) end - files.push uri + files.push jid end files end diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb index 8c5b199..fcc363f 100644 --- a/devops-service/db/mongo/connectors/report.rb +++ b/devops-service/db/mongo/connectors/report.rb @@ -51,8 +51,18 @@ module Connectors collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)} end - def set_report_status(jid, status) - collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}}) + def add_report_subreports(jid, subreports) + collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}}) + end + + def set_report_status(jid, status, job_result_code) + set = {"status" => status, "updated_at" => Time.new} + set["job_result_code"] = job_result_code unless job_result_code.nil? + collection.update({"_id" => jid}, {"$set" => set}) + end + + def set_report_status(jid, status, job_result_code) + collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}}) end def set_report_server_data id, chef_node_name, host diff --git a/devops-service/db/mongo/models/report.rb b/devops-service/db/mongo/models/report.rb index c7ca2c4..c557b22 100644 --- a/devops-service/db/mongo/models/report.rb +++ b/devops-service/db/mongo/models/report.rb @@ -12,7 +12,7 @@ module Devops DEPLOY_STACK_TYPE = 6 DELETE_SERVER_TYPE = 7 - attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack + attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code def initialize r self.id = r["_id"] @@ -21,13 +21,15 @@ module Devops self.project = r["project"] self.deploy_env = r["deploy_env"] self.type = r["type"] - self.created_at = r["created_at"] +# self.created_at = r["created_at"] self.chef_node_name = r["chef_node_name"] self.host = r["host"] self.stack = r["stack"] self.created_at = r["created_at"].localtime unless r["created_at"].nil? self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil? self.status = r["status"] + self.subreports = r["subreports"] + self.job_result_code = r["job_result_code"] end def to_hash_without_id @@ -42,7 +44,9 @@ module Devops "chef_node_name" => self.chef_node_name, "host" => self.host, "status" => self.status, - "stack" => self.stack + "stack" => self.stack, + "subreports" => self.subreports, + "job_result_code" => self.job_result_code } end diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index 224e952..bda584e 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -31,7 +31,7 @@ class MongoConnector [:user_auth, :user, :users, :users_names, :user_insert, :user_delete, :user_update, :create_root_user, :check_user_privileges] => :users_connector, [:keys, :key, :key_insert, :key_delete] => :keys_connector, - [:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector, + [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector, [:insert_statistic, :search_statistic] => :statistics_connector, [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector ) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 73aff74..52644f8 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -319,6 +319,7 @@ module Provider 'key_name' => instance["keyName"], 'private_ip' => instance["privateIpAddress"], 'public_ip' => instance["ipAddress"], + 'tags' => instance["tagSet"] } end end diff --git a/devops-service/workers/bootstrap_worker.rb b/devops-service/workers/bootstrap_worker.rb index ba8b2c6..a261900 100644 --- a/devops-service/workers/bootstrap_worker.rb +++ b/devops-service/workers/bootstrap_worker.rb @@ -1,20 +1,18 @@ require File.join(File.dirname(__FILE__), "worker") require "lib/executors/server_executor" -require "providers/provider_factory" require "db/mongo/models/server" require "db/mongo/models/report" class BootstrapWorker < Worker def perform(options) - provider_name = options.fetch('provider_name') server_attrs = options.fetch('server_attrs') # bootstrap_template = options.fetch('bootstrap_template') owner = options.fetch('owner') options = convert_config(options) - call(provider_name) do |provider, out, file| + call() do |out, file| server = Devops::Model::Server.new(server_attrs) report = save_report(file, owner, server) diff --git a/devops-service/workers/create_server_worker.rb b/devops-service/workers/create_server_worker.rb index 192de26..bc2655a 100644 --- a/devops-service/workers/create_server_worker.rb +++ b/devops-service/workers/create_server_worker.rb @@ -1,6 +1,5 @@ require File.join(File.dirname(__FILE__), "worker") -require "providers/provider_factory" require "db/mongo/models/server" require "db/mongo/models/report" require "lib/executors/server_executor" @@ -11,7 +10,7 @@ class CreateServerWorker < Worker server_attrs = options.fetch('server_attrs') owner = options.fetch('owner') - call(nil) do |not_used, out, file| + call() do |out, file| project = mongo.project(server_attrs["project"]) env = project.deploy_env(server_attrs["deploy_env"]) report = save_report(file, project, env, owner) diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb index 66c8f48..c8fe476 100644 --- a/devops-service/workers/delete_server_worker.rb +++ b/devops-service/workers/delete_server_worker.rb @@ -8,7 +8,7 @@ class DeleteServerWorker < Worker chef_node_name = options.fetch('server_chef_node_name') puts "Expire server '#{chef_node_name}'." - call(nil) do |not_used, out, file| + call() do |out, file| server = mongo.server_by_chef_node_name(chef_node_name) report = save_report(file, server) diff --git a/devops-service/workers/deploy_worker.rb b/devops-service/workers/deploy_worker.rb index e47d384..9916ee2 100644 --- a/devops-service/workers/deploy_worker.rb +++ b/devops-service/workers/deploy_worker.rb @@ -12,7 +12,7 @@ class DeployWorker < Worker tags = options.fetch('tags') deploy_info = options.fetch('deploy_info') - call(nil) do |not_used, out, file| + call() do |out, file| server = Devops::Model::Server.new(server_attrs) report = save_report(file, owner, server) diff --git a/devops-service/workers/project_test_worker.rb b/devops-service/workers/project_test_worker.rb index 5f70be4..760b06c 100644 --- a/devops-service/workers/project_test_worker.rb +++ b/devops-service/workers/project_test_worker.rb @@ -15,7 +15,7 @@ class ProjectTestWorker < Worker project_name = params.fetch('project') deploy_env_name = params.fetch('deploy_env') - call(nil) do |not_used, out, file| + call() do |out, file| DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})" project = mongo.project(project_name) env = project.deploy_env(deploy_env_name) diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 780606a..442e793 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -1,4 +1,3 @@ -require "providers/provider_factory" require "commands/stack" require "db/mongo/models/stack/stack_factory" require "db/mongo/models/project" @@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker include StackCommands def perform(options) - provider_name = options.fetch('provider_name') stack_attrs = options.fetch('stack_attributes') - call(provider_name) do |provider, out, file| + call() do |out, file| @out = out without_bootstrap = stack_attrs.delete('without_bootstrap') @out.puts "Received 'without_bootstrap' option" if without_bootstrap @@ -23,14 +21,23 @@ class StackBootstrapWorker < Worker report = save_report(file, stack_attrs) begin - stack = create_stack(provider_name, stack_attrs) + stack = create_stack(stack_attrs) rescue StackCreatingError return 1 end + #TODO: errors begin - servers = persist_stack_servers!(stack, provider) - bootstrap_servers!(servers, report) unless without_bootstrap + servers_with_priority = persist_stack_servers!(stack) + unless without_bootstrap + sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x} + sorted_keys.each do |key| + @out.puts "Servers with priority '#{key}':" + bootstrap_servers!(servers_with_priority[key], report) + end + end + @out.puts "Done." + 0 rescue BootstrapingStackServerError @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback." rollback_stack!(stack) @@ -57,8 +64,8 @@ class StackBootstrapWorker < Worker @out.puts "Rollback has been completed" end - def create_stack(provider_name, stack_attrs) - stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out) + def create_stack(stack_attrs) + stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) mongo.stack_insert(stack) operation_result = sync_stack_proc.call(@out, stack, mongo) @@ -76,15 +83,40 @@ class StackBootstrapWorker < Worker def bootstrap_servers!(servers, report) @out << "\nStart bootstraping stack servers\n" - bootstraping_results = {} + + subreports = [] + data = {} servers.each do |server| - executor = Devops::Executor::ServerExecutor.new(server, @out) - executor.report = report - #TODO: can stack choose bootstrap template? - bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'}) - @out.flush + sjid = Worker.start_async(BootstrapWorker, + server_attrs: server.to_mongo_hash, + bootstrap_template: 'omnibus', + owner: server.created_by + ) + subreports << sjid + @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}" + data[server.id] = sjid end - check_bootstrap_results!(bootstraping_results) + @out.puts + @out.flush + mongo.add_report_subreports(jid, subreports) + results = [] + data.each do |server_id, subreport_id| + begin + sleep(5) + subreport = mongo.report(subreport_id) + status = subreport.status + if status == Worker::STATUS::COMPLETED + @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}" + break + elsif status == Worker::STATUS::FAILED + results << subreport.job_result_code + @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'" + break + end + end while(true) + end + @out.flush + results.empty? ? 0 : -5 end def check_bootstrap_results!(results) @@ -124,42 +156,60 @@ class StackBootstrapWorker < Worker "created_by" => stack_attrs['owner'], "project" => stack_attrs["project"], "deploy_env" => stack_attrs["deploy_env"], - "type" => ::Devops::Model::Report::STACK_TYPE + "type" => ::Devops::Model::Report::STACK_TYPE, + "subreports" => [], + "stack" => stack_attrs['name'] ) mongo.save_report(report) report end - def persist_stack_servers!(stack, provider) + # returns + # { + # "priority" => [Servers] + # } + def persist_stack_servers!(stack) @out.puts "Start syncing stack servers with CID" @out.flush project = mongo.project(stack.project) deploy_env = project.deploy_env(stack.deploy_env) + provider = stack.provider_instance - servers = provider.stack_servers(stack).map do |extended_info| - server_attrs = { - 'provider' => provider.name, - 'project' => project.id, - 'deploy_env' => deploy_env.identifier, - 'remote_user' => mongo.image(deploy_env.image).remote_user, - 'key' => extended_info["key_name"] || provider.ssh_key, - '_id' => extended_info["id"], - 'chef_node_name' => extended_info["name"], - 'private_ip' => extended_info["private_ip"], - 'public_ip' => extended_info["public_ip"], - 'created_by' => stack.owner, - 'run_list' => stack.run_list || [], - 'stack' => stack.name - } + stack_servers = provider.stack_servers(stack) + stack_servers.each do |info| + info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i + end + stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]} + stack_servers_with_priority = {} + stack_servers_info.each do |priority, info_array| + stack_servers_with_priority[priority] = info_array.map do |extended_info| + server_attrs = { + 'provider' => provider.name, + 'project' => project.id, + 'deploy_env' => deploy_env.identifier, + 'remote_user' => mongo.image(deploy_env.image).remote_user, + 'key' => extended_info["key_name"] || provider.ssh_key, + '_id' => extended_info["id"], + 'chef_node_name' => extended_info["name"], + 'private_ip' => extended_info["private_ip"], + 'public_ip' => extended_info["public_ip"], + 'created_by' => stack.owner, + 'run_list' => stack.run_list || [], + 'stack' => stack.name + } - server = ::Devops::Model::Server.new(server_attrs) - mongo.server_insert(server) - # server.chef_node_name = provider.create_default_chef_node_name(server) - server + server = ::Devops::Model::Server.new(server_attrs) + mongo.server_insert(server) + # server.chef_node_name = provider.create_default_chef_node_name(server) + server + end end @out.puts "Stack servers have been synced with CID" + stack_servers_with_priority.each do |priority, servers| + @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" + end @out.flush - servers + stack_servers_with_priority end end diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb index d550e30..3c80a63 100644 --- a/devops-service/workers/worker.rb +++ b/devops-service/workers/worker.rb @@ -28,14 +28,16 @@ class Worker IN_QUEUE = "queued" end - def self.start_async(worker_class, request, job_options) - jid = worker_class.perform_async(job_options) + def self.start_async(worker_class, job_options) + jid = worker_class.perform_async(job_options.dup) Worker.set_status jid, Worker::STATUS::IN_QUEUE DevopsLogger.logger.info "Job '#{jid}' has been queued" - + jid +=begin uri = URI.parse(request.url) uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}" uri.to_s +=end end def self.start_sync(worker_class, request, job_options, out) @@ -54,14 +56,13 @@ class Worker Sidekiq.redis {|con| con.hset "devops", id, status} end - def call provider_name, &block + def call &block begin - initialize_devops(provider_name) - provider = ::Provider::ProviderFactory.get(provider_name) if provider_name + initialize_devops() if jid - call_async(provider, &block) + call_async(&block) else - call_sync(provider, &block) + call_sync(&block) end rescue StandardError => e DevopsLogger.logger.error "#{e.message}:\n#{e.backtrace.join("\n")}" @@ -70,7 +71,7 @@ class Worker private - def initialize_devops(provider_name) + def initialize_devops() DevopsLogger.logger = logger DevopsConfig.read DevopsService.init @@ -78,33 +79,33 @@ class Worker end # outputs to file - def call_async(provider) + def call_async() dir = DevopsConfig[:report_dir_v2] # directory is created on server start in config.ru file = File.join(dir, jid) - update_job_status(STATUS::INIT) + update_job_status(STATUS::INIT, nil) File.open(file, "w") do |out| begin - update_job_status(STATUS::RUNNING) + update_job_status(STATUS::RUNNING, nil) self.out = out - job_result = yield(provider, out, file) + job_result = yield(out, file) canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED) - update_job_status(canonical_status) + update_job_status(canonical_status, job_result) rescue StandardError, RecordNotFound => e out << "\n #{e.class}\n #{e.message}\n" out << e.backtrace.join("\n") - update_job_status(STATUS::FAILED) + update_job_status(STATUS::FAILED, -100) end end end # outputs to STDOUT - def call_sync(provider) + def call_sync() out = STDOUT begin - yield(provider, out, '') + yield(out, '') rescue StandardError, RecordNotFound => e out << "\n" out << e.message @@ -117,17 +118,12 @@ class Worker ::Devops::Db.connector end - def update_job_status(status) + def update_job_status(status, job_result_code) set_status(jid, status) - mongo.set_report_status(jid, status) + mongo.set_report_status(jid, status, job_result_code) status end - def init_provider(provider_name) - ::Provider::ProviderFactory.init(DevopsConfig.config) - ::Provider::ProviderFactory.get(provider_name) if provider_name - end - def convert_config conf config = {} conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}