diff --git a/devops-client/lib/devops-client/handler/stack.rb b/devops-client/lib/devops-client/handler/stack.rb index d681a45..6729c0d 100644 --- a/devops-client/lib/devops-client/handler/stack.rb +++ b/devops-client/lib/devops-client/handler/stack.rb @@ -41,30 +41,31 @@ class Stack < Handler end def create_handler - q = {} - - q[:without_bootstrap] = options[:without_bootstrap] - # q[:provider] = options[:provider] || resources_selector.select_available_provider - q[:project] = options[:project] || resources_selector.select_available_project - q[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(q[:project]) - env = fetcher.fetch_project(q[:project])['deploy_envs'].detect {|env| env['identifier'] == q[:deploy_env]} - q[:provider] = env['provider'] + attrs = {} + attrs[:project] = options[:project] || resources_selector.select_available_project + attrs[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(attrs[:project]) + env = fetcher.fetch_project(attrs[:project])['deploy_envs'].detect {|env| env['identifier'] == attrs[:deploy_env]} + attrs[:provider] = env['provider'] params_filepath = options[:parameters_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.parameters_file')) if params_filepath.empty? - q[:parameters] = {} + attrs[:parameters] = {} else - q[:parameters] = JSON.parse(File.read(params_filepath)) + attrs[:parameters] = JSON.parse(File.read(params_filepath)) end tags_filepath = options[:tags_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.tags_file')) if tags_filepath.empty? - q[:tags] = {} + attrs[:tags] = {} else - q[:tags] = JSON.parse(File.read(tags_filepath)) + attrs[:tags] = JSON.parse(File.read(tags_filepath)) end - json = JSON.pretty_generate(q) + json = JSON.pretty_generate( + without_bootstrap: options[:without_bootstrap] || false, + skip_rollback: options[:skip_rollback] || false, + stack_attributes: attrs + ) if question(I18n.t("handler.stack.question.create")) {puts json} job_ids = post_body "/stack", json reports_urls(job_ids) diff --git a/devops-client/lib/devops-client/options/stack_options.rb b/devops-client/lib/devops-client/options/stack_options.rb index 435f855..a33a33a 100644 --- a/devops-client/lib/devops-client/options/stack_options.rb +++ b/devops-client/lib/devops-client/options/stack_options.rb @@ -31,6 +31,7 @@ class StackOptions < CommonOptions parser.recognize_option_value(:tags_file) parser.recognize_option_value(:run_list) parser.recognize_option_value(:without_bootstrap, type: :switch, switch_value: true) + parser.recognize_option_value(:skip_rollback, type: :switch, switch_value: true) end end diff --git a/devops-service/app/api2/handlers/provider_notification.rb b/devops-service/app/api2/handlers/provider_notification.rb new file mode 100644 index 0000000..27a786f --- /dev/null +++ b/devops-service/app/api2/handlers/provider_notification.rb @@ -0,0 +1,22 @@ +require 'workers/stack_sync_worker' +require_relative "request_handler" + +module Devops + module API2_0 + module Handler + class ProviderNotification < RequestHandler + + def autoscaling_groups_change(group_id, provider_account) + provider = ::Provider::ProviderFactory.get('ec2', provider_account) + stack_id = provider.stack_id_of_autoscaling_group(group_id) + stack = ::Devops::Db.connector.stack_by_id(stack_id) + jid = Worker.start_async(StackSyncWorker, stack_name: stack.name) + puts jid + jid + end + + end + end + end +end + diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 917f736..06e9249 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -21,16 +21,16 @@ module Devops def create_stack object = parser.create - project = Devops::Db.connector.project(object["project"]) - env = project.deploy_env(object["deploy_env"]) + stack_attrs = object['stack_attributes'] + project = Devops::Db.connector.project(stack_attrs['project']) + env = project.deploy_env(stack_attrs['deploy_env']) raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil? - object["stack_template"] = env.stack_template - object["owner"] = parser.current_user - object["provider"] = env.provider - object["provider_account"] = env.provider_account + add_stack_attributes(stack_attrs, env, parser) jid = Worker.start_async(StackBootstrapWorker, - stack_attributes: object + stack_attributes: stack_attrs, + without_bootstrap: object['without_bootstrap'], + skip_rollback: object['skip_rollback'] ) [jid] end @@ -48,7 +48,7 @@ module Devops def sync id stack = self.stack(id) - stack.sync! + stack.sync_status_and_events! Devops::Db.connector.stack_update(stack) stack @@ -201,6 +201,15 @@ module Devops stack.change_stack_template!(stack_template_id) end + private + + def add_stack_attributes(stack_attrs, env, parser) + stack_attrs['stack_template'] = env.stack_template + stack_attrs['owner'] = parser.current_user + stack_attrs['provider'] = env.provider + stack_attrs['provider_account'] = env.provider_account + end + end end end diff --git a/devops-service/app/api2/parsers/stack.rb b/devops-service/app/api2/parsers/stack.rb index de1ee1c..322ef85 100644 --- a/devops-service/app/api2/parsers/stack.rb +++ b/devops-service/app/api2/parsers/stack.rb @@ -7,10 +7,11 @@ module Devops def create @body ||= create_object_from_json_body - project_name = check_string(@body["project"], "Parameter 'project' must be a not empty string") - env_name = check_string(@body["deploy_env"], "Parameter 'deploy_env' must be a not empty string") - check_string(@body["name"], "Parameter 'name' must be a not empty string", true, false) - list = check_array(@body["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true) + stack_attributes = @body.fetch('stack_attributes') + project_name = check_string(stack_attributes["project"], "Parameter 'project' must be a not empty string") + env_name = check_string(stack_attributes["deploy_env"], "Parameter 'deploy_env' must be a not empty string") + check_string(stack_attributes["name"], "Parameter 'name' must be a not empty string", true, false) + list = check_array(stack_attributes["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true) Validators::Helpers::RunList.new(list).validate! unless list.nil? @body end diff --git a/devops-service/app/api2/routes/provider_notification.rb b/devops-service/app/api2/routes/provider_notification.rb new file mode 100644 index 0000000..4a61695 --- /dev/null +++ b/devops-service/app/api2/routes/provider_notification.rb @@ -0,0 +1,24 @@ +module Devops + module API2_0 + module Routes + module ProviderNotificationRoutes + + def self.registered(app) + + # * *Request* + # - method : POST + # Checks if given autoscaling group is launched within stack that is handled by CID. + # If so, starts syncing that stack. Otherwise returns 404 error. + # + # * *Returns* : + # report_id + app.post_with_headers '/provider_notifications/aws/:provider_account/autoscaling_groups/:id/changes', :headers => [:accept] do |provider_account, group_id| + check_privileges("stack", "r") + json Devops::API2_0::Handler::ProviderNotification.new(request).autoscaling_groups_change(group_id, provider_account) + end + + end + end + end + end +end \ No newline at end of file diff --git a/devops-service/app/api2/routes/stack.rb b/devops-service/app/api2/routes/stack.rb index ede1968..2dc235c 100644 --- a/devops-service/app/api2/routes/stack.rb +++ b/devops-service/app/api2/routes/stack.rb @@ -30,16 +30,19 @@ module Devops # - Content-Type: application/json # - body : # { - # "without_bootstrap": null, - # "project": "project_name", - # "deploy_env": "test", - # "provider": "ec2", - # "tags": { - # "tagName": "tagValue" - # }, - # "parameters": { - # "KeyName": "Value" + # "stack_attributes": { + # "project": "project_name", + # "deploy_env": "test", + # "provider": "ec2", + # "tags": { + # "tagName": "tagValue" + # }, + # "parameters": { + # "KeyName": "Value" + # } # } + # "without_bootstrap": false, + # "skip_rollback": false # } # # * *Returns* : diff --git a/devops-service/app/devops-api2.rb b/devops-service/app/devops-api2.rb index 37ba7ac..9833fb3 100644 --- a/devops-service/app/devops-api2.rb +++ b/devops-service/app/devops-api2.rb @@ -19,9 +19,9 @@ module Devops require_relative "api2/handlers/server" require_relative "api2/handlers/image" require_relative "api2/handlers/project" - require_relative "api2/handlers/stack" require_relative "api2/handlers/stack_template" + require_relative "api2/handlers/provider_notification" require 'lib/stubber' end @@ -70,6 +70,7 @@ module Devops require_relative "api2/routes/stack_template" require_relative "api2/routes/statistic" require_relative "api2/routes/report" + require_relative "api2/routes/provider_notification" routes = Devops::API2_0::Routes.constants.collect{|s| Devops::API2_0::Routes.const_get(s)}.select {|const| const.class == Module} routes.each do |r| diff --git a/devops-service/db/mongo/connectors/stack.rb b/devops-service/db/mongo/connectors/stack.rb index e28b189..5280b64 100644 --- a/devops-service/db/mongo/connectors/stack.rb +++ b/devops-service/db/mongo/connectors/stack.rb @@ -21,6 +21,14 @@ module Connectors collection.update({"_id" => id}, {"$set" => {"run_list" => run_list}}) end + def lock_persisting_stack(id) + collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => true}}) + end + + def unlock_persisting_stack(id) + collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => false}}) + end + def stack(name, options={}) query = {'name' => name}.merge(options) bson = collection.find(query).to_a.first @@ -28,6 +36,13 @@ module Connectors model_from_bson(bson) end + def stack_by_id(id) + query = {'_id' => id} + bson = collection.find(query).to_a.first + raise RecordNotFound.new("'#{id}' not found") unless bson + model_from_bson(bson) + end + private def model_from_bson(bson) diff --git a/devops-service/db/mongo/models/report.rb b/devops-service/db/mongo/models/report.rb index e0d03db..0115314 100644 --- a/devops-service/db/mongo/models/report.rb +++ b/devops-service/db/mongo/models/report.rb @@ -12,6 +12,10 @@ module Devops DEPLOY_STACK_TYPE = 6 DELETE_SERVER_TYPE = 7 EXPIRE_SERVER_TYPE = 8 + SYNC_STACK_TYPE = 9 + + + SYSTEM_OWNER = 'SYSTEM' attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index 4f44613..169a375 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -7,7 +7,7 @@ module Devops include ModelWithProvider - attr_accessor :id, :name, :project, :deploy_env, :stack_template, :parameters, :events, :owner, :run_list, :stack_status + attr_accessor :parameters, :events, :stack_status, :persisting_is_locked set_field_validators :id, [::Validators::FieldValidator::NotNil, ::Validators::FieldValidator::FieldType::String, @@ -56,6 +56,7 @@ module Devops self.run_list = attrs['run_list'] || [] self.tags = attrs['tags'] || {} self.stack_status = attrs['stack_status'] + self.persisting_is_locked = attrs['persisting_is_locked'] self end @@ -70,7 +71,8 @@ module Devops stack_status: stack_status, owner: owner, run_list: run_list, - tags: tags + tags: tags, + persisting_is_locked: persisting_is_locked }.merge(provider_hash) end @@ -87,7 +89,7 @@ module Devops provider_instance.delete_stack(self) end - def sync! + def sync_status_and_events! self.stack_status = provider_instance.stack_details(self)[:stack_status] self.events = provider_instance.stack_events(self) rescue Fog::AWS::CloudFormation::NotFound @@ -111,6 +113,16 @@ module Devops Devops::Db.connector.stack_template(stack_template) end + def lock_persisting! + self.persisting_is_locked = true + Devops::Db.connector.lock_persisting_stack(id) + end + + def unlock_persisting! + self.persisting_is_locked = false + Devops::Db.connector.unlock_persisting_stack(id) + end + class << self # attrs should include: @@ -122,7 +134,7 @@ module Devops def create(attrs, out) model = new(attrs) model.create_stack_in_cloud!(out) - model.sync! + model.sync_status_and_events! model end diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index cc44133..bf8f5eb 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -17,7 +17,8 @@ class MongoConnector delegate( [:images, :image, :image_insert, :image_delete, :image_update] => :images_connector, [:stack_templates, :stack_template, :stack_template_insert, :stack_template_delete, :stack_template_update] => :stack_templates_connector, - [:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list] => :stacks_connector, + [:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list, + :stack_by_id, :lock_persisting_stack, :unlock_persisting_stack] => :stacks_connector, [:available_images, :add_available_images, :delete_available_images] => :filters_connector, [:project, :projects_all, :projects, :project_names_with_envs, :projects_by_image, :projects_by_user, :project_insert, :project_update, diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 15fadd7..00e1ce4 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -131,7 +131,7 @@ module Devops res = self.run_hook(:before_bootstrap, @out) @out << "Done\n" if @server.private_ip.nil? - @out << "Error: Private IP is null" + @out.puts "Error: Private IP is null" return error_code(:server_bootstrap_private_ip_unset) end ja = { @@ -238,13 +238,15 @@ module Devops @out << "Server #{@server.chef_node_name} is created" else @out.puts "Can not find client or node on chef-server" - roll_back + @out.puts "Skip rollback because :skip_rollback option is set" + roll_back unless options[:skip_rollback] @out.flush mongo.server_delete @server.id return error_code(:server_not_in_chef_nodes) end else - roll_back + @out.puts "Skip rollback because :skip_rollback option is set" + roll_back unless options[:skip_rollback] mongo.server_delete @server.id msg = "Failed while bootstraping server with id '#{@server.id}'\n" msg << "Bootstraping operation result was #{bootstrap_status}" diff --git a/devops-service/lib/executors/stack_executor.rb b/devops-service/lib/executors/stack_executor.rb index acb1d1e..26684de 100644 --- a/devops-service/lib/executors/stack_executor.rb +++ b/devops-service/lib/executors/stack_executor.rb @@ -4,6 +4,7 @@ module Devops; module Executor; class StackExecutor; end; end; end # predeclare require_relative "stack_executor/stack_creation_waiter" require_relative "stack_executor/stack_servers_persister" require_relative "stack_executor/prioritized_groups_bootstrapper" +require_relative "stack_executor/stack_servers_fetcher" module Devops module Executor @@ -14,38 +15,52 @@ module Devops def initialize(options) @out = options.fetch(:out) @stack = options[:stack] - end - - def wait_till_stack_is_created - wait_result = StackCreationWaiter.new(stack, out).sync - - if wait_result.ok? - puts_and_flush "\nStack '#{stack.name}' has been created" - true - else - puts_and_flush "An error ocurred during stack creation: #{wait_result.reason}" - false - end + self.just_persisted_by_priority = {} end def create_stack(stack_attrs) + stack_attrs.merge!('persisting_is_locked' => true) @stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, out) mongo.stack_insert(@stack) end - def persist_stack_servers - puts_and_flush 'Start saving stack servers into CID database.' - persister = StackServersPersister.new(stack, out) - persister.persist_new_servers - puts_and_flush "Stack servers have been saved." - { - just_persisted_by_priority: persister.just_persisted_by_priority, - deleted: persister.deleted - } + def wait_till_stack_is_created + wait_result = waiter.wait + stack.unlock_persisting! + wait_result.ok? end - def bootstrap_servers_by_priority(servers_by_priorities, jid) - PrioritizedGroupsBootstrapper.new(out, jid, servers_by_priorities).bootstrap_servers_by_priority + def persist_new_servers + reload_stack + raise 'It seems that stack is synchronizing at the moment' if stack.persisting_is_locked + begin + stack.lock_persisting! + fetcher.new_servers_by_priorities.each do |priority, provider_infos| + servers = provider_infos.map {|info| persister.persist(info) } + just_persisted_by_priority[priority] = servers + end + + just_persisted_by_priority.values.flatten.each do |server| + puts_and_flush "\n\nPersisted server #{server.id}: #{JSON.pretty_generate(server.to_hash)}" + end + ensure + stack.unlock_persisting! + end + end + + def bootstrap_just_persisted(jid) + puts_and_flush "Bootstrapping just persisted servers" if just_persisted_by_priority.values.flatten.any? + just_persisted_by_priority.each do |priority, servers| + puts_and_flush "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" + end + PrioritizedGroupsBootstrapper.new(out, jid, just_persisted_by_priority).bootstrap_servers_by_priority + end + + def delete_stale_servers + fetcher.stale_servers.each do |server| + server_executor = Devops::Executor::ServerExecutor.new(server, out) + server_executor.delete_server + end end def delete_stack @@ -56,10 +71,28 @@ module Devops private + attr_accessor :just_persisted_by_priority + def mongo Devops::Db.connector end + def reload_stack + @stack = mongo.stack(@stack.name) + end + + def fetcher + @fetcher ||= StackServersFetcher.new(stack, out) + end + + def persister + @persister ||= StackServersPersister.new(@stack) + end + + def waiter + StackCreationWaiter.new(stack, out) + end + end end end diff --git a/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb b/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb index 1be2d67..bddfb3d 100644 --- a/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb +++ b/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb @@ -25,7 +25,8 @@ class Devops::Executor::StackExecutor def initialize(attrs) @server_info = attrs[:provider_server_info] @project, @env = attrs[:project_id], attrs[:env_id] - @mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK + @mask = @server_info['tags']['cid:node-name-mask'] if @server_info['tags'] + @mask ||= DEFAULT_MASK end diff --git a/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb b/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb index 16aae48..098eba0 100644 --- a/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb +++ b/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb @@ -28,7 +28,7 @@ class Devops::Executor::StackExecutor @server_bootstrap_jobs.map do |server_id, job_id| result = wait_for_bootstrap_job(job_id) - puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id) + puts_and_flush Devops::Messages.t("stack_executor.servers_bootstrapper.result.#{result.reason}", server_id: server_id, job_id: job_id) result end end @@ -41,7 +41,8 @@ class Devops::Executor::StackExecutor job_id = Worker.start_async(::BootstrapWorker, server_attrs: server.to_mongo_hash, bootstrap_template: 'omnibus', - owner: server.created_by + owner: server.created_by, + skip_rollback: true ) @out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})." @server_bootstrap_jobs[server.id] = job_id diff --git a/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb b/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb index 7df3529..0e63a61 100644 --- a/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb +++ b/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb @@ -22,12 +22,12 @@ class Devops::Executor::StackExecutor @sync_result = nil end - def sync + def wait puts_and_flush "Syncing stack '#{stack.id}'..." sleep_times.detect do |sleep_time| sleep sleep_time - stack.sync! + stack.sync_status_and_events! print_new_events update_stack_status if stack_status_changed? @@ -75,7 +75,7 @@ class Devops::Executor::StackExecutor end def print_result_message - puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}", + puts_and_flush Devops::Messages.t("stack_executor.stack_creation_waiter.result.#{@sync_result.reason}", stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+)) end diff --git a/devops-service/lib/executors/stack_executor/stack_servers_fetcher.rb b/devops-service/lib/executors/stack_executor/stack_servers_fetcher.rb new file mode 100644 index 0000000..6df318f --- /dev/null +++ b/devops-service/lib/executors/stack_executor/stack_servers_fetcher.rb @@ -0,0 +1,97 @@ +class Devops::Executor::StackExecutor + class StackServersFetcher + include PutsAndFlush + attr_reader :out + + NEW = :new + PERSISTED = :persisted + STALE = :stale + + def initialize(stack, out) + @stack, @out = stack, out + @by_state = {NEW => [], PERSISTED => [], STALE => []} + fetch + end + + + def new_servers_by_priorities + servers_with_priority = {} + @by_state[NEW].each do |provider_info| + priority = priority_from_info(provider_info) + servers_with_priority[priority] ||= [] + servers_with_priority[priority] << provider_info + end + servers_with_priority + end + + def servers_to_persist + @by_state[NEW] + end + + def already_persisted_servers + @by_state[PERSISTED] + end + + def stale_servers + @by_state[STALE] + end + + private + + def fetch + @servers_info = @stack.provider_instance.stack_servers(@stack) + divide_into_states + output_fetched_servers + end + + def priority_from_info(provider_info) + if provider_info['tags'] + priority = provider_info['tags']['cid:priority'].to_i + else + 0 + end + end + + def divide_into_states + persisted = Devops::Db.connector.stack_servers(@stack.name) + persisted_ids = persisted.map(&:id) + in_cloud_ids = @servers_info.map {|info| info['id']} + new_ids = in_cloud_ids - persisted_ids + deleted_ids = persisted_ids - in_cloud_ids + + @servers_info.each do |info| + if new_ids.include?(info['id']) + @by_state[NEW] << info + else + @by_state[PERSISTED] << info + end + end + + @by_state[STALE] = persisted.select { |server| deleted_ids.include?(server.id) } + end + + # Do not move it to stack executor till set events handling properly. + # For now there may be already persisted servers on creation because of too early events from aws. + def output_fetched_servers + if servers_to_persist.any? + out.puts 'Servers to persist:' + servers_to_persist.each { |info| out.puts JSON.pretty_generate(info) } + else + out.puts 'There are no servers to persist.' + end + + if already_persisted_servers.any? + out.puts "\nAlready persisted servers:" + already_persisted_servers.each { |info| out.puts JSON.pretty_generate(info) } + end + + if stale_servers.any? + out.puts "\nStale servers:" + stale_servers.each { |server| out.puts JSON.pretty_generate(server.to_hash) } + else + out.puts "\nThere are no stale servers." + end + out.flush + end + end +end \ No newline at end of file diff --git a/devops-service/lib/executors/stack_executor/stack_servers_persister.rb b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb index 162e76a..1ee3599 100644 --- a/devops-service/lib/executors/stack_executor/stack_servers_persister.rb +++ b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb @@ -1,102 +1,43 @@ require_relative 'chef_node_name_builder' -# Fetches info about stack servers from provider and then persist them in mongo. class Devops::Executor::StackExecutor class StackServersPersister - include PutsAndFlush - attr_reader :stack, :out, :deleted, :servers_info + attr_reader :stack - NEW = 'new' - DELETED = 'deleted' - PERSISTED = 'persisted' - JUST_PERSISTED = 'just_persisted' - - def initialize(stack, out) - @stack, @out = stack, out + def initialize(stack) + @stack = stack @project = mongo.project(stack.project) @deploy_env = @project.deploy_env(stack.deploy_env) - fetch_provider_servers_info - set_servers_states end - def persist_new_servers - with_state(NEW).each do |info| - info[:server] = persist_stack_server(info[:provider_info]) - info[:state] = JUST_PERSISTED - puts_and_flush "Persisted server with id '#{info[:id]}' and name '#{info[:server].chef_node_name}'" - end - end - - # returns: { priority_as_integer => [Servers] } - def just_persisted_by_priority - stack_servers_with_priority = {} - with_state(JUST_PERSISTED).each do |info| - stack_servers_with_priority[info[:priority]] ||= [] - stack_servers_with_priority[info[:priority]] << info[:server] - end - stack_servers_with_priority - end - - private - - def fetch_provider_servers_info - @servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info| - { - id: provider_info['id'], - provider_info: provider_info, - priority: provider_info['tags']['cid:priority'].to_i - } - end - end - - def set_servers_states - persisted = Devops::Db.connector.stack_servers(stack.id) - persisted_ids = persisted.map(&:id) - in_cloud_ids = @servers_info.map {|info| info[:id]} - new_ids = in_cloud_ids - persisted_ids - deleted_ids = persisted_ids - in_cloud_ids - - @servers_info.each do |info| - if new_ids.include?(info[:id]) - info[:state] = NEW - else - info[:state] = PERSISTED - end - end - - @deleted = persisted.select { |server| deleted_ids.include?(server.id) } - end - - def with_state(state) - @servers_info.select { |info| info[:state] == state } - end - - # takes a hash, returns Server model - def persist_stack_server(server_info) + def persist(provider_info) server_attrs = { - '_id' => server_info['id'], - 'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values), + '_id' => provider_info['id'], + 'chef_node_name' => get_name_builder(provider_info).build_node_name!(incrementers_values), 'created_by' => stack.owner, 'deploy_env' => @deploy_env.identifier, - 'key' => server_info['key_name'] || stack.provider_instance.ssh_key, + 'key' => provider_info['key_name'] || stack.provider_instance.ssh_key, 'project' => @project.id, 'provider' => @stack.provider, 'provider_account' => @stack.provider_account, 'remote_user' => mongo.image(@deploy_env.image).remote_user, - 'private_ip' => server_info['private_ip'], - 'public_ip' => server_info['public_ip'], + 'private_ip' => provider_info['private_ip'], + 'public_ip' => provider_info['public_ip'], 'run_list' => stack.run_list || [], 'stack' => stack.name } server = ::Devops::Model::Server.new(server_attrs) mongo.server_insert(server) + # here custom insert method is used and it doesn't return server model server end - def get_name_builder(server_info) + private + + def get_name_builder(provider_info) ChefNodeNameBuilder.new( - provider_server_info: server_info, + provider_server_info: provider_info, project_id: @project.id, env_id: @deploy_env.identifier, owner: stack.owner diff --git a/devops-service/lib/sync_stack_lambda.py b/devops-service/lib/sync_stack_lambda.py index b796565..9269d46 100644 --- a/devops-service/lib/sync_stack_lambda.py +++ b/devops-service/lib/sync_stack_lambda.py @@ -8,10 +8,11 @@ import urllib2 def lambda_handler(event, context): print("Received event: " + json.dumps(event, indent=2)) - stack_id = event['Records'][0]['Sns']['Message'] - url = 'http://example.com/v2.0/stack/%s/sync' % stack_id - username = 'root' - password = 'pass' + host = 'http://CHANGE_ME' + group_name = event['detail']['AutoScalingGroupName'] + url = '%s/v2.0/provider_notifications/aws/first/autoscaling_groups/%s/changes' % (host, group_name) + username = 'CHANGE_ME' + password = 'CHANGE_ME' request = urllib2.Request(url, '') base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') diff --git a/devops-service/messages/en.yml b/devops-service/messages/en.yml index 240fe95..6cdac42 100644 --- a/devops-service/messages/en.yml +++ b/devops-service/messages/en.yml @@ -13,17 +13,26 @@ en: Stack was launched, but an error occured during deploying stack servers. You can redeploy stack after fixing the error. timeout_reached: Bootstrap or deploy wasn't completed due to timeout. + stack_sync: + bootstrap_result: + ok: "Stack has been successfully synced." + bootstrap_error: An error occured during new stack servers bootstrapp. + deploy_error: An error occured during new stack servers deploy. + timeout_reached: Bootstrap or deploy of new servers wasn't completed due to timeout. + stack_executor: servers_bootstrapper: - bootstrap_servers: + result: ok: "Server '%{server_id}' has been bootstraped (job %{job_id})." timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached." bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})." deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})." - stack_creation_waiter: - result: - ok: "Stack '%{stack_id}' status is now %{status}" - stack_rolled_back: "Stack '%{stack_id}' status is now %{status}" - stack_deleted: "Stack '%{stack_id}' status is now %{status}" - stack_not_found: "Stack '%{stack_id}' status is now %{status}" - unkown_status: "Unknown stack status: '%{status}'" - timeout: "Stack hasn't been synced in %{seconds} seconds." \ No newline at end of file + stack_creation_waiter: + result: + ok: | + Stack '%{stack_id}' status is now %{status} + Stack has been created + stack_rolled_back: "Stack '%{stack_id}' status is now %{status}" + stack_deleted: "Stack '%{stack_id}' status is now %{status}" + stack_not_found: "Stack '%{stack_id}' status is now %{status}" + unkown_status: "Unknown stack status: '%{status}'" + timeout: "Stack hasn't been synced in %{seconds} seconds." diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 5aeaeb9..47e1445 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -211,10 +211,6 @@ module Provider connection_compute(connection_options) end - def cloud_formation - @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options) - end - def create_stack(stack, out) begin out << "Creating stack for project '#{stack.project}' and environment '#{stack.deploy_env}'...\n" @@ -291,26 +287,20 @@ module Provider cloud_formation.describe_stack_events(stack.name).body['StackEvents'].map{|se| {"timestamp" => se["Timestamp"], "stack_name" => se["StackName"], "stack_id" => se["StackId"], "event_id" => se["EventId"], "reason" => se["ResourceStatusReason"], "status" => se["ResourceStatus"]}}.sort{|se1, se2| se1["timestamp"] <=> se2["timestamp"]} end - # не работает, не используется - # def stack_resource(stack, resource_id) - # physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id - # compute.servers.get(physical_id) - # end - def stack_servers(stack) - # orchestration.describe_stack_resources возвращает мало информации resources = compute.describe_instances( 'tag-key' => 'aws:cloudformation:stack-id', 'tag-value' => stack.id ).body["reservationSet"] - # В ресурсах могут лежать не только конкретные инстансы, но и MasterNodesGroup, которые управляют - # несколькими инстансами. Обрабатываем эту ситуацию. + + # There may be not only instances but autoscaling groups with several instances. + # Handle such situations. instances = resources.map { |resource| resource["instancesSet"] }.flatten + instances.delete_if {|i| i['instanceState']['name'] == 'terminated'} instances.map do |instance| { - # 'name' => instance["tagSet"]["Name"], 'name' => [stack.name, instance_name(instance)].join('-'), 'id' => instance["instanceId"], 'key_name' => instance["keyName"], @@ -325,6 +315,18 @@ module Provider "stack-#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}".gsub('_', '-') end + def stack_id_of_autoscaling_group(id) + response = auto_scaling.describe_auto_scaling_groups('AutoScalingGroupNames' => [id]) + tags = response.body['DescribeAutoScalingGroupsResult']['AutoScalingGroups'].first['Tags'] + stack_id_tag = tags.find {|t| t['Key'] == 'aws:cloudformation:stack-id'} + if stack_id_tag + stack_id_tag['Value'] + end + rescue StandardError => e + puts "An error occured while analyzing group '#{id}': #{[e.name, e.message, e.backtrace].join("\n")}" + nil + end + def describe_vpcs self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } } end @@ -391,14 +393,18 @@ module Provider r end - def orchestration - @orchestration ||= Fog::AWS::CloudFormation.new(connection_options) - end - def storage @storage ||= Fog::Storage.new(connection_options) end + def cloud_formation + @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options) + end + + def auto_scaling + @auto_scaling ||= Fog::AWS::AutoScaling.new(connection_options) + end + def stack_templates_bucket bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplatesnibrdev' bucket = storage.directories.get(bucket_name) diff --git a/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb b/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb index 44542a5..3bb9f6c 100644 --- a/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb +++ b/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb @@ -7,7 +7,7 @@ class Devops::Executor::StackExecutor let(:syncer) { described_class.new(stack, out) } before do - allow(stack).to receive(:sync!) + allow(stack).to receive(:sync_status_and_events!) allow(stack).to receive(:events).and_return( [{'event_id' => 1}] ) allow(syncer).to receive(:sleep) allow(stubbed_connector).to receive(:stack_update) @@ -15,17 +15,17 @@ class Devops::Executor::StackExecutor def setup_statuses(statuses_array) statuses = statuses_array.to_enum - allow(stack).to receive(:sync!) { + allow(stack).to receive(:sync_status_and_events!) { stack.stack_status = statuses.next } end - describe '#sync', stubbed_logger: true do + describe '#wait', stubbed_logger: true do it 'waits for stack creating to be finished' do setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']) expect(syncer).to receive(:sleep).at_least(10).times - expect(stack).to receive(:sync!).at_least(10).times - syncer.sync + expect(stack).to receive(:sync_status_and_events!).at_least(10).times + syncer.wait end it 'prints each message only once' do @@ -39,13 +39,13 @@ class Devops::Executor::StackExecutor expect(out).to receive(:puts).with(/t1/).once.ordered expect(out).to receive(:puts).with(/t2/).once.ordered expect(out).to receive(:puts).with(/t3/).once.ordered - syncer.sync + syncer.wait end it 'updates stack when status is changed' do setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) expect(stubbed_connector).to receive(:stack_update).exactly(3).times - syncer.sync + syncer.wait end context 'when stack creating was successful' do @@ -53,7 +53,7 @@ class Devops::Executor::StackExecutor setup_statuses(['CREATE_COMPLETE']) expect(stubbed_connector).to receive(:stack_update).with(stack) expect(out).to receive(:puts).with(/CREATE_COMPLETE/) - expect(syncer.sync).to be_ok + expect(syncer.wait).to be_ok end end @@ -61,7 +61,7 @@ class Devops::Executor::StackExecutor it 'returns 1 (:stack_rolled_back)' do setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/) - expect(syncer.sync).to be_stack_rolled_back + expect(syncer.wait).to be_stack_rolled_back end end @@ -69,7 +69,7 @@ class Devops::Executor::StackExecutor it 'returns 2 (:unkown_status)' do setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) expect(out).to receive(:puts).with(/unknown/) - expect(syncer.sync).to be_unkown_status + expect(syncer.wait).to be_unkown_status end end @@ -77,7 +77,7 @@ class Devops::Executor::StackExecutor it 'returns 3 (:timeout)' do allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} expect(out).to receive(:puts).with(/hasn't been synced/) - expect(syncer.sync).to be_timeout + expect(syncer.wait).to be_timeout end end @@ -85,7 +85,7 @@ class Devops::Executor::StackExecutor it 'returns 5 (:error)' do setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) allow(stubbed_connector).to receive(:stack_update) { raise } - expect(syncer.sync.code).to eq 5 + expect(syncer.wait.code).to eq 5 end end end diff --git a/devops-service/spec/executors/stack_executor/stack_servers_fetcher_spec.rb b/devops-service/spec/executors/stack_executor/stack_servers_fetcher_spec.rb new file mode 100644 index 0000000..2ada0a7 --- /dev/null +++ b/devops-service/spec/executors/stack_executor/stack_servers_fetcher_spec.rb @@ -0,0 +1,55 @@ +require 'lib/executors/stack_executor/stack_servers_fetcher' + +class Devops::Executor::StackExecutor + RSpec.describe StackServersFetcher, stubbed_connector: true do + def info_hash_for_id(id) + { + 'id' => id, + 'name' => 'server_name', + 'key_name' => 'key', + 'private_ip' => '127.0.0.1', + 'public_ip' => '127.0.0.2', + 'tags' => { + 'cid:priority' => '3', + 'Name' => 'server1' + } + } + end + + let(:out) { double('out', puts: nil, flush: nil) } + let(:stack) { build(:stack_ec2) } + let(:fetcher) { described_class.new(stack, out) } + let(:new_server_info) { info_hash_for_id('i-new') } + let(:persisted_server_info) { info_hash_for_id('i-persisted') } + let(:stale_server) { build(:server, id: 'i-deleted') } + + before do + allow(stubbed_connector).to receive(:stack_servers) { [stale_server, build(:server, id: 'i-persisted')] } + provider = instance_double(Provider::Ec2, name: 'ec2') + allow(stack).to receive(:provider_instance) { provider } + allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]} + end + + describe '#new_servers_by_priorities' do + let(:new_servers_by_priorities) { fetcher.new_servers_by_priorities } + + it 'returns new servers info by :new key divided by priorities' do + expect(new_servers_by_priorities).to be_a(Hash) + expect(new_servers_by_priorities.length).to eq 1 + expect(new_servers_by_priorities[3].length).to eq 1 + expect(new_servers_by_priorities[3].first).to eq new_server_info + end + + it "sets priority to 0 if it's absent" do + new_server_info['tags'].delete('cid:priority') + expect(new_servers_by_priorities[0].first).to eq new_server_info + end + end + + describe '#stale_servers' do + it 'returns array of stale servers' do + expect(fetcher.stale_servers).to match_array [stale_server] + end + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb b/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb index 919e4dc..ebfe44e 100644 --- a/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb +++ b/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb @@ -2,9 +2,9 @@ require 'lib/executors/stack_executor/stack_servers_persister' class Devops::Executor::StackExecutor RSpec.describe StackServersPersister, stubbed_connector: true do - def info_hash_for_id(id) + let(:provider_info) { { - 'id' => id, + 'id' => 'i-new', 'name' => 'server_name', 'key_name' => 'key', 'private_ip' => '127.0.0.1', @@ -14,154 +14,88 @@ class Devops::Executor::StackExecutor 'Name' => 'server1' } } - end + } - let(:out) { double(:out, puts: nil, flush: nil) } let(:run_list) { ['role[asd]'] } + let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) } let(:project) { build(:project, id: 'name') } - let(:persister) { described_class.new(stack, out) } - let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } - let(:new_server_info) { info_hash_for_id('i-new') } - let(:deleted_server_info) { info_hash_for_id('i-deleted') } - let(:persisted_server_info) { info_hash_for_id('i-persisted') } + let(:persister) { described_class.new(stack) } before do allow(stubbed_connector).to receive(:project) { project } allow(stubbed_connector).to receive(:image) { instance_double(Devops::Model::Image, remote_user: 'user') } - allow(stubbed_connector).to receive(:server_insert) - allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] } + allow(stubbed_connector).to receive(:server_insert) {|server| server} allow(stack).to receive(:provider_instance) { provider } - allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]} end - describe '#initialize' do - it 'fetches stack servers info' do - expect(provider).to receive(:stack_servers).with(stack) - described_class.new(stack, out) + describe '#persist' do + let(:persist) { persister.persist(provider_info) } + + it 'persist record to mongo' do + expect(stubbed_connector).to receive(:server_insert) + persist end - it "doesn't raise error if cid:priority tag is absent" do - new_server_info['tags'].delete('cid:priority') - expect { described_class.new(stack, out) }.not_to raise_error - end - - it 'sets proper statuses' do - infos = described_class.new(stack, out).servers_info - expect(infos.length).to eq 2 - expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted' - expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new' - end - - it 'sets deleted servers' do - deleted = described_class.new(stack, out).deleted - expect(deleted.length).to eq 1 - expect(deleted.first.id).to eq 'i-deleted' - end - end - - describe '#persist_new_servers' do it 'takes id, key_name, private_ip and public_ip attrs from info hash' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.id).to eq 'i-new' - expect(server.key).to eq 'key' - expect(server.private_ip).to eq '127.0.0.1' - expect(server.public_ip).to eq '127.0.0.2' - end - persister.persist_new_servers + expect(persist.to_hash).to include( + 'id' => 'i-new', + 'key' => 'key', + 'private_ip' => '127.0.0.1', + 'public_ip' => '127.0.0.2' + ) end it 'takes created_by, run_list and stack attrs from stack' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.created_by).to eq 'root' - expect(server.run_list).to eq run_list - expect(server.stack).to eq 'iamstack' - end - persister.persist_new_servers + expect(persist.to_hash).to include( + 'created_by' => 'root', + 'run_list' => run_list, + 'stack' => 'iamstack' + ) end it 'takes remote_user from image user' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.remote_user).to eq 'user' - end - persister.persist_new_servers + expect(persist.remote_user).to eq 'user' end it "takes deploy_env from project's deploy_env identifier" do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.deploy_env).to eq 'foo' - end - persister.persist_new_servers + expect(persist.deploy_env).to eq 'foo' end it "takes default provider's ssh key if info doesn't contain it" do allow(provider).to receive(:ssh_key) { 'default_key' } - new_server_info.delete('key_name') - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.key).to eq 'default_key' - end - persister.persist_new_servers + provider_info.delete('key_name') + expect(persist.key).to eq 'default_key' end it "sets server's run list to empty array if stack's run_list is nil" do stack.run_list = nil - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.run_list).to eq [] - end - persister.persist_new_servers + expect(persist.run_list).to eq [] end it 'build chef_node_name with default mask ":project-:env-:instanceid"' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'name-foo-i-new' - end - persister.persist_new_servers + expect(persist.chef_node_name).to eq 'name-foo-i-new' end it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do - new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123' - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'name-server1-123' - end - persister.persist_new_servers + provider_info['tags']['cid:node-name-mask'] = ':project-:instancename-123' + expect(persist.chef_node_name).to eq 'name-server1-123' end it "sets provider and provider account from stack" do stack.provider_account = 'foo' - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.provider).to eq 'ec2' - expect(server.provider_account).to eq 'foo' - end - persister.persist_new_servers + expect(persist.provider_account).to eq 'foo' end describe 'incremented variables' do it 'substitutes :increment-groupid: with incrementing numbers' do - allow(provider).to receive(:stack_servers) {[ - {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}, - {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'} - ]} - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'node-01-dev' - end.ordered - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'node-02-dev' - end - persister.persist_new_servers + provider_info['tags']['cid:node-name-mask'] = 'node-:increment-group1:-dev' + expect(persister.persist(provider_info).chef_node_name).to eq 'node-01-dev' + expect(persister.persist(provider_info).chef_node_name).to eq 'node-02-dev' end end end - - describe '#just_persisted_by_priority' do - it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do - persister.persist_new_servers - result = persister.just_persisted_by_priority - expect(result).to be_a(Hash) - expect(result.size).to eq 1 - expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1) - end - end end end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor_spec.rb b/devops-service/spec/executors/stack_executor_spec.rb index 2fc1517..2dbe783 100644 --- a/devops-service/spec/executors/stack_executor_spec.rb +++ b/devops-service/spec/executors/stack_executor_spec.rb @@ -6,51 +6,119 @@ class Devops::Executor::StackExecutor let(:stack) { build(:stack) } let(:executor_without_stack) { described_class.new(out: out) } let(:executor_with_stack) { described_class.new(out: out, stack: stack) } + let(:new_servers_by_priorities) { + { + 0 => [{id: 1}], + 2 => [{id: 2}] + } + } + let(:just_persisted_by_priority) { + { + 0 => [double('info 1', id: 1)], + 2 => [double('info 2', id: 2)] + } + } + let(:fetcher) { + instance_double(StackServersFetcher, + new_servers_by_priorities: new_servers_by_priorities, + stale_servers: build_list(:server, 2), + fetch: nil + ) + } + + before do + allow(executor_with_stack).to receive(:fetcher) { fetcher } + end + describe '#wait_till_stack_is_created' do + let(:waiter) { instance_double(StackCreationWaiter, wait: double("creation_result", ok?: true)) } + + before do + allow(executor_with_stack).to receive(:waiter) { waiter } + allow(stack).to receive(:unlock_persisting!) + end + + it 'waites till stack is created, then fetches stack servers and unlocks stack persisting' do + expect(waiter).to receive(:wait).ordered + expect(fetcher).to receive(:fetch).ordered + expect(stack).to receive(:unlock_persisting!).ordered + executor_with_stack.wait_till_stack_is_created + end + it "return true if syncer returns ok" do - allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) } expect(executor_with_stack.wait_till_stack_is_created).to be true end it "return false if syncer returns not ok" do - allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') } + allow(waiter).to receive(:wait) { double("creation_result", ok?: false, reason: '') } expect(executor_with_stack.wait_till_stack_is_created).to be false end end - describe '#create_stack', stubbed_connector: true do + describe '#create_stack' do + before { expect(stubbed_connector).to receive(:stack_insert) } + it 'initiate creation in cloud and persists stack' do expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out) - expect(stubbed_connector).to receive(:stack_insert) + executor_with_stack.create_stack({'provider' => 'ec2'}) + end + + it 'locks persisting on create' do + expect(Devops::Model::StackFactory).to receive(:create) do |_, params| + expect(params).to include('persisting_is_locked' => true) + end executor_with_stack.create_stack({'provider' => 'ec2'}) end end - describe '#persist_stack_servers' do - let(:persister) { - instance_double(StackServersPersister, - persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil - ) - } - before { allow(StackServersPersister).to receive(:new).and_return(persister) } + describe '#persist_new_servers' do + let(:persister) { instance_double(StackServersPersister, persist: build(:server)) } - it 'calls StackServersPersister#persist_new_servers' do - expect(persister).to receive(:persist_new_servers) - executor_with_stack.persist_stack_servers + before do + allow(executor_with_stack).to receive(:persister) { persister } + allow(stack).to receive(:lock_persisting!) + allow(stack).to receive(:unlock_persisting!) + allow(stubbed_connector).to receive(:stack) { stack } end - it 'returns hash with :just_persisted_by_priority and :deleted keys' do - expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil) + it 'calls StackServersPersister#persist for each server' do + expect(persister).to receive(:persist).exactly(2).times + executor_with_stack.persist_new_servers + end + + it 'locks persisting of a stack before start and unlocks after finish' do + expect(stack).to receive(:lock_persisting!).ordered + expect(persister).to receive(:persist).ordered + expect(stack).to receive(:unlock_persisting!).ordered + executor_with_stack.persist_new_servers + end + + it 'unlocks persisting even in case of failures' do + allow(persister).to receive(:persist) { raise } + expect(stack).to receive(:unlock_persisting!) + expect { executor_with_stack.persist_new_servers }.to raise_error StandardError end end - describe '#bootstrap_servers_by_priority' do + describe '#bootstrap_just_persisted' do it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do result = double('bootstrap_result') + allow(executor_with_stack).to receive(:just_persisted_by_priority) { just_persisted_by_priority } allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result } expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) - expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result + expect(executor_with_stack.bootstrap_just_persisted(1000)).to eq result + end + end + + describe '#delete_stale_servers' do + it 'builds server executor per stale server and properly delete them' do + executor1 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil) + executor2 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil) + allow(Devops::Executor::ServerExecutor).to receive(:new).and_return(executor1, executor2) + expect(executor1).to receive(:delete_server).ordered + expect(executor2).to receive(:delete_server).ordered + executor_with_stack.delete_stale_servers end end @@ -61,7 +129,6 @@ class Devops::Executor::StackExecutor expect(stubbed_connector).to receive(:stack_delete).ordered executor_with_stack.delete_stack end - end end end \ No newline at end of file diff --git a/devops-service/spec/models/stack/stack_ec2_spec.rb b/devops-service/spec/models/stack/stack_ec2_spec.rb index ddfb952..579f7d4 100644 --- a/devops-service/spec/models/stack/stack_ec2_spec.rb +++ b/devops-service/spec/models/stack/stack_ec2_spec.rb @@ -80,7 +80,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do end - describe '#sync!' do + describe '#sync_status_and_events!' do let(:fresh_events) { double('fresh_events') } let(:provider) { instance_double('Provider::Ec2', @@ -95,12 +95,12 @@ RSpec.describe Devops::Model::StackEc2, type: :model do it "get fresh stack details and updates stack status" do expect(provider).to receive(:stack_details) - expect {stack.sync!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE') + expect {stack.sync_status_and_events!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE') end it "get fresh stack events and stores it in @events" do expect(stack).to receive_message_chain('provider_instance.stack_events') - expect {stack.sync!}.to change {stack.events}.from(nil).to(fresh_events) + expect {stack.sync_status_and_events!}.to change {stack.events}.from(nil).to(fresh_events) end end @@ -140,7 +140,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do before do allow_any_instance_of(described_class).to receive(:create_stack_in_cloud!) - allow_any_instance_of(described_class).to receive(:sync!) + allow_any_instance_of(described_class).to receive(:sync_status_and_events!) end it "returns instance of #{described_class.name}" do @@ -153,7 +153,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do end it 'synchronizes details' do - expect_any_instance_of(described_class).to receive(:sync!) + expect_any_instance_of(described_class).to receive(:sync_status_and_events!) subject end end diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb index 3891cf2..b0c17ad 100644 --- a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb @@ -3,15 +3,16 @@ require 'lib/executors/stack_executor' RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys } - let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } - let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) } let(:worker) { described_class.new } + let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } + let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs, 'without_bootstrap' => true) } let(:executor) { instance_double(Devops::Executor::StackExecutor, wait_till_stack_is_created: true, create_stack: Devops::Model::StackEc2.new(stack_attrs), - persist_stack_servers: nil, - delete_stack: nil + persist_new_servers: nil, + delete_stack: nil, + bootstrap_just_persisted: bootstrap_result(:ok) ) } @@ -22,8 +23,6 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini before do allow(worker).to receive(:update_report) allow(worker).to receive(:executor) { executor } - allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} } - allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:ok) } end @@ -39,7 +38,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini it 'updates report about operation, creates stack and persists stack servers' do expect(worker).to receive(:update_report).ordered expect(executor).to receive(:create_stack).ordered - expect(worker).to receive(:persist_stack_servers).ordered + expect(executor).to receive(:persist_new_servers).ordered perform_without_bootstrap end @@ -55,7 +54,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini context 'if without_bootstrap is true' do it "doesn't bootstrap servers" do - expect(worker).not_to receive(:bootstrap_servers_by_priority) + expect(worker).not_to receive(:bootstrap_or_rollback_if_failed) perform_without_bootstrap end @@ -70,27 +69,27 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini end it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do - allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:bootstrap_error) } + allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:bootstrap_error) } expect(executor).to receive(:delete_stack) perform_with_bootstrap expect(perform_with_bootstrap).to eq 2 end it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do - allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:deploy_error) } + allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:deploy_error) } expect(worker).not_to receive(:rollback_stack!) expect(perform_with_bootstrap).to eq 3 end it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do - allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:timeout_reached) } + allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:timeout_reached) } expect(worker).not_to receive(:rollback_stack!) expect(perform_with_bootstrap).to eq 4 end it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do error = StandardError.new - allow(worker).to receive(:bootstrap_servers_by_priority) { raise error } + allow(executor).to receive(:bootstrap_just_persisted) { raise error } expect(worker).to receive(:rollback_stack!) expect{perform_with_bootstrap}.to raise_error(error) end diff --git a/devops-service/workers/bootstrap_worker.rb b/devops-service/workers/bootstrap_worker.rb index 1e40e05..b959d56 100644 --- a/devops-service/workers/bootstrap_worker.rb +++ b/devops-service/workers/bootstrap_worker.rb @@ -6,10 +6,19 @@ require "db/mongo/models/report" class BootstrapWorker < Worker - # options must contain 'server_attrs', 'owner' + # @options + # 'server_attrs': required + # 'owner': required + # 'skip_rollback': optional + # 'deploy_info': optional + # 'deployers': optional + # 'bootstrap_template': optional + # 'chef_environment': optional + # 'config': optional, whatever this parameter really means def perform(options) call do owner = options.fetch('owner') + skip_rollback = options['skip_rollback'] converted_options = convert_config(options) server = Devops::Model::Server.new(options.fetch('server_attrs')) diff --git a/devops-service/workers/delete_expired_server_worker.rb b/devops-service/workers/delete_expired_server_worker.rb index 0d2ee7f..63bc074 100644 --- a/devops-service/workers/delete_expired_server_worker.rb +++ b/devops-service/workers/delete_expired_server_worker.rb @@ -22,7 +22,7 @@ class DeleteExpiredServerWorker < Worker def save_report(server) update_report( - "created_by" => 'SYSTEM', + "created_by" => Devops::Model::Report::SYSTEM_OWNER, "project" => server.project, "deploy_env" => server.deploy_env, "type" => Devops::Model::Report::EXPIRE_SERVER_TYPE diff --git a/devops-service/workers/run_workers.rb b/devops-service/workers/run_workers.rb index 6e25e39..98e3d1b 100644 --- a/devops-service/workers/run_workers.rb +++ b/devops-service/workers/run_workers.rb @@ -7,6 +7,7 @@ require File.join(root, "project_test_worker") require File.join(root, "stack_bootstrap_worker") require File.join(root, "delete_server_worker") require File.join(root, "delete_expired_server_worker") - +require File.join(root, "stack_sync_worker") +require 'byebug' config = {} #require File.join(root, "../proxy") diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 2fb1018..c174919 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -2,14 +2,16 @@ require 'lib/executors/stack_executor' class StackBootstrapWorker < Worker - # options must contain 'stack_attributes' + # @options: + # 'stack_attributes', required + # 'without_bootstrap', optional. false by default + # 'skip_rollback', optional. false by default def perform(options) call do + puts_and_flush JSON.pretty_generate(options) stack_attrs = options.fetch('stack_attributes') - - without_bootstrap = stack_attrs.delete('without_bootstrap') - skip_rollback = false # take it from options in future - @out.puts "Received 'without_bootstrap' option" if without_bootstrap + without_bootstrap = options['without_bootstrap'] || false + skip_rollback = options['skip_rollback'] || false save_report(stack_attrs) @@ -20,7 +22,7 @@ class StackBootstrapWorker < Worker end begin - persist_stack_servers + executor.persist_new_servers if without_bootstrap 0 else @@ -40,17 +42,13 @@ class StackBootstrapWorker < Worker @executor ||= Devops::Executor::StackExecutor.new(out: out) end + # let it stay inside method to improve readability of #perform method def create_stack(stack_attrs) @stack = executor.create_stack(stack_attrs) end - def persist_stack_servers - @servers_by_priorities = executor.persist_stack_servers[:just_persisted_by_priority] - end - - # options should contain :skip_rollback def bootstrap_or_rollback_if_failed(options) - bootstrap_result = bootstrap_servers_by_priority + bootstrap_result = executor.bootstrap_just_persisted(jid) puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}") if bootstrap_result.bootstrap_error? && !options[:skip_rollback] rollback_stack! @@ -58,14 +56,6 @@ class StackBootstrapWorker < Worker bootstrap_result.code end - def bootstrap_servers_by_priority - out.puts "Bootstrapping just persisted servers" - @servers_by_priorities.each do |priority, servers| - out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" - end - out.flush - executor.bootstrap_servers_by_priority(@servers_by_priorities, jid) - end def rollback_stack! puts_and_flush "\nStart rollback of a stack" diff --git a/devops-service/workers/stack_sync_worker.rb b/devops-service/workers/stack_sync_worker.rb new file mode 100644 index 0000000..76e44cb --- /dev/null +++ b/devops-service/workers/stack_sync_worker.rb @@ -0,0 +1,62 @@ +require 'lib/executors/stack_executor' + +class StackSyncWorker < Worker + + MAX_LOCK_WAITING_TIME = 15000 + + # @options: + # 'stack_name' required + # 'created_by' optional + def perform(options) + call do + stack_name = options.fetch('stack_name') + created_by = options['created_by'] || ::Devops::Model::Report::SYSTEM_OWNER + @stack = mongo.stack(stack_name) + + save_report(created_by) + wait_until_stack_is_unlocked + puts_and_flush 'Persisting new servers.' + executor.persist_new_servers + + puts_and_flush "\n\nDeleting stale servers." + executor.delete_stale_servers + + puts_and_flush "\n\nBootstrapping just persisted servers." + bootstrap_result = executor.bootstrap_just_persisted(jid) + puts_and_flush Devops::Messages.t("worker.stack_sync.bootstrap_result.#{bootstrap_result.reason}") + bootstrap_result.code + end + end + + private + + def executor + @executor ||= Devops::Executor::StackExecutor.new(out: out, stack: @stack) + end + + def wait_until_stack_is_unlocked + return unless @stack.persisting_is_locked + puts_and_flush 'Stack is locked, waiting...' + waiting_time = 0 + sleep_time = 10 + loop do + sleep(sleep_time) + @stack = mongo.stack(@stack.name) + return unless @stack.persisting_is_locked + + waiting_time += sleep_time + raise "Stack has been locked for too long" if waiting_time > MAX_LOCK_WAITING_TIME + end + end + + def save_report(created_by) + update_report( + "created_by" => created_by, + "project" => @stack.project, + "deploy_env" => @stack.deploy_env, + "type" => ::Devops::Model::Report::SYNC_STACK_TYPE, + "subreports" => [], + "stack" => @stack.name + ) + end +end