diff --git a/devops-service/lib/executors/stack_executor.rb b/devops-service/lib/executors/stack_executor.rb index fff04e5..acb1d1e 100644 --- a/devops-service/lib/executors/stack_executor.rb +++ b/devops-service/lib/executors/stack_executor.rb @@ -1,5 +1,6 @@ require "db/mongo/models/stack/stack_factory" require "lib/puts_and_flush" +module Devops; module Executor; class StackExecutor; end; end; end # predeclare for visibility require_relative "stack_executor/stack_creation_waiter" require_relative "stack_executor/stack_servers_persister" require_relative "stack_executor/prioritized_groups_bootstrapper" diff --git a/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb b/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb index a57e590..1be2d67 100644 --- a/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb +++ b/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb @@ -1,67 +1,70 @@ # Builds node name from mask. Mask could be passed in server's +cid:node-name-mask+ tag, # default mask is used otherwise. Mask is a string with possible several variables inserted, e.g. -# +':project/:env/:instanceid/:instancename/:increment-group1:'+ +# ':project/:env/:instanceid/:instancename/:increment-group1:' # Variables meanings: -# - +:project+ is replaced with project name (given in constructor) -# - +:env+ is replaced with env name (given in constructor) -# - +:time+ is replaced with current timestamp -# - +:instanceid+ is replaced with provider instance id (fetched from server info) -# - +:instancename+ is replaced with value of Name tag (fetched from server info) -# - +:increment-groupname:+ is replaced with incremented number tied to group name. There could be several groups in one stack. +# - :project is replaced with project name (given in constructor) +# - :env is replaced with env name (given in constructor) +# - :time is replaced with current timestamp +# - :instanceid is replaced with provider instance id (fetched from server info) +# - :instancename is replaced with value of Name tag (fetched from server info) +# - :increment-groupname: is replaced with incremented number tied to group name. There could be several groups in one stack. # P.S. Colons are used instead of dollar signs, because stacks don't support dollar signs in tags (unlike EC2 instances), # but it's convenient to set mask tag directly to a stack (not in template): you set tag once and it propagates to all instances. -class ChefNodeNameBuilder - # We need to ensure that node name is uniq by default. We can't use :time to provide uniqueness because - # several servers are persisting at once on stack creating at it's likely that Time.now.to_i will give similar values - # to different servers. So (by default) we should use :instanceid. - DEFAULT_MASK = ':project-:env-:instanceid' +class Devops::Executor::StackExecutor + class ChefNodeNameBuilder + # We need to ensure that node name is uniq by default. We can't use :time to provide uniqueness because + # several servers are persisting at once on stack creating at it's likely that Time.now.to_i will give similar values + # to different servers. So (by default) we should use :instanceid. + DEFAULT_MASK = ':project-:env-:instanceid' - # @param attrs [Hash] should contain - # +:provider_server_info+ - # +:project_id+ - # +:env_id+ - def initialize(attrs) - @server_info = attrs[:provider_server_info] - @project, @env = attrs[:project_id], attrs[:env_id] - @mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK - end + # @param attrs [Hash] should contain + # :provider_server_info + # :project_id + # :env_id + def initialize(attrs) + @server_info = attrs[:provider_server_info] + @project, @env = attrs[:project_id], attrs[:env_id] + @mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK + end - # @param incrementers_values [Hash] is a hash in which key is name of a variable and value is last substituted number for that var. - # This method modifies +incrementers_values+, updating values for substituted variables. - # - # Examples (assume mask is set to +':project-master-:increment-group1:'+): - # incremeters_values = {} - # builder.build_node_name!(incremeters_values) # returns 'mpda-master-01' - # puts incremeters_values # {'group1' => 1} - # builder.build_node_name!(incremeters_values) # returns 'mpda-master-02' - # puts incremeters_values # {'group1' => 2} - def build_node_name!(incrementers_values) - result = @mask.dup - replace_variables!(result) - replace_incrementers!(result, incrementers_values) - result - end + # @param incrementers_values [Hash] is a hash in which key is name of a variable + # and value is last substituted number for that var. + # This method modifies incrementers_values, updating values for substituted variables. + # + # Examples (assume mask is set to +':project-master-:increment-group1:'+): + # incremeters_values = {} + # builder.build_node_name!(incremeters_values) # returns 'mpda-master-01' + # puts incremeters_values # {'group1' => 1} + # builder.build_node_name!(incremeters_values) # returns 'mpda-master-02' + # puts incremeters_values # {'group1' => 2} + def build_node_name!(incrementers_values) + result = @mask.dup + replace_variables!(result) + replace_incrementers!(result, incrementers_values) + result + end - private + private - def replace_variables!(result) - result.gsub!(':project', @project) - result.gsub!(':env', @env) - result.gsub!(':instanceid', @server_info['id']) - result.gsub!(':instancename', @server_info['tags']['Name'] || '') - result.gsub!(':time', Time.now.to_i.to_s) - end + def replace_variables!(result) + result.gsub!(':project', @project) + result.gsub!(':env', @env) + result.gsub!(':instanceid', @server_info['id']) + result.gsub!(':instancename', @server_info['tags']['Name'] || '') + result.gsub!(':time', Time.now.to_i.to_s) + end - def replace_incrementers!(result, incrementers_values) - groupname_regexp = /(?<=:increment-)\w+(?=:)/ - result.gsub!(/:increment-\w+:/) do |incrementer| - group_name = groupname_regexp.match(incrementer)[0] - prev_value = incrementers_values[group_name] || 0 - current_value = prev_value + 1 - incrementers_values[group_name] = current_value - current_value.to_s.rjust(2, '0') + def replace_incrementers!(result, incrementers_values) + groupname_regexp = /(?<=:increment-)\w+(?=:)/ + result.gsub!(/:increment-\w+:/) do |incrementer| + group_name = groupname_regexp.match(incrementer)[0] + prev_value = incrementers_values[group_name] || 0 + current_value = prev_value + 1 + incrementers_values[group_name] = current_value + current_value.to_s.rjust(2, '0') + end end end end \ No newline at end of file diff --git a/devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb b/devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb index 5cc68d0..9fde355 100644 --- a/devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb +++ b/devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb @@ -2,35 +2,37 @@ require_relative 'servers_bootstrapper' # Bootstrap groups of servers based on priorities: higher first. # Doesn't start bootstrap of next group if bootstrap of previous group failed. -class PrioritizedGroupsBootstrapper - include PutsAndFlush - attr_reader :out +class Devops::Executor::StackExecutor + class PrioritizedGroupsBootstrapper + include PutsAndFlush + attr_reader :out - def initialize(out, jid, servers_with_priorities) - @out, @jid, @servers_with_priorities = out, jid, servers_with_priorities - end - - # @param servers_with_priorities [Hash] is a Hash like - # {1 => [server1, server2]} - # Starts bootstrapping another group only after successful bootstrapping of previous. - def bootstrap_servers_by_priority - sorted_priorities.each do |priority| - puts_and_flush "Bootstrap servers with priority '#{priority}':" - bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority]) - bootstrap_results = bootstrapper.bootstrap_group - error = most_critical_error(bootstrap_results) - return error if error + def initialize(out, jid, servers_with_priorities) + @out, @jid, @servers_with_priorities = out, jid, servers_with_priorities end - ServersBootstrapper::Result.from_reason(:ok) - end - private + # @param servers_with_priorities [Hash] is a Hash like + # {1 => [server1, server2]} + # Starts bootstrapping another group only after successful bootstrapping of previous. + def bootstrap_servers_by_priority + sorted_priorities.each do |priority| + puts_and_flush "Bootstrap servers with priority '#{priority}':" + bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority]) + bootstrap_results = bootstrapper.bootstrap_group + error = most_critical_error(bootstrap_results) + return error if error + end + ServersBootstrapper::Result.from_reason(:ok) + end - def sorted_priorities - @servers_with_priorities.keys.sort.reverse - end + private - def most_critical_error(results) - results.detect(&:bootstrap_error?) || results.detect(&:failed?) + def sorted_priorities + @servers_with_priorities.keys.sort.reverse + end + + def most_critical_error(results) + results.detect(&:bootstrap_error?) || results.detect(&:failed?) + end end end \ No newline at end of file diff --git a/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb b/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb index 399e2fa..16aae48 100644 --- a/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb +++ b/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb @@ -1,72 +1,74 @@ +require 'lib/helpers/job_waiter' require 'workers/bootstrap_worker' -require 'workers/helpers/job_waiter' # Starts bootstrap workers for each server in group and wait for them to end (synchroniously). -class ServersBootstrapper - include PutsAndFlush - attr_reader :out +class Devops::Executor::StackExecutor + class ServersBootstrapper + include PutsAndFlush + attr_reader :out - class Result < Devops::Helpers::ResultObject - set_result_codes( - ok: 0, - bootstrap_error: 2, - deploy_error: 3, - timeout_reached: 4 - ) - end - - def initialize(out, jid, servers) - @out, @jid, @servers = out, jid, servers - @server_bootstrap_jobs = {} - end - - # returns array of Results - def bootstrap_group - start_workers! - ::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values) - - @server_bootstrap_jobs.map do |server_id, job_id| - result = wait_for_bootstrap_job(job_id) - puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id) - result - end - end - - private - - # returns hash: {server_id => worker_job_id} - def start_workers! - @servers.each do |server| - job_id = Worker.start_async(::BootstrapWorker, - server_attrs: server.to_mongo_hash, - bootstrap_template: 'omnibus', - owner: server.created_by + class Result < Devops::Helpers::ResultObject + set_result_codes( + ok: 0, + bootstrap_error: 2, + deploy_error: 3, + timeout_reached: 4 ) - @out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})." - @server_bootstrap_jobs[server.id] = job_id end - puts_and_flush "\n\n\n" - end - def result(reason) - Result.from_reason(reason) - end + def initialize(out, jid, servers) + @out, @jid, @servers = out, jid, servers + @server_bootstrap_jobs = {} + end - def wait_for_bootstrap_job(job_id) - result_code = JobWaiter.new(job_id).wait - result_from_job_code(result_code) - rescue JobWaiter::TimeoutReached - result(:timeout_reached) - end + # returns array of Results + def bootstrap_group + start_workers! + ::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values) - def result_from_job_code(result_code) - job_result = Devops::Executor::ServerOperationResult.new(result_code) - if job_result.ok? - result(:ok) - elsif job_result.one_of_bootstrap_errors? - result(:bootstrap_error) - else - result(:deploy_error) + @server_bootstrap_jobs.map do |server_id, job_id| + result = wait_for_bootstrap_job(job_id) + puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id) + result + end + end + + private + + # returns hash: {server_id => worker_job_id} + def start_workers! + @servers.each do |server| + job_id = Worker.start_async(::BootstrapWorker, + server_attrs: server.to_mongo_hash, + bootstrap_template: 'omnibus', + owner: server.created_by + ) + @out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})." + @server_bootstrap_jobs[server.id] = job_id + end + puts_and_flush "\n\n\n" + end + + def result(reason) + Result.from_reason(reason) + end + + def wait_for_bootstrap_job(job_id) + result_code = Devops::Helpers::JobWaiter.new(job_id).wait + result_from_job_code(result_code) + rescue Devops::Helpers::JobWaiter::TimeoutReached + result(:timeout_reached) + end + + def result_from_job_code(result_code) + job_result = Devops::Executor::ServerOperationResult.new(result_code) + if job_result.ok? + result(:ok) + elsif job_result.one_of_bootstrap_errors? + result(:bootstrap_error) + else + result(:deploy_error) + end end end end \ No newline at end of file diff --git a/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb b/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb index 1cc9a53..7df3529 100644 --- a/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb +++ b/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb @@ -1,99 +1,101 @@ # Polling stack status until it's completed or failed. -class StackCreationWaiter - include PutsAndFlush - attr_reader :out, :stack +class Devops::Executor::StackExecutor + class StackCreationWaiter + include PutsAndFlush + attr_reader :out, :stack - class SyncResult < Devops::Helpers::ResultObject - set_result_codes( - ok: 0, - stack_rolled_back: 1, - unkown_status: 2, - timeout: 3, - error: 5, - stack_deleted: 6, - stack_not_found: 7 - ) - end - - def initialize(stack, out) - @stack, @out = stack, out - @printed_events = [] - @sync_result = nil - end - - def sync - puts_and_flush "Syncing stack '#{stack.id}'..." - - sleep_times.detect do |sleep_time| - sleep sleep_time - stack.sync! - print_new_events - - update_stack_status if stack_status_changed? - stack_is_already_created_or_failed? + class SyncResult < Devops::Helpers::ResultObject + set_result_codes( + ok: 0, + stack_rolled_back: 1, + unkown_status: 2, + timeout: 3, + error: 5, + stack_deleted: 6, + stack_not_found: 7 + ) end - @sync_result ||= result(:timeout) - print_result_message - @sync_result - rescue StandardError => e - DevopsLogger.logger.error e.message - puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" - result(:error) - end - - private - - def stack_is_already_created_or_failed? - case stack.stack_status - when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' - false - when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' - @sync_result = result_for_provider_status(stack.stack_status) - else - @sync_result = result(:unkown_status) + def initialize(stack, out) + @stack, @out = stack, out + @printed_events = [] + @sync_result = nil end - end - def sleep_times - [5]*5 + [10]*400 - end + def sync + puts_and_flush "Syncing stack '#{stack.id}'..." - def result(reason) - SyncResult.from_reason(reason) - end + sleep_times.detect do |sleep_time| + sleep sleep_time + stack.sync! + print_new_events - def result_for_provider_status(status) - provider_status_mapping = { - 'CREATE_COMPLETE' => :ok, - 'ROLLBACK_COMPLETE' => :stack_rolled_back, - 'DELETE_COMPLETE' => :stack_deleted, - 'NOT_FOUND' => :stack_not_found - } - result(provider_status_mapping.fetch(status)) - end + update_stack_status if stack_status_changed? + stack_is_already_created_or_failed? + end - def print_result_message - puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}", - stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+)) - end + @sync_result ||= result(:timeout) + print_result_message + @sync_result + rescue StandardError => e + DevopsLogger.logger.error e.message + puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" + result(:error) + end - def update_stack_status - ::Devops::Db.connector.stack_update(stack) - @last_status = stack.stack_status - end + private - def stack_status_changed? - @last_status != stack.stack_status - end - - def print_new_events - stack.events.each do |event| - unless @printed_events.include?(event["event_id"]) - @printed_events << event["event_id"] - out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}" + def stack_is_already_created_or_failed? + case stack.stack_status + when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' + false + when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' + @sync_result = result_for_provider_status(stack.stack_status) + else + @sync_result = result(:unkown_status) end end - out.flush + + def sleep_times + [5]*5 + [10]*400 + end + + def result(reason) + SyncResult.from_reason(reason) + end + + def result_for_provider_status(status) + provider_status_mapping = { + 'CREATE_COMPLETE' => :ok, + 'ROLLBACK_COMPLETE' => :stack_rolled_back, + 'DELETE_COMPLETE' => :stack_deleted, + 'NOT_FOUND' => :stack_not_found + } + result(provider_status_mapping.fetch(status)) + end + + def print_result_message + puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}", + stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+)) + end + + def update_stack_status + ::Devops::Db.connector.stack_update(stack) + @last_status = stack.stack_status + end + + def stack_status_changed? + @last_status != stack.stack_status + end + + def print_new_events + stack.events.each do |event| + unless @printed_events.include?(event["event_id"]) + @printed_events << event["event_id"] + out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}" + end + end + out.flush + end end end \ No newline at end of file diff --git a/devops-service/lib/executors/stack_executor/stack_servers_persister.rb b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb index 99424ed..8b70438 100644 --- a/devops-service/lib/executors/stack_executor/stack_servers_persister.rb +++ b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb @@ -1,111 +1,113 @@ require_relative 'chef_node_name_builder' # Fetches info about stack servers from provider and then persist them in mongo. -class StackServersPersister - include PutsAndFlush - attr_reader :stack, :out, :deleted, :servers_info +class Devops::Executor::StackExecutor + class StackServersPersister + include PutsAndFlush + attr_reader :stack, :out, :deleted, :servers_info - NEW = 'new' - DELETED = 'deleted' - PERSISTED = 'persisted' - JUST_PERSISTED = 'just_persisted' + NEW = 'new' + DELETED = 'deleted' + PERSISTED = 'persisted' + JUST_PERSISTED = 'just_persisted' - def initialize(stack, out) - @stack, @out = stack, out - @project = mongo.project(stack.project) - @deploy_env = @project.deploy_env(stack.deploy_env) - fetch_provider_servers_info - set_servers_states - end - - def persist_new_servers - with_state(NEW).each do |info| - info[:server] = persist_stack_server(info[:provider_info]) - info[:state] = JUST_PERSISTED + def initialize(stack, out) + @stack, @out = stack, out + @project = mongo.project(stack.project) + @deploy_env = @project.deploy_env(stack.deploy_env) + fetch_provider_servers_info + set_servers_states end - end - # returns: { priority_as_integer => [Servers] } - def just_persisted_by_priority - stack_servers_with_priority = {} - with_state(JUST_PERSISTED).each do |info| - stack_servers_with_priority[info[:priority]] ||= [] - stack_servers_with_priority[info[:priority]] << info[:server] - end - stack_servers_with_priority - end - - private - - def fetch_provider_servers_info - @servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info| - { - id: provider_info['id'], - provider_info: provider_info, - priority: provider_info['tags']['cid:priority'].to_i - } - end - end - - def set_servers_states - persisted = Devops::Db.connector.stack_servers(stack.id) - persisted_ids = persisted.map(&:id) - in_cloud_ids = @servers_info.map {|info| info[:id]} - new_ids = in_cloud_ids - persisted_ids - deleted_ids = persisted_ids - in_cloud_ids - - @servers_info.each do |info| - if new_ids.include?(info[:id]) - info[:state] = NEW - else - info[:state] = PERSISTED + def persist_new_servers + with_state(NEW).each do |info| + info[:server] = persist_stack_server(info[:provider_info]) + info[:state] = JUST_PERSISTED end end - @deleted = persisted.select { |server| deleted_ids.include?(server.id) } - end + # returns: { priority_as_integer => [Servers] } + def just_persisted_by_priority + stack_servers_with_priority = {} + with_state(JUST_PERSISTED).each do |info| + stack_servers_with_priority[info[:priority]] ||= [] + stack_servers_with_priority[info[:priority]] << info[:server] + end + stack_servers_with_priority + end - def with_state(state) - @servers_info.select { |info| info[:state] == state } - end + private - # takes a hash, returns Server model - def persist_stack_server(server_info) - server_attrs = { - '_id' => server_info['id'], - 'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values), - 'created_by' => stack.owner, - 'deploy_env' => @deploy_env.identifier, - 'key' => server_info['key_name'] || stack.provider_instance.ssh_key, - 'project' => @project.id, - 'provider' => @stack.provider, - 'provider_account' => @stack.provider_account, - 'remote_user' => mongo.image(@deploy_env.image).remote_user, - 'private_ip' => server_info['private_ip'], - 'public_ip' => server_info['public_ip'], - 'run_list' => stack.run_list || [], - 'stack' => stack.name - } + def fetch_provider_servers_info + @servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info| + { + id: provider_info['id'], + provider_info: provider_info, + priority: provider_info['tags']['cid:priority'].to_i + } + end + end - server = ::Devops::Model::Server.new(server_attrs) - mongo.server_insert(server) - server - end + def set_servers_states + persisted = Devops::Db.connector.stack_servers(stack.id) + persisted_ids = persisted.map(&:id) + in_cloud_ids = @servers_info.map {|info| info[:id]} + new_ids = in_cloud_ids - persisted_ids + deleted_ids = persisted_ids - in_cloud_ids - def get_name_builder(server_info) - ChefNodeNameBuilder.new( - provider_server_info: server_info, - project_id: @project.id, - env_id: @deploy_env.identifier, - owner: stack.owner - ) - end + @servers_info.each do |info| + if new_ids.include?(info[:id]) + info[:state] = NEW + else + info[:state] = PERSISTED + end + end - def incrementers_values - @incrementers_values ||= {} - end + @deleted = persisted.select { |server| deleted_ids.include?(server.id) } + end - def mongo - Devops::Db.connector + def with_state(state) + @servers_info.select { |info| info[:state] == state } + end + + # takes a hash, returns Server model + def persist_stack_server(server_info) + server_attrs = { + '_id' => server_info['id'], + 'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values), + 'created_by' => stack.owner, + 'deploy_env' => @deploy_env.identifier, + 'key' => server_info['key_name'] || stack.provider_instance.ssh_key, + 'project' => @project.id, + 'provider' => @stack.provider, + 'provider_account' => @stack.provider_account, + 'remote_user' => mongo.image(@deploy_env.image).remote_user, + 'private_ip' => server_info['private_ip'], + 'public_ip' => server_info['public_ip'], + 'run_list' => stack.run_list || [], + 'stack' => stack.name + } + + server = ::Devops::Model::Server.new(server_attrs) + mongo.server_insert(server) + server + end + + def get_name_builder(server_info) + ChefNodeNameBuilder.new( + provider_server_info: server_info, + project_id: @project.id, + env_id: @deploy_env.identifier, + owner: stack.owner + ) + end + + def incrementers_values + @incrementers_values ||= {} + end + + def mongo + Devops::Db.connector + end end end \ No newline at end of file diff --git a/devops-service/lib/helpers/job_waiter.rb b/devops-service/lib/helpers/job_waiter.rb new file mode 100644 index 0000000..e30ff22 --- /dev/null +++ b/devops-service/lib/helpers/job_waiter.rb @@ -0,0 +1,27 @@ +module Devops + module Helpers + class JobWaiter + class TimeoutReached < StandardError; end + + INTERVAL = 5 + + def initialize(job_id, timeout=5000) + @job_id, @timeout = job_id, timeout + end + + def wait + (@timeout / INTERVAL).times do + sleep(INTERVAL) + report = ::Devops::Db.connector.report(@job_id) + case report.status + when Worker::STATUS::COMPLETED + return 0 + when Worker::STATUS::FAILED + return report.job_result_code + end + end + raise TimeoutReached + end + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/helpers/job_waiter_spec.rb b/devops-service/spec/executors/helpers/job_waiter_spec.rb new file mode 100644 index 0000000..b01c15b --- /dev/null +++ b/devops-service/spec/executors/helpers/job_waiter_spec.rb @@ -0,0 +1,36 @@ +require 'lib/helpers/job_waiter' +require "db/mongo/models/report" + +module Devops::Helpers + RSpec.describe JobWaiter, stubbed_connector: true do + let(:job_waiter) { described_class.new('job_id') } + let(:report) { build(:report) } + + before do + allow(stubbed_connector).to receive(:report) { report } + allow(job_waiter).to receive(:sleep) + end + + it 'it returns 0 when job become completed' do + allow(report).to receive(:status) { 'completed' } + expect(job_waiter.wait).to eq 0 + end + + it 'returns error code when job failes' do + allow(report).to receive(:status) { 'failed' } + allow(report).to receive(:job_result_code) { 1 } + expect(job_waiter.wait).to eq 1 + end + + it 'sleeps until something happens' do + allow(report).to receive(:status).and_return('running', 'running', 'running', 'completed') + expect(job_waiter).to receive(:sleep).exactly(4).times + job_waiter.wait + end + + it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do + allow(report).to receive(:status) { 'running' } + expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached) + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/chef_node_name_builder_spec.rb b/devops-service/spec/executors/stack_executor/chef_node_name_builder_spec.rb new file mode 100644 index 0000000..6af378d --- /dev/null +++ b/devops-service/spec/executors/stack_executor/chef_node_name_builder_spec.rb @@ -0,0 +1,74 @@ +require 'lib/executors/stack_executor/chef_node_name_builder' +class Devops::Executor::StackExecutor + RSpec.describe ChefNodeNameBuilder do + # test with real response to ensure it is processed correctly + let(:server_info) do + { + "name"=>"stack-achuchkalov-aws-test-1455976199-master01", + "id"=>"i-fac32c7e", + "key_name"=>"achuchkalov", + "private_ip"=>"172.31.11.30", + "public_ip"=>"52.90.250.51", + "tags" => { + "Name"=>"master01", + "aws:cloudformation:logical-id"=>"EC2Instance1", + "aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199", + "StackTemplate"=>"1inst", + "aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6", + "cid:deployEnv"=>"test", + "cid:project"=>"aws", + "cid:user"=>"root", + "cid:priority"=>0 + } + } + end + let(:node_name_builder) { + ChefNodeNameBuilder.new( + provider_server_info: server_info, + project_id: 'proj', + env_id: 'dev' + ) + } + let(:build_node_name) { node_name_builder.build_node_name!({}) } + + def set_mask(mask) + server_info['tags']['cid:node-name-mask'] = mask + end + + describe '#build_node_name' do + it 'uses default mask (":project-:env-:instanceid")' do + expect(build_node_name).to eq 'proj-dev-i-fac32c7e' + end + + it 'substitutes :project, :env, :instanceid and :instancename' do + set_mask(':project/:env/:instanceid/:instancename') + expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01' + end + + it 'substitutes :time' do + set_mask(':project-:time') + expect(build_node_name).to match /proj-\d+/ + end + + describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do + it 'starts with 01 for empty hash' do + set_mask('node-:increment-slave:') + expect(node_name_builder.build_node_name!({})).to eq 'node-01' + end + + it "continues with next values if hash isn't empty" do + set_mask('node-:increment-slave:') + expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01' + expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02' + expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51' + end + + it 'could substitute different incrementers at once' do + set_mask('node-:increment-slave:-:increment-master:') + expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04' + end + end + + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/prioritized_groups_bootstrapper_spec.rb b/devops-service/spec/executors/stack_executor/prioritized_groups_bootstrapper_spec.rb new file mode 100644 index 0000000..04039c4 --- /dev/null +++ b/devops-service/spec/executors/stack_executor/prioritized_groups_bootstrapper_spec.rb @@ -0,0 +1,50 @@ +require 'lib/executors/stack_executor/prioritized_groups_bootstrapper' + +class Devops::Executor::StackExecutor + RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:jid) { 1000 } + let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) } + before do + @array1 = []; @array2 = []; @array3 = [] + @servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3} + end + + describe '#bootstrap_servers_by_priority' do + subject { groups_bootstrapper.bootstrap_servers_by_priority } + + it 'bootstraps servers in order by priorities, separately' do + allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) } + expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered + expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered + expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered + expect(subject).to be_ok + end + + it 'it returns :bootstrap_error result if error occured during bootstrap' do + allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { + [ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)] + } + expect(subject).to be_bootstrap_error + end + + it 'it returns :deploy_error result if error occured during deploy' do + allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { + [ServersBootstrapper::Result.from_reason(:deploy_error)] + } + expect(subject).to be_deploy_error + end + + it "doesn't bootstrap group if previous one failed" do + allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { + [ServersBootstrapper::Result.from_reason(:deploy_error)] + } + allow(ServersBootstrapper).to receive(:new).and_call_original + expect(ServersBootstrapper).to receive(:new).once + subject + end + + + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/servers_bootstrapper_spec.rb b/devops-service/spec/executors/stack_executor/servers_bootstrapper_spec.rb new file mode 100644 index 0000000..c5ce466 --- /dev/null +++ b/devops-service/spec/executors/stack_executor/servers_bootstrapper_spec.rb @@ -0,0 +1,43 @@ +require 'lib/executors/stack_executor/servers_bootstrapper' +class Devops::Executor::StackExecutor + RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:jid) { 1000 } + let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] } + let(:bootstrapper) { described_class.new(out, jid, servers ) } + let(:bootstrap_job_ids) { %w(100 200) } + + describe '#bootstrap_group' do + before do + allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids) + allow(stubbed_connector).to receive(:add_report_subreports) + allow(stubbed_connector).to receive(:report) + allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) { 0 } + end + + it 'start bootstrap workers and add subreports' do + expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner)) + expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids) + bootstrapper.bootstrap_group + end + + it 'returns :ok result if everything ok' do + expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok + end + + it 'returns proper error results' do + waiter = instance_double(Devops::Helpers::JobWaiter) + allow(waiter).to receive(:wait).and_return(2, 8) + allow(Devops::Helpers::JobWaiter).to receive(:new) { waiter } + expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error] + end + + it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do + allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) { + raise Devops::Helpers::JobWaiter::TimeoutReached + } + expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached + end + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb b/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb new file mode 100644 index 0000000..44542a5 --- /dev/null +++ b/devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb @@ -0,0 +1,94 @@ +require 'lib/executors/stack_executor/stack_creation_waiter' + +class Devops::Executor::StackExecutor + RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:stack) { build(:stack) } + let(:syncer) { described_class.new(stack, out) } + + before do + allow(stack).to receive(:sync!) + allow(stack).to receive(:events).and_return( [{'event_id' => 1}] ) + allow(syncer).to receive(:sleep) + allow(stubbed_connector).to receive(:stack_update) + end + + def setup_statuses(statuses_array) + statuses = statuses_array.to_enum + allow(stack).to receive(:sync!) { + stack.stack_status = statuses.next + } + end + + describe '#sync', stubbed_logger: true do + it 'waits for stack creating to be finished' do + setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']) + expect(syncer).to receive(:sleep).at_least(10).times + expect(stack).to receive(:sync!).at_least(10).times + syncer.sync + end + + it 'prints each message only once' do + event1 = {'event_id' => 1, 'timestamp' => 't1'} + event2 = {'event_id' => 2, 'timestamp' => 't2'} + event3 = {'event_id' => 3, 'timestamp' => 't3'} + + allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3]) + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) + expect(syncer).to receive(:sleep).exactly(4).times + expect(out).to receive(:puts).with(/t1/).once.ordered + expect(out).to receive(:puts).with(/t2/).once.ordered + expect(out).to receive(:puts).with(/t3/).once.ordered + syncer.sync + end + + it 'updates stack when status is changed' do + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) + expect(stubbed_connector).to receive(:stack_update).exactly(3).times + syncer.sync + end + + context 'when stack creating was successful' do + it 'updates stack in DB when stack creating is finished and returns 0' do + setup_statuses(['CREATE_COMPLETE']) + expect(stubbed_connector).to receive(:stack_update).with(stack) + expect(out).to receive(:puts).with(/CREATE_COMPLETE/) + expect(syncer.sync).to be_ok + end + end + + context 'when stack was rollbacked' do + it 'returns 1 (:stack_rolled_back)' do + setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) + expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/) + expect(syncer.sync).to be_stack_rolled_back + end + end + + context 'when unkown stack status was found' do + it 'returns 2 (:unkown_status)' do + setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) + expect(out).to receive(:puts).with(/unknown/) + expect(syncer.sync).to be_unkown_status + end + end + + context "when stack hasn't been synced in an hour" do + it 'returns 3 (:timeout)' do + allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} + expect(out).to receive(:puts).with(/hasn't been synced/) + expect(syncer.sync).to be_timeout + end + end + + context 'when an error occured during syncing', stubbed_logger: true do + it 'returns 5 (:error)' do + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) + allow(stubbed_connector).to receive(:stack_update) { raise } + expect(syncer.sync.code).to eq 5 + end + end + end + + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb b/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb new file mode 100644 index 0000000..919e4dc --- /dev/null +++ b/devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb @@ -0,0 +1,167 @@ +require 'lib/executors/stack_executor/stack_servers_persister' + +class Devops::Executor::StackExecutor + RSpec.describe StackServersPersister, stubbed_connector: true do + def info_hash_for_id(id) + { + 'id' => id, + 'name' => 'server_name', + 'key_name' => 'key', + 'private_ip' => '127.0.0.1', + 'public_ip' => '127.0.0.2', + 'tags' => { + 'cid:priority' => '3', + 'Name' => 'server1' + } + } + end + + let(:out) { double(:out, puts: nil, flush: nil) } + let(:run_list) { ['role[asd]'] } + let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) } + let(:project) { build(:project, id: 'name') } + let(:persister) { described_class.new(stack, out) } + let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } + let(:new_server_info) { info_hash_for_id('i-new') } + let(:deleted_server_info) { info_hash_for_id('i-deleted') } + let(:persisted_server_info) { info_hash_for_id('i-persisted') } + + before do + allow(stubbed_connector).to receive(:project) { project } + allow(stubbed_connector).to receive(:image) { + instance_double(Devops::Model::Image, remote_user: 'user') + } + allow(stubbed_connector).to receive(:server_insert) + allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] } + allow(stack).to receive(:provider_instance) { provider } + allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]} + end + + describe '#initialize' do + it 'fetches stack servers info' do + expect(provider).to receive(:stack_servers).with(stack) + described_class.new(stack, out) + end + + it "doesn't raise error if cid:priority tag is absent" do + new_server_info['tags'].delete('cid:priority') + expect { described_class.new(stack, out) }.not_to raise_error + end + + it 'sets proper statuses' do + infos = described_class.new(stack, out).servers_info + expect(infos.length).to eq 2 + expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted' + expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new' + end + + it 'sets deleted servers' do + deleted = described_class.new(stack, out).deleted + expect(deleted.length).to eq 1 + expect(deleted.first.id).to eq 'i-deleted' + end + end + + describe '#persist_new_servers' do + it 'takes id, key_name, private_ip and public_ip attrs from info hash' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.id).to eq 'i-new' + expect(server.key).to eq 'key' + expect(server.private_ip).to eq '127.0.0.1' + expect(server.public_ip).to eq '127.0.0.2' + end + persister.persist_new_servers + end + + it 'takes created_by, run_list and stack attrs from stack' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.created_by).to eq 'root' + expect(server.run_list).to eq run_list + expect(server.stack).to eq 'iamstack' + end + persister.persist_new_servers + end + + it 'takes remote_user from image user' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.remote_user).to eq 'user' + end + persister.persist_new_servers + end + + it "takes deploy_env from project's deploy_env identifier" do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.deploy_env).to eq 'foo' + end + persister.persist_new_servers + end + + it "takes default provider's ssh key if info doesn't contain it" do + allow(provider).to receive(:ssh_key) { 'default_key' } + new_server_info.delete('key_name') + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.key).to eq 'default_key' + end + persister.persist_new_servers + end + + it "sets server's run list to empty array if stack's run_list is nil" do + stack.run_list = nil + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.run_list).to eq [] + end + persister.persist_new_servers + end + + it 'build chef_node_name with default mask ":project-:env-:instanceid"' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'name-foo-i-new' + end + persister.persist_new_servers + end + + it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do + new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123' + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'name-server1-123' + end + persister.persist_new_servers + end + + it "sets provider and provider account from stack" do + stack.provider_account = 'foo' + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.provider).to eq 'ec2' + expect(server.provider_account).to eq 'foo' + end + persister.persist_new_servers + end + + describe 'incremented variables' do + it 'substitutes :increment-groupid: with incrementing numbers' do + allow(provider).to receive(:stack_servers) {[ + {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}, + {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'} + ]} + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'node-01-dev' + end.ordered + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'node-02-dev' + end + persister.persist_new_servers + end + end + end + + describe '#just_persisted_by_priority' do + it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do + persister.persist_new_servers + result = persister.just_persisted_by_priority + expect(result).to be_a(Hash) + expect(result.size).to eq 1 + expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1) + end + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/stack_executor_spec.rb b/devops-service/spec/executors/stack_executor_spec.rb index 65c8e8c..2fc1517 100644 --- a/devops-service/spec/executors/stack_executor_spec.rb +++ b/devops-service/spec/executors/stack_executor_spec.rb @@ -1,65 +1,67 @@ require 'lib/executors/stack_executor' -RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do - let(:out) { double('out', puts: nil, flush: nil) } - let(:stack) { build(:stack) } - let(:executor_without_stack) { described_class.new(out: out) } - let(:executor_with_stack) { described_class.new(out: out, stack: stack) } +class Devops::Executor::StackExecutor + RSpec.describe self, type: :executor, stubbed_connector: true, stubbed_logger: true do + let(:out) { double('out', puts: nil, flush: nil) } + let(:stack) { build(:stack) } + let(:executor_without_stack) { described_class.new(out: out) } + let(:executor_with_stack) { described_class.new(out: out, stack: stack) } - describe '#wait_till_stack_is_created' do - it "return true if syncer returns ok" do - allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) } - expect(executor_with_stack.wait_till_stack_is_created).to be true + describe '#wait_till_stack_is_created' do + it "return true if syncer returns ok" do + allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) } + expect(executor_with_stack.wait_till_stack_is_created).to be true + end + + it "return false if syncer returns not ok" do + allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') } + expect(executor_with_stack.wait_till_stack_is_created).to be false + end end - it "return false if syncer returns not ok" do - allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') } - expect(executor_with_stack.wait_till_stack_is_created).to be false + describe '#create_stack', stubbed_connector: true do + it 'initiate creation in cloud and persists stack' do + expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out) + expect(stubbed_connector).to receive(:stack_insert) + executor_with_stack.create_stack({'provider' => 'ec2'}) + end + end + + describe '#persist_stack_servers' do + let(:persister) { + instance_double(StackServersPersister, + persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil + ) + } + before { allow(StackServersPersister).to receive(:new).and_return(persister) } + + it 'calls StackServersPersister#persist_new_servers' do + expect(persister).to receive(:persist_new_servers) + executor_with_stack.persist_stack_servers + end + + it 'returns hash with :just_persisted_by_priority and :deleted keys' do + expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil) + end + end + + describe '#bootstrap_servers_by_priority' do + it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do + result = double('bootstrap_result') + allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result } + expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) + expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result + end + end + + describe '#delete_stack', stubbed_connector: true do + it 'deletes stack from cloud, then deletes stack servers, and then deletes stack itself' do + expect(stack).to receive(:delete_stack_in_cloud!).ordered + expect(stubbed_connector).to receive(:stack_servers_delete).ordered + expect(stubbed_connector).to receive(:stack_delete).ordered + executor_with_stack.delete_stack + end + end end - - describe '#create_stack', stubbed_connector: true do - it 'initiate creation in cloud and persists stack' do - expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out) - expect(stubbed_connector).to receive(:stack_insert) - executor_with_stack.create_stack({'provider' => 'ec2'}) - end - end - - describe '#persist_stack_servers' do - let(:persister) { - instance_double(StackServersPersister, persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil) - } - before { allow(StackServersPersister).to receive(:new).and_return(persister) } - - it 'calls StackServersPersister#persist_new_servers' do - expect(persister).to receive(:persist_new_servers) - executor_with_stack.persist_stack_servers - end - - it 'returns hash with :just_persisted_by_priority and :deleted keys' do - expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil) - end - end - - describe '#bootstrap_servers_by_priority' do - it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do - result = double('bootstrap_result') - allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result } - expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) - expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result - end - end - - describe '#delete_stack', stubbed_connector: true do - it 'deletes stack from cloud, then deletes stack servers, and then deletes stack itself' do - expect(stack).to receive(:delete_stack_in_cloud!).ordered - expect(stubbed_connector).to receive(:stack_servers_delete).ordered - expect(stubbed_connector).to receive(:stack_delete).ordered - executor_with_stack.delete_stack - end - - end - - end \ No newline at end of file diff --git a/devops-service/spec/workers/helpers/job_waiter_spec.rb b/devops-service/spec/workers/helpers/job_waiter_spec.rb deleted file mode 100644 index 767e59b..0000000 --- a/devops-service/spec/workers/helpers/job_waiter_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -require 'workers/helpers/job_waiter' -require "db/mongo/models/report" - -RSpec.describe JobWaiter, stubbed_connector: true do - let(:job_waiter) { described_class.new('job_id') } - - before do - @report_double = instance_double(Devops::Model::Report) - allow(stubbed_connector).to receive(:report) { @report_double } - allow(job_waiter).to receive(:sleep) - end - - it 'it returns 0 when job become completed' do - allow(@report_double).to receive(:status) { 'completed' } - expect(job_waiter.wait).to eq 0 - end - - it 'returns error code when job failes' do - allow(@report_double).to receive(:status) { 'failed' } - allow(@report_double).to receive(:job_result_code) { 1 } - expect(job_waiter.wait).to eq 1 - end - - it 'sleeps until something happens' do - allow(@report_double).to receive(:status).and_return('running', 'running', 'running', 'completed') - expect(job_waiter).to receive(:sleep).exactly(4).times - job_waiter.wait - end - - it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do - allow(@report_double).to receive(:status) { 'running' } - expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached) - end - -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb b/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb deleted file mode 100644 index cd4e864..0000000 --- a/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb +++ /dev/null @@ -1,72 +0,0 @@ -require 'lib/executors/stack_executor/chef_node_name_builder' -RSpec.describe ChefNodeNameBuilder do - # test with real response to ensure it is processed correctly - let(:server_info) do - { - "name"=>"stack-achuchkalov-aws-test-1455976199-master01", - "id"=>"i-fac32c7e", - "key_name"=>"achuchkalov", - "private_ip"=>"172.31.11.30", - "public_ip"=>"52.90.250.51", - "tags" => { - "Name"=>"master01", - "aws:cloudformation:logical-id"=>"EC2Instance1", - "aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199", - "StackTemplate"=>"1inst", - "aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6", - "cid:deployEnv"=>"test", - "cid:project"=>"aws", - "cid:user"=>"root", - "cid:priority"=>0 - } - } - end - let(:node_name_builder) { - ChefNodeNameBuilder.new( - provider_server_info: server_info, - project_id: 'proj', - env_id: 'dev' - ) - } - let(:build_node_name) { node_name_builder.build_node_name!({}) } - - def set_mask(mask) - server_info['tags']['cid:node-name-mask'] = mask - end - - describe '#build_node_name' do - it 'uses default mask (":project-:env-:instanceid")' do - expect(build_node_name).to eq 'proj-dev-i-fac32c7e' - end - - it 'substitutes :project, :env, :instanceid and :instancename' do - set_mask(':project/:env/:instanceid/:instancename') - expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01' - end - - it 'substitutes :time' do - set_mask(':project-:time') - expect(build_node_name).to match /proj-\d+/ - end - - describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do - it 'starts with 01 for empty hash' do - set_mask('node-:increment-slave:') - expect(node_name_builder.build_node_name!({})).to eq 'node-01' - end - - it "continues with next values if hash isn't empty" do - set_mask('node-:increment-slave:') - expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01' - expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02' - expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51' - end - - it 'could substitute different incrementers at once' do - set_mask('node-:increment-slave:-:increment-master:') - expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04' - end - end - - end -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb b/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb deleted file mode 100644 index f23f923..0000000 --- a/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'lib/executors/stack_executor/prioritized_groups_bootstrapper' - -RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do - let(:out) { double(:out, puts: nil, flush: nil) } - let(:jid) { 1000 } - let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) } - before do - @array1 = []; @array2 = []; @array3 = [] - @servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3} - end - - describe '#bootstrap_servers_by_priority' do - subject { groups_bootstrapper.bootstrap_servers_by_priority } - - it 'bootstraps servers in order by priorities, separately' do - allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) } - expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered - expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered - expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered - expect(subject).to be_ok - end - - it 'it returns :bootstrap_error result if error occured during bootstrap' do - allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { - [ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)] - } - expect(subject).to be_bootstrap_error - end - - it 'it returns :deploy_error result if error occured during deploy' do - allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { - [ServersBootstrapper::Result.from_reason(:deploy_error)] - } - expect(subject).to be_deploy_error - end - - it "doesn't bootstrap group if previous one failed" do - allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) { - [ServersBootstrapper::Result.from_reason(:deploy_error)] - } - allow(ServersBootstrapper).to receive(:new).and_call_original - expect(ServersBootstrapper).to receive(:new).once - subject - end - - - end -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb b/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb deleted file mode 100644 index 056707a..0000000 --- a/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'lib/executors/stack_executor/servers_bootstrapper' - -RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do - let(:out) { double(:out, puts: nil, flush: nil) } - let(:jid) { 1000 } - let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] } - let(:bootstrapper) { described_class.new(out, jid, servers ) } - let(:bootstrap_job_ids) { %w(100 200) } - - describe '#bootstrap_group' do - before do - allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids) - allow(stubbed_connector).to receive(:add_report_subreports) - allow(stubbed_connector).to receive(:report) - allow_any_instance_of(JobWaiter).to receive(:wait) { 0 } - end - - it 'start bootstrap workers and add subreports' do - expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner)) - expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids) - bootstrapper.bootstrap_group - end - - it 'returns :ok result if everything ok' do - expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok - end - - it 'returns proper error results' do - waiter = instance_double(JobWaiter) - allow(waiter).to receive(:wait).and_return(2, 8) - allow(JobWaiter).to receive(:new) { waiter } - expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error] - end - - it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do - allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached } - expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached - end - end -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb deleted file mode 100644 index 17556c5..0000000 --- a/devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb +++ /dev/null @@ -1,91 +0,0 @@ -require 'lib/executors/stack_executor/stack_creation_waiter' -RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do - let(:out) { double(:out, puts: nil, flush: nil) } - let(:stack) { build(:stack) } - let(:syncer) { described_class.new(stack, out) } - - before do - allow(stack).to receive(:sync!) - allow(stack).to receive(:events).and_return( [{'event_id' => 1}] ) - allow(syncer).to receive(:sleep) - allow(stubbed_connector).to receive(:stack_update) - end - - def setup_statuses(statuses_array) - statuses = statuses_array.to_enum - allow(stack).to receive(:sync!) { - stack.stack_status = statuses.next - } - end - - describe '#sync', stubbed_logger: true do - it 'waits for stack creating to be finished' do - setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']) - expect(syncer).to receive(:sleep).at_least(10).times - expect(stack).to receive(:sync!).at_least(10).times - syncer.sync - end - - it 'prints each message only once' do - event1 = {'event_id' => 1, 'timestamp' => 't1'} - event2 = {'event_id' => 2, 'timestamp' => 't2'} - event3 = {'event_id' => 3, 'timestamp' => 't3'} - - allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3]) - setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) - expect(syncer).to receive(:sleep).exactly(4).times - expect(out).to receive(:puts).with(/t1/).once.ordered - expect(out).to receive(:puts).with(/t2/).once.ordered - expect(out).to receive(:puts).with(/t3/).once.ordered - syncer.sync - end - - it 'updates stack when status is changed' do - setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) - expect(stubbed_connector).to receive(:stack_update).exactly(3).times - syncer.sync - end - - context 'when stack creating was successful' do - it 'updates stack in DB when stack creating is finished and returns 0' do - setup_statuses(['CREATE_COMPLETE']) - expect(stubbed_connector).to receive(:stack_update).with(stack) - expect(out).to receive(:puts).with(/CREATE_COMPLETE/) - expect(syncer.sync).to be_ok - end - end - - context 'when stack was rollbacked' do - it 'returns 1 (:stack_rolled_back)' do - setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) - expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/) - expect(syncer.sync).to be_stack_rolled_back - end - end - - context 'when unkown stack status was found' do - it 'returns 2 (:unkown_status)' do - setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) - expect(out).to receive(:puts).with(/unknown/) - expect(syncer.sync).to be_unkown_status - end - end - - context "when stack hasn't been synced in an hour" do - it 'returns 3 (:timeout)' do - allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} - expect(out).to receive(:puts).with(/hasn't been synced/) - expect(syncer.sync).to be_timeout - end - end - - context 'when an error occured during syncing', stubbed_logger: true do - it 'returns 5 (:error)' do - setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) - allow(stubbed_connector).to receive(:stack_update) { raise } - expect(syncer.sync.code).to eq 5 - end - end - end - -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb deleted file mode 100644 index 1127614..0000000 --- a/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb +++ /dev/null @@ -1,166 +0,0 @@ -require 'lib/executors/stack_executor/stack_servers_persister' - -RSpec.describe StackServersPersister, stubbed_connector: true do - def info_hash_for_id(id) - { - 'id' => id, - 'name' => 'server_name', - 'key_name' => 'key', - 'private_ip' => '127.0.0.1', - 'public_ip' => '127.0.0.2', - 'tags' => { - 'cid:priority' => '3', - 'Name' => 'server1' - } - } - end - - let(:out) { double(:out, puts: nil, flush: nil) } - let(:run_list) { ['role[asd]'] } - let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) } - let(:project) { build(:project, id: 'name') } - let(:persister) { described_class.new(stack, out) } - let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } - let(:new_server_info) { info_hash_for_id('i-new') } - let(:deleted_server_info) { info_hash_for_id('i-deleted') } - let(:persisted_server_info) { info_hash_for_id('i-persisted') } - - before do - allow(stubbed_connector).to receive(:project) { project } - allow(stubbed_connector).to receive(:image) { - instance_double(Devops::Model::Image, remote_user: 'user') - } - allow(stubbed_connector).to receive(:server_insert) - allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] } - allow(stack).to receive(:provider_instance) { provider } - allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]} - end - - describe '#initialize' do - it 'fetches stack servers info' do - expect(provider).to receive(:stack_servers).with(stack) - described_class.new(stack, out) - end - - it "doesn't raise error if cid:priority tag is absent" do - new_server_info['tags'].delete('cid:priority') - expect { described_class.new(stack, out) }.not_to raise_error - end - - it 'sets proper statuses' do - infos = described_class.new(stack, out).servers_info - expect(infos.length).to eq 2 - expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted' - expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new' - end - - it 'sets deleted servers' do - deleted = described_class.new(stack, out).deleted - expect(deleted.length).to eq 1 - expect(deleted.first.id).to eq 'i-deleted' - end - end - - describe '#persist_new_servers' do - it 'takes id, key_name, private_ip and public_ip attrs from info hash' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.id).to eq 'i-new' - expect(server.key).to eq 'key' - expect(server.private_ip).to eq '127.0.0.1' - expect(server.public_ip).to eq '127.0.0.2' - end - persister.persist_new_servers - end - - it 'takes created_by, run_list and stack attrs from stack' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.created_by).to eq 'root' - expect(server.run_list).to eq run_list - expect(server.stack).to eq 'iamstack' - end - persister.persist_new_servers - end - - it 'takes remote_user from image user' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.remote_user).to eq 'user' - end - persister.persist_new_servers - end - - it "takes deploy_env from project's deploy_env identifier" do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.deploy_env).to eq 'foo' - end - persister.persist_new_servers - end - - it "takes default provider's ssh key if info doesn't contain it" do - allow(provider).to receive(:ssh_key) { 'default_key' } - new_server_info.delete('key_name') - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.key).to eq 'default_key' - end - persister.persist_new_servers - end - - it "sets server's run list to empty array if stack's run_list is nil" do - stack.run_list = nil - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.run_list).to eq [] - end - persister.persist_new_servers - end - - it 'build chef_node_name with default mask ":project-:env-:instanceid"' do - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'name-foo-i-new' - end - persister.persist_new_servers - end - - it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do - new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123' - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'name-server1-123' - end - persister.persist_new_servers - end - - it "sets provider and provider account from stack" do - stack.provider_account = 'foo' - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.provider).to eq 'ec2' - expect(server.provider_account).to eq 'foo' - end - persister.persist_new_servers - end - - describe 'incremented variables' do - it 'substitutes :increment-groupid: with incrementing numbers' do - allow(provider).to receive(:stack_servers) {[ - {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}, - {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'} - ]} - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'node-01-dev' - end.ordered - expect(stubbed_connector).to receive(:server_insert) do |server| - expect(server.chef_node_name).to eq 'node-02-dev' - end - persister.persist_new_servers - end - end - end - - describe '#just_persisted_by_priority' do - it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do - persister.persist_new_servers - result = persister.just_persisted_by_priority - expect(result).to be_a(Hash) - expect(result.size).to eq 1 - expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1) - end - end - -end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb index bc5d632..3891cf2 100644 --- a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb @@ -16,7 +16,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini } def bootstrap_result(reason) - ServersBootstrapper::Result.from_reason(reason) + Devops::Executor::StackExecutor::ServersBootstrapper::Result.from_reason(reason) end before do diff --git a/devops-service/workers/helpers/job_waiter.rb b/devops-service/workers/helpers/job_waiter.rb deleted file mode 100644 index 72bbede..0000000 --- a/devops-service/workers/helpers/job_waiter.rb +++ /dev/null @@ -1,23 +0,0 @@ -class JobWaiter - class TimeoutReached < StandardError; end - - INTERVAL = 5 - - def initialize(job_id, timeout=5000) - @job_id, @timeout = job_id, timeout - end - - def wait - (@timeout / INTERVAL).times do - sleep(INTERVAL) - report = ::Devops::Db.connector.report(@job_id) - case report.status - when Worker::STATUS::COMPLETED - return 0 - when Worker::STATUS::FAILED - return report.job_result_code - end - end - raise TimeoutReached - end -end \ No newline at end of file