From 28b4099f27d3c3144ad1d5308757406754949774 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 30 Mar 2016 12:05:59 +0300 Subject: [PATCH] move stack operations to StackExecutor --- .../lib/executors/stack_executor.rb | 64 ++++++++++++++ .../stack_executor}/chef_node_name_builder.rb | 0 .../prioritized_groups_bootstrapper.rb | 0 .../stack_executor}/servers_bootstrapper.rb | 0 .../stack_executor}/stack_creation_waiter.rb | 0 .../stack_servers_persister.rb | 2 +- .../expiration_scheduler_spec.rb | 0 .../spec/executors/stack_executor_spec.rb | 65 ++++++++++++++ .../chef_node_name_builder_spec.rb | 2 +- .../prioritized_groups_bootstrapper_spec.rb | 2 +- .../servers_bootstrapper_spec.rb | 2 +- ..._spec.rb => stack_creation_waiter_spec.rb} | 2 +- .../stack_servers_persister_spec.rb | 2 +- .../workers/stack_bootstrap_worker_spec.rb | 84 +++++++------------ .../workers/stack_bootstrap_worker.rb | 76 ++++++++--------- 15 files changed, 198 insertions(+), 103 deletions(-) create mode 100644 devops-service/lib/executors/stack_executor.rb rename devops-service/{workers/stack_bootstrap => lib/executors/stack_executor}/chef_node_name_builder.rb (100%) rename devops-service/{workers/stack_bootstrap => lib/executors/stack_executor}/prioritized_groups_bootstrapper.rb (100%) rename devops-service/{workers/stack_bootstrap => lib/executors/stack_executor}/servers_bootstrapper.rb (100%) rename devops-service/{workers/stack_bootstrap => lib/executors/stack_executor}/stack_creation_waiter.rb (100%) rename devops-service/{workers/stack_bootstrap => lib/executors/stack_executor}/stack_servers_persister.rb (98%) rename devops-service/spec/executors/{ => server_executor}/expiration_scheduler_spec.rb (100%) create mode 100644 devops-service/spec/executors/stack_executor_spec.rb rename devops-service/spec/workers/stack_bootstrap/{stack_synchronizer_spec.rb => stack_creation_waiter_spec.rb} (98%) diff --git a/devops-service/lib/executors/stack_executor.rb b/devops-service/lib/executors/stack_executor.rb new file mode 100644 index 0000000..fff04e5 --- /dev/null +++ b/devops-service/lib/executors/stack_executor.rb @@ -0,0 +1,64 @@ +require "db/mongo/models/stack/stack_factory" +require "lib/puts_and_flush" +require_relative "stack_executor/stack_creation_waiter" +require_relative "stack_executor/stack_servers_persister" +require_relative "stack_executor/prioritized_groups_bootstrapper" + +module Devops + module Executor + class StackExecutor + include PutsAndFlush + attr_reader :out, :stack + + def initialize(options) + @out = options.fetch(:out) + @stack = options[:stack] + end + + def wait_till_stack_is_created + wait_result = StackCreationWaiter.new(stack, out).sync + + if wait_result.ok? + puts_and_flush "\nStack '#{stack.name}' has been created" + true + else + puts_and_flush "An error ocurred during stack creation: #{wait_result.reason}" + false + end + end + + def create_stack(stack_attrs) + @stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, out) + mongo.stack_insert(@stack) + end + + def persist_stack_servers + puts_and_flush 'Start saving stack servers into CID database.' + persister = StackServersPersister.new(stack, out) + persister.persist_new_servers + puts_and_flush "Stack servers have been saved." + { + just_persisted_by_priority: persister.just_persisted_by_priority, + deleted: persister.deleted + } + end + + def bootstrap_servers_by_priority(servers_by_priorities, jid) + PrioritizedGroupsBootstrapper.new(out, jid, servers_by_priorities).bootstrap_servers_by_priority + end + + def delete_stack + stack.delete_stack_in_cloud! + mongo.stack_servers_delete(stack.name) + mongo.stack_delete(stack.id) + end + + private + + def mongo + Devops::Db.connector + end + + end + end +end diff --git a/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb b/devops-service/lib/executors/stack_executor/chef_node_name_builder.rb similarity index 100% rename from devops-service/workers/stack_bootstrap/chef_node_name_builder.rb rename to devops-service/lib/executors/stack_executor/chef_node_name_builder.rb diff --git a/devops-service/workers/stack_bootstrap/prioritized_groups_bootstrapper.rb b/devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb similarity index 100% rename from devops-service/workers/stack_bootstrap/prioritized_groups_bootstrapper.rb rename to devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb diff --git a/devops-service/workers/stack_bootstrap/servers_bootstrapper.rb b/devops-service/lib/executors/stack_executor/servers_bootstrapper.rb similarity index 100% rename from devops-service/workers/stack_bootstrap/servers_bootstrapper.rb rename to devops-service/lib/executors/stack_executor/servers_bootstrapper.rb diff --git a/devops-service/workers/stack_bootstrap/stack_creation_waiter.rb b/devops-service/lib/executors/stack_executor/stack_creation_waiter.rb similarity index 100% rename from devops-service/workers/stack_bootstrap/stack_creation_waiter.rb rename to devops-service/lib/executors/stack_executor/stack_creation_waiter.rb diff --git a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb similarity index 98% rename from devops-service/workers/stack_bootstrap/stack_servers_persister.rb rename to devops-service/lib/executors/stack_executor/stack_servers_persister.rb index 1fbcdd7..99424ed 100644 --- a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb +++ b/devops-service/lib/executors/stack_executor/stack_servers_persister.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/chef_node_name_builder' +require_relative 'chef_node_name_builder' # Fetches info about stack servers from provider and then persist them in mongo. class StackServersPersister diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/server_executor/expiration_scheduler_spec.rb similarity index 100% rename from devops-service/spec/executors/expiration_scheduler_spec.rb rename to devops-service/spec/executors/server_executor/expiration_scheduler_spec.rb diff --git a/devops-service/spec/executors/stack_executor_spec.rb b/devops-service/spec/executors/stack_executor_spec.rb new file mode 100644 index 0000000..65c8e8c --- /dev/null +++ b/devops-service/spec/executors/stack_executor_spec.rb @@ -0,0 +1,65 @@ +require 'lib/executors/stack_executor' + +RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do + let(:out) { double('out', puts: nil, flush: nil) } + let(:stack) { build(:stack) } + let(:executor_without_stack) { described_class.new(out: out) } + let(:executor_with_stack) { described_class.new(out: out, stack: stack) } + + describe '#wait_till_stack_is_created' do + it "return true if syncer returns ok" do + allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) } + expect(executor_with_stack.wait_till_stack_is_created).to be true + end + + it "return false if syncer returns not ok" do + allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') } + expect(executor_with_stack.wait_till_stack_is_created).to be false + end + end + + describe '#create_stack', stubbed_connector: true do + it 'initiate creation in cloud and persists stack' do + expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out) + expect(stubbed_connector).to receive(:stack_insert) + executor_with_stack.create_stack({'provider' => 'ec2'}) + end + end + + describe '#persist_stack_servers' do + let(:persister) { + instance_double(StackServersPersister, persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil) + } + before { allow(StackServersPersister).to receive(:new).and_return(persister) } + + it 'calls StackServersPersister#persist_new_servers' do + expect(persister).to receive(:persist_new_servers) + executor_with_stack.persist_stack_servers + end + + it 'returns hash with :just_persisted_by_priority and :deleted keys' do + expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil) + end + end + + describe '#bootstrap_servers_by_priority' do + it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do + result = double('bootstrap_result') + allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result } + expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) + expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result + end + end + + describe '#delete_stack', stubbed_connector: true do + it 'deletes stack from cloud, then deletes stack servers, and then deletes stack itself' do + expect(stack).to receive(:delete_stack_in_cloud!).ordered + expect(stubbed_connector).to receive(:stack_servers_delete).ordered + expect(stubbed_connector).to receive(:stack_delete).ordered + executor_with_stack.delete_stack + end + + end + + +end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb b/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb index 2187de8..cd4e864 100644 --- a/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/chef_node_name_builder' +require 'lib/executors/stack_executor/chef_node_name_builder' RSpec.describe ChefNodeNameBuilder do # test with real response to ensure it is processed correctly let(:server_info) do diff --git a/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb b/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb index 183a1f7..f23f923 100644 --- a/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/prioritized_groups_bootstrapper' +require 'lib/executors/stack_executor/prioritized_groups_bootstrapper' RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do let(:out) { double(:out, puts: nil, flush: nil) } diff --git a/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb b/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb index 0fb4d49..056707a 100644 --- a/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/servers_bootstrapper' +require 'lib/executors/stack_executor/servers_bootstrapper' RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do let(:out) { double(:out, puts: nil, flush: nil) } diff --git a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb similarity index 98% rename from devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb rename to devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb index e7c5497..17556c5 100644 --- a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/stack_creation_waiter' +require 'lib/executors/stack_executor/stack_creation_waiter' RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do let(:out) { double(:out, puts: nil, flush: nil) } let(:stack) { build(:stack) } diff --git a/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb index 0a611aa..1127614 100644 --- a/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb @@ -1,4 +1,4 @@ -require 'workers/stack_bootstrap/stack_servers_persister' +require 'lib/executors/stack_executor/stack_servers_persister' RSpec.describe StackServersPersister, stubbed_connector: true do def info_hash_for_id(id) diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb index ee9f79f..bc5d632 100644 --- a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb @@ -1,21 +1,29 @@ require 'workers/stack_bootstrap_worker' +require 'lib/executors/stack_executor' RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys } let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) } let(:worker) { described_class.new } + let(:executor) { + instance_double(Devops::Executor::StackExecutor, + wait_till_stack_is_created: true, + create_stack: Devops::Model::StackEc2.new(stack_attrs), + persist_stack_servers: nil, + delete_stack: nil + ) + } + + def bootstrap_result(reason) + ServersBootstrapper::Result.from_reason(reason) + end before do - allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) - allow(worker).to receive(:update_report) - allow(worker).to receive(:wait_till_stack_is_created) { true } + allow(worker).to receive(:executor) { executor } allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} } - allow(worker).to receive(:bootstrap_servers_by_priority) { ServersBootstrapper::Result.new(0) } - - allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) } - allow(Devops::Model::StackEc2).to receive(:create) + allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:ok) } end @@ -29,13 +37,22 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini end it 'updates report about operation, creates stack and persists stack servers' do - allow(worker).to receive(:create_stack).and_call_original expect(worker).to receive(:update_report).ordered - expect(worker).to receive(:create_stack).ordered + expect(executor).to receive(:create_stack).ordered expect(worker).to receive(:persist_stack_servers).ordered perform_without_bootstrap end + it "waits for stack creation to be completed" do + expect(executor).to receive(:wait_till_stack_is_created) + perform_with_bootstrap + end + + it "returns 1 if waiting wasn't successful" do + allow(executor).to receive(:wait_till_stack_is_created) { false } + expect(perform_with_bootstrap).to eq 1 + end + context 'if without_bootstrap is true' do it "doesn't bootstrap servers" do expect(worker).not_to receive(:bootstrap_servers_by_priority) @@ -53,28 +70,20 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini end it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do - allow(worker).to receive(:bootstrap_servers_by_priority) { - ServersBootstrapper::Result.from_reason(:bootstrap_error) - } - expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!) - expect(stubbed_connector).to receive(:stack_servers_delete) - expect(stubbed_connector).to receive(:stack_delete) + allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:bootstrap_error) } + expect(executor).to receive(:delete_stack) perform_with_bootstrap expect(perform_with_bootstrap).to eq 2 end it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do - allow(worker).to receive(:bootstrap_servers_by_priority) { - ServersBootstrapper::Result.from_reason(:deploy_error) - } + allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:deploy_error) } expect(worker).not_to receive(:rollback_stack!) expect(perform_with_bootstrap).to eq 3 end it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do - allow(worker).to receive(:bootstrap_servers_by_priority) { - ServersBootstrapper::Result.from_reason(:timeout_reached) - } + allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:timeout_reached) } expect(worker).not_to receive(:rollback_stack!) expect(perform_with_bootstrap).to eq 4 end @@ -82,41 +91,8 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do error = StandardError.new allow(worker).to receive(:bootstrap_servers_by_priority) { raise error } - allow(worker).to receive(:rollback_stack!) expect(worker).to receive(:rollback_stack!) expect{perform_with_bootstrap}.to raise_error(error) end end - - context "without stubbing methods", stubbed_connector: true do - before do - allow(worker).to receive(:wait_till_stack_is_created).and_call_original - allow(worker).to receive(:persist_stack_servers).and_call_original - allow(worker).to receive(:bootstrap_servers_by_priority).and_call_original - allow(StackServersPersister).to receive(:new) { - instance_double(StackServersPersister, - persist_new_servers: nil, - just_persisted_by_priority: {1 => build_list(:server, 2)} - ) - } - allow(PrioritizedGroupsBootstrapper).to receive(:new) { - instance_double(PrioritizedGroupsBootstrapper, bootstrap_servers_by_priority: ServersBootstrapper::Result.new(0)) - } - end - - it "return 0 if syncer returns ok" do - allow(StackCreationWaiter).to receive(:new) { - instance_double(StackCreationWaiter, sync: StackCreationWaiter::SyncResult.new(0)) - } - expect(StackServersPersister).to receive(:new).with(instance_of(Devops::Model::StackEc2), anything) - expect(perform_with_bootstrap).to eq 0 - end - - it 'returns 1 if syncer returns error' do - allow(StackCreationWaiter).to receive(:new) { - instance_double(StackCreationWaiter, sync: StackCreationWaiter::SyncResult.new(5)) - } - expect(perform_with_bootstrap).to eq 1 - end - end end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index a4900e8..2fb1018 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -1,8 +1,4 @@ -require "db/mongo/models/stack/stack_factory" -require "db/mongo/models/project" -require "workers/stack_bootstrap/stack_creation_waiter" -require "workers/stack_bootstrap/servers_bootstrapper" -require "workers/stack_bootstrap/stack_servers_persister" +require 'lib/executors/stack_executor' class StackBootstrapWorker < Worker @@ -12,27 +8,27 @@ class StackBootstrapWorker < Worker stack_attrs = options.fetch('stack_attributes') without_bootstrap = stack_attrs.delete('without_bootstrap') + skip_rollback = false # take it from options in future @out.puts "Received 'without_bootstrap' option" if without_bootstrap save_report(stack_attrs) - @stack = create_stack(stack_attrs) - if !wait_till_stack_is_created + create_stack(stack_attrs) + if !executor.wait_till_stack_is_created puts_and_flush "Stack creating error" return 1 end begin persist_stack_servers - return 0 if without_bootstrap - - bootstrap_result = bootstrap_servers_by_priority - puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}") - rollback_stack!(@stack) if bootstrap_result.bootstrap_error? - bootstrap_result.code + if without_bootstrap + 0 + else + bootstrap_or_rollback_if_failed(skip_rollback: skip_rollback) + end rescue StandardError => e puts_and_flush "\nAn error occured." - rollback_stack!(@stack) + rollback_stack! unless skip_rollback raise e end end @@ -40,12 +36,26 @@ class StackBootstrapWorker < Worker private + def executor + @executor ||= Devops::Executor::StackExecutor.new(out: out) + end + + def create_stack(stack_attrs) + @stack = executor.create_stack(stack_attrs) + end + def persist_stack_servers - puts_and_flush 'Start saving stack servers into CID database.' - persister = StackServersPersister.new(@stack, out) - persister.persist_new_servers - @servers_by_priorities = persister.just_persisted_by_priority - puts_and_flush "Stack servers have been saved." + @servers_by_priorities = executor.persist_stack_servers[:just_persisted_by_priority] + end + + # options should contain :skip_rollback + def bootstrap_or_rollback_if_failed(options) + bootstrap_result = bootstrap_servers_by_priority + puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}") + if bootstrap_result.bootstrap_error? && !options[:skip_rollback] + rollback_stack! + end + bootstrap_result.code end def bootstrap_servers_by_priority @@ -54,35 +64,15 @@ class StackBootstrapWorker < Worker out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" end out.flush - PrioritizedGroupsBootstrapper.new(out, jid, @servers_by_priorities).bootstrap_servers_by_priority + executor.bootstrap_servers_by_priority(@servers_by_priorities, jid) end - # builds and persist stack model, initiate stack creating in cloud - def create_stack(stack_attrs) - stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) - mongo.stack_insert(stack) - end - - def wait_till_stack_is_created - wait_result = StackCreationWaiter.new(@stack, out).sync - - if wait_result.ok? - puts_and_flush "\nStack '#{@stack.name}' has been created" - true - else - puts_and_flush "An error ocurred during stack creation: #{wait_result.reason}" - false - end - end - - def rollback_stack!(stack) + def rollback_stack! puts_and_flush "\nStart rollback of a stack" begin - stack.delete_stack_in_cloud! - Devops::Db.connector.stack_servers_delete(stack.name) - Devops::Db.connector.stack_delete(stack.id) + executor.delete_stack puts_and_flush "Stack rollback has been completed" - rescue StandardError, Sinatra::NotFound # Sinatra::NotFound is often raised in tests + rescue StandardError puts_and_flush "Stack rollback failed" end end