refactore ResultObject and ServersBootstrapper

devops-service/Guardfile

@@ -43,6 +43,7 @@ guard :rspec, cmd: "rspec" do
   # Devops files
   watch(%r{db/.+\.rb}) { rspec.spec_dir }
   watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
+  watch(%r{lib/helpers/.+\.rb}) { ["#{rspec.spec_dir}/executors", "#{rspec.spec_dir}/workers"] }
   watch(%r{workers/.+\.rb}) { "#{rspec.spec_dir}/workers" }
   watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" }
 end

devops-service/core/devops-messages.rb

@@ -15,7 +15,6 @@ module Devops
       end
 
       def merge file
-        puts "Trying to merge messages with file '#{file}'"
         lang = lang_key
         messages = read_file(file)[lang]
         raise "It is undefined main key '#{lang}' in file '#{file}'" if messages.nil?
@@ -45,7 +44,8 @@ module Devops
       end
 
       def lang_key
-        DevopsConfig.config["messages.lang"] || "en"
+        locale = DevopsConfig.config && DevopsConfig.config['messages.lang']
+        locale || "en"
       end
     end
 

devops-service/lib/executors/server_operation_result.rb

@@ -4,23 +4,21 @@ module Devops
   module Executor
     class ServerOperationResult < Helpers::ResultObject
 
-      def occured_during_bootstrap?
-        [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
-      end
+      set_result_codes(
+        ok: 0,
+        server_bootstrap_fail: 2,
+        server_cannot_update_tags: 3,
+        server_bootstrap_private_ip_unset: 4,
+        server_not_in_chef_nodes: 5,
+        server_bootstrap_unknown_error: 7,
+        deploy_unknown_error: 6,
+        deploy_failed: 8,
+        creating_server_unknown_error: 9,
+        creating_server_in_cloud_failed: 10
+      )
 
-      def self.result_codes
-        {
-          ok: 0,
-          server_bootstrap_fail: 2,
-          server_cannot_update_tags: 3,
-          server_bootstrap_private_ip_unset: 4,
-          server_not_in_chef_nodes: 5,
-          server_bootstrap_unknown_error: 7,
-          deploy_unknown_error: 6,
-          deploy_failed: 8,
-          creating_server_unknown_error: 9,
-          creating_server_in_cloud_failed: 10
-        }
+      def one_of_bootstrap_errors?
+        [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
       end
 
     end

devops-service/lib/helpers/result_object.rb

@@ -1,11 +1,6 @@
 module Devops
   module Helpers
     class ResultObject
-      # this method should be overrided in descendents
-      def self.result_codes
-        {ok: 0}
-      end
-
       attr_reader :code
 
       def initialize(code)
@@ -16,17 +11,40 @@ module Devops
         @code == 0
       end
 
+      def failed?
+        !ok?
+      end
+
       def reason
         self.class.result_codes.key(@code) || :unknown_error
       end
 
-      def self.code_of_reason(reason)
-        result_codes.fetch(reason)
+      class << self
+        def result_codes
+          @result_codes || {ok: 0}
+        end
+
+        def code_of_reason(reason)
+          result_codes.fetch(reason)
+        end
+
+        def from_reason(reason)
+          new(code_of_reason(reason))
+        end
+
+        private
+
+        # defines methods like :bootstrap_error?
+        def set_result_codes(new_result_codes)
+          @result_codes = new_result_codes
+          @result_codes.each do |pretendent_reason, pretendent_code|
+            define_method "#{pretendent_reason}?" do
+              code == pretendent_code
+            end
+          end
+        end
       end
 
-      def self.from_reason(reason)
-        new(code_of_reason(reason))
-      end
 
     end
   end

devops-service/messages/en.yml

@@ -4,3 +4,18 @@ en:
       validation:
         users:
           not_exist: "These users are missing in mongo: '%{users}'"
+  worker:
+    stack_bootstrap:
+      bootstrap_result:
+        ok: "All servers have been successfully bootstrapped."
+        bootstrap_error: An error occured during bootstraping stack servers.
+        deploy_error: |
+          Stack was launched, but an error occured during deploying stack servers.
+          You can redeploy stack after fixing the error.
+        timeout_reached: Bootstrap or deploy wasn't completed due to timeout.
+    servers_bootstrapper:
+      bootstrap_servers:
+        ok: "Server '%{server_id}' has been bootstraped (job %{job_id})."
+        timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
+        bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
+        deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."

devops-service/spec/shared_contexts/init_messages.rb

@@ -0,0 +1,5 @@
+RSpec.shared_context 'init messages', init_messages: true do
+  before(:all) do
+    Devops::Messages.init
+  end
+end

devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb

@@ -0,0 +1,48 @@
+require 'workers/stack_bootstrap/prioritized_groups_bootstrapper'
+
+RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:jid) { 1000 }
+  let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
+  before do
+    @array1 = []; @array2 = []; @array3 = []
+    @servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
+  end
+
+  describe '#bootstrap_servers_by_priority' do
+    subject { groups_bootstrapper.bootstrap_servers_by_priority }
+
+    it 'bootstraps servers in order by priorities, separately' do
+      allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
+      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
+      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
+      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
+      expect(subject).to be_ok
+    end
+
+    it 'it returns :bootstrap_error result if error occured during bootstrap' do
+      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+        [ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
+      }
+      expect(subject).to be_bootstrap_error
+    end
+
+    it 'it returns :deploy_error result if error occured during deploy' do
+      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+        [ServersBootstrapper::Result.from_reason(:deploy_error)]
+      }
+      expect(subject).to be_deploy_error
+    end
+
+    it "doesn't bootstrap group if previous one failed" do
+      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+        [ServersBootstrapper::Result.from_reason(:deploy_error)]
+      }
+      allow(ServersBootstrapper).to receive(:new).and_call_original
+      expect(ServersBootstrapper).to receive(:new).once
+      subject
+    end
+
+
+  end
+end

devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb

@@ -0,0 +1,40 @@
+require 'workers/stack_bootstrap/servers_bootstrapper'
+
+RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:jid) { 1000 }
+  let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
+  let(:bootstrapper) { described_class.new(out, jid, servers ) }
+  let(:bootstrap_job_ids) { %w(100 200) }
+
+  describe '#bootstrap_group' do
+    before do
+      allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
+      allow(stubbed_connector).to receive(:add_report_subreports)
+      allow(stubbed_connector).to receive(:report)
+      allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
+    end
+
+    it 'start bootstrap workers and add subreports' do
+      expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
+      expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
+      bootstrapper.bootstrap_group
+    end
+
+    it 'returns :ok result if everything ok' do
+      expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
+    end
+
+    it 'returns proper error results' do
+      waiter = instance_double(JobWaiter)
+      allow(waiter).to receive(:wait).and_return(2, 8)
+      allow(JobWaiter).to receive(:new) { waiter }
+      expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
+    end
+
+    it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
+      allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
+      expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
+    end
+  end
+end

devops-service/spec/workers/stack_bootstrap/stack_servers_bootstrapper_spec.rb

@@ -1,58 +0,0 @@
-require 'workers/stack_bootstrap_worker'
-
-RSpec.describe StackServersBootstrapper, stubbed_connector: true do
-  let(:out) { double(:out, puts: nil, flush: nil) }
-  let(:jid) { 1000 }
-  let(:bootstrapper) { described_class.new(out, jid) }
-  let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
-  let(:bootstrap_job_ids) { %w(100 200) }
-  let(:subreport1) { build(:report, id: bootstrap_job_ids.first) }
-  let(:subreport2) { build(:report, id: bootstrap_job_ids.last) }
-
-  describe '#bootstrap' do
-    let(:bootstrap!) { bootstrapper.bootstrap(servers) }
-
-    before do
-      allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
-      allow(stubbed_connector).to receive(:add_report_subreports)
-      allow(stubbed_connector).to receive(:report) do |subreport_id|
-        subreport_id == '100' ? subreport1 : subreport2
-      end
-      allow(bootstrapper).to receive(:get_bootstrap_result) { 0 }
-    end
-
-    it 'start bootstrap workers' do
-      expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
-      bootstrap!
-    end
-
-    it 'add subreports' do
-      expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
-      bootstrap!
-    end
-
-    it 'delegates waiting to JobWaiter' do
-      allow(bootstrapper).to receive(:get_bootstrap_result).and_call_original
-      allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
-      expect_any_instance_of(JobWaiter).to receive(:wait)
-      bootstrapper.bootstrap(build_list(:server, 1))
-    end
-
-    it 'raises StackServerBootstrapError if an error occured during bootstrap' do
-      allow(bootstrapper).to receive(:get_bootstrap_result) { 2 }
-      expect { bootstrap! }.to raise_error StackServerBootstrapError
-    end
-
-    it 'raises StackServerDeployError if an error occured during deploy' do
-      allow(bootstrapper).to receive(:get_bootstrap_result) { 8 }
-      expect { bootstrap! }.to raise_error StackServerDeployError
-    end
-
-    it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do
-      allow(bootstrapper).to receive(:get_bootstrap_result).and_call_original
-      allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
-      expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout
-    end
-  end
-
-end

devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb

@@ -53,29 +53,28 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
       it 'updates stack in DB when stack creating is finished and returns 0' do
         setup_statuses(['CREATE_COMPLETE'])
         expect(stubbed_connector).to receive(:stack_update).with(stack)
-        expect(syncer.sync.code).to eq 0
+        expect(syncer.sync).to be_ok
       end
     end
 
     context 'when stack was rollbacked' do
       it 'returns 1 (:stack_rolled_back)' do
         setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
-        expect(syncer.sync.code).to eq 1
+        expect(syncer.sync).to be_stack_rolled_back
       end
     end
 
     context 'when unkown stack status was found' do
       it 'returns 2 (:unkown_status)' do
         setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
-
-        expect(syncer.sync.code).to eq 2
+        expect(syncer.sync).to be_unkown_status
       end
     end
 
     context "when stack hasn't been synced in an hour" do
       it 'returns 3 (:timeout)' do
         allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
-        expect(syncer.sync.code).to eq 3
+        expect(syncer.sync).to be_timeout
       end
     end
 

devops-service/spec/workers/stack_bootstrap_worker_spec.rb

@@ -1,21 +1,18 @@
 require 'workers/stack_bootstrap_worker'
 
-RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
+RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do
   let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
   let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
   let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
-  let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) }
-  let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
-  let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
   let(:worker) { described_class.new }
 
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
 
     allow(worker).to receive(:update_report)
-    allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
-    allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
-    allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
+    allow(worker).to receive(:sync_stack) { true }
+    allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} }
+    allow(worker).to receive(:bootstrap_servers_by_priority) { ServersBootstrapper::Result.new(0) }
 
     allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) }
     allow(Devops::Model::StackEc2).to receive(:create)
@@ -35,13 +32,13 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
     allow(worker).to receive(:create_stack).and_call_original
     expect(worker).to receive(:update_report).ordered
     expect(worker).to receive(:create_stack).ordered
-    expect(stack_servers_persister).to receive(:persist).ordered
+    expect(worker).to receive(:persist_stack_servers).ordered
     perform_without_bootstrap
   end
 
   context 'if without_bootstrap is true' do
     it "doesn't bootstrap servers" do
-      expect(stack_servers_bootstrapper).not_to receive(:bootstrap)
+      expect(worker).not_to receive(:bootstrap_servers_by_priority)
       perform_without_bootstrap
     end
 
@@ -51,72 +48,72 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
   end
 
   context 'if without_bootstrap is false or not set' do
-    it 'bootstraps servers in order by priorities, separately' do
-      first_servers = build_list(:server, 2)
-      last_servers = build_list(:server, 3)
-      allow(stack_servers_persister).to receive(:persist) {
-        {3 => first_servers, 1 => last_servers}
+    it 'returns 0 when bootstraping servers was successful' do
+      expect(perform_with_bootstrap).to eq 0
+    end
+
+    it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do
+      allow(worker).to receive(:bootstrap_servers_by_priority) {
+        ServersBootstrapper::Result.from_reason(:bootstrap_error)
       }
-      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered
-      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered
+      expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
+      expect(stubbed_connector).to receive(:stack_servers_delete)
+      expect(stubbed_connector).to receive(:stack_delete)
       perform_with_bootstrap
+      expect(perform_with_bootstrap).to eq 2
     end
 
-    context 'when bootstraping servers was successful' do
-      it 'returns 0' do
-        expect(perform_with_bootstrap).to eq 0
-      end
+    it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do
+      allow(worker).to receive(:bootstrap_servers_by_priority) {
+        ServersBootstrapper::Result.from_reason(:deploy_error)
+      }
+      expect(worker).not_to receive(:rollback_stack!)
+      expect(perform_with_bootstrap).to eq 3
     end
 
-    context 'when a known error occured during servers bootstrap' do
-      before do
-        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError }
-      end
-
-      it 'rollbacks stack and returns 2' do
-        expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
-        expect(stubbed_connector).to receive(:stack_servers_delete)
-        expect(stubbed_connector).to receive(:stack_delete)
-        perform_with_bootstrap
-      end
-
-      it 'returns 2' do
-        allow(worker).to receive(:rollback_stack!)
-        expect(perform_with_bootstrap).to eq 2
-      end
+    it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do
+      allow(worker).to receive(:bootstrap_servers_by_priority) {
+        ServersBootstrapper::Result.from_reason(:timeout_reached)
+      }
+      expect(worker).not_to receive(:rollback_stack!)
+      expect(perform_with_bootstrap).to eq 4
     end
 
-    context 'when a known error occured during servers deploy' do
-      it "doesn't rollback stack and returns 3" do
-        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError }
-        expect(worker).not_to receive(:rollback_stack!)
-        expect(perform_with_bootstrap).to eq 3
-      end
-    end
-
-    context "when a servers bootstrap & deploy haven't been finished due to timeout" do
-      it "doesn't rollback stack and returns 3" do
-        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout }
-        expect(worker).not_to receive(:rollback_stack!)
-        expect(perform_with_bootstrap).to eq 4
-      end
-    end
-
-    context 'when an unknown error occured during servers bootsrap and deploy' do
-      it 'rollbacks stack and reraises that error' do
-        error = StandardError.new
-        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error }
-        allow(worker).to receive(:rollback_stack!)
-        expect(worker).to receive(:rollback_stack!)
-        expect{perform_with_bootstrap}.to raise_error(error)
-      end
+    it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do
+      error = StandardError.new
+      allow(worker).to receive(:bootstrap_servers_by_priority) { raise error }
+      allow(worker).to receive(:rollback_stack!)
+      expect(worker).to receive(:rollback_stack!)
+      expect{perform_with_bootstrap}.to raise_error(error)
     end
   end
 
-  context "when stack creation wasn't successful" do
-    it 'returns 1' do
-      allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) }
-      expect(perform_without_bootstrap).to eq 1
+  context "without stubbing methods", stubbed_connector: true do
+    before do
+      allow(worker).to receive(:sync_stack).and_call_original
+      allow(worker).to receive(:persist_stack_servers).and_call_original
+      allow(worker).to receive(:bootstrap_servers_by_priority).and_call_original
+      allow(StackServersPersister).to receive(:new) {
+        instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)})
+      }
+      allow(PrioritizedGroupsBootstrapper).to receive(:new) {
+        instance_double(PrioritizedGroupsBootstrapper, bootstrap_servers_by_priority: ServersBootstrapper::Result.new(0))
+      }
+    end
+
+    it "return 0 if syncer returns ok" do
+      allow(StackSynchronizer).to receive(:new) {
+        instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0))
+      }
+      expect(StackServersPersister).to receive(:new).with(instance_of(Devops::Model::StackEc2), anything)
+      expect(perform_with_bootstrap).to eq 0
+    end
+
+    it 'returns 1 if syncer returns error' do
+      allow(StackSynchronizer).to receive(:new) {
+        instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(5))
+      }
+      expect(perform_with_bootstrap).to eq 1
     end
   end
 end

devops-service/workers/stack_bootstrap/errors.rb

@@ -1,3 +0,0 @@
-class StackServerBootstrapError < StandardError; end
-class StackServerDeployError < StandardError; end
-class StackServerBootstrapDeployTimeout < StandardError; end

devops-service/workers/stack_bootstrap/prioritized_groups_bootstrapper.rb

@@ -0,0 +1,36 @@
+require_relative 'servers_bootstrapper'
+
+# Bootstrap groups of servers based on priorities: higher first.
+# Doesn't start bootstrap of next group if bootstrap of previous group failed.
+class PrioritizedGroupsBootstrapper
+  include PutsAndFlush
+  attr_reader :out
+
+  def initialize(out, jid, servers_with_priorities)
+    @out, @jid, @servers_with_priorities = out, jid, servers_with_priorities
+  end
+
+  # @param servers_with_priorities [Hash] is a Hash like
+  #   {1 => [server1, server2]}
+  # Starts bootstrapping another group only after successful bootstrapping of previous.
+  def bootstrap_servers_by_priority
+    sorted_priorities.each do |priority|
+      puts_and_flush "Bootstrap servers with priority '#{priority}':"
+      bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority])
+      bootstrap_results = bootstrapper.bootstrap_group
+      error = most_critical_error(bootstrap_results)
+      return error if error
+    end
+    ServersBootstrapper::Result.from_reason(:ok)
+  end
+
+  private
+
+  def sorted_priorities
+    @servers_with_priorities.keys.sort.reverse
+  end
+
+  def most_critical_error(results)
+    results.detect(&:bootstrap_error?) || results.detect(&:failed?)
+  end
+end

devops-service/workers/stack_bootstrap/servers_bootstrapper.rb

@@ -0,0 +1,72 @@
+require 'workers/bootstrap_worker'
+require 'workers/helpers/job_waiter'
+
+# Starts bootstrap workers for each server in group and wait for them to end (synchroniously).
+class ServersBootstrapper
+  include PutsAndFlush
+  attr_reader :out
+
+  class Result < Devops::Helpers::ResultObject
+    set_result_codes(
+      ok: 0,
+      bootstrap_error: 2,
+      deploy_error: 3,
+      timeout_reached: 4
+    )
+  end
+
+  def initialize(out, jid, servers)
+    @out, @jid, @servers = out, jid, servers
+    @server_bootstrap_jobs = {}
+  end
+
+  # returns array of Results
+  def bootstrap_group
+    start_workers!
+    ::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values)
+
+    @server_bootstrap_jobs.map do |server_id, job_id|
+      result = wait_for_bootstrap_job(job_id)
+      puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
+      result
+    end
+  end
+
+  private
+
+  # returns hash: {server_id => worker_job_id}
+  def start_workers!
+    @servers.each do |server|
+      job_id = Worker.start_async(::BootstrapWorker,
+        server_attrs: server.to_mongo_hash,
+        bootstrap_template: 'omnibus',
+        owner: server.created_by
+      )
+      @out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
+      @server_bootstrap_jobs[server.id] = job_id
+    end
+    puts_and_flush "\n\n\n"
+  end
+
+  def result(reason)
+    Result.from_reason(reason)
+  end
+
+  def wait_for_bootstrap_job(job_id)
+    result_code = JobWaiter.new(job_id).wait
+    result_from_job_code(result_code)
+  rescue JobWaiter::TimeoutReached
+    result(:timeout_reached)
+  end
+
+  def result_from_job_code(result_code)
+    job_result = Devops::Executor::ServerOperationResult.new(result_code)
+    if job_result.ok?
+      result(:ok)
+    elsif job_result.one_of_bootstrap_errors?
+      result(:bootstrap_error)
+    else
+      result(:deploy_error)
+    end
+  end
+end

devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb

@@ -1,69 +0,0 @@
-require 'workers/bootstrap_worker'
-require 'workers/stack_bootstrap/errors'
-require 'workers/helpers/job_waiter'
-
-class StackServersBootstrapper
-  include PutsAndFlush
-  attr_reader :out
-
-  def initialize(out, jid)
-    @out, @jid = out, jid
-  end
-
-  def bootstrap(servers)
-    @servers = servers
-    puts_and_flush "\nStart bootstraping stack servers"
-
-    servers_jobs_ids = start_workers
-    ::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values)
-
-    out.puts
-    servers_jobs_ids.each do |server_id, job_id|
-      bootstrap_result_code = get_bootstrap_result(server_id, job_id)
-      check_bootstrap_result!(server_id, bootstrap_result_code, job_id)
-    end
-    puts_and_flush "Stack servers have been bootstraped"
-  end
-
-  private
-
-  def check_bootstrap_result!(server_id, result_code, job_id)
-    operation_result = Devops::Executor::ServerOperationResult.new(result_code)
-
-    if operation_result.ok?
-      puts_and_flush "Server '#{server_id}' has been bootstraped (job #{job_id})."
-      return
-    end
-
-    puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
-
-    if operation_result.occured_during_bootstrap?
-      raise StackServerBootstrapError # will cause rollback of a stack
-    else
-      raise StackServerDeployError # will not cause rollback of a stack
-    end
-  end
-
-  def get_bootstrap_result(server_id, job_id)
-    JobWaiter.new(job_id).wait
-  rescue JobWaiter::TimeoutReached
-    puts_and_flush "Waiting for job #{job_id} halted: timeout reached."
-    raise StackServerBootstrapDeployTimeout
-  end
-
-  # returns hash: {server_id => worker_job_id}
-  def start_workers
-    servers_jobs_ids = {}
-    @servers.each do |server|
-      job_id = Worker.start_async(::BootstrapWorker,
-        server_attrs: server.to_mongo_hash,
-        bootstrap_template: 'omnibus',
-        owner: server.created_by
-      )
-      @out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}"
-      servers_jobs_ids[server.id] = job_id
-    end
-    puts_and_flush "\n"
-    servers_jobs_ids
-  end
-end

devops-service/workers/stack_bootstrap/stack_servers_persister.rb

@@ -1,5 +1,6 @@
 require 'workers/stack_bootstrap/chef_node_name_builder'
 
+# Fetches info about stack servers from provider and then persist them in mongo.
 class StackServersPersister
   include PutsAndFlush
   attr_reader :stack, :out

devops-service/workers/stack_bootstrap/stack_synchronizer.rb

@@ -1,19 +1,18 @@
+# Polling stack status until it's completed or failed.
 class StackSynchronizer
   include PutsAndFlush
   attr_reader :out, :stack
 
   class SyncResult < Devops::Helpers::ResultObject
-    def self.result_codes
-      {
-        ok: 0,
-        stack_rolled_back: 1,
-        unkown_status: 2,
-        timeout: 3,
-        error: 5,
-        stack_deleted: 6,
-        stack_not_found: 7
-      }
-    end
+    set_result_codes(
+      ok: 0,
+      stack_rolled_back: 1,
+      unkown_status: 2,
+      timeout: 3,
+      error: 5,
+      stack_deleted: 6,
+      stack_not_found: 7
+    )
   end
 
   def initialize(stack, out)

devops-service/workers/stack_bootstrap_worker.rb

@@ -1,11 +1,8 @@
 require "db/mongo/models/stack/stack_factory"
 require "db/mongo/models/project"
-require "db/mongo/models/report"
 require "workers/stack_bootstrap/stack_synchronizer"
-require "workers/stack_bootstrap/stack_servers_bootstrapper"
+require "workers/stack_bootstrap/servers_bootstrapper"
 require "workers/stack_bootstrap/stack_servers_persister"
-require "workers/stack_bootstrap/errors"
-
 
 class StackBootstrapWorker < Worker
 
@@ -26,20 +23,13 @@ class StackBootstrapWorker < Worker
       end
 
       begin
-        @servers_with_priority = stack_servers_persister.persist
-        bootstrap_in_priority_order unless without_bootstrap
-        0
-      rescue StackServerBootstrapError
-        puts_and_flush "\nAn error occured during bootstraping stack servers."
-        rollback_stack!(@stack)
-        2
-      rescue StackServerDeployError
-        out.puts "\nStack was launched, but an error occured during deploying stack servers."
-        puts_and_flush "You can redeploy stack after fixing the error."
-        3
-      rescue StackServerBootstrapDeployTimeout
-        puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
-        4
+        @servers_by_priorities = persist_stack_servers
+        return 0 if without_bootstrap
+
+        bootstrap_result = bootstrap_servers_by_priority
+        puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}")
+        rollback_stack!(@stack) if bootstrap_result.bootstrap_error?
+        bootstrap_result.code
       rescue StandardError => e
         puts_and_flush "\nAn error occured."
         rollback_stack!(@stack)
@@ -50,16 +40,12 @@ class StackBootstrapWorker < Worker
 
   private
 
-  def stack_synchronizer
-    @stack_synchronizer ||= StackSynchronizer.new(@stack, out)
+  def persist_stack_servers
+    StackServersPersister.new(@stack, out).persist
   end
 
-  def stack_servers_persister
-    @stack_servers_persister ||= StackServersPersister.new(@stack, out)
-  end
-
-  def stack_servers_bootstrapper
-    @stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid)
+  def bootstrap_servers_by_priority
+    PrioritizedGroupsBootstrapper.new(out, jid, @servers_by_priorities).bootstrap_servers_by_priority
   end
 
   # builds and persist stack model, initiate stack creating in cloud
@@ -69,7 +55,7 @@ class StackBootstrapWorker < Worker
   end
 
   def sync_stack
-    sync_result = stack_synchronizer.sync
+    sync_result = StackSynchronizer.new(@stack, out).sync
 
     if sync_result.ok?
       puts_and_flush "\nStack '#{@stack.name}' has been created"
@@ -80,22 +66,16 @@ class StackBootstrapWorker < Worker
     end
   end
 
-  # Bootstrap servers with high priorities first
-  def bootstrap_in_priority_order
-    sorted_priorities = @servers_with_priority.keys.sort.reverse
-    sorted_priorities.each do |priority|
-      @out.puts "Servers with priority '#{priority}':"
-      stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority])
-    end
-    puts_and_flush "Done."
-  end
-
   def rollback_stack!(stack)
     puts_and_flush "\nStart rollback of a stack"
-    stack.delete_stack_in_cloud!
-    Devops::Db.connector.stack_servers_delete(stack.name)
-    Devops::Db.connector.stack_delete(stack.id)
-    puts_and_flush "Rollback has been completed"
+    begin
+      stack.delete_stack_in_cloud!
+      Devops::Db.connector.stack_servers_delete(stack.name)
+      Devops::Db.connector.stack_delete(stack.id)
+      puts_and_flush "Stack rollback has been completed"
+    rescue StandardError, Sinatra::NotFound # Sinatra::NotFound often raised in tests
+      puts_and_flush "Stack rollback failed"
+    end
   end
 
   def save_report(stack_attrs)

devops-service/workers/worker.rb

@@ -10,6 +10,7 @@ require "core/devops-service"
 require "core/devops-config"
 require "core/devops-logger"
 require "core/devops-db"
+require "db/mongo/models/report"
 require "providers/provider_factory"
 require "lib/knife/knife_factory"
 require "lib/puts_and_flush"