add possibility to set chef_node_name mask for stacks; cover StackBootstrapWorker with specs

devops-service/Guardfile

@@ -43,4 +43,5 @@ guard :rspec, cmd: "rspec" do
   # Devops files
   watch(%r{db/.+\.rb}) { rspec.spec_dir }
   watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
+  watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" }
 end

devops-service/commands/stack.rb

@@ -1,62 +0,0 @@
-module StackCommands
-  extend self
-
-  RESULT_CODES = {
-    stack_rolled_back: 1,
-    unkown_status: 2,
-    timeout: 3,
-    error: 5
-  }
-
-  def self.result_codes
-    RESULT_CODES
-  end
-
-  def self.result_code(code)
-    result_codes.fetch(code)
-  end
-
-  def sync_stack_proc
-    lambda do |out, stack, mongo|
-      # 5 tries each 5 seconds, then 200 tries each 10 seconds
-      sleep_times = [5]*5 + [10]*200
-
-      begin
-        out << "Syncing stack '#{stack.id}'...\n"
-        events_keys = []
-        sleep_times.each do |sleep_time|
-          sleep sleep_time
-          stack.sync_details!
-          stack.events.each do |event|
-            unless events_keys.include?(event["event_id"])
-              events_keys << event["event_id"]
-              out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
-            end
-          end
-          case stack.stack_status
-          when 'CREATE_IN_PROGRESS'
-            out.flush
-          when 'CREATE_COMPLETE'
-            mongo.stack_update(stack)
-            out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n"
-            out.flush
-            return 0
-          when 'ROLLBACK_COMPLETE'
-            out << "\nStack '#{stack.id}' status is rolled back\n"
-            return StackCommands.result_code(:stack_rolled_back)
-          else
-            out.puts "\nUnknown stack status: '#{stack.stack_status}'"
-            return StackCommands.result_code(:unkown_status)
-          end
-        end
-        out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds."
-        return StackCommands.result_code(:timeout)
-      rescue StandardError => e
-        logger.error e.message
-        out << "Error: #{e.message}\n"
-        return StackCommands.result_code(:error)
-      end
-    end
-  end
-
-end

devops-service/db/validators/base.rb

@@ -13,7 +13,6 @@ module Validators
       raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}")
     end
 
-    # :nocov:
     def valid?
       raise 'override me'
     end
@@ -21,7 +20,6 @@ module Validators
     def message
       raise 'override me'
     end
-    # :nocov:
 
     class << self
       private

devops-service/lib/executors/server_executor.rb

@@ -53,16 +53,20 @@ module Devops
         @current_user = options[:current_user]
       end
 
-      def self.error_code(symbolic_code)
-        ERROR_CODES.fetch(symbolic_code)
+      def self.error_code(reason)
+        ERROR_CODES.fetch(reason)
       end
 
-      def self.symbolic_error_code(integer_code)
+      def self.reason_from_error_code(integer_code)
         ERROR_CODES.key(integer_code) || :unknown_error
       end
 
-      def error_code(symbolic_code)
-        self.class.error_code(symbolic_code)
+      def self.bootstrap_errors_reasons
+        [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error]
+      end
+
+      def error_code(reason)
+        self.class.error_code(reason)
       end
 
       def create_server_object options
@@ -511,8 +515,6 @@ module Devops
         cmd
       end
 
-      # to simplify testing
-      # :nocov:
       def execute_system_command(cmd)
         `#{cmd}`
       end
@@ -524,7 +526,6 @@ module Devops
       def knife_instance
         @knife_instance ||= KnifeFactory.instance
       end
-      # :nocov:
 
     end
   end

devops-service/lib/puts_and_flush.rb

@@ -0,0 +1,9 @@
+module PutsAndFlush
+  private
+
+  # out stream should be defined
+  def puts_and_flush(message)
+    out.puts message
+    out.flush
+  end
+end

devops-service/spec/executors/server_executor_spec.rb

@@ -53,13 +53,13 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   end
 
 
-  describe '.symbolic_error_code' do
+  describe '.reason_from_error_code' do
     it 'returns symbol given an integer' do
-      expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail
+      expect(described_class.reason_from_error_code(2)).to eq :server_bootstrap_fail
     end
 
     it "returns :unknown_error if can't recognize error code" do
-      expect(described_class.symbolic_error_code(123)).to eq :unknown_error
+      expect(described_class.reason_from_error_code(123)).to eq :unknown_error
     end
   end
 

devops-service/spec/shared_contexts/stubbed_logger.rb

@@ -1,8 +1,6 @@
 RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do
   before do
-    allow(DevopsLogger).to receive_message_chain('logger.debug')
-    allow(DevopsLogger).to receive_message_chain('logger.info')
-    allow(DevopsLogger).to receive_message_chain('logger.error')
-    allow(DevopsLogger).to receive_message_chain('logger.warn')
+    logger = double('logger', debug: nil, info: nil, error: nil, warn: nil)
+    allow(DevopsLogger).to receive(:logger) { logger }
   end
 end

devops-service/spec/workers/chef_node_name_builder_spec.rb

@@ -0,0 +1,45 @@
+require 'workers/stack_bootstrap/chef_node_name_builder'
+RSpec.describe ChefNodeNameBuilder do
+  let(:server_info) do
+    {
+      'id' => 'server1',
+      'name' => 'server_name',
+      'key_name' => 'key',
+      'private_ip' => '127.0.0.1',
+      'public_ip' => '127.0.0.2',
+      'tags' => {
+        'cid:priority' => '3'
+      }
+    }
+  end
+  let(:project) { build(:project, id: 'proj', with_deploy_env_identifiers: %w(dev)) }
+  let(:env) { project.deploy_env('dev') }
+  let(:name_builder) { described_class.new(server_info, project, env) }
+  let(:build_name) { name_builder.build_node_name }
+
+  def set_mask(mask)
+    server_info['tags']['cid:node-name-mask'] = mask
+  end
+
+  describe '#build_node_name' do
+    it 'uses default mask ("$project-$nodename-$env")' do
+      expect(build_name).to eq 'proj-server1-dev'
+    end
+
+    it 'substitutes project, env and nodename' do
+      set_mask('$project/$env/$nodename')
+      expect(build_name).to eq 'proj/dev/server1'
+    end
+
+    it 'substitutes $time' do
+      set_mask('$nodename-$time')
+      expect(build_name).to match /server1-\d+/
+    end
+
+    it 'substitutes underscores to dashes' do
+      server_info['id'] = 'server_1'
+      expect(build_name).to match 'proj-server-1-dev'
+    end
+  end
+
+end

devops-service/spec/workers/stack_bootstrap_worker_spec.rb

@@ -0,0 +1,128 @@
+require 'workers/stack_bootstrap_worker'
+
+RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:file) { 'temp.txt' }
+
+  let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
+  let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
+  let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
+  let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) }
+  let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
+  let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
+  let(:worker) { described_class.new }
+
+  before do
+    allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
+    allow(stubbed_connector).to receive(:save_report)
+    allow(stubbed_connector).to receive(:stack_insert)
+
+    allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
+    allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
+    allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
+    allow(worker).to receive(:call).and_yield(out, file)
+    allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
+  end
+
+
+  it 'requires "stack_attributes" in options' do
+    expect{
+      worker.perform({})
+    }.to raise_error KeyError
+  end
+
+  it 'saves report about operation' do
+    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report))
+    perform_without_bootstrap
+  end
+
+  it 'saves report about operation, creates stack and persists stack servers' do
+    allow(worker).to receive(:create_stack).and_call_original
+    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)).ordered
+    expect(worker).to receive(:create_stack).ordered
+    expect(stack_servers_persister).to receive(:persist).ordered
+    perform_without_bootstrap
+  end
+
+  context 'if without_bootstrap is true' do
+    it "doesn't bootstrap servers" do
+      expect(stack_servers_bootstrapper).not_to receive(:bootstrap)
+      perform_without_bootstrap
+    end
+
+    it 'returns 0' do
+      expect(perform_without_bootstrap).to eq 0
+    end
+  end
+
+  context 'if without_bootstrap is false or not set' do
+    it 'bootstraps servers in order by priorities, separately' do
+      first_servers = build_list(:server, 2)
+      last_servers = build_list(:server, 3)
+      allow(stack_servers_persister).to receive(:persist) {
+        {3 => first_servers, 1 => last_servers}
+      }
+      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered
+      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered
+      perform_with_bootstrap
+    end
+
+    context 'when bootstraping servers was successful' do
+      it 'returns 0' do
+        expect(perform_with_bootstrap).to eq 0
+      end
+    end
+
+    context 'when a known error occured during servers bootstrap' do
+      before do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError }
+      end
+
+      it 'rollbacks stack and returns 2' do
+        expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
+        expect(stubbed_connector).to receive(:stack_servers_delete)
+        expect(stubbed_connector).to receive(:stack_delete)
+        perform_with_bootstrap
+      end
+
+      it 'returns 2' do
+        allow(worker).to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 2
+      end
+    end
+
+    context 'when a known error occured during servers deploy' do
+      it "doesn't rollback stack and returns 3" do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError }
+        expect(worker).not_to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 3
+      end
+    end
+
+    context "when a servers bootstrap & deploy haven't been finished due to timeout" do
+      it "doesn't rollback stack and returns 3" do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout }
+        expect(worker).not_to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 4
+      end
+    end
+
+    context 'when an unknown error occured during servers bootsrap and deploy' do
+      it 'rollbacks stack and reraises that error' do
+        error = StandardError.new
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error }
+        allow(worker).to receive(:rollback_stack!)
+        expect(worker).to receive(:rollback_stack!)
+        expect{perform_with_bootstrap}.to raise_error(error)
+      end
+    end
+  end
+
+  context "when stack creation wasn't successful" do
+    it 'returns 1' do
+      allow(stack_synchronizer).to receive(:sync) { 5 }
+      allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
+      expect(perform_without_bootstrap).to eq 1
+    end
+  end
+end

devops-service/spec/workers/stack_servers_bootstrapper_spec.rb

@@ -0,0 +1,59 @@
+require 'workers/stack_bootstrap_worker'
+
+RSpec.describe StackServersBootstrapper, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:jid) { 1000 }
+  let(:bootstrapper) { described_class.new(out, jid) }
+  let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
+  let(:bootstrap_job_ids) { %w(100 200) }
+  let(:subreport1) { build(:report, id: bootstrap_job_ids.first) }
+  let(:subreport2) { build(:report, id: bootstrap_job_ids.last) }
+
+  describe '#bootstrap' do
+    let(:bootstrap!) { bootstrapper.bootstrap(servers) }
+
+    before do
+      allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
+      allow(stubbed_connector).to receive(:add_report_subreports)
+      allow(stubbed_connector).to receive(:report) do |subreport_id|
+        subreport_id == '100' ? subreport1 : subreport2
+      end
+      allow(bootstrapper).to receive(:sleep)
+    end
+
+    it 'start bootstrap workers' do
+      expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
+      bootstrap!
+    end
+
+    it 'add subreports' do
+      expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
+      bootstrap!
+    end
+
+    it 'waits for job to end' do
+      allow(subreport1).to receive(:status).and_return('running', 'running', 'running', 'completed')
+      allow(subreport2).to receive(:status).and_return('running', 'running', 'running', 'completed')
+      expect(bootstrapper).to receive(:sleep).exactly(2*4).times
+      bootstrap!
+    end
+
+    it 'raises StackServerBootstrapError if an error occured during bootstrap' do
+      allow(subreport1).to receive(:status) {'failed'}
+      allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:server_bootstrap_fail) }
+      expect { bootstrap! }.to raise_error StackServerBootstrapError
+    end
+
+    it 'raises StackServerDeployError if an error occured during deploy' do
+      allow(subreport1).to receive(:status) {'failed'}
+      allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:deploy_failed) }
+      expect { bootstrap! }.to raise_error StackServerDeployError
+    end
+
+    it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do
+      allow(subreport1).to receive(:status) {'running'}
+      expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout
+    end
+  end
+
+end

devops-service/spec/workers/stack_servers_persister_spec.rb

@@ -0,0 +1,116 @@
+require 'workers/stack_bootstrap/stack_servers_persister'
+
+RSpec.describe StackServersPersister, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:run_list) { ['role[asd]'] }
+  let(:stack) { build(:stack, deploy_env: 'foo', run_list: run_list) }
+  let(:project) { build(:project, id: 'name') }
+  let(:persister) { described_class.new(stack, out) }
+  let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
+  let(:server_info_hash) do
+    {
+      'id' => 'server1',
+      'name' => 'server_name',
+      'key_name' => 'key',
+      'private_ip' => '127.0.0.1',
+      'public_ip' => '127.0.0.2',
+      'tags' => {
+        'cid:priority' => '3'
+      }
+    }
+  end
+
+  before do
+    allow(stubbed_connector).to receive(:project) { project }
+    allow(stubbed_connector).to receive(:image) {
+      instance_double(Devops::Model::Image, remote_user: 'user')
+    }
+    allow(stubbed_connector).to receive(:server_insert)
+    allow(stack).to receive(:provider_instance) { provider }
+    allow(provider).to receive(:stack_servers) {[server_info_hash]}
+  end
+
+  describe '#persist' do
+    it 'fetches stack servers info' do
+      expect(provider).to receive(:stack_servers).with(stack)
+      persister.persist
+    end
+
+    it "doesn't raise error if cid:priority tag is absent" do
+      server_info_hash['tags'].delete('cid:priority')
+      expect {persister.persist}.not_to raise_error
+    end
+
+    it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
+      result = persister.persist
+      expect(result).to be_a(Hash)
+      expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
+    end
+
+    it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.id).to eq 'server1'
+        expect(server.key).to eq 'key'
+        expect(server.private_ip).to eq '127.0.0.1'
+        expect(server.public_ip).to eq '127.0.0.2'
+      end
+      persister.persist
+    end
+
+    it 'takes created_by, run_list and stack attrs from stack' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.created_by).to eq 'root'
+        expect(server.run_list).to eq run_list
+        expect(server.stack).to eq 'iamstack'
+      end
+      persister.persist
+    end
+
+    it 'takes remote_user from image user' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.remote_user).to eq 'user'
+      end
+      persister.persist
+    end
+
+    it "takes deploy_env from project's deploy_env identifier" do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.deploy_env).to eq 'foo'
+      end
+      persister.persist
+    end
+
+    it "takes default provider's ssh key if info doesn't contain it" do
+      allow(provider).to receive(:ssh_key) { 'default_key' }
+      server_info_hash.delete('key_name')
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.key).to eq 'default_key'
+      end
+      persister.persist
+    end
+
+    it "sets server's run list to empty array if stack's run_list is nil" do
+      stack.run_list = nil
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.run_list).to eq []
+      end
+      persister.persist
+    end
+
+    it 'build chef_node_name with default mask "$project-$nodename-$env"' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.chef_node_name).to eq 'name-server1-foo'
+      end
+      persister.persist
+    end
+
+    it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
+      server_info_hash['tags']['cid:node-name-mask'] = '$project-$nodename-123'
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.chef_node_name).to eq 'name-server1-123'
+      end
+      persister.persist
+    end
+  end
+
+end

devops-service/spec/workers/stack_synchronizer_spec.rb

@@ -0,0 +1,82 @@
+require 'workers/stack_bootstrap/stack_synchronizer'
+RSpec.describe StackSynchronizer, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:stack) { build(:stack) }
+  let(:syncer) { described_class.new(stack, out) }
+
+  before do
+    allow(stack).to receive(:sync_details!)
+    allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
+    allow(syncer).to receive(:sleep)
+    allow(stubbed_connector).to receive(:stack_update)
+
+    lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
+    allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
+  end
+
+  describe '#sync' do
+    it 'waits for stack creating to be finished' do
+      expect(syncer).to receive(:sleep).at_least(10).times
+      expect(stack).to receive(:sync_details!).at_least(10).times
+      syncer.sync
+    end
+
+    it 'prints each message only once' do
+      event1 = {'event_id' => 1, 'timestamp' => 't1'}
+      event2 = {'event_id' => 2, 'timestamp' => 't2'}
+      event3 = {'event_id' => 3, 'timestamp' => 't3'}
+
+      allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
+      syncer.sync
+      expect(out).to have_received(:puts).with(/t1/).once.ordered
+      expect(out).to have_received(:puts).with(/t2/).once.ordered
+      expect(out).to have_received(:puts).with(/t3/).once.ordered
+    end
+
+    context 'when stack creating was successful' do
+      it 'updates stack in DB when stack creating is finished and returns 0' do
+        expect(stubbed_connector).to receive(:stack_update).with(stack)
+        expect(syncer.sync).to eq 0
+      end
+    end
+
+    context 'when stack was rollbacked' do
+      it 'returns 1 (:stack_rolled_back)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE')
+        expect(syncer.sync).to eq 1
+      end
+    end
+
+    context 'when unkown stack status was found' do
+      it 'returns 2 (:unkown_status)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown')
+        expect(syncer.sync).to eq 2
+      end
+    end
+
+    context "when stack hasn't been synced in an hour" do
+      it 'returns 3 (:timeout)' do
+        allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
+        expect(syncer.sync).to eq 3
+      end
+    end
+
+    context 'when an error occured during syncing', stubbed_logger: true do
+      it 'returns 5 (:error)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+        allow(stubbed_connector).to receive(:stack_update) { raise }
+        expect(syncer.sync).to eq 5
+      end
+    end
+  end
+
+  describe '#reason_from_error_code' do
+    it 'returns reason as symbol for integer error_code' do
+      expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
+      expect(syncer.reason_from_error_code(2)).to eq :unkown_status
+      expect(syncer.reason_from_error_code(3)).to eq :timeout
+      expect(syncer.reason_from_error_code(5)).to eq :error
+    end
+  end
+
+end

devops-service/workers/stack_bootstrap/chef_node_name_builder.rb

@@ -0,0 +1,15 @@
+class ChefNodeNameBuilder
+  def initialize(server_info, project, env)
+    @server_info, @project, @env = server_info, project, env
+    @mask = server_info['tags']['cid:node-name-mask'] || '$project-$nodename-$env'
+  end
+
+  def build_node_name
+    @mask.gsub!('$project', @project.id)
+    @mask.gsub!('$env', @env.identifier)
+    @mask.gsub!('$nodename', @server_info['id'])
+    @mask.gsub!('$time', Time.now.to_i.to_s)
+    @mask.gsub!('_', '-')
+    @mask
+  end
+end

devops-service/workers/stack_bootstrap/errors.rb

@@ -0,0 +1,4 @@
+class StackCreatingError < StandardError; end
+class StackServerBootstrapError < StandardError; end
+class StackServerDeployError < StandardError; end
+class StackServerBootstrapDeployTimeout < StandardError; end

devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb

@@ -0,0 +1,78 @@
+require 'workers/bootstrap_worker'
+require "workers/stack_bootstrap/errors"
+
+class StackServersBootstrapper
+  include PutsAndFlush
+  attr_reader :out
+
+  def initialize(out, jid)
+    @out, @jid = out, jid
+  end
+
+  def bootstrap(servers)
+    @servers = servers
+    puts_and_flush "\nStart bootstraping stack servers"
+
+    servers_jobs_ids = start_workers
+    ::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values)
+
+    out.puts
+    servers_jobs_ids.each do |server_id, subreport_id|
+      job_result_code = wait_for_job(server_id, subreport_id)
+      check_job_result!(server_id, job_result_code)
+    end
+    puts_and_flush "Stack servers have been bootstraped"
+  end
+
+  private
+
+  def check_job_result!(server_id, job_result_code)
+    return if job_result_code == 0
+
+    reason = Devops::Executor::ServerExecutor.reason_from_error_code(job_result_code)
+    puts_and_flush "Operation result for #{server_id}: #{reason}"
+
+    if error_occured_during_bootstrap?(reason)
+      raise StackServerBootstrapError # will cause rollback of a stack
+    else
+      raise StackServerDeployError #will not cause rollback of a stack
+    end
+  end
+
+  def error_occured_during_bootstrap?(reason)
+    Devops::Executor::ServerExecutor.bootstrap_errors_reasons.include?(reason)
+  end
+
+  def wait_for_job(server_id, subreport_id)
+    1000.times do
+      sleep(5)
+      subreport = ::Devops::Db.connector.report(subreport_id)
+      case subreport.status
+      when Worker::STATUS::COMPLETED
+        puts_and_flush "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
+        return 0
+      when Worker::STATUS::FAILED
+        puts_and_flush "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
+        return subreport.job_result_code
+      end
+    end
+    puts_and_flush "Waiting for job #{subreport_id} halted: timeout reached."
+    raise StackServerBootstrapDeployTimeout
+  end
+
+  # returns hash: {server_id => worker_job_id}
+  def start_workers
+    servers_jobs_ids = {}
+    @servers.each do |server|
+      job_id = Worker.start_async(::BootstrapWorker,
+        server_attrs: server.to_mongo_hash,
+        bootstrap_template: 'omnibus',
+        owner: server.created_by
+      )
+      @out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}"
+      servers_jobs_ids[server.id] = job_id
+    end
+    puts_and_flush "\n"
+    servers_jobs_ids
+  end
+end

devops-service/workers/stack_bootstrap/stack_servers_persister.rb

@@ -0,0 +1,69 @@
+require 'workers/stack_bootstrap/chef_node_name_builder'
+
+class StackServersPersister
+  include PutsAndFlush
+  attr_reader :stack, :out
+
+  def initialize(stack, out)
+    @stack, @out = stack, out
+    @project = mongo.project(stack.project)
+    @deploy_env = @project.deploy_env(stack.deploy_env)
+    @provider = stack.provider_instance
+  end
+
+  # returns: {  priority_as_integer => [Servers] }
+  def persist
+    puts_and_flush 'Start syncing stack servers with CID'
+
+    stack_servers_with_priority = {}
+    stack_servers_info.each do |priority, info_array|
+      stack_servers_with_priority[priority] = info_array.map do |info_hash|
+        out.puts "Instance '#{info_hash["id"]}' has been launched with stack."
+        persist_stack_server(info_hash)
+      end
+    end
+    puts_and_flush "Stack servers have been synced with CID"
+    stack_servers_with_priority.each do |priority, servers|
+      out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
+    end
+    out.flush
+    stack_servers_with_priority
+  end
+
+  private
+
+  # returns: {priority_as_integer => array_of_sersvers_info}
+  def stack_servers_info
+    stack_servers = @provider.stack_servers(stack)
+    stack_servers.each do |info|
+      info['tags']['cid:priority'] = info['tags']['cid:priority'].to_i
+    end
+    stack_servers.group_by{|info| info['tags']['cid:priority']}
+  end
+
+  # takes a hash, returns Server model
+  def persist_stack_server(info_hash)
+    server_attrs = {
+      '_id' => info_hash['id'],
+      'chef_node_name' => ChefNodeNameBuilder.new(info_hash, @project, @deploy_env).build_node_name,
+      'created_by' => stack.owner,
+      'deploy_env' => @deploy_env.identifier,
+      'key' => info_hash['key_name'] || @provider.ssh_key,
+      'project' => @project.id,
+      'provider' => @provider.name,
+      'remote_user' => mongo.image(@deploy_env.image).remote_user,
+      'private_ip' => info_hash['private_ip'],
+      'public_ip' => info_hash['public_ip'],
+      'run_list' => stack.run_list || [],
+      'stack' => stack.name
+    }
+
+    server = ::Devops::Model::Server.new(server_attrs)
+    mongo.server_insert(server)
+    server
+  end
+
+  def mongo
+    Devops::Db.connector
+  end
+end

devops-service/workers/stack_bootstrap/stack_synchronizer.rb

@@ -0,0 +1,71 @@
+class StackSynchronizer
+  include PutsAndFlush
+  attr_reader :out, :stack
+
+  def initialize(stack, out)
+    @stack, @out = stack, out
+    @printed_events = []
+  end
+
+  def sync
+    puts_and_flush "Syncing stack '#{stack.id}'..."
+
+    # 5 tries each 5 seconds, then 200 tries each 10 seconds
+    sleep_times = [5]*5 + [10]*400
+
+    sleep_times.each do |sleep_time|
+      sleep sleep_time
+      stack.sync_details!
+      print_new_events
+      case stack.stack_status
+      when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS'
+      when 'CREATE_COMPLETE'
+        ::Devops::Db.connector.stack_update(stack)
+        puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
+        return 0
+      when 'ROLLBACK_COMPLETE'
+        puts_and_flush "Stack '#{stack.id}' status is rolled back"
+        return error_code(:stack_rolled_back)
+      else
+        puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
+        return error_code(:unkown_status)
+      end
+    end
+    puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
+    error_code(:timeout)
+  rescue StandardError => e
+    DevopsLogger.logger.error e.message
+    puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}"
+    error_code(:error)
+  end
+
+  def reason_from_error_code(code)
+    error_codes.key(code)
+  end
+
+  private
+
+  def error_code(reason)
+    error_codes.fetch(reason)
+  end
+
+  def error_codes
+    {
+      stack_rolled_back: 1,
+      unkown_status: 2,
+      timeout: 3,
+      error: 5
+    }
+  end
+
+
+  def print_new_events
+    stack.events.each do |event|
+      unless @printed_events.include?(event["event_id"])
+        @printed_events << event["event_id"]
+        out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
+      end
+    end
+    out.flush
+  end
+end

devops-service/workers/stack_bootstrap_worker.rb

@@ -1,14 +1,13 @@
-require "commands/stack"
 require "db/mongo/models/stack/stack_factory"
 require "db/mongo/models/project"
 require "db/mongo/models/report"
+require "workers/stack_bootstrap/stack_synchronizer"
+require "workers/stack_bootstrap/stack_servers_bootstrapper"
+require "workers/stack_bootstrap/stack_servers_persister"
+require "workers/stack_bootstrap/errors"
 
-class StackCreatingError < StandardError; end
-class BootstrapingStackServerError < StandardError; end
-class DeployingStackServerError < StandardError; end
 
 class StackBootstrapWorker < Worker
-  include StackCommands
 
   def perform(options)
     stack_attrs = options.fetch('stack_attributes')
@@ -18,38 +17,33 @@ class StackBootstrapWorker < Worker
       without_bootstrap = stack_attrs.delete('without_bootstrap')
       @out.puts "Received 'without_bootstrap' option" if without_bootstrap
 
-      report = save_report(file, stack_attrs)
+      save_report(file, stack_attrs)
 
       begin
-        stack = create_stack(stack_attrs)
+        @stack = create_stack(stack_attrs)
 
-        #TODO: errors
         begin
-          servers_with_priority = persist_stack_servers!(stack)
-          unless without_bootstrap
-            sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
-            sorted_keys.each do |key|
-              @out.puts "Servers with priority '#{key}':"
-              bootstrap_servers!(servers_with_priority[key], report)
-            end
-          end
-          @out.puts "Done."
+          @servers_with_priority = stack_servers_persister.persist
+          bootstrap_in_priority_order unless without_bootstrap
           0
-        rescue BootstrapingStackServerError
-          @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
-          rollback_stack!(stack)
+        rescue StackServerBootstrapError
+          puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
+          rollback_stack!(@stack)
           2
-        rescue DeployingStackServerError => e
-          @out.puts "\nStack was launched, but an error occured during deploying stack servers."
-          @out.puts "You can redeploy stack after fixing the error."
+        rescue StackServerDeployError => e
+          out.puts "\nStack was launched, but an error occured during deploying stack servers."
+          puts_and_flush "You can redeploy stack after fixing the error."
           3
+        rescue StackServerBootstrapDeployTimeout
+          puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
+          4
         rescue StandardError => e
-          @out.puts "\nAn error occured. Initiating stack rollback."
-          rollback_stack!(stack)
+          puts_and_flush "\nAn error occured. Initiating stack rollback."
+          rollback_stack!(@stack)
           raise e
         end
       rescue StackCreatingError
-        @out.puts "Stack creating error"
+        puts_and_flush "Stack creating error"
         1
       end
     end
@@ -57,97 +51,52 @@ class StackBootstrapWorker < Worker
 
   private
 
-  def rollback_stack!(stack)
-    @out.puts "\nStart rollback of a stack"
-    stack.delete_stack_in_cloud!
-    Devops::Db.connector.stack_servers_delete(stack.name)
-    Devops::Db.connector.stack_delete(stack.id)
-    @out.puts "Rollback has been completed"
+  def stack_synchronizer(stack)
+    StackSynchronizer.new(stack, out)
   end
 
+  def stack_servers_persister
+    @stack_servers_persister ||= StackServersPersister.new(@stack, out)
+  end
+
+  def stack_servers_bootstrapper
+    @stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid)
+  end
+
+  # builds and persist stack model, initiate stack creating in cloud
   def create_stack(stack_attrs)
     stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
     mongo.stack_insert(stack)
-    operation_result = sync_stack_proc.call(@out, stack, mongo)
+    synchronizer = stack_synchronizer(stack)
+    operation_result = synchronizer.sync
 
     if operation_result == 0
-      @out.puts "\nStack '#{stack.name}' has been created"
-      @out.flush
+      puts_and_flush "\nStack '#{stack.name}' has been created"
       stack
     else
-      human_readable_code = StackCommands.result_codes.key(operation_result)
-      @out.puts "An error ocurred during stack creating"
-      @out.puts "Stack creating operation result was #{human_readable_code}"
+      human_readable_code = synchronizer.reason_from_error_code(operation_result)
+      out.puts "An error ocurred during stack creating"
+      puts_and_flush "Stack creating operation result was #{human_readable_code}"
       raise StackCreatingError
     end
   end
 
-  def bootstrap_servers!(servers, report)
-    @out << "\nStart bootstraping stack servers\n"
-
-    subreports = []
-    data = {}
-    servers.each do |server|
-      sjid = Worker.start_async(BootstrapWorker,
-        server_attrs: server.to_mongo_hash,
-        bootstrap_template: 'omnibus',
-        owner: server.created_by
-      )
-      subreports << sjid
-      @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
-      data[server.id] = sjid
+  # Bootstrap servers with high priorities first
+  def bootstrap_in_priority_order
+    sorted_priorities = @servers_with_priority.keys.sort.reverse
+    sorted_priorities.each do |priority|
+      @out.puts "Servers with priority '#{priority}':"
+      stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority])
     end
-    @out.puts
-    @out.flush
-    mongo.add_report_subreports(jid, subreports)
-    results = []
-    data.each do |server_id, subreport_id|
-      begin
-        sleep(5)
-        subreport = mongo.report(subreport_id)
-        status = subreport.status
-        if status == Worker::STATUS::COMPLETED
-          @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
-          break
-        elsif status == Worker::STATUS::FAILED
-          results << subreport.job_result_code
-          @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
-          break
-        end
-      end while(true)
-    end
-    @out.flush
-    results.empty? ? 0 : -5
+    puts_and_flush "Done."
   end
 
-  def check_bootstrap_results!(results)
-    if results.values.all?(&:zero?)
-      # everything is OK
-      @out.puts "Stack servers have been bootstraped"
-      @out.flush
-      return 0
-    end
-
-    @out.puts
-    results.each do |chef_node_name, code|
-      human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code)
-      @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
-    end
-
-    if errors_in_bootstrapping_present?(results.values)
-      raise BootstrapingStackServerError # will cause rollback of a stack
-    else
-      raise DeployingStackServerError #will not cause rollback of a stack
-    end
-  end
-
-  def errors_in_bootstrapping_present?(result_codes)
-    bootstrap_error_codes = []
-    [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
-      bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code)
-    end
-
-    (bootstrap_error_codes & result_codes).size > 0
+  def rollback_stack!(stack)
+    puts_and_flush "\nStart rollback of a stack"
+    stack.delete_stack_in_cloud!
+    Devops::Db.connector.stack_servers_delete(stack.name)
+    Devops::Db.connector.stack_delete(stack.id)
+    puts_and_flush "Rollback has been completed"
   end
 
   def save_report(file, stack_attrs)
@@ -164,54 +113,4 @@ class StackBootstrapWorker < Worker
     mongo.save_report(report)
     report
   end
-
-  # returns
-  # {
-  #   "priority" => [Servers]
-  # }
-  def persist_stack_servers!(stack)
-    @out.puts "Start syncing stack servers with CID"
-    @out.flush
-    project = mongo.project(stack.project)
-    deploy_env = project.deploy_env(stack.deploy_env)
-    provider = stack.provider_instance
-
-    stack_servers = provider.stack_servers(stack)
-    stack_servers.each do |info|
-      info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
-    end
-    stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
-    stack_servers_with_priority = {}
-    stack_servers_info.each do |priority, info_array|
-      stack_servers_with_priority[priority] = info_array.map do |extended_info|
-        @out.puts "Instance '#{extended_info["id"]}' has been launched with stack."
-        server_attrs = {
-          'provider' => provider.name,
-          'project' => project.id,
-          'deploy_env' => deploy_env.identifier,
-          'remote_user' => mongo.image(deploy_env.image).remote_user,
-          'key' => extended_info["key_name"] || provider.ssh_key,
-          '_id' => extended_info["id"],
-          'chef_node_name' => extended_info["name"],
-          'private_ip' => extended_info["private_ip"],
-          'public_ip' => extended_info["public_ip"],
-          'created_by' => stack.owner,
-          'run_list' => stack.run_list || [],
-          'stack' => stack.name
-        }
-
-        server = ::Devops::Model::Server.new(server_attrs)
-        mongo.server_insert(server)
-        # server.chef_node_name =  provider.create_default_chef_node_name(server)
-        server
-      end
-    end
-    @out.puts "Stack servers have been synced with CID"
-    stack_servers_with_priority.each do |priority, servers|
-      @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
-    end
-    @out.flush
-    stack_servers_with_priority
-  end
-
 end

devops-service/workers/worker.rb

@@ -12,11 +12,12 @@ require "core/devops-logger"
 require "core/devops-db"
 require "providers/provider_factory"
 require "lib/knife/knife_factory"
-
+require "lib/puts_and_flush"
 
 # All options keys MUST be a symbol!!!
 class Worker
   include Sidekiq::Worker
+  include PutsAndFlush
 
   attr_accessor :out