put executor helper classes inside modules

devops-service/lib/executors/stack_executor.rb

@@ -1,5 +1,6 @@
 require "db/mongo/models/stack/stack_factory"
 require "lib/puts_and_flush"
+module Devops; module Executor; class StackExecutor; end; end; end # predeclare for visibility
 require_relative "stack_executor/stack_creation_waiter"
 require_relative "stack_executor/stack_servers_persister"
 require_relative "stack_executor/prioritized_groups_bootstrapper"

devops-service/lib/executors/stack_executor/chef_node_name_builder.rb

@@ -1,26 +1,27 @@
 # Builds node name from mask. Mask could be passed in server's +cid:node-name-mask+ tag,
 # default mask is used otherwise. Mask is a string with possible several variables inserted, e.g.
-# +':project/:env/:instanceid/:instancename/:increment-group1:'+
+# ':project/:env/:instanceid/:instancename/:increment-group1:'
 # Variables meanings:
-# - +:project+ is replaced with project name (given in constructor)
+# - :project is replaced with project name (given in constructor)
-# - +:env+ is replaced with env name (given in constructor)
+# - :env is replaced with env name (given in constructor)
-# - +:time+ is replaced with current timestamp
+# - :time is replaced with current timestamp
-# - +:instanceid+ is replaced with provider instance id (fetched from server info)
+# - :instanceid is replaced with provider instance id (fetched from server info)
-# - +:instancename+ is replaced with value of Name tag (fetched from server info)
+# - :instancename is replaced with value of Name tag (fetched from server info)
-# - +:increment-groupname:+ is replaced with incremented number tied to group name. There could be several groups in one stack.
+# - :increment-groupname: is replaced with incremented number tied to group name. There could be several groups in one stack.
 # P.S. Colons are used instead of dollar signs, because stacks don't support dollar signs in tags (unlike EC2 instances),
 # but it's convenient to set mask tag directly to a stack (not in template): you set tag once and it propagates to all instances.
 
-class ChefNodeNameBuilder
+class Devops::Executor::StackExecutor
+  class ChefNodeNameBuilder
     # We need to ensure that node name is uniq by default. We can't use :time to provide uniqueness because
     # several servers are persisting at once on stack creating at it's likely that Time.now.to_i will give similar values
     # to different servers. So (by default) we should use :instanceid.
     DEFAULT_MASK = ':project-:env-:instanceid'
 
     # @param attrs [Hash] should contain
-  #   +:provider_server_info+
+    #   :provider_server_info
-  #   +:project_id+
+    #   :project_id
-  #   +:env_id+
+    #   :env_id
     def initialize(attrs)
       @server_info = attrs[:provider_server_info]
       @project, @env = attrs[:project_id], attrs[:env_id]
@@ -28,8 +29,9 @@ class ChefNodeNameBuilder
     end
 
 
-  # @param incrementers_values [Hash] is a hash in which key is name of a variable and value is last substituted number for that var.
+    # @param incrementers_values [Hash] is a hash in which key is name of a variable
-  # This method modifies +incrementers_values+, updating values for substituted variables.
+    # and value is last substituted number for that var.
+    # This method modifies incrementers_values, updating values for substituted variables.
     #
     # Examples (assume mask is set to +':project-master-:increment-group1:'+):
     #   incremeters_values = {}
@@ -64,4 +66,5 @@ class ChefNodeNameBuilder
         current_value.to_s.rjust(2, '0')
       end
     end
+  end
 end

devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb

@@ -2,7 +2,8 @@ require_relative 'servers_bootstrapper'
 
 # Bootstrap groups of servers based on priorities: higher first.
 # Doesn't start bootstrap of next group if bootstrap of previous group failed.
-class PrioritizedGroupsBootstrapper
+class Devops::Executor::StackExecutor
+  class PrioritizedGroupsBootstrapper
     include PutsAndFlush
     attr_reader :out
 
@@ -33,4 +34,5 @@ class PrioritizedGroupsBootstrapper
     def most_critical_error(results)
       results.detect(&:bootstrap_error?) || results.detect(&:failed?)
     end
+  end
 end

devops-service/lib/executors/stack_executor/servers_bootstrapper.rb

@@ -1,8 +1,9 @@
+require 'lib/helpers/job_waiter'
 require 'workers/bootstrap_worker'
-require 'workers/helpers/job_waiter'
 
 # Starts bootstrap workers for each server in group and wait for them to end (synchroniously).
-class ServersBootstrapper
+class Devops::Executor::StackExecutor
+  class ServersBootstrapper
     include PutsAndFlush
     attr_reader :out
 
@@ -53,9 +54,9 @@ class ServersBootstrapper
     end
 
     def wait_for_bootstrap_job(job_id)
-    result_code = JobWaiter.new(job_id).wait
+      result_code = Devops::Helpers::JobWaiter.new(job_id).wait
       result_from_job_code(result_code)
-  rescue JobWaiter::TimeoutReached
+    rescue Devops::Helpers::JobWaiter::TimeoutReached
       result(:timeout_reached)
     end
 
@@ -69,4 +70,5 @@ class ServersBootstrapper
         result(:deploy_error)
       end
     end
+  end
 end

devops-service/lib/executors/stack_executor/stack_creation_waiter.rb

@@ -1,5 +1,6 @@
 # Polling stack status until it's completed or failed.
-class StackCreationWaiter
+class Devops::Executor::StackExecutor
+  class StackCreationWaiter
     include PutsAndFlush
     attr_reader :out, :stack
 
@@ -96,4 +97,5 @@ class StackCreationWaiter
       end
       out.flush
     end
+  end
 end

devops-service/lib/executors/stack_executor/stack_servers_persister.rb

@@ -1,7 +1,8 @@
 require_relative 'chef_node_name_builder'
 
 # Fetches info about stack servers from provider and then persist them in mongo.
-class StackServersPersister
+class Devops::Executor::StackExecutor
+  class StackServersPersister
     include PutsAndFlush
     attr_reader :stack, :out, :deleted, :servers_info
 
@@ -108,4 +109,5 @@ class StackServersPersister
     def mongo
       Devops::Db.connector
     end
+  end
 end

devops-service/lib/helpers/job_waiter.rb

@@ -0,0 +1,27 @@
+module Devops
+  module Helpers
+    class JobWaiter
+      class TimeoutReached < StandardError; end
+
+      INTERVAL = 5
+
+      def initialize(job_id, timeout=5000)
+        @job_id, @timeout = job_id, timeout
+      end
+
+      def wait
+        (@timeout / INTERVAL).times do
+          sleep(INTERVAL)
+          report = ::Devops::Db.connector.report(@job_id)
+          case report.status
+          when Worker::STATUS::COMPLETED
+            return 0
+          when Worker::STATUS::FAILED
+            return report.job_result_code
+          end
+        end
+        raise TimeoutReached
+      end
+    end
+  end
+end

devops-service/spec/executors/helpers/job_waiter_spec.rb

@@ -0,0 +1,36 @@
+require 'lib/helpers/job_waiter'
+require "db/mongo/models/report"
+
+module Devops::Helpers
+  RSpec.describe JobWaiter, stubbed_connector: true do
+    let(:job_waiter) { described_class.new('job_id') }
+    let(:report) { build(:report) }
+
+    before do
+      allow(stubbed_connector).to receive(:report) { report }
+      allow(job_waiter).to receive(:sleep)
+    end
+
+    it 'it returns 0 when job become completed' do
+      allow(report).to receive(:status) { 'completed' }
+      expect(job_waiter.wait).to eq 0
+    end
+
+    it 'returns error code when job failes' do
+      allow(report).to receive(:status) { 'failed' }
+      allow(report).to receive(:job_result_code) { 1 }
+      expect(job_waiter.wait).to eq 1
+    end
+
+    it 'sleeps until something happens' do
+      allow(report).to receive(:status).and_return('running', 'running', 'running', 'completed')
+      expect(job_waiter).to receive(:sleep).exactly(4).times
+      job_waiter.wait
+    end
+
+    it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do
+      allow(report).to receive(:status) { 'running' }
+      expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached)
+    end
+  end
+end

devops-service/spec/executors/stack_executor/chef_node_name_builder_spec.rb

@@ -0,0 +1,74 @@
+require 'lib/executors/stack_executor/chef_node_name_builder'
+class Devops::Executor::StackExecutor
+  RSpec.describe ChefNodeNameBuilder do
+    # test with real response to ensure it is processed correctly
+    let(:server_info) do
+      {
+        "name"=>"stack-achuchkalov-aws-test-1455976199-master01",
+        "id"=>"i-fac32c7e",
+        "key_name"=>"achuchkalov",
+        "private_ip"=>"172.31.11.30",
+        "public_ip"=>"52.90.250.51",
+        "tags" => {
+          "Name"=>"master01",
+          "aws:cloudformation:logical-id"=>"EC2Instance1",
+          "aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199",
+          "StackTemplate"=>"1inst",
+          "aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6",
+          "cid:deployEnv"=>"test",
+          "cid:project"=>"aws",
+          "cid:user"=>"root",
+          "cid:priority"=>0
+        }
+      }
+    end
+    let(:node_name_builder) {
+      ChefNodeNameBuilder.new(
+        provider_server_info: server_info,
+        project_id: 'proj',
+        env_id: 'dev'
+      )
+    }
+    let(:build_node_name) { node_name_builder.build_node_name!({}) }
+
+    def set_mask(mask)
+      server_info['tags']['cid:node-name-mask'] = mask
+    end
+
+    describe '#build_node_name' do
+      it 'uses default mask (":project-:env-:instanceid")' do
+        expect(build_node_name).to eq 'proj-dev-i-fac32c7e'
+      end
+
+      it 'substitutes :project, :env, :instanceid and :instancename' do
+        set_mask(':project/:env/:instanceid/:instancename')
+        expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01'
+      end
+
+      it 'substitutes :time' do
+        set_mask(':project-:time')
+        expect(build_node_name).to match /proj-\d+/
+      end
+
+      describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do
+        it 'starts with 01 for empty hash' do
+          set_mask('node-:increment-slave:')
+          expect(node_name_builder.build_node_name!({})).to eq 'node-01'
+        end
+
+        it "continues with next values if hash isn't empty" do
+          set_mask('node-:increment-slave:')
+          expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01'
+          expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02'
+          expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51'
+        end
+
+        it 'could substitute different incrementers at once' do
+          set_mask('node-:increment-slave:-:increment-master:')
+          expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04'
+        end
+      end
+
+    end
+  end
+end

devops-service/spec/executors/stack_executor/prioritized_groups_bootstrapper_spec.rb

@@ -0,0 +1,50 @@
+require 'lib/executors/stack_executor/prioritized_groups_bootstrapper'
+
+class Devops::Executor::StackExecutor
+  RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
+    let(:out) { double(:out, puts: nil, flush: nil) }
+    let(:jid) { 1000 }
+    let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
+    before do
+      @array1 = []; @array2 = []; @array3 = []
+      @servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
+    end
+
+    describe '#bootstrap_servers_by_priority' do
+      subject { groups_bootstrapper.bootstrap_servers_by_priority }
+
+      it 'bootstraps servers in order by priorities, separately' do
+        allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
+        expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
+        expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
+        expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
+        expect(subject).to be_ok
+      end
+
+      it 'it returns :bootstrap_error result if error occured during bootstrap' do
+        allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+          [ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
+        }
+        expect(subject).to be_bootstrap_error
+      end
+
+      it 'it returns :deploy_error result if error occured during deploy' do
+        allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+          [ServersBootstrapper::Result.from_reason(:deploy_error)]
+        }
+        expect(subject).to be_deploy_error
+      end
+
+      it "doesn't bootstrap group if previous one failed" do
+        allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
+          [ServersBootstrapper::Result.from_reason(:deploy_error)]
+        }
+        allow(ServersBootstrapper).to receive(:new).and_call_original
+        expect(ServersBootstrapper).to receive(:new).once
+        subject
+      end
+
+
+    end
+  end
+end

devops-service/spec/executors/stack_executor/servers_bootstrapper_spec.rb

@@ -0,0 +1,43 @@
+require 'lib/executors/stack_executor/servers_bootstrapper'
+class Devops::Executor::StackExecutor
+  RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
+    let(:out) { double(:out, puts: nil, flush: nil) }
+    let(:jid) { 1000 }
+    let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
+    let(:bootstrapper) { described_class.new(out, jid, servers ) }
+    let(:bootstrap_job_ids) { %w(100 200) }
+
+    describe '#bootstrap_group' do
+      before do
+        allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
+        allow(stubbed_connector).to receive(:add_report_subreports)
+        allow(stubbed_connector).to receive(:report)
+        allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) { 0 }
+      end
+
+      it 'start bootstrap workers and add subreports' do
+        expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
+        expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
+        bootstrapper.bootstrap_group
+      end
+
+      it 'returns :ok result if everything ok' do
+        expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
+      end
+
+      it 'returns proper error results' do
+        waiter = instance_double(Devops::Helpers::JobWaiter)
+        allow(waiter).to receive(:wait).and_return(2, 8)
+        allow(Devops::Helpers::JobWaiter).to receive(:new) { waiter }
+        expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
+      end
+
+      it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
+        allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) {
+          raise Devops::Helpers::JobWaiter::TimeoutReached
+        }
+        expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
+      end
+    end
+  end
+end

devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb

@@ -0,0 +1,94 @@
+require 'lib/executors/stack_executor/stack_creation_waiter'
+
+class Devops::Executor::StackExecutor
+  RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do
+    let(:out) { double(:out, puts: nil, flush: nil) }
+    let(:stack) { build(:stack) }
+    let(:syncer) { described_class.new(stack, out) }
+
+    before do
+      allow(stack).to receive(:sync!)
+      allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
+      allow(syncer).to receive(:sleep)
+      allow(stubbed_connector).to receive(:stack_update)
+    end
+
+    def setup_statuses(statuses_array)
+      statuses = statuses_array.to_enum
+      allow(stack).to receive(:sync!) {
+        stack.stack_status = statuses.next
+      }
+    end
+
+    describe '#sync', stubbed_logger: true do
+      it 'waits for stack creating to be finished' do
+        setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
+        expect(syncer).to receive(:sleep).at_least(10).times
+        expect(stack).to receive(:sync!).at_least(10).times
+        syncer.sync
+      end
+
+      it 'prints each message only once' do
+        event1 = {'event_id' => 1, 'timestamp' => 't1'}
+        event2 = {'event_id' => 2, 'timestamp' => 't2'}
+        event3 = {'event_id' => 3, 'timestamp' => 't3'}
+
+        allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
+        setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
+        expect(syncer).to receive(:sleep).exactly(4).times
+        expect(out).to receive(:puts).with(/t1/).once.ordered
+        expect(out).to receive(:puts).with(/t2/).once.ordered
+        expect(out).to receive(:puts).with(/t3/).once.ordered
+        syncer.sync
+      end
+
+      it 'updates stack when status is changed' do
+        setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
+        expect(stubbed_connector).to receive(:stack_update).exactly(3).times
+        syncer.sync
+      end
+
+      context 'when stack creating was successful' do
+        it 'updates stack in DB when stack creating is finished and returns 0' do
+          setup_statuses(['CREATE_COMPLETE'])
+          expect(stubbed_connector).to receive(:stack_update).with(stack)
+          expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
+          expect(syncer.sync).to be_ok
+        end
+      end
+
+      context 'when stack was rollbacked' do
+        it 'returns 1 (:stack_rolled_back)' do
+          setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
+          expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
+          expect(syncer.sync).to be_stack_rolled_back
+        end
+      end
+
+      context 'when unkown stack status was found' do
+        it 'returns 2 (:unkown_status)' do
+          setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
+          expect(out).to receive(:puts).with(/unknown/)
+          expect(syncer.sync).to be_unkown_status
+        end
+      end
+
+      context "when stack hasn't been synced in an hour" do
+        it 'returns 3 (:timeout)' do
+          allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
+          expect(out).to receive(:puts).with(/hasn't been synced/)
+          expect(syncer.sync).to be_timeout
+        end
+      end
+
+      context 'when an error occured during syncing', stubbed_logger: true do
+        it 'returns 5 (:error)' do
+          setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
+          allow(stubbed_connector).to receive(:stack_update) { raise }
+          expect(syncer.sync.code).to eq 5
+        end
+      end
+    end
+
+  end
+end

devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb

@@ -0,0 +1,167 @@
+require 'lib/executors/stack_executor/stack_servers_persister'
+
+class Devops::Executor::StackExecutor
+  RSpec.describe StackServersPersister, stubbed_connector: true do
+    def info_hash_for_id(id)
+      {
+        'id' => id,
+        'name' => 'server_name',
+        'key_name' => 'key',
+        'private_ip' => '127.0.0.1',
+        'public_ip' => '127.0.0.2',
+        'tags' => {
+          'cid:priority' => '3',
+          'Name' => 'server1'
+        }
+      }
+    end
+
+    let(:out) { double(:out, puts: nil, flush: nil) }
+    let(:run_list) { ['role[asd]'] }
+    let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
+    let(:project) { build(:project, id: 'name') }
+    let(:persister) { described_class.new(stack, out) }
+    let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
+    let(:new_server_info) { info_hash_for_id('i-new') }
+    let(:deleted_server_info) { info_hash_for_id('i-deleted') }
+    let(:persisted_server_info) { info_hash_for_id('i-persisted') }
+
+    before do
+      allow(stubbed_connector).to receive(:project) { project }
+      allow(stubbed_connector).to receive(:image) {
+        instance_double(Devops::Model::Image, remote_user: 'user')
+      }
+      allow(stubbed_connector).to receive(:server_insert)
+      allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
+      allow(stack).to receive(:provider_instance) { provider }
+      allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
+    end
+
+    describe '#initialize' do
+      it 'fetches stack servers info' do
+        expect(provider).to receive(:stack_servers).with(stack)
+        described_class.new(stack, out)
+      end
+
+      it "doesn't raise error if cid:priority tag is absent" do
+        new_server_info['tags'].delete('cid:priority')
+        expect { described_class.new(stack, out) }.not_to raise_error
+      end
+
+      it 'sets proper statuses' do
+        infos = described_class.new(stack, out).servers_info
+        expect(infos.length).to eq 2
+        expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
+        expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
+      end
+
+      it 'sets deleted servers' do
+        deleted = described_class.new(stack, out).deleted
+        expect(deleted.length).to eq 1
+        expect(deleted.first.id).to eq 'i-deleted'
+      end
+    end
+
+    describe '#persist_new_servers' do
+      it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.id).to eq 'i-new'
+          expect(server.key).to eq 'key'
+          expect(server.private_ip).to eq '127.0.0.1'
+          expect(server.public_ip).to eq '127.0.0.2'
+        end
+        persister.persist_new_servers
+      end
+
+      it 'takes created_by, run_list and stack attrs from stack' do
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.created_by).to eq 'root'
+          expect(server.run_list).to eq run_list
+          expect(server.stack).to eq 'iamstack'
+        end
+        persister.persist_new_servers
+      end
+
+      it 'takes remote_user from image user' do
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.remote_user).to eq 'user'
+        end
+        persister.persist_new_servers
+      end
+
+      it "takes deploy_env from project's deploy_env identifier" do
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.deploy_env).to eq 'foo'
+        end
+        persister.persist_new_servers
+      end
+
+      it "takes default provider's ssh key if info doesn't contain it" do
+        allow(provider).to receive(:ssh_key) { 'default_key' }
+        new_server_info.delete('key_name')
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.key).to eq 'default_key'
+        end
+        persister.persist_new_servers
+      end
+
+      it "sets server's run list to empty array if stack's run_list is nil" do
+        stack.run_list = nil
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.run_list).to eq []
+        end
+        persister.persist_new_servers
+      end
+
+      it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.chef_node_name).to eq 'name-foo-i-new'
+        end
+        persister.persist_new_servers
+      end
+
+      it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
+        new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.chef_node_name).to eq 'name-server1-123'
+        end
+        persister.persist_new_servers
+      end
+
+      it "sets provider and provider account from stack" do
+        stack.provider_account = 'foo'
+        expect(stubbed_connector).to receive(:server_insert) do |server|
+          expect(server.provider).to eq 'ec2'
+          expect(server.provider_account).to eq 'foo'
+        end
+        persister.persist_new_servers
+      end
+
+      describe 'incremented variables' do
+        it 'substitutes :increment-groupid: with incrementing numbers' do
+          allow(provider).to receive(:stack_servers) {[
+            {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
+            {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
+          ]}
+          expect(stubbed_connector).to receive(:server_insert) do |server|
+            expect(server.chef_node_name).to eq 'node-01-dev'
+          end.ordered
+          expect(stubbed_connector).to receive(:server_insert) do |server|
+            expect(server.chef_node_name).to eq 'node-02-dev'
+          end
+          persister.persist_new_servers
+        end
+      end
+    end
+
+    describe '#just_persisted_by_priority' do
+      it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
+        persister.persist_new_servers
+        result = persister.just_persisted_by_priority
+        expect(result).to be_a(Hash)
+        expect(result.size).to eq 1
+        expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
+      end
+    end
+  end
+end

devops-service/spec/executors/stack_executor_spec.rb

@@ -1,6 +1,7 @@
 require 'lib/executors/stack_executor'
 
-RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do
+class Devops::Executor::StackExecutor
+  RSpec.describe self, type: :executor, stubbed_connector: true, stubbed_logger: true do
     let(:out) { double('out', puts: nil, flush: nil) }
     let(:stack) { build(:stack) }
     let(:executor_without_stack) { described_class.new(out: out) }
@@ -28,7 +29,9 @@ RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connect
 
     describe '#persist_stack_servers' do
       let(:persister) {
-      instance_double(StackServersPersister, persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil)
+        instance_double(StackServersPersister,
+          persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil
+        )
       }
       before { allow(StackServersPersister).to receive(:new).and_return(persister) }
 
@@ -60,6 +63,5 @@ RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connect
       end
 
     end
-
+  end
-
 end

devops-service/spec/workers/helpers/job_waiter_spec.rb

@@ -1,35 +0,0 @@
-require 'workers/helpers/job_waiter'
-require "db/mongo/models/report"
-
-RSpec.describe JobWaiter, stubbed_connector: true do
-  let(:job_waiter) { described_class.new('job_id') }
-
-  before do
-    @report_double = instance_double(Devops::Model::Report)
-    allow(stubbed_connector).to receive(:report) { @report_double }
-    allow(job_waiter).to receive(:sleep)
-  end
-
-  it 'it returns 0 when job become completed' do
-    allow(@report_double).to receive(:status) { 'completed' }
-    expect(job_waiter.wait).to eq 0
-  end
-
-  it 'returns error code when job failes' do
-    allow(@report_double).to receive(:status) { 'failed' }
-    allow(@report_double).to receive(:job_result_code) { 1 }
-    expect(job_waiter.wait).to eq 1
-  end
-
-  it 'sleeps until something happens' do
-    allow(@report_double).to receive(:status).and_return('running', 'running', 'running', 'completed')
-    expect(job_waiter).to receive(:sleep).exactly(4).times
-    job_waiter.wait
-  end
-
-  it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do
-    allow(@report_double).to receive(:status) { 'running' }
-    expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached)
-  end
-
-end

devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb

@@ -1,72 +0,0 @@
-require 'lib/executors/stack_executor/chef_node_name_builder'
-RSpec.describe ChefNodeNameBuilder do
-  # test with real response to ensure it is processed correctly
-  let(:server_info) do
-    {
-      "name"=>"stack-achuchkalov-aws-test-1455976199-master01",
-      "id"=>"i-fac32c7e",
-      "key_name"=>"achuchkalov",
-      "private_ip"=>"172.31.11.30",
-      "public_ip"=>"52.90.250.51",
-      "tags" => {
-        "Name"=>"master01",
-        "aws:cloudformation:logical-id"=>"EC2Instance1",
-        "aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199",
-        "StackTemplate"=>"1inst",
-        "aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6",
-        "cid:deployEnv"=>"test",
-        "cid:project"=>"aws",
-        "cid:user"=>"root",
-        "cid:priority"=>0
-      }
-    }
-  end
-  let(:node_name_builder) {
-    ChefNodeNameBuilder.new(
-      provider_server_info: server_info,
-      project_id: 'proj',
-      env_id: 'dev'
-    )
-  }
-  let(:build_node_name) { node_name_builder.build_node_name!({}) }
-
-  def set_mask(mask)
-    server_info['tags']['cid:node-name-mask'] = mask
-  end
-
-  describe '#build_node_name' do
-    it 'uses default mask (":project-:env-:instanceid")' do
-      expect(build_node_name).to eq 'proj-dev-i-fac32c7e'
-    end
-
-    it 'substitutes :project, :env, :instanceid and :instancename' do
-      set_mask(':project/:env/:instanceid/:instancename')
-      expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01'
-    end
-
-    it 'substitutes :time' do
-      set_mask(':project-:time')
-      expect(build_node_name).to match /proj-\d+/
-    end
-
-    describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do
-      it 'starts with 01 for empty hash' do
-        set_mask('node-:increment-slave:')
-        expect(node_name_builder.build_node_name!({})).to eq 'node-01'
-      end
-
-      it "continues with next values if hash isn't empty" do
-        set_mask('node-:increment-slave:')
-        expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01'
-        expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02'
-        expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51'
-      end
-
-      it 'could substitute different incrementers at once' do
-        set_mask('node-:increment-slave:-:increment-master:')
-        expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04'
-      end
-    end
-
-  end
-end

devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb

@@ -1,48 +0,0 @@
-require 'lib/executors/stack_executor/prioritized_groups_bootstrapper'
-
-RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
-  let(:out) { double(:out, puts: nil, flush: nil) }
-  let(:jid) { 1000 }
-  let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
-  before do
-    @array1 = []; @array2 = []; @array3 = []
-    @servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
-  end
-
-  describe '#bootstrap_servers_by_priority' do
-    subject { groups_bootstrapper.bootstrap_servers_by_priority }
-
-    it 'bootstraps servers in order by priorities, separately' do
-      allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
-      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
-      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
-      expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
-      expect(subject).to be_ok
-    end
-
-    it 'it returns :bootstrap_error result if error occured during bootstrap' do
-      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
-        [ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
-      }
-      expect(subject).to be_bootstrap_error
-    end
-
-    it 'it returns :deploy_error result if error occured during deploy' do
-      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
-        [ServersBootstrapper::Result.from_reason(:deploy_error)]
-      }
-      expect(subject).to be_deploy_error
-    end
-
-    it "doesn't bootstrap group if previous one failed" do
-      allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
-        [ServersBootstrapper::Result.from_reason(:deploy_error)]
-      }
-      allow(ServersBootstrapper).to receive(:new).and_call_original
-      expect(ServersBootstrapper).to receive(:new).once
-      subject
-    end
-
-
-  end
-end

devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb

@@ -1,40 +0,0 @@
-require 'lib/executors/stack_executor/servers_bootstrapper'
-
-RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
-  let(:out) { double(:out, puts: nil, flush: nil) }
-  let(:jid) { 1000 }
-  let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
-  let(:bootstrapper) { described_class.new(out, jid, servers ) }
-  let(:bootstrap_job_ids) { %w(100 200) }
-
-  describe '#bootstrap_group' do
-    before do
-      allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
-      allow(stubbed_connector).to receive(:add_report_subreports)
-      allow(stubbed_connector).to receive(:report)
-      allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
-    end
-
-    it 'start bootstrap workers and add subreports' do
-      expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
-      expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
-      bootstrapper.bootstrap_group
-    end
-
-    it 'returns :ok result if everything ok' do
-      expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
-    end
-
-    it 'returns proper error results' do
-      waiter = instance_double(JobWaiter)
-      allow(waiter).to receive(:wait).and_return(2, 8)
-      allow(JobWaiter).to receive(:new) { waiter }
-      expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
-    end
-
-    it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
-      allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
-      expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
-    end
-  end
-end

devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb

@@ -1,91 +0,0 @@
-require 'lib/executors/stack_executor/stack_creation_waiter'
-RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do
-  let(:out) { double(:out, puts: nil, flush: nil) }
-  let(:stack) { build(:stack) }
-  let(:syncer) { described_class.new(stack, out) }
-
-  before do
-    allow(stack).to receive(:sync!)
-    allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
-    allow(syncer).to receive(:sleep)
-    allow(stubbed_connector).to receive(:stack_update)
-  end
-
-  def setup_statuses(statuses_array)
-    statuses = statuses_array.to_enum
-    allow(stack).to receive(:sync!) {
-      stack.stack_status = statuses.next
-    }
-  end
-
-  describe '#sync', stubbed_logger: true do
-    it 'waits for stack creating to be finished' do
-      setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
-      expect(syncer).to receive(:sleep).at_least(10).times
-      expect(stack).to receive(:sync!).at_least(10).times
-      syncer.sync
-    end
-
-    it 'prints each message only once' do
-      event1 = {'event_id' => 1, 'timestamp' => 't1'}
-      event2 = {'event_id' => 2, 'timestamp' => 't2'}
-      event3 = {'event_id' => 3, 'timestamp' => 't3'}
-
-      allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
-      setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
-      expect(syncer).to receive(:sleep).exactly(4).times
-      expect(out).to receive(:puts).with(/t1/).once.ordered
-      expect(out).to receive(:puts).with(/t2/).once.ordered
-      expect(out).to receive(:puts).with(/t3/).once.ordered
-      syncer.sync
-    end
-
-    it 'updates stack when status is changed' do
-      setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
-      expect(stubbed_connector).to receive(:stack_update).exactly(3).times
-      syncer.sync
-    end
-
-    context 'when stack creating was successful' do
-      it 'updates stack in DB when stack creating is finished and returns 0' do
-        setup_statuses(['CREATE_COMPLETE'])
-        expect(stubbed_connector).to receive(:stack_update).with(stack)
-        expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
-        expect(syncer.sync).to be_ok
-      end
-    end
-
-    context 'when stack was rollbacked' do
-      it 'returns 1 (:stack_rolled_back)' do
-        setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
-        expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
-        expect(syncer.sync).to be_stack_rolled_back
-      end
-    end
-
-    context 'when unkown stack status was found' do
-      it 'returns 2 (:unkown_status)' do
-        setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
-        expect(out).to receive(:puts).with(/unknown/)
-        expect(syncer.sync).to be_unkown_status
-      end
-    end
-
-    context "when stack hasn't been synced in an hour" do
-      it 'returns 3 (:timeout)' do
-        allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
-        expect(out).to receive(:puts).with(/hasn't been synced/)
-        expect(syncer.sync).to be_timeout
-      end
-    end
-
-    context 'when an error occured during syncing', stubbed_logger: true do
-      it 'returns 5 (:error)' do
-        setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
-        allow(stubbed_connector).to receive(:stack_update) { raise }
-        expect(syncer.sync.code).to eq 5
-      end
-    end
-  end
-
-end

devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb

@@ -1,166 +0,0 @@
-require 'lib/executors/stack_executor/stack_servers_persister'
-
-RSpec.describe StackServersPersister, stubbed_connector: true do
-  def info_hash_for_id(id)
-    {
-      'id' => id,
-      'name' => 'server_name',
-      'key_name' => 'key',
-      'private_ip' => '127.0.0.1',
-      'public_ip' => '127.0.0.2',
-      'tags' => {
-        'cid:priority' => '3',
-        'Name' => 'server1'
-      }
-    }
-  end
-
-  let(:out) { double(:out, puts: nil, flush: nil) }
-  let(:run_list) { ['role[asd]'] }
-  let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
-  let(:project) { build(:project, id: 'name') }
-  let(:persister) { described_class.new(stack, out) }
-  let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
-  let(:new_server_info) { info_hash_for_id('i-new') }
-  let(:deleted_server_info) { info_hash_for_id('i-deleted') }
-  let(:persisted_server_info) { info_hash_for_id('i-persisted') }
-
-  before do
-    allow(stubbed_connector).to receive(:project) { project }
-    allow(stubbed_connector).to receive(:image) {
-      instance_double(Devops::Model::Image, remote_user: 'user')
-    }
-    allow(stubbed_connector).to receive(:server_insert)
-    allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
-    allow(stack).to receive(:provider_instance) { provider }
-    allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
-  end
-
-  describe '#initialize' do
-    it 'fetches stack servers info' do
-      expect(provider).to receive(:stack_servers).with(stack)
-      described_class.new(stack, out)
-    end
-
-    it "doesn't raise error if cid:priority tag is absent" do
-      new_server_info['tags'].delete('cid:priority')
-      expect { described_class.new(stack, out) }.not_to raise_error
-    end
-
-    it 'sets proper statuses' do
-      infos = described_class.new(stack, out).servers_info
-      expect(infos.length).to eq 2
-      expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
-      expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
-    end
-
-    it 'sets deleted servers' do
-      deleted = described_class.new(stack, out).deleted
-      expect(deleted.length).to eq 1
-      expect(deleted.first.id).to eq 'i-deleted'
-    end
-  end
-
-  describe '#persist_new_servers' do
-    it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.id).to eq 'i-new'
-        expect(server.key).to eq 'key'
-        expect(server.private_ip).to eq '127.0.0.1'
-        expect(server.public_ip).to eq '127.0.0.2'
-      end
-      persister.persist_new_servers
-    end
-
-    it 'takes created_by, run_list and stack attrs from stack' do
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.created_by).to eq 'root'
-        expect(server.run_list).to eq run_list
-        expect(server.stack).to eq 'iamstack'
-      end
-      persister.persist_new_servers
-    end
-
-    it 'takes remote_user from image user' do
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.remote_user).to eq 'user'
-      end
-      persister.persist_new_servers
-    end
-
-    it "takes deploy_env from project's deploy_env identifier" do
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.deploy_env).to eq 'foo'
-      end
-      persister.persist_new_servers
-    end
-
-    it "takes default provider's ssh key if info doesn't contain it" do
-      allow(provider).to receive(:ssh_key) { 'default_key' }
-      new_server_info.delete('key_name')
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.key).to eq 'default_key'
-      end
-      persister.persist_new_servers
-    end
-
-    it "sets server's run list to empty array if stack's run_list is nil" do
-      stack.run_list = nil
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.run_list).to eq []
-      end
-      persister.persist_new_servers
-    end
-
-    it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.chef_node_name).to eq 'name-foo-i-new'
-      end
-      persister.persist_new_servers
-    end
-
-    it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
-      new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.chef_node_name).to eq 'name-server1-123'
-      end
-      persister.persist_new_servers
-    end
-
-    it "sets provider and provider account from stack" do
-      stack.provider_account = 'foo'
-      expect(stubbed_connector).to receive(:server_insert) do |server|
-        expect(server.provider).to eq 'ec2'
-        expect(server.provider_account).to eq 'foo'
-      end
-      persister.persist_new_servers
-    end
-
-    describe 'incremented variables' do
-      it 'substitutes :increment-groupid: with incrementing numbers' do
-        allow(provider).to receive(:stack_servers) {[
-          {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
-          {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
-        ]}
-        expect(stubbed_connector).to receive(:server_insert) do |server|
-          expect(server.chef_node_name).to eq 'node-01-dev'
-        end.ordered
-        expect(stubbed_connector).to receive(:server_insert) do |server|
-          expect(server.chef_node_name).to eq 'node-02-dev'
-        end
-        persister.persist_new_servers
-      end
-    end
-  end
-
-  describe '#just_persisted_by_priority' do
-    it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
-      persister.persist_new_servers
-      result = persister.just_persisted_by_priority
-      expect(result).to be_a(Hash)
-      expect(result.size).to eq 1
-      expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
-    end
-  end
-
-end

devops-service/spec/workers/stack_bootstrap_worker_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
   }
 
   def bootstrap_result(reason)
-    ServersBootstrapper::Result.from_reason(reason)
+    Devops::Executor::StackExecutor::ServersBootstrapper::Result.from_reason(reason)
   end
 
   before do

devops-service/workers/helpers/job_waiter.rb

@@ -1,23 +0,0 @@
-class JobWaiter
-  class TimeoutReached < StandardError; end
-
-  INTERVAL = 5
-
-  def initialize(job_id, timeout=5000)
-    @job_id, @timeout = job_id, timeout
-  end
-
-  def wait
-    (@timeout / INTERVAL).times do
-      sleep(INTERVAL)
-      report = ::Devops::Db.connector.report(@job_id)
-      case report.status
-      when Worker::STATUS::COMPLETED
-        return 0
-      when Worker::STATUS::FAILED
-        return report.job_result_code
-      end
-    end
-    raise TimeoutReached
-  end
-end