From 062aebe5f2c24bfd61d15ea2dc649ac6b90498f2 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <achuchkalov@ggasoftware.com>
Date: Thu, 10 Dec 2015 18:47:36 +0400
Subject: [PATCH 01/35] start writing provider_accounts_connector specs

---
 devops-service/app/api2/handlers/provider.rb  |  4 +--
 ...ovider_accounts.rb => provider_account.rb} |  2 +-
 .../provider_accounts/provider_account.rb     |  6 ++++
 devops-service/db/mongo/mongo_connector.rb    |  4 +--
 .../provider_accounts_connector_spec.rb       | 35 +++++++++++++++++++
 .../tester_connector/provider_account.rb      |  6 ++++
 6 files changed, 52 insertions(+), 5 deletions(-)
 rename devops-service/db/mongo/connectors/{provider_accounts.rb => provider_account.rb} (96%)
 create mode 100644 devops-service/spec/connectors/provider_accounts_connector_spec.rb
 create mode 100644 devops-service/spec/connectors/tester_connector/provider_account.rb

diff --git a/devops-service/app/api2/handlers/provider.rb b/devops-service/app/api2/handlers/provider.rb
index 8e5445d..7e74f27 100644
--- a/devops-service/app/api2/handlers/provider.rb
+++ b/devops-service/app/api2/handlers/provider.rb
@@ -27,14 +27,14 @@ module Devops
         def add_account provider
           account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account)
           account.validate_fields!
-          Devops::Db.connector.provider_accounts_insert(account)
+          Devops::Db.connector.provider_account_insert(account)
           ::Provider::ProviderFactory.add_account(provider, account)
           account.to_hash
         end
 
         def delete_account name, provider
           account = Devops::Db.connector.provider_account(provider, name)
-          Devops::Db.connector.provider_accounts_delete(name)
+          Devops::Db.connector.provider_account_delete(name)
           ::Provider::ProviderFactory.delete_account(provider, account)
           account.to_hash
         end
diff --git a/devops-service/db/mongo/connectors/provider_accounts.rb b/devops-service/db/mongo/connectors/provider_account.rb
similarity index 96%
rename from devops-service/db/mongo/connectors/provider_accounts.rb
rename to devops-service/db/mongo/connectors/provider_account.rb
index 4bd235a..5d51558 100644
--- a/devops-service/db/mongo/connectors/provider_accounts.rb
+++ b/devops-service/db/mongo/connectors/provider_account.rb
@@ -1,5 +1,5 @@
 module Connectors
-  class ProviderAccounts < Base
+  class ProviderAccount < Base
 
     include Helpers::InsertCommand,
             Helpers::DeleteCommand
diff --git a/devops-service/db/mongo/models/provider_accounts/provider_account.rb b/devops-service/db/mongo/models/provider_accounts/provider_account.rb
index d3c9c42..efc6dd1 100644
--- a/devops-service/db/mongo/models/provider_accounts/provider_account.rb
+++ b/devops-service/db/mongo/models/provider_accounts/provider_account.rb
@@ -61,6 +61,12 @@ module Devops
         }
       end
 
+      # absent of "id" attribute can cause some inconviniences.
+      # for example, we have "record.id" call in InsertCommand
+      def id
+        account_name
+      end
+
     end
   end
 end
diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb
index baa47ff..60deaa7 100644
--- a/devops-service/db/mongo/mongo_connector.rb
+++ b/devops-service/db/mongo/mongo_connector.rb
@@ -33,7 +33,7 @@ class MongoConnector
     [:keys, :key, :key_insert, :key_delete] => :keys_connector,
     [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
     [:insert_statistic, :search_statistic] => :statistics_connector,
-    [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_account] => :provider_accounts_connector
+    [:provider_accounts, :provider_account_insert, :provider_account_delete, :provider_account] => :provider_accounts_connector
   )
 
   def initialize(db, host, port=27017, user=nil, password=nil)
@@ -48,7 +48,7 @@ class MongoConnector
   private
 
   def provider_accounts_connector
-    @provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db)
+    @provider_accounts_connector ||= Connectors::ProviderAccount.new(@db)
   end
 
   def images_connector
diff --git a/devops-service/spec/connectors/provider_accounts_connector_spec.rb b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
new file mode 100644
index 0000000..994ca89
--- /dev/null
+++ b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
@@ -0,0 +1,35 @@
+require 'db/mongo/models/provider_accounts/ec2_provider_account'
+require 'db/mongo/models/provider_accounts/openstack_provider_account'
+require 'db/mongo/connectors/provider_account'
+require 'spec/connectors/tester_connector/provider_account'
+
+RSpec.describe Connectors::ProviderAccount, type: :connector do
+  set_tester_connector TesterConnector::ProviderAccount
+
+  include_examples 'mongo connector', {
+    model_name: :provider_account,
+    factory_name: :ec2_provider_account,
+    only: [:insert, :delete],
+    field_to_update: :description
+  }
+
+  describe '#provider_accounts', cleanup_after: :all do
+    before(:all) do
+      @tester_connector.create(account_name: 'foo', provider: 'ec2')
+      @tester_connector.create(account_name: 'bar', provider: 'openstack')
+    end
+
+    it 'returns array of Ec2ProviderAccount if @provider is ec2' do
+      expect(
+        @connector.provider_accounts('ec2')
+      ).to be_an_array_of(Devops::Model::Ec2ProviderAccount).and have_size(1)
+    end
+
+    it 'returns array of Ec2ProviderAccount if @provider is openstack' do
+      expect(
+        @connector.provider_accounts('openstack')
+      ).to be_an_array_of(Devops::Model::OpenstackProviderAccount).and have_size(1)
+    end
+  end
+
+end
diff --git a/devops-service/spec/connectors/tester_connector/provider_account.rb b/devops-service/spec/connectors/tester_connector/provider_account.rb
new file mode 100644
index 0000000..38a8feb
--- /dev/null
+++ b/devops-service/spec/connectors/tester_connector/provider_account.rb
@@ -0,0 +1,6 @@
+require_relative 'base'
+
+module TesterConnector
+  class ProviderAccount < Base
+  end
+end
\ No newline at end of file

From ba25c41082fd58febccaa6a17ebcfee2f6ebb3c7 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <achuchkalov@ggasoftware.com>
Date: Mon, 14 Dec 2015 17:27:07 +0400
Subject: [PATCH 02/35] add coverage tool

---
 devops-service/Gemfile             |  1 +
 devops-service/Gemfile.lock        | 10 +++++
 devops-service/spec/spec_helper.rb | 61 +++++++++++++++++++-----------
 3 files changed, 50 insertions(+), 22 deletions(-)

diff --git a/devops-service/Gemfile b/devops-service/Gemfile
index 13b88c3..d6628eb 100644
--- a/devops-service/Gemfile
+++ b/devops-service/Gemfile
@@ -36,4 +36,5 @@ end
 group :devepoment do
   gem 'byebug'
   gem 'guard-rspec', require: false
+  gem 'simplecov', require: false
 end
diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock
index 31da4a0..d9ca5a3 100644
--- a/devops-service/Gemfile.lock
+++ b/devops-service/Gemfile.lock
@@ -60,6 +60,7 @@ GEM
       gherkin (~> 2.12.0)
     daemons (1.2.3)
     diff-lcs (1.2.5)
+    docile (1.1.5)
     em-websocket (0.3.8)
       addressable (>= 2.1.1)
       eventmachine (>= 0.12.9)
@@ -291,6 +292,11 @@ GEM
       json
       redis (>= 3.0.6)
       redis-namespace (>= 1.3.1)
+    simplecov (0.11.1)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
     sinatra (1.4.5)
       rack (~> 1.4)
       rack-protection (~> 1.4)
@@ -352,9 +358,13 @@ DEPENDENCIES
   rspec (~> 3.3)
   rspec_junit_formatter
   sidekiq (= 3.2.6)
+  simplecov
   sinatra (= 1.4.5)
   sinatra-contrib
   sinatra-websocket
   test-unit
   thin (~> 1.5.1)
   wisper
+
+BUNDLED WITH
+   1.10.6
diff --git a/devops-service/spec/spec_helper.rb b/devops-service/spec/spec_helper.rb
index 9f2500f..5dd1d0a 100644
--- a/devops-service/spec/spec_helper.rb
+++ b/devops-service/spec/spec_helper.rb
@@ -4,32 +4,49 @@ require 'factory_girl'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/inflector'
 
-# setup load_path and require support files
-root = File.join(File.dirname(__FILE__), "..")
-$LOAD_PATH.push root unless $LOAD_PATH.include? root
-
-# suppress output
-original_stdout = $stdout
-$stdout = File.open(File::NULL, "w")
-
-Dir[("./spec/support/**/*.rb")].each { |f| require f }
-Dir[("./spec/shared_contexts/**/*.rb")].each { |f| require f }
-
-# Factory girl configuration
-FactoryGirl.define do
-  # do not try to persist, but raise validation errors
-  to_create { |model| model.validate! }
+def suppress_output!
+  original_stdout = $stdout
+  $stdout = File.open(File::NULL, "w")
+  RSpec.configure do |config|
+    config.after(:all) do
+      $stdout = original_stdout
+    end
+  end
 end
-FactoryGirl.find_definitions
+
+def check_coverage
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter { |src| src.filename =~ /spec\// }
+  end
+end
+
+def require_support_files
+  root = File.join(File.dirname(__FILE__), "..")
+  $LOAD_PATH.push root unless $LOAD_PATH.include?(root)
+  Dir[("#{root}/spec/support/**/*.rb")].each { |f| require f }
+  Dir[("#{root}/spec/shared_contexts/**/*.rb")].each { |f| require f }
+end
+
+def setup_factory_girl
+  FactoryGirl.define do
+    # do not persist, but raise validation errors
+    to_create { |model| model.validate! }
+  end
+  FactoryGirl.find_definitions
+  RSpec.configure { |config| config.include FactoryGirl::Syntax::Methods }
+end
+
+
+# extra configuration
+suppress_output!
+check_coverage if ENV['COVERAGE']
+require_support_files
+setup_factory_girl
+
 
 # RSpec configuration
 RSpec.configure do |config|
-  config.include FactoryGirl::Syntax::Methods
-
-  config.after(:all) do
-    $stdout = original_stdout
-  end
-
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest
   # assertions if you prefer.

From 9aaa66e7c62f49e0ed51b70964ed49eedbabb0d1 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 15 Dec 2015 15:49:23 +0300
Subject: [PATCH 03/35] finish provider_account connector spec

---
 .../db/mongo/connectors/provider_account.rb   |  2 +-
 .../provider_accounts_connector_spec.rb       | 27 +++++++++++++++++--
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/devops-service/db/mongo/connectors/provider_account.rb b/devops-service/db/mongo/connectors/provider_account.rb
index 5d51558..931f99f 100644
--- a/devops-service/db/mongo/connectors/provider_account.rb
+++ b/devops-service/db/mongo/connectors/provider_account.rb
@@ -16,7 +16,7 @@ module Connectors
     def provider_account provider, account
       c = Provider::ProviderFactory.get_account_class(provider)
       bson = collection.find({provider: provider, _id: account}).to_a.first
-      raise RecordNotFound.new("'Account #{account}' for provider '#{provider}' not found") unless bson
+      raise RecordNotFound.new("Account '#{account}' for provider '#{provider}' not found") unless bson
       c.build_from_bson(bson)
     end
 
diff --git a/devops-service/spec/connectors/provider_accounts_connector_spec.rb b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
index 994ca89..2500179 100644
--- a/devops-service/spec/connectors/provider_accounts_connector_spec.rb
+++ b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
@@ -15,8 +15,8 @@ RSpec.describe Connectors::ProviderAccount, type: :connector do
 
   describe '#provider_accounts', cleanup_after: :all do
     before(:all) do
-      @tester_connector.create(account_name: 'foo', provider: 'ec2')
-      @tester_connector.create(account_name: 'bar', provider: 'openstack')
+      @tester_connector.create(id: 'foo', provider: 'ec2')
+      @tester_connector.create(id: 'bar', provider: 'openstack')
     end
 
     it 'returns array of Ec2ProviderAccount if @provider is ec2' do
@@ -32,4 +32,27 @@ RSpec.describe Connectors::ProviderAccount, type: :connector do
     end
   end
 
+  describe '#provider_account', cleanup_after: :all do
+    before(:all) do
+      @tester_connector.create(id: 'foo', provider: 'ec2')
+      @tester_connector.create(id: 'bar', provider: 'openstack')
+    end
+
+    it 'returns ec2 provider account' do
+      acc = @connector.provider_account('ec2', 'foo')
+      expect(acc).to be_a(Devops::Model::Ec2ProviderAccount)
+      expect(acc.account_name).to eq 'foo'
+    end
+
+    it 'returns openstack provider account' do
+      acc = @connector.provider_account('openstack', 'bar')
+      expect(acc).to be_a(Devops::Model::OpenstackProviderAccount)
+      expect(acc.account_name).to eq 'bar'
+    end
+
+    it 'raises error if account is missing' do
+      expect{@connector.provider_account('ec2', 'missing')}.to raise_error(RecordNotFound)
+    end
+  end
+
 end

From fe81076887cec9a24f10301289c8e801e598c327 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <achuchkalov@ggasoftware.com>
Date: Mon, 21 Dec 2015 19:23:17 +0400
Subject: [PATCH 04/35] start server executor specs

---
 .../spec/executors/server_executor_spec.rb    | 50 +++++++++++++++++++
 .../spec/support/instance_variable_matcher.rb |  6 +++
 2 files changed, 56 insertions(+)
 create mode 100644 devops-service/spec/executors/server_executor_spec.rb
 create mode 100644 devops-service/spec/support/instance_variable_matcher.rb

diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
new file mode 100644
index 0000000..6202f1f
--- /dev/null
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -0,0 +1,50 @@
+require 'lib/executors/server_executor'
+
+RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true do
+  let(:project) { build(:project) }
+  let(:deploy_env) { project.deploy_env('foo') }
+  let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
+  let(:executor) { described_class.new(server, '') }
+
+  before do
+    allow(stubbed_connector).to receive(:project) { project }
+  end
+
+  describe '#initialize' do
+
+    it 'sets server, project, deploy_env, out instance variables' do
+      expect(executor).to have_instance_variable_value(:server, server)
+      expect(executor).to have_instance_variable_value(:project, project)
+      expect(executor).to have_instance_variable_value(:deploy_env, deploy_env)
+      expect(executor).to have_instance_variable_value(:out, '')
+    end
+
+    it 'set knife_instance instance variable' do
+      allow(KnifeFactory).to receive(:instance)
+      expect(executor).to be_instance_variable_defined(:@knife_instance)
+    end
+
+    it 'defines :flush method on @out if it is absent' do
+      out = Class.new.new
+      expect(out).not_to respond_to(:flush)
+      described_class.new(server, out)
+      expect(out).to respond_to(:flush)
+    end
+
+    it 'sets current_user from options' do
+      user = double
+      executor = described_class.new(server, '', {current_user: user})
+      expect(executor).to have_instance_variable_value(:current_user, user)
+    end
+  end
+
+  describe '#create_server_object' do
+    it 'builds Server object' do
+      server = executor.create_server_object('created_by' => 'me')
+      expect(server).to be_a(Devops::Model::Server)
+      expect(server.project).to eq 'my_project'
+      expect(server.deploy_env).to eq 'foo'
+      expect(server.created_by).to eq 'me'
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/support/instance_variable_matcher.rb b/devops-service/spec/support/instance_variable_matcher.rb
new file mode 100644
index 0000000..51f5097
--- /dev/null
+++ b/devops-service/spec/support/instance_variable_matcher.rb
@@ -0,0 +1,6 @@
+RSpec::Matchers.define :have_instance_variable_value do |name, value|
+  match do |actual|
+    actual.instance_variable_defined?("@#{name}") &&
+      actual.instance_variable_get("@#{name}") == value
+  end
+end
\ No newline at end of file

From 7273228833d4c286be5fa1b7060ef9cae96a768b Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <achuchkalov@ggasoftware.com>
Date: Thu, 24 Dec 2015 19:51:52 +0400
Subject: [PATCH 05/35] add specs for ServerExecutor#create_server

---
 .../lib/executors/server_executor.rb          |  18 +--
 .../spec/executors/server_executor_spec.rb    | 128 +++++++++++++++++-
 .../spec/shared_contexts/stubbed_connector.rb |   2 +-
 3 files changed, 130 insertions(+), 18 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 7ff7fdc..fda02fb 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -13,7 +13,8 @@ module Devops
         server_not_in_chef_nodes: 5,
         server_bootstrap_unknown_error: 7,
         deploy_unknown_error: 6,
-        deploy_failed: 8
+        deploy_failed: 8,
+        creating_server_unknown_error: 9
       }
 
       # waiting for 5*60 seconds (5 min)
@@ -35,6 +36,9 @@ module Devops
 
       before_deploy :create_run_list
 
+
+      attr_accessor :server, :deploy_env
+
       def initialize server, out, options={}
         if server
           @project = Devops::Db.connector.project(server.project)
@@ -67,14 +71,6 @@ module Devops
         @project = p
       end
 
-      def deploy_env= e
-        @deploy_env = e
-      end
-
-      def server
-        @server
-      end
-
       def create_server_object options
         Devops::Model::Server.new({
           "project" => @project.id,
@@ -133,8 +129,7 @@ module Devops
           DevopsLogger.logger.error e.message
           roll_back
           mongo.server_delete @server.id
-          # return 5
-          return result_code(:server_not_in_chef_nodes)
+          result_code(:creating_server_unknown_error)
         end
       end
 
@@ -277,7 +272,6 @@ module Devops
 
         # deploy phase. Assume that all servers are bootstraped successfully here.
         begin
-          #raise "hello"
           @out << "\n"
           run_list = compute_run_list
           @out << "\nComputed run list: #{run_list.join(", ")}"
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 6202f1f..e2fc1b2 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -4,19 +4,19 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   let(:project) { build(:project) }
   let(:deploy_env) { project.deploy_env('foo') }
   let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
-  let(:executor) { described_class.new(server, '') }
+  let(:output) { File.open(File::NULL, "w") }
+  let(:executor) { described_class.new(server, output) }
 
   before do
     allow(stubbed_connector).to receive(:project) { project }
   end
 
   describe '#initialize' do
-
     it 'sets server, project, deploy_env, out instance variables' do
-      expect(executor).to have_instance_variable_value(:server, server)
+      expect(executor.server).to eq server
+      expect(executor.deploy_env).to eq deploy_env
       expect(executor).to have_instance_variable_value(:project, project)
-      expect(executor).to have_instance_variable_value(:deploy_env, deploy_env)
-      expect(executor).to have_instance_variable_value(:out, '')
+      expect(executor).to have_instance_variable_value(:out, output)
     end
 
     it 'set knife_instance instance variable' do
@@ -47,4 +47,122 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       expect(server.created_by).to eq 'me'
     end
   end
+
+  describe '#create_server', stubbed_connector: true, stubbed_logger: true do
+    let!(:without_bootstrap) { @without_bootstrap = true }
+    let!(:run_list) { @run_list = %w(role[asd]) }
+    let!(:key) { @key = 'key' }
+
+    subject {
+      executor.create_server(
+        'created_by' => 'user',
+        'run_list' => @run_list,
+        'name' => 'node_name',
+        'key' => @key,
+        'without_bootstrap' => @without_bootstrap
+      )
+    }
+
+    before do
+      @provider = double('Provider instance')
+      allow(executor.deploy_env).to receive(:provider_instance) { @provider }
+      allow(@provider).to receive(:create_server) { true }
+
+      @image = double('Image instance')
+      allow(@image).to receive(:remote_user) { 'remote_user' }
+
+      allow(stubbed_connector).to receive(:image) { @image}
+      allow(stubbed_connector).to receive(:server_insert)
+    end
+
+    it 'builds server model from given options' do
+      subject
+      expect(executor.server.created_by).to eq 'user'
+      expect(executor.server.chef_node_name).to eq 'node_name'
+      expect(executor.server.key).to eq @key
+      expect(executor.server.run_list).to eq @run_list
+    end
+
+    it 'sets run list to an empty array by default' do
+      @run_list = nil
+      subject
+      expect(executor.server.run_list).to eq []
+    end
+
+    it 'sets key to default provider ssh key by default' do
+      @key = nil
+      allow(@provider).to receive(:ssh_key) { 'default_key' }
+      subject
+      expect(executor.server.key).to eq 'default_key'
+    end
+
+    it 'runs hooks' do
+      expect(executor).to receive(:run_hook).with(:before_create).ordered
+      expect(executor).to receive(:run_hook).with(:after_create).ordered
+      subject
+    end
+
+    it 'creates server in cloud' do
+      expect(@provider).to receive(:create_server).with(
+        an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output
+      )
+      subject
+    end
+
+    it 'inserts built server into mongo' do
+      expect(stubbed_connector).to receive(:server_insert)
+      subject
+    end
+
+    it 'schedules expiration for server' do
+      expect(executor).to receive(:schedule_expiration).with(an_instance_of(Devops::Model::Server))
+      subject
+    end
+
+    context 'without_bootstrap option is false' do
+      it 'launches bootstrap' do
+        @without_bootstrap = false
+        allow(@image).to receive(:bootstrap_template) { 'template' }
+        allow(executor).to receive(:two_phase_bootstrap)
+        expect(executor).to receive(:two_phase_bootstrap)
+        subject
+      end
+    end
+
+    context 'without_bootstrap option is nil' do
+      it 'launches bootstrap' do
+        @without_bootstrap = nil
+        allow(@image).to receive(:bootstrap_template) { 'template' }
+        allow(executor).to receive(:two_phase_bootstrap)
+        expect(executor).to receive(:two_phase_bootstrap)
+        subject
+      end
+    end
+
+    context 'without_bootstrap option is true' do
+      it "doesn't launch bootstrap" do
+        @without_bootstrap = true
+        expect(executor).not_to receive(:two_phase_bootstrap)
+        subject
+      end
+    end
+
+    context 'if error has been raised during execution' do
+      before do
+        allow(stubbed_connector).to receive(:server_delete)
+        allow(@provider).to receive(:create_server) { raise }
+      end
+
+      it 'rollbacks server creating' do
+        expect(executor).to receive(:roll_back)
+        subject
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete)
+        subject
+      end
+
+    end
+  end
 end
\ No newline at end of file
diff --git a/devops-service/spec/shared_contexts/stubbed_connector.rb b/devops-service/spec/shared_contexts/stubbed_connector.rb
index 7d6dabb..89ec7f9 100644
--- a/devops-service/spec/shared_contexts/stubbed_connector.rb
+++ b/devops-service/spec/shared_contexts/stubbed_connector.rb
@@ -1,5 +1,5 @@
 RSpec.shared_context 'stubbed calls to connector', stubbed_connector: true do
-  let(:stubbed_connector) { double() }
+  let(:stubbed_connector) { double('Connector') }
   before do
     allow(Devops::Db).to receive(:connector) { stubbed_connector }
   end

From cf19294772f87d190ce1656fc36e9531311d860c Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Mon, 28 Dec 2015 13:07:13 +0300
Subject: [PATCH 06/35] add specs for ServerExecutor#bootstrap

---
 devops-service/Guardfile                      |   1 +
 .../lib/executors/server_executor.rb          |  82 ++++++++----
 .../spec/executors/server_executor_spec.rb    | 126 +++++++++++++++---
 .../spec/shared_contexts/stubbed_knife.rb     |   6 +
 4 files changed, 172 insertions(+), 43 deletions(-)
 create mode 100644 devops-service/spec/shared_contexts/stubbed_knife.rb

diff --git a/devops-service/Guardfile b/devops-service/Guardfile
index 13d5834..a66e388 100644
--- a/devops-service/Guardfile
+++ b/devops-service/Guardfile
@@ -42,4 +42,5 @@ guard :rspec, cmd: "rspec" do
 
   # Devops files
   watch(%r{db/.+\.rb}) { rspec.spec_dir }
+  watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
 end
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index fda02fb..23c6420 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -10,11 +10,13 @@ module Devops
 
       RESULT_CODES = {
         server_bootstrap_fail: 2,
+        server_bootstrap_private_ip_unset: 3,
         server_not_in_chef_nodes: 5,
         server_bootstrap_unknown_error: 7,
         deploy_unknown_error: 6,
         deploy_failed: 8,
-        creating_server_unknown_error: 9
+        creating_server_unknown_error: 9,
+        creating_server_in_cloud_failed: 10
       }
 
       # waiting for 5*60 seconds (5 min)
@@ -44,7 +46,6 @@ module Devops
           @project = Devops::Db.connector.project(server.project)
           @deploy_env = @project.deploy_env(server.deploy_env)
         end
-        @knife_instance = KnifeFactory.instance
         @server = server
         @out = out
         @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
@@ -103,7 +104,9 @@ module Devops
           res[:before] = self.run_hook :before_create
           @out << "Done\n"
 
-          return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
+          unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
+            return result_code(:creating_server_in_cloud_failed)
+          end
           mongo.server_insert @server
 
           @out << "\nAfter create hooks...\n"
@@ -133,6 +136,11 @@ module Devops
         end
       end
 
+      # options:
+      #   :run_list (optional)
+      #   :bootstrap_template (optional)
+      #   :chef_environment (optional)
+      #   :config (optional)
       def bootstrap options
         @out << "\n\nBootstrap...\n"
         @out.flush
@@ -144,7 +152,7 @@ module Devops
         @out << "Done\n"
         if @server.private_ip.nil?
           @out << "Error: Private IP is null"
-          return false
+          return result_code(:server_bootstrap_private_ip_unset)
         end
         ja = {
           :provider => @server.provider,
@@ -159,13 +167,7 @@ module Devops
 
         address = "#{@server.remote_user}@#{ip}"
 
-        cmd = 'ssh '
-        cmd << "-i #{cert_path} "
-        cmd << '-q '
-        cmd << '-o StrictHostKeyChecking=no '
-        cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
-        cmd << "#{address} 'exit'"
-        cmd << " 2>&1"
+        cmd = check_ssh_command(cert_path, address)
 
         @out << "\nWaiting for SSH..."
         @out << "\nTest command: '#{cmd}'\n"
@@ -174,16 +176,16 @@ module Devops
         retries_amount = 0
         begin
           sleep(5)
-          res = `#{cmd}`
+          res = execute_system_command(cmd)
           retries_amount += 1
-          if retries_amount > MAX_SSH_RETRIES_AMOUNT
+          if retries_amount >= MAX_SSH_RETRIES_AMOUNT
             @out.puts "Can not connect to #{address}"
             @out.puts res
             @out.flush
             DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
             return result_code(:server_bootstrap_fail)
           end
-          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
+          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless connected_successfully?
         rescue ArgumentError => e
           @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
           @out.flush
@@ -193,7 +195,7 @@ module Devops
         provider = @server.provider_instance
         @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
 
-        r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
+        r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
 
         if r == 0
           @out << "Chef node name: #{@server.chef_node_name}\n"
@@ -214,6 +216,12 @@ module Devops
         end
       end
 
+      # options:
+      #   :cert_path (required)
+      #   :run_list (optional)
+      #   :bootstrap_template (optional)
+      #   :chef_environment (optional)
+      #   :config (optional)
       def bootstrap_options attributes, options
         bootstrap_options = [
           "-x #{@server.remote_user}",
@@ -276,7 +284,7 @@ module Devops
           run_list = compute_run_list
           @out << "\nComputed run list: #{run_list.join(", ")}"
           @out.flush
-          @knife_instance.set_run_list(@server.chef_node_name, run_list)
+          knife_instance.set_run_list(@server.chef_node_name, run_list)
           deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
           deploy_status = deploy_server(deploy_info)
           if deploy_status == 0
@@ -296,7 +304,7 @@ module Devops
       end
 
       def check_server
-        @knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
+        knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
       def unbootstrap
@@ -330,13 +338,13 @@ module Devops
         old_tags_str = nil
         new_tags_str = nil
         unless tags.empty?
-          old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
+          old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
           @out << "Server tags: #{old_tags_str}\n"
-          @knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
+          knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
 
           new_tags_str = tags.join(" ")
           @out << "Server new tags: #{new_tags_str}\n"
-          cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
+          cmd = knife_instance.tags_create(@server.chef_node_name, new_tags_str)
           unless cmd[1]
             m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
             DevopsLogger.logger.error(m)
@@ -350,9 +358,9 @@ module Devops
 
         unless tags.empty?
           @out << "Restore tags\n"
-          cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
+          cmd = knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
           DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
-          cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
+          cmd = knife_instance.tags_create(@server.chef_node_name, old_tags_str)
           DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
         end
         return r
@@ -393,7 +401,7 @@ module Devops
         end
         @out.flush
         k = Devops::Db.connector.key(@server.key)
-        lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
+        lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
         r = /Chef\sClient\sfinished/i
 
         if lline && lline[r]
@@ -413,8 +421,8 @@ module Devops
 
       def delete_from_chef_server node_name
         {
-          :chef_node => @knife_instance.chef_node_delete(node_name),
-          :chef_client => @knife_instance.chef_client_delete(node_name)
+          :chef_node => knife_instance.chef_node_delete(node_name),
+          :chef_client => knife_instance.chef_client_delete(node_name)
         }
       end
 
@@ -533,6 +541,30 @@ module Devops
         end
       end
 
+      def check_ssh_command(cert_path, address)
+        cmd = 'ssh '
+        cmd << "-i #{cert_path} "
+        cmd << '-q '
+        cmd << '-o StrictHostKeyChecking=no '
+        cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
+        cmd << "#{address} 'exit'"
+        cmd << " 2>&1"
+        cmd
+      end
+
+      # to simplify testing
+      def execute_system_command(cmd)
+        `#{cmd}`
+      end
+
+      def connected_successfully?
+        $?.success?
+      end
+
+      def knife_instance
+        @knife_instance ||= KnifeFactory.instance
+      end
+
     end
   end
 end
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index e2fc1b2..de2c118 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -5,10 +5,14 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   let(:deploy_env) { project.deploy_env('foo') }
   let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
   let(:output) { File.open(File::NULL, "w") }
+  let(:provider) { double('Provider instance') }
   let(:executor) { described_class.new(server, output) }
 
+
   before do
     allow(stubbed_connector).to receive(:project) { project }
+    allow(executor.deploy_env).to receive(:provider_instance) { provider }
+    allow(server).to receive(:provider_instance) { provider }
   end
 
   describe '#initialize' do
@@ -19,11 +23,6 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       expect(executor).to have_instance_variable_value(:out, output)
     end
 
-    it 'set knife_instance instance variable' do
-      allow(KnifeFactory).to receive(:instance)
-      expect(executor).to be_instance_variable_defined(:@knife_instance)
-    end
-
     it 'defines :flush method on @out if it is absent' do
       out = Class.new.new
       expect(out).not_to respond_to(:flush)
@@ -52,6 +51,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     let!(:without_bootstrap) { @without_bootstrap = true }
     let!(:run_list) { @run_list = %w(role[asd]) }
     let!(:key) { @key = 'key' }
+    let!(:image) { double('Image instance', remote_user: 'remote_user') }
 
     subject {
       executor.create_server(
@@ -64,14 +64,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     }
 
     before do
-      @provider = double('Provider instance')
-      allow(executor.deploy_env).to receive(:provider_instance) { @provider }
-      allow(@provider).to receive(:create_server) { true }
-
-      @image = double('Image instance')
-      allow(@image).to receive(:remote_user) { 'remote_user' }
-
-      allow(stubbed_connector).to receive(:image) { @image}
+      allow(provider).to receive(:create_server) { true }
+      allow(stubbed_connector).to receive(:image) { image }
       allow(stubbed_connector).to receive(:server_insert)
     end
 
@@ -91,7 +85,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
     it 'sets key to default provider ssh key by default' do
       @key = nil
-      allow(@provider).to receive(:ssh_key) { 'default_key' }
+      allow(provider).to receive(:ssh_key) { 'default_key' }
       subject
       expect(executor.server.key).to eq 'default_key'
     end
@@ -103,7 +97,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
 
     it 'creates server in cloud' do
-      expect(@provider).to receive(:create_server).with(
+      expect(provider).to receive(:create_server).with(
         an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output
       )
       subject
@@ -122,7 +116,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context 'without_bootstrap option is false' do
       it 'launches bootstrap' do
         @without_bootstrap = false
-        allow(@image).to receive(:bootstrap_template) { 'template' }
+        allow(image).to receive(:bootstrap_template) { 'template' }
         allow(executor).to receive(:two_phase_bootstrap)
         expect(executor).to receive(:two_phase_bootstrap)
         subject
@@ -132,7 +126,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context 'without_bootstrap option is nil' do
       it 'launches bootstrap' do
         @without_bootstrap = nil
-        allow(@image).to receive(:bootstrap_template) { 'template' }
+        allow(image).to receive(:bootstrap_template) { 'template' }
         allow(executor).to receive(:two_phase_bootstrap)
         expect(executor).to receive(:two_phase_bootstrap)
         subject
@@ -150,7 +144,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context 'if error has been raised during execution' do
       before do
         allow(stubbed_connector).to receive(:server_delete)
-        allow(@provider).to receive(:create_server) { raise }
+        allow(provider).to receive(:create_server) { raise }
       end
 
       it 'rollbacks server creating' do
@@ -162,7 +156,103 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         expect(stubbed_connector).to receive(:server_delete)
         subject
       end
+    end
 
+    context "if creating server in cloud wasn't successful" do
+      it 'returns proper error code' do
+        allow(provider).to receive(:create_server) { false }
+        expect(subject).to eq 10
+      end
+    end
+  end
+
+
+
+  describe '#bootstrap', stubbed_logger: true, stubbed_knife: true do
+    subject { executor.bootstrap({}) }
+    let(:image) { double('Key instance', path: 'path') }
+
+    before do
+      allow(executor).to receive(:sleep)
+      allow(executor).to receive(:connected_successfully?).and_return(true)
+      allow(executor).to receive(:execute_system_command)
+      allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node')
+      allow(stubbed_connector).to receive(:key).and_return(image)
+      allow(stubbed_connector).to receive(:server_set_chef_node_name)
+      allow(stubbed_knife).to receive(:knife_bootstrap).and_return(0)
+    end
+
+    it 'run before hook' do
+      expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered
+      expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered
+      subject
+    end
+
+    context "when server's private ip is unset" do
+      it 'returns proper error code' do
+        server.private_ip = nil
+        expect(subject).to eq 3
+      end
+    end
+
+    it 'tries to ssh to server' do
+      expect(executor).to receive(:execute_system_command).with(/ssh/)
+      subject
+    end
+
+    context "couldn't ssh to server" do
+      before { allow(executor).to receive(:connected_successfully?) { false } }
+
+      it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do
+        max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT
+        expect(executor).to receive(:execute_system_command).exactly(max_retries).times
+        subject
+      end
+
+      it 'returns proper error code' do
+        expect(subject).to eq 2
+      end
+    end
+
+
+    context 'after successful ssh check' do
+      before { allow(executor).to receive(:connected_successfully?).and_return(false, true) }
+
+      it "sets default chef node name if it's nil" do
+        executor.server.chef_node_name = nil
+        expect {subject}.to change {executor.server.chef_node_name}.to 'chef_node'
+      end
+
+      it 'executes knife bootstrap' do
+        expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array))
+        subject
+      end
+
+      it "bootstraps to public ip if it's set" do
+        server.public_ip = '8.8.8.8'
+        expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array))
+        subject
+      end
+
+      context 'after successful bootstrap' do
+        it "updates server's chef node name in db" do
+          expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server))
+          subject
+        end
+      end
+
+      context "if bootstraping wasn't successful" do
+        before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) }
+
+        it 'returns proper code' do
+          expect(subject).to eq 2
+        end
+
+        it "doesn't run after hook" do
+          expect(executor).to receive(:run_hook).with(:before_bootstrap, output)
+          subject
+        end
+      end
     end
   end
 end
\ No newline at end of file
diff --git a/devops-service/spec/shared_contexts/stubbed_knife.rb b/devops-service/spec/shared_contexts/stubbed_knife.rb
new file mode 100644
index 0000000..cd49d05
--- /dev/null
+++ b/devops-service/spec/shared_contexts/stubbed_knife.rb
@@ -0,0 +1,6 @@
+RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do
+  let(:stubbed_knife) { double('KnifeCommands') }
+  before do
+    allow(KnifeFactory).to receive(:instance) { stubbed_knife }
+  end
+end
\ No newline at end of file

From ac5b9fd4ac0cf86c4226bc8b4647626688719347 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 29 Dec 2015 16:10:05 +0300
Subject: [PATCH 07/35] add specs for ServerExecutor#two_phase_bootstrap

---
 .../lib/executors/server_executor.rb          |  10 +-
 .../spec/executors/server_executor_spec.rb    | 125 +++++++++++++++++-
 2 files changed, 124 insertions(+), 11 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 23c6420..a544990 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -243,6 +243,7 @@ module Devops
         @out << "Done\n"
       end
 
+      # essentially, it just bootstrap and then deploy
       def two_phase_bootstrap options
         prepare_two_phase_bootstrap(options)
         # bootstrap phase
@@ -253,7 +254,7 @@ module Devops
           bootstrap_status = bootstrap(options)
 
           if bootstrap_status == 0
-            if check_server
+            if check_server_on_chef_server
               @out << "Server #{@server.chef_node_name} is created"
             else
               @out.puts "Can not find client or node on chef-server"
@@ -280,9 +281,8 @@ module Devops
 
         # deploy phase. Assume that all servers are bootstraped successfully here.
         begin
-          @out << "\n"
           run_list = compute_run_list
-          @out << "\nComputed run list: #{run_list.join(", ")}"
+          @out << "\n\nComputed run list: #{run_list.join(", ")}"
           @out.flush
           knife_instance.set_run_list(@server.chef_node_name, run_list)
           deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
@@ -303,7 +303,7 @@ module Devops
         end
       end
 
-      def check_server
+      def check_server_on_chef_server
         knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
@@ -553,6 +553,7 @@ module Devops
       end
 
       # to simplify testing
+      # :nocov:
       def execute_system_command(cmd)
         `#{cmd}`
       end
@@ -564,6 +565,7 @@ module Devops
       def knife_instance
         @knife_instance ||= KnifeFactory.instance
       end
+      # :nocov:
 
     end
   end
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index de2c118..3a2f3fe 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -1,6 +1,6 @@
 require 'lib/executors/server_executor'
 
-RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true do
+RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do
   let(:project) { build(:project) }
   let(:deploy_env) { project.deploy_env('foo') }
   let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
@@ -47,7 +47,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
-  describe '#create_server', stubbed_connector: true, stubbed_logger: true do
+  describe '#create_server' do
     let!(:without_bootstrap) { @without_bootstrap = true }
     let!(:run_list) { @run_list = %w(role[asd]) }
     let!(:key) { @key = 'key' }
@@ -159,7 +159,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
 
     context "if creating server in cloud wasn't successful" do
-      it 'returns proper error code' do
+      it 'returns creating_server_in_cloud_failed error code' do
         allow(provider).to receive(:create_server) { false }
         expect(subject).to eq 10
       end
@@ -168,7 +168,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
 
-  describe '#bootstrap', stubbed_logger: true, stubbed_knife: true do
+  describe '#bootstrap', stubbed_knife: true do
     subject { executor.bootstrap({}) }
     let(:image) { double('Key instance', path: 'path') }
 
@@ -189,7 +189,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
 
     context "when server's private ip is unset" do
-      it 'returns proper error code' do
+      it 'returns server_bootstrap_private_ip_unset error code' do
         server.private_ip = nil
         expect(subject).to eq 3
       end
@@ -209,7 +209,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         subject
       end
 
-      it 'returns proper error code' do
+      it 'returns server_bootstrap_fail error code' do
         expect(subject).to eq 2
       end
     end
@@ -244,7 +244,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       context "if bootstraping wasn't successful" do
         before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) }
 
-        it 'returns proper code' do
+        it 'returns :server_bootstrap_fail code' do
           expect(subject).to eq 2
         end
 
@@ -255,4 +255,115 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       end
     end
   end
+
+  describe '#two_phase_bootstrap', stubbed_knife: true do
+    subject { executor.two_phase_bootstrap({}) }
+
+    before do
+      allow(provider).to receive(:run_list) {[]}
+      allow(stubbed_connector).to receive(:server_delete)
+    end
+
+    context 'when bootstrap was successful' do
+      before do
+        allow(executor).to receive(:bootstrap) { 0 }
+        allow(executor).to receive(:check_server_on_chef_server) { false }
+      end
+
+      context 'if node presents on chef server' do
+        before do
+          allow(executor).to receive(:check_server_on_chef_server) { true }
+          allow(executor).to receive(:deploy_server)
+          allow(stubbed_knife).to receive(:set_run_list)
+        end
+
+        it 'builds run list' do
+          expect(executor).to receive(:compute_run_list)
+          subject
+        end
+
+        it 'sets run list to chef node' do
+          expect(stubbed_knife).to receive(:set_run_list)
+          subject
+        end
+
+        it 'deploys server' do
+          expect(executor).to receive(:deploy_server)
+          subject
+        end
+
+        context 'if deploy was successful' do
+          it 'returns 0' do
+            allow(executor).to receive(:deploy_server) { 0 }
+            expect(subject).to eq 0
+          end
+        end
+
+        context "if deploy wasn't successful" do
+          it 'returns :deploy_failed code' do
+            allow(executor).to receive(:deploy_server) { 1 }
+            expect(subject).to eq 8
+          end
+        end
+
+        context 'when an error occured during deploy' do
+          it 'returns :deploy_unknown_error code' do
+            allow(executor).to receive(:deploy_server) { raise }
+            expect(subject).to eq 6
+          end
+        end
+      end
+
+      context "if node doesn't present on chef server" do
+        it 'roll backs and then deletes server from mongo' do
+          allow(executor).to receive(:check_server_on_chef_server) { false }
+          allow(executor).to receive(:roll_back)
+          allow(stubbed_connector).to receive(:server_delete)
+          expect(executor).to receive(:roll_back).ordered
+          expect(stubbed_connector).to receive(:server_delete).ordered
+          subject
+        end
+      end
+    end
+
+    context "when bootstrap wasn't successful" do
+      it 'returns :server_bootstrap_fail error code' do
+        allow(executor).to receive(:bootstrap) { 1 }
+        expect(subject).to eq 2
+      end
+    end
+
+    context 'when an error occured during bootstrap' do
+      it 'returns :server_bootstrap_unknown_error error code' do
+        allow(executor).to receive(:bootstrap) { raise }
+        expect(subject).to eq 7
+      end
+    end
+  end
+
+  describe '#check_server_on_chef_server', stubbed_knife: true do
+    subject { executor.check_server_on_chef_server }
+
+    before do
+      server.chef_node_name = 'a'
+      allow(stubbed_knife).to receive(:chef_node_list) { @node_list }
+      allow(stubbed_knife).to receive(:chef_client_list) { @client_list }
+    end
+
+    it 'returns true when node_name in node list and in client list' do
+      @node_list = %w(a); @client_list = %w(a)
+      expect(subject).to be true
+    end
+
+    it "returns false if node name isn't in node list" do
+      @node_list = []; @client_list = %w(a)
+      expect(subject).to be false
+    end
+
+    it "returns false if node name isn't in node list" do
+      @node_list = %w(a); @client_list = []
+      expect(subject).to be false
+    end
+  end
+
 end
\ No newline at end of file

From 970d845801d25437d3fd2ed2d41825432766a833 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 29 Dec 2015 16:59:43 +0300
Subject: [PATCH 08/35] update .gitignore

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 49cc974..c742b09 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 devops-service/tests/features/support/config.yml
 .devops_files/
 devops-service/plugins
-devops-service/spec/examples.txt
\ No newline at end of file
+devops-service/spec/examples.txt
+devops-service/coverage
+devops-service/tmp
\ No newline at end of file

From 170863809f32fa2e5b0630a0e904c0abb6a621b2 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 14 Jan 2016 13:43:42 +0300
Subject: [PATCH 09/35] add specs

---
 devops-service/commands/knife_commands.rb     |  7 +++
 .../lib/executors/server_executor.rb          | 47 +++++++---------
 .../spec/executors/server_executor_spec.rb    | 56 ++++++++++++++++++-
 .../openstack_provider_account_spec.rb        |  1 -
 .../spec/shared_contexts/stubbed_knife.rb     |  2 +-
 5 files changed, 84 insertions(+), 29 deletions(-)

diff --git a/devops-service/commands/knife_commands.rb b/devops-service/commands/knife_commands.rb
index 45c4835..73c41bf 100644
--- a/devops-service/commands/knife_commands.rb
+++ b/devops-service/commands/knife_commands.rb
@@ -38,6 +38,13 @@ class KnifeCommands
     knife("tag delete #{name} #{tagsStr}")
   end
 
+  # extracted from server_executor.rb
+  def swap_tags(name, from, to)
+    tags_delete(name, from)
+    result = tags_create(name, to)
+    !!result[1]
+  end
+
   def create_role role_name, project, env
     file = "/tmp/new_role.json"
     File.open(file, "w") do |f|
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index a544990..c28356a 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -10,7 +10,8 @@ module Devops
 
       RESULT_CODES = {
         server_bootstrap_fail: 2,
-        server_bootstrap_private_ip_unset: 3,
+        server_cannot_update_tags: 3,
+        server_bootstrap_private_ip_unset: 4,
         server_not_in_chef_nodes: 5,
         server_bootstrap_unknown_error: 7,
         deploy_unknown_error: 6,
@@ -307,6 +308,7 @@ module Devops
         knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
+      # there were changes in unbootstrap in other branches; leave it for now
       def unbootstrap
         k = Devops::Db.connector.key(@server.key)
         cert_path = k.path
@@ -335,35 +337,28 @@ module Devops
       end
 
       def deploy_server_with_tags tags, deploy_info
-        old_tags_str = nil
-        new_tags_str = nil
-        unless tags.empty?
-          old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
-          @out << "Server tags: #{old_tags_str}\n"
-          knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
+        return deploy_server(deploy_info) if tags.empty?
 
-          new_tags_str = tags.join(" ")
-          @out << "Server new tags: #{new_tags_str}\n"
-          cmd = knife_instance.tags_create(@server.chef_node_name, new_tags_str)
-          unless cmd[1]
-            m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
-            DevopsLogger.logger.error(m)
-            @out << m + "\n"
-            return 3
-          end
-          DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
+        old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
+        new_tags_str = tags.join(" ")
+
+        @out.puts "Temporarily changing tags (#{old_tags_str}) to (#{new_tags_str})"
+        unless knife_instance.swap_tags(@server.chef_node_name, old_tags_str, new_tags_str)
+          m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
+          DevopsLogger.logger.error(m)
+          @out.puts m
+          return result_code(:server_cannot_update_tags)
         end
+        DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
 
-        r = deploy_server deploy_info
-
-        unless tags.empty?
-          @out << "Restore tags\n"
-          cmd = knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
-          DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
-          cmd = knife_instance.tags_create(@server.chef_node_name, old_tags_str)
-          DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
+        begin
+          deploy_result = deploy_server deploy_info
+        ensure
+          @out.puts "Restoring tags"
+          knife_instance.swap_tags(@server.chef_node_name, new_tags_str, old_tags_str)
+          DevopsLogger.logger.info("Restoring tags for #{@server.chef_node_name}: from #{new_tags_str} back to (#{old_tags_str})")
         end
-        return r
+        deploy_result
       end
 
       def deploy_server deploy_info
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 3a2f3fe..16c8441 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -191,7 +191,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context "when server's private ip is unset" do
       it 'returns server_bootstrap_private_ip_unset error code' do
         server.private_ip = nil
-        expect(subject).to eq 3
+        expect(subject).to eq 4
       end
     end
 
@@ -366,4 +366,58 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+  describe '#deploy_server_with_tags', stubbed_knife: true do
+    let(:current_tags) { @current_tags }
+    let(:initial_tags) { %w(a b) }
+    let(:joined_initial_tags) { initial_tags.join(' ') }
+    let(:given_tags) { %w(c d) }
+    let(:joined_given_tags) { given_tags.join(' ') }
+    subject { executor.deploy_server_with_tags(given_tags, {}) }
+
+    before do
+      @current_tags = initial_tags.dup
+      allow(stubbed_knife).to receive(:tags_list) { @current_tags }
+      allow(stubbed_knife).to receive(:swap_tags) do |_, tags_to_delete, tags_to_add|
+        @current_tags -= tags_to_delete.split
+        @current_tags += tags_to_add.split
+      end
+      allow(executor).to receive(:deploy_server)
+    end
+
+    context 'when tags are empty' do
+      it 'just deploys server' do
+        expect(executor).to receive(:deploy_server)
+        expect(stubbed_knife).not_to receive(:swap_tags)
+        executor.deploy_server_with_tags([], {})
+      end
+    end
+
+    context 'when tags are not empty' do
+      it 'temporarily swaps current_tags with given ones, deploys server and then restores tags' do
+        expect(stubbed_knife).to receive(:tags_list).ordered
+        expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_initial_tags, joined_given_tags).ordered
+        expect(executor).to receive(:deploy_server).ordered
+        expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_given_tags, joined_initial_tags).ordered
+        subject
+      end
+    end
+
+    context 'if error occures during deploy' do
+      it 'restores tags anyway' do
+        allow(executor).to receive(:deploy_server) { raise }
+        expect {
+          subject
+        }.to raise_error StandardError
+        expect(current_tags).to eq initial_tags
+      end
+    end
+
+    context 'if cannot add tags to server' do
+      it 'returns :server_cannot_update_tags code' do
+        allow(stubbed_knife).to receive(:swap_tags) { false }
+        expect(subject).to eq 3
+      end
+    end
+  end
+
 end
\ No newline at end of file
diff --git a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
index e601778..97aabf6 100644
--- a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
+++ b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
@@ -1,6 +1,5 @@
 require 'spec_helper'
 
-# не пытайся выделить в shared_specs, фигня выйдет
 RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do
   let(:provider_account) { build(:openstack_provider_account) }
 
diff --git a/devops-service/spec/shared_contexts/stubbed_knife.rb b/devops-service/spec/shared_contexts/stubbed_knife.rb
index cd49d05..042bcc7 100644
--- a/devops-service/spec/shared_contexts/stubbed_knife.rb
+++ b/devops-service/spec/shared_contexts/stubbed_knife.rb
@@ -1,5 +1,5 @@
 RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do
-  let(:stubbed_knife) { double('KnifeCommands') }
+  let(:stubbed_knife) { instance_double(KnifeCommands) }
   before do
     allow(KnifeFactory).to receive(:instance) { stubbed_knife }
   end

From 33b460d50ba3b509bd109d53cf170bf2177f339c Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 13:36:29 +0300
Subject: [PATCH 10/35] add specs for ServerExecutor#deploy

---
 .../lib/executors/server_executor.rb          |   2 +-
 .../spec/executors/server_executor_spec.rb    | 137 ++++++++++++++++++
 devops-service/spec/factories/key.rb          |   2 +-
 devops-service/spec/support/spec_support.rb   |  13 +-
 .../spec/support/{ => templates}/blank_file   |   0
 5 files changed, 151 insertions(+), 3 deletions(-)
 rename devops-service/spec/support/{ => templates}/blank_file (100%)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index c28356a..468e354 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -381,7 +381,7 @@ module Devops
               f.write json
             end
           end
-          @out << "Deploy Input Parameters:\n"
+          @out.puts "Deploy Input Parameters:"
           @out.puts json
           @out.flush
           cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 16c8441..3c3b267 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -420,4 +420,141 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+  describe '#deploy_server', stubbed_knife: true, stubbed_connector: true do
+    let(:deploy_info) { @deploy_info }
+    let(:json_file_name) { 'json.json' }
+    let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) }
+
+    subject { executor.deploy_server(deploy_info) }
+
+    before do
+      allow(executor).to receive(:run_hook).with(:before_deploy, any_args)
+      allow(executor).to receive(:run_hook).with(:after_deploy, any_args)
+      allow(stubbed_knife).to receive(:ssh_stream) { 'Chef Client finished'}
+      allow(stubbed_connector).to receive(:key) { double('Key', path: 'path_to_key') }
+      allow(stubbed_connector).to receive(:server_update)
+      @deploy_info = {}
+    end
+
+
+    it 'runs before_deploy and after_deploy hooks' do
+      expect(executor).to receive(:run_hook).with(:before_deploy, any_args).ordered
+      expect(executor).to receive(:run_hook).with(:after_deploy, any_args).ordered
+      subject
+    end
+
+    context 'when uses json file' do
+      before(:all) do
+        @tmp_files_at_start = Dir.entries(SpecSupport.tmp_dir)
+      end
+      before do
+        allow(DevopsConfig).to receive(:config).and_return({
+          project_info_dir: SpecSupport.tmp_dir,
+          address: 'host.com',
+          port: '8080',
+          url_prefix: 'api'
+        })
+        deploy_info['use_json_file'] = true
+      end
+
+      after(:all) do
+        diff = Dir.entries(SpecSupport.tmp_dir) - @tmp_files_at_start
+        diff.each do |file|
+          FileUtils.rm(File.join(SpecSupport.tmp_dir, file))
+        end
+      end
+
+
+      it 'writes deploy_info to json file if it not exists' do
+        expect { subject }.to change { Dir.entries(SpecSupport.tmp_dir)}
+      end
+
+      it "writes deploy_info to given json file name if it doesn't exist" do
+        FileUtils.rm(json_file_path) if File.exists?(json_file_path)
+        deploy_info['json_file'] = json_file_name
+        expect { subject }.to change {
+          Dir.entries(SpecSupport.tmp_dir)
+        }
+        FileUtils.rm(json_file_path)
+      end
+
+      it 'reads json from file if it exists' do
+        deploy_info['json_file'] = json_file_name
+        File.open(json_file_path, 'w') { |file| file.puts '{"foo": "bar"'}
+        expect { subject }.not_to change {
+          Dir.entries(SpecSupport.tmp_dir)
+        }
+        FileUtils.rm(json_file_path)
+      end
+
+      it 'adds link to json to deploy command' do
+        deploy_info['json_file'] = json_file_name
+        regexp = %r(-j http://host.com:8080/api/v2.0/deploy/data/#{json_file_name})
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, regexp, any_args)
+        subject
+      end
+    end
+
+    context "doesn't use json file" do
+      before do
+        deploy_info['use_json_file'] = false
+        deploy_info['run_list'] = %w(foo bar)
+      end
+
+      it "adds run list to command if server's stack is set" do
+        server.stack = 'stack'
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, %r(-r foo,bar), any_args)
+        subject
+      end
+
+      it "doesn't add run list to command if server's stack is unset" do
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args)
+        subject
+      end
+    end
+
+    it "uses server's key" do
+      expect(stubbed_connector).to receive(:key).with('key_id')
+      expect(stubbed_knife).to receive(:ssh_stream).with(any_args, 'path_to_key')
+      subject
+    end
+
+    it "uses public ip if it's set" do
+      server.public_ip = '127.0.0.1'
+      expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, '127.0.0.1', any_args)
+      subject
+    end
+
+    it "uses private_ip if public_ip isn't set" do
+      expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, server.private_ip, any_args)
+      subject
+    end
+
+    context 'if deploy was successful' do
+      it "updates server's last operation" do
+        expect(server).to receive(:set_last_operation).with('deploy', anything)
+        expect(stubbed_connector).to receive(:server_update).with(server)
+        subject
+      end
+
+      it 'returns 0' do
+        expect(subject).to eq 0
+      end
+    end
+
+    context "when deploy wasn't successful" do
+      before { allow(stubbed_knife).to receive(:ssh_stream) { 'fail'} }
+      it "doesn't run after_deploy hook" do
+        expect(executor).to receive(:run_hook).with(:before_deploy, any_args)
+        expect(executor).not_to receive(:run_hook).with(:after_deploy, any_args)
+        subject
+      end
+
+      it 'returns 1' do
+        expect(subject).to eq 1
+      end
+    end
+
+  end
+
 end
\ No newline at end of file
diff --git a/devops-service/spec/factories/key.rb b/devops-service/spec/factories/key.rb
index 06fb96e..ccc0085 100644
--- a/devops-service/spec/factories/key.rb
+++ b/devops-service/spec/factories/key.rb
@@ -3,7 +3,7 @@ require 'db/mongo/models/key'
 FactoryGirl.define do
   factory :key, class: Devops::Model::Key do
     id 'user_key'
-    path SpecSupport::BLANK_FILE
+    path SpecSupport.blank_file
     scope 'user'
   end
 end
\ No newline at end of file
diff --git a/devops-service/spec/support/spec_support.rb b/devops-service/spec/support/spec_support.rb
index ab47bc6..6b96ed8 100644
--- a/devops-service/spec/support/spec_support.rb
+++ b/devops-service/spec/support/spec_support.rb
@@ -2,7 +2,14 @@ require 'yaml'
 
 module SpecSupport
   ROOT = File.join(__dir__, '../../')
-  BLANK_FILE = File.join(ROOT, 'spec/support/blank_file')
+
+  def self.blank_file
+    File.join(ROOT, 'spec/support/templates/blank_file')
+  end
+
+  def self.tmp_dir
+    File.join(ROOT, 'tmp/')
+  end
 
   def self.db_params
     @db_params ||= begin
@@ -32,4 +39,8 @@ module SpecSupport
       end
     end
   end
+
+  def self.root
+    File.join(__dir__, '../../')
+  end
 end
\ No newline at end of file
diff --git a/devops-service/spec/support/blank_file b/devops-service/spec/support/templates/blank_file
similarity index 100%
rename from devops-service/spec/support/blank_file
rename to devops-service/spec/support/templates/blank_file

From f46feedcbe0b94af8022dafc8e3b67d2d9fd729e Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 14:18:50 +0300
Subject: [PATCH 11/35] add missing spec for Server.build_from_bson method

---
 devops-service/spec/models/server_spec.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/devops-service/spec/models/server_spec.rb b/devops-service/spec/models/server_spec.rb
index d49f328..961da4e 100644
--- a/devops-service/spec/models/server_spec.rb
+++ b/devops-service/spec/models/server_spec.rb
@@ -44,6 +44,14 @@ RSpec.describe Devops::Model::Server, type: :model do
     end
   end
 
+  describe '.build_from_bson' do
+    it 'takes a hash and returns instance of Server model' do
+      model = described_class.build_from_bson('id' => 'foo')
+      expect(model).to be_an_instance_of(described_class)
+      expect(model.id).to eq 'foo'
+    end
+  end
+
   it '#to_hash_without_id returns not nil fields' do
     server = described_class.new('run_list' => [], 'project' =>  'asd')
     expect(server.to_hash_without_id.keys).to match_array(%w(run_list project))

From a88fa8e2f6b6a2486e77c31ed71fb31e2f76691a Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 14:29:34 +0300
Subject: [PATCH 12/35] rename return_codes to error_codes

---
 .../lib/executors/server_executor.rb          | 36 +++++++++----------
 .../spec/executors/server_executor_spec.rb    | 10 ++++++
 .../workers/stack_bootstrap_worker.rb         |  4 +--
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 468e354..6d488fb 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -8,7 +8,7 @@ module Devops
     class ServerExecutor
       include Hooks
 
-      RESULT_CODES = {
+      ERROR_CODES = {
         server_bootstrap_fail: 2,
         server_cannot_update_tags: 3,
         server_bootstrap_private_ip_unset: 4,
@@ -53,16 +53,16 @@ module Devops
         @current_user = options[:current_user]
       end
 
-      def self.result_code(symbolic_code)
-        RESULT_CODES.fetch(symbolic_code)
+      def self.error_code(symbolic_code)
+        ERROR_CODES.fetch(symbolic_code)
       end
 
-      def self.symbolic_result_code(integer_code)
-        RESULT_CODES.key(integer_code) || :unknown_error
+      def self.symbolic_error_code(integer_code)
+        ERROR_CODES.key(integer_code) || :unknown_error
       end
 
-      def result_code(symbolic_code)
-        self.class.result_code(symbolic_code)
+      def error_code(symbolic_code)
+        self.class.error_code(symbolic_code)
       end
 
       def report= r
@@ -106,7 +106,7 @@ module Devops
           @out << "Done\n"
 
           unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
-            return result_code(:creating_server_in_cloud_failed)
+            return error_code(:creating_server_in_cloud_failed)
           end
           mongo.server_insert @server
 
@@ -133,7 +133,7 @@ module Devops
           DevopsLogger.logger.error e.message
           roll_back
           mongo.server_delete @server.id
-          result_code(:creating_server_unknown_error)
+          error_code(:creating_server_unknown_error)
         end
       end
 
@@ -153,7 +153,7 @@ module Devops
         @out << "Done\n"
         if @server.private_ip.nil?
           @out << "Error: Private IP is null"
-          return result_code(:server_bootstrap_private_ip_unset)
+          return error_code(:server_bootstrap_private_ip_unset)
         end
         ja = {
           :provider => @server.provider,
@@ -184,7 +184,7 @@ module Devops
             @out.puts res
             @out.flush
             DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
-            return result_code(:server_bootstrap_fail)
+            return error_code(:server_bootstrap_fail)
           end
           raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless connected_successfully?
         rescue ArgumentError => e
@@ -213,7 +213,7 @@ module Devops
         else
           @out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
           @out.flush
-          result_code(:server_bootstrap_fail)
+          error_code(:server_bootstrap_fail)
         end
       end
 
@@ -262,7 +262,7 @@ module Devops
               roll_back
               @out.flush
               mongo.server_delete @server.id
-              return result_code(:server_not_in_chef_nodes)
+              return error_code(:server_not_in_chef_nodes)
             end
           else
             # @out << roll_back
@@ -272,12 +272,12 @@ module Devops
             DevopsLogger.logger.error msg
             @out.puts msg
             @out.flush
-            return result_code(:server_bootstrap_fail)
+            return error_code(:server_bootstrap_fail)
           end
         rescue => e
           @out << "\nError: #{e.message}\n"
           @out.flush
-          return result_code(:server_bootstrap_unknown_error)
+          return error_code(:server_bootstrap_unknown_error)
         end
 
         # deploy phase. Assume that all servers are bootstraped successfully here.
@@ -295,12 +295,12 @@ module Devops
             msg << "\nDeploing server operation status was #{deploy_status}"
             DevopsLogger.logger.error msg
             @out << "\n" + msg + "\n"
-            result_code(:deploy_failed)
+            error_code(:deploy_failed)
           end
         rescue => e
           @out << "\nError: #{e.message}\n"
           DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
-          result_code(:deploy_unknown_error)
+          error_code(:deploy_unknown_error)
         end
       end
 
@@ -347,7 +347,7 @@ module Devops
           m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
           DevopsLogger.logger.error(m)
           @out.puts m
-          return result_code(:server_cannot_update_tags)
+          return error_code(:server_cannot_update_tags)
         end
         DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
 
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 3c3b267..e2abc2a 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -37,6 +37,16 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+  describe '.symbolic_error_code' do
+    it 'returns symbol given an integer' do
+      expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail
+    end
+
+    it "returns :unknown_error if can't recognize error code" do
+      expect(described_class.symbolic_error_code(123)).to eq :unknown_error
+    end
+  end
+
   describe '#create_server_object' do
     it 'builds Server object' do
       server = executor.create_server_object('created_by' => 'me')
diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb
index 6ad25d7..da7f712 100644
--- a/devops-service/workers/stack_bootstrap_worker.rb
+++ b/devops-service/workers/stack_bootstrap_worker.rb
@@ -130,7 +130,7 @@ class StackBootstrapWorker < Worker
 
     @out.puts
     results.each do |chef_node_name, code|
-      human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code)
+      human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code)
       @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
     end
 
@@ -144,7 +144,7 @@ class StackBootstrapWorker < Worker
   def errors_in_bootstrapping_present?(result_codes)
     bootstrap_error_codes = []
     [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
-      bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
+      bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code)
     end
 
     (bootstrap_error_codes & result_codes).size > 0

From db1ccc2e3343118b7f8569e26f2c1baa7027f338 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 16:17:06 +0300
Subject: [PATCH 13/35] add specs for ServerExecutor#delete_server and
 #roll_back

---
 .../spec/executors/server_executor_spec.rb    | 131 ++++++++++++++++++
 .../spec/shared_contexts/stubbed_connector.rb |   2 +-
 .../spec/shared_contexts/stubbed_logger.rb    |   1 +
 3 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index e2abc2a..a7ed918 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -37,6 +37,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '.symbolic_error_code' do
     it 'returns symbol given an integer' do
       expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail
@@ -47,6 +49,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '#create_server_object' do
     it 'builds Server object' do
       server = executor.create_server_object('created_by' => 'me')
@@ -57,6 +61,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '#create_server' do
     let!(:without_bootstrap) { @without_bootstrap = true }
     let!(:run_list) { @run_list = %w(role[asd]) }
@@ -266,6 +272,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '#two_phase_bootstrap', stubbed_knife: true do
     subject { executor.two_phase_bootstrap({}) }
 
@@ -376,6 +384,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '#deploy_server_with_tags', stubbed_knife: true do
     let(:current_tags) { @current_tags }
     let(:initial_tags) { %w(a b) }
@@ -430,6 +440,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+
   describe '#deploy_server', stubbed_knife: true, stubbed_connector: true do
     let(:deploy_info) { @deploy_info }
     let(:json_file_name) { 'json.json' }
@@ -564,7 +576,126 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         expect(subject).to eq 1
       end
     end
+  end
 
+
+  describe '#delete_from_chef_server', stubbed_knife: true do
+    subject { executor.delete_from_chef_server('foo') }
+    before do
+      allow(stubbed_knife).to receive(:chef_client_delete)
+      allow(stubbed_knife).to receive(:chef_node_delete)
+      subject
+    end
+
+    it 'returns hash with :chef_node and :chef_client keys' do
+      expect(subject).to be_a(Hash).and include(:chef_node, :chef_client)
+    end
+
+    it 'calls to :chef_node_delete and :chef_client_delete' do
+      expect(stubbed_knife).to have_received(:chef_client_delete)
+      expect(stubbed_knife).to have_received(:chef_node_delete)
+    end
+  end
+
+
+  describe '#delete_server' do
+    let(:delete_server) { executor.delete_server }
+
+    context 'when server is static' do
+      before do
+        server.provider = 'static'
+        allow(stubbed_connector).to receive(:server_delete).with(server.id)
+        allow(executor).to receive(:unbootstrap)
+      end
+
+      it 'performs unbootstrap' do
+        expect(executor).to receive(:unbootstrap)
+        delete_server
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete).with(server.id)
+        delete_server
+      end
+
+      it 'returns message and nil' do
+        expect(delete_server.first).to be_a(String)
+        expect(delete_server.last).to be nil
+      end
+
+      it "doesn't try to remove it from cloud" do
+        expect{delete_server}.not_to raise_error
+      end
+    end
+
+    context "when server isn't static", stubbed_knife: true do
+      before do
+        allow(server).to receive_message_chain('provider_instance.delete_server')
+        allow(stubbed_connector).to receive(:server_delete).with(server.id)
+        allow(stubbed_knife).to receive(:chef_node_delete)
+        allow(stubbed_knife).to receive(:chef_client_delete)
+      end
+
+      it 'deletes from info about note chef server' do
+        allow(executor).to receive(:delete_from_chef_server).and_call_original
+        expect(executor).to receive(:delete_from_chef_server)
+        delete_server
+      end
+
+      it "doesn't unbootstrap server" do
+        expect(executor).not_to receive(:unbootstrap)
+        delete_server
+      end
+
+      it 'deletes server from cloud' do
+        expect(server).to receive_message_chain('provider_instance.delete_server').with(server)
+        delete_server
+      end
+
+      it "doesn't raise error if server wasn't found in cloud" do
+        allow(server).to receive_message_chain('provider_instance.name')
+        allow(server).to receive_message_chain('provider_instance.delete_server') {
+          raise Fog::Compute::OpenStack::NotFound
+        }
+        expect { delete_server }.not_to raise_error
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete).with(server.id)
+        delete_server
+      end
+
+      it 'returns message and hash with :chef_node, :chef_client and :server keys' do
+        expect(delete_server.first).to be_a(String)
+        expect(delete_server.last).to be_a(Hash).and include(:chef_client, :chef_node, :server)
+      end
+    end
+  end
+
+
+  describe '#rollback' do
+    before do
+      allow(executor).to receive(:delete_from_chef_server) { {} }
+      allow(server).to receive_message_chain('provider_instance.delete_server')
+    end
+
+    it "does nothing if server.id is nil" do
+      server.id = nil
+      expect(executor).not_to receive(:delete_from_chef_server)
+      expect(server).not_to receive(:provider_instance)
+      executor.roll_back
+    end
+
+    it 'deletes node from chef server and instance from cloud' do
+      expect(executor).to receive(:delete_from_chef_server)
+      expect(server).to receive_message_chain('provider_instance.delete_server')
+      executor.roll_back
+    end
+
+    it "doesn't raise if deleting server in cloud raises an error" do
+      allow(server).to receive_message_chain('provider_instance.delete_server') { raise }
+      expect { executor.roll_back }.not_to raise_error
+    end
   end
 
 end
\ No newline at end of file
diff --git a/devops-service/spec/shared_contexts/stubbed_connector.rb b/devops-service/spec/shared_contexts/stubbed_connector.rb
index 89ec7f9..49b202a 100644
--- a/devops-service/spec/shared_contexts/stubbed_connector.rb
+++ b/devops-service/spec/shared_contexts/stubbed_connector.rb
@@ -1,5 +1,5 @@
 RSpec.shared_context 'stubbed calls to connector', stubbed_connector: true do
-  let(:stubbed_connector) { double('Connector') }
+  let(:stubbed_connector) { instance_double(MongoConnector) }
   before do
     allow(Devops::Db).to receive(:connector) { stubbed_connector }
   end
diff --git a/devops-service/spec/shared_contexts/stubbed_logger.rb b/devops-service/spec/shared_contexts/stubbed_logger.rb
index f2d79e9..486caa1 100644
--- a/devops-service/spec/shared_contexts/stubbed_logger.rb
+++ b/devops-service/spec/shared_contexts/stubbed_logger.rb
@@ -3,5 +3,6 @@ RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do
     allow(DevopsLogger).to receive_message_chain('logger.debug')
     allow(DevopsLogger).to receive_message_chain('logger.info')
     allow(DevopsLogger).to receive_message_chain('logger.error')
+    allow(DevopsLogger).to receive_message_chain('logger.warn')
   end
 end
\ No newline at end of file

From 48572f6875880a26605645ab3952230d641431ab Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 17:00:21 +0300
Subject: [PATCH 14/35] add specs for
 ServerExecutor#add_run_list_to_deploy_info and #compute_run_list

---
 .../lib/executors/server_executor.rb          | 20 ++------
 .../spec/executors/server_executor_spec.rb    | 49 +++++++++++++++++++
 2 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 6d488fb..79f78dc 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -37,7 +37,7 @@ module Devops
       define_hook :before_bootstrap
       define_hook :after_bootstrap
 
-      before_deploy :create_run_list
+      before_deploy :add_run_list_to_deploy_info
 
 
       attr_accessor :server, :deploy_env
@@ -471,7 +471,7 @@ module Devops
         end
       end
 
-      def create_run_list out, deploy_info
+      def add_run_list_to_deploy_info out, deploy_info
         out << "\nGenerate run list hook...\n"
         if deploy_info["run_list"]
           out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
@@ -480,14 +480,6 @@ module Devops
         out << "Project run list: #{@project.run_list.join(", ")}\n"
         out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
         out << "Server run list: #{@server.run_list.join(", ")}\n"
-=begin
-        rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
-        if @server.stack
-          stack = Devops::Db.connector.stack(@server.stack)
-          out << "Stack run list: #{stack.run_list.join(", ")}\n"
-          rlist.merge(stack.run_list)
-        end
-=end
         deploy_info["run_list"] = compute_run_list
         out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
       end
@@ -495,16 +487,14 @@ module Devops
       def compute_run_list
         rlist = []
         [@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
-          rlist += sub_run_list if sub_run_list.is_a?(Array)
+          rlist += sub_run_list if sub_run_list
         end
-        rlist = Set.new(rlist)
         if @server.stack
           stack = Devops::Db.connector.stack(@server.stack)
-#          out << "Stack run list: #{stack.run_list.join(", ")}\n"
           srl = stack.run_list
-          rlist.merge(srl) if srl.is_a?(Array)
+          rlist += srl if srl
         end
-        rlist.to_a
+        rlist.uniq
       end
 
       private
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index a7ed918..2b7dbed 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -698,4 +698,53 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
   end
 
+
+  describe '#add_run_list_to_deploy_info' do
+    it "doesn't change deploy info if it already includes run list" do
+      deploy_info = {'run_list' => %w(foo)}
+      expect {
+        executor.add_run_list_to_deploy_info(output, deploy_info)
+      }.not_to change { deploy_info }
+    end
+
+    it 'computes and adds run_list to deploy_info' do
+      deploy_info = {}
+      allow(executor).to receive(:compute_run_list) { %w(foo) }
+      expect(executor).to receive(:compute_run_list)
+      executor.add_run_list_to_deploy_info(output, deploy_info)
+      expect(deploy_info['run_list']).to eq %w(foo)
+    end
+  end
+
+  describe '#compute_run_list' do
+    before do
+      allow(deploy_env).to receive_message_chain('provider_instance.run_list') { %w(a) }
+      project.run_list = %w(b)
+      deploy_env.run_list = %w(c)
+      server.run_list = %w(d)
+    end
+
+    it "returns array with run list merged from provider's, project's, env's and server's run lists" do
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d))
+    end
+
+    it "includes stack's run list if stack is set", stubbed_connector: true do
+      server.stack = 'stack'
+      allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: %w(e)) }
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d e))
+    end
+
+    it "doesn't contain nils" do
+      server.run_list = nil
+      server.stack = 'stack'
+      allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: nil) }
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c))
+    end
+
+    it 'returns uniq elements' do
+      project.run_list = %w(a)
+      deploy_env.run_list = %w(a)
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a d))
+    end
+  end
 end
\ No newline at end of file

From 5b364ea41b2c104a9d945da1e9681eee47738ffe Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 17:44:10 +0300
Subject: [PATCH 15/35] add specs for ServerExecutor#report=, project= and
 expiration scheduling

---
 .../lib/executors/server_executor.rb          | 10 +-------
 .../spec/executors/server_executor_spec.rb    | 24 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 79f78dc..ef0b9c9 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -40,7 +40,7 @@ module Devops
       before_deploy :add_run_list_to_deploy_info
 
 
-      attr_accessor :server, :deploy_env
+      attr_accessor :server, :deploy_env, :report, :project
 
       def initialize server, out, options={}
         if server
@@ -65,14 +65,6 @@ module Devops
         self.class.error_code(symbolic_code)
       end
 
-      def report= r
-        @report = r
-      end
-
-      def project= p
-        @project = p
-      end
-
       def create_server_object options
         Devops::Model::Server.new({
           "project" => @project.id,
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 2b7dbed..f3e8ff1 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -38,6 +38,20 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   end
 
 
+  describe '#report=' do
+    it 'sets report instance variable' do
+      executor.report= 'foo'
+      expect(executor).to have_instance_variable_value(:report, 'foo')
+    end
+  end
+
+  describe '#project=' do
+    it 'sets project instance variable' do
+      executor.project= 'foo'
+      expect(executor).to have_instance_variable_value(:project, 'foo')
+    end
+  end
+
 
   describe '.symbolic_error_code' do
     it 'returns symbol given an integer' do
@@ -125,7 +139,15 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
 
     it 'schedules expiration for server' do
-      expect(executor).to receive(:schedule_expiration).with(an_instance_of(Devops::Model::Server))
+      deploy_env.expires = '2m'
+      allow(DeleteServerWorker).to receive(:perform_in)
+      expect(DeleteServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
+      subject
+    end
+
+    it "doesn't schedule expiration if deploy_env.expires is nil" do
+      deploy_env.expires = nil
+      expect(DeleteServerWorker).not_to receive(:perform_in)
       subject
     end
 

From 87811cca25bc7657f248e545ccf02f8d31cb9062 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 17:45:51 +0300
Subject: [PATCH 16/35] move users permissions updater to another dir

---
 devops-service/{lib => migrations}/users_permissions_updater.rb | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename devops-service/{lib => migrations}/users_permissions_updater.rb (100%)

diff --git a/devops-service/lib/users_permissions_updater.rb b/devops-service/migrations/users_permissions_updater.rb
similarity index 100%
rename from devops-service/lib/users_permissions_updater.rb
rename to devops-service/migrations/users_permissions_updater.rb

From 83cbfd7a48728eb3d28008f9da1c430482a3237a Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 18:06:09 +0300
Subject: [PATCH 17/35] move string helper methods to core extensions

---
 .../connectors/helpers/delete_command.rb      |  4 +--
 .../connectors/helpers/insert_command.rb      |  6 ++--
 .../mongo/connectors/helpers/list_command.rb  |  6 +---
 .../mongo/connectors/helpers/show_command.rb  |  4 +--
 .../connectors/helpers/update_command.rb      |  5 +--
 devops-service/lib/stack_presets/base.rb      |  3 +-
 devops-service/lib/string_ext.rb              | 18 ++++++++++
 devops-service/lib/string_helper.rb           | 35 -------------------
 8 files changed, 25 insertions(+), 56 deletions(-)
 delete mode 100644 devops-service/lib/string_helper.rb

diff --git a/devops-service/db/mongo/connectors/helpers/delete_command.rb b/devops-service/db/mongo/connectors/helpers/delete_command.rb
index b66be9c..8841aaa 100644
--- a/devops-service/db/mongo/connectors/helpers/delete_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/delete_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module DeleteCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
+        resource_name = base.to_s.underscore_class
         method_name = "#{resource_name}_delete".to_sym
         alias_method method_name, :delete
       end
diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb
index be380a9..446ca5f 100644
--- a/devops-service/db/mongo/connectors/helpers/insert_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module InsertCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
+        resource_name = base.to_s.underscore_class
         method_name = "#{resource_name}_insert".to_sym
         alias_method method_name, :insert
       end
@@ -24,7 +22,7 @@ module Connectors
         rescue Mongo::OperationFailure => e
           # exception's message doesn't always start from error code
           if e.message =~ /11000/
-            resource_name = StringHelper.underscore_class(record.class)
+            resource_name = record.class.to_s.underscore_class
             raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
           end
         end
diff --git a/devops-service/db/mongo/connectors/helpers/list_command.rb b/devops-service/db/mongo/connectors/helpers/list_command.rb
index aaab3b8..a53dcb0 100644
--- a/devops-service/db/mongo/connectors/helpers/list_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/list_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module ListCommand
@@ -9,9 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base).to_sym
-        method_name = StringHelper.pluralize(resource_name)
-        alias_method method_name, :list
+        alias_method base.to_s.underscore_class.pluralize, :list
       end
 
       # query options is needed, for example, for fields limiting
diff --git a/devops-service/db/mongo/connectors/helpers/show_command.rb b/devops-service/db/mongo/connectors/helpers/show_command.rb
index cb06d74..2889d72 100644
--- a/devops-service/db/mongo/connectors/helpers/show_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/show_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module ShowCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        method_name = StringHelper.underscore_class(base).to_sym
+        method_name = base.to_s.underscore_class.to_sym
         alias_method method_name, :show
       end
 
diff --git a/devops-service/db/mongo/connectors/helpers/update_command.rb b/devops-service/db/mongo/connectors/helpers/update_command.rb
index 674eb60..a8f88b6 100644
--- a/devops-service/db/mongo/connectors/helpers/update_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/update_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module UpdateCommand
@@ -8,8 +6,7 @@ module Connectors
       # We need second method name to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
-        method_name = "#{resource_name}_update".to_sym
+        method_name = "#{base.to_s.underscore_class}_update".to_sym
         alias_method method_name, :update
       end
 
diff --git a/devops-service/lib/stack_presets/base.rb b/devops-service/lib/stack_presets/base.rb
index ae09ac7..7465b28 100644
--- a/devops-service/lib/stack_presets/base.rb
+++ b/devops-service/lib/stack_presets/base.rb
@@ -1,4 +1,3 @@
-require 'lib/string_helper'
 require 'db/mongo/models/stack_template/stack_template_factory'
 
 module Devops
@@ -6,7 +5,7 @@ module Devops
     class Base
 
       def id
-        StringHelper.underscore_class(self.class)
+        self.class.to_s.underscore_class
       end
 
       def to_hash
diff --git a/devops-service/lib/string_ext.rb b/devops-service/lib/string_ext.rb
index ce2e9ca..54475e3 100644
--- a/devops-service/lib/string_ext.rb
+++ b/devops-service/lib/string_ext.rb
@@ -6,4 +6,22 @@ class String
   def blank?
     empty?
   end
+
+  # from ActiveSupport
+  def underscore
+    gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    tr("-", "_").
+    downcase
+  end
+
+  def underscore_class
+    split('::').last.underscore
+  end
+
+  # rough simplification
+  def pluralize
+    "#{self}s"
+  end
 end
\ No newline at end of file
diff --git a/devops-service/lib/string_helper.rb b/devops-service/lib/string_helper.rb
deleted file mode 100644
index 74a6727..0000000
--- a/devops-service/lib/string_helper.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-module StringHelper
-  extend self
-
-  # from Rails' ActiveSupport
-  def underscore(string)
-    string.gsub(/::/, '/').
-    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-    gsub(/([a-z\d])([A-Z])/,'\1_\2').
-    tr("-", "_").
-    downcase
-  end
-
-  def underscore_class(klass, without_ancestors=true)
-    class_name = if without_ancestors
-      klass.to_s.split('::').last
-    else
-      klass.to_s
-    end
-    StringHelper.underscore(class_name)
-  end
-
-  # from Rails' ActiveSupport
-  def camelize(term)
-    string = term.to_s
-    string = string.sub(/^[a-z\d]*/) { $&.capitalize }
-    string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
-    string.gsub!(/\//, '::')
-    string
-  end
-
-  # rough simplification
-  def pluralize(string)
-    "#{string}s"
-  end
-end

From 2055d8bd369772ecd5490374f166d9c9da71b239 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 19 Jan 2016 18:11:07 +0300
Subject: [PATCH 18/35] move core ext files into separate dir

---
 devops-service/core/devops-service.rb                       | 6 +++---
 devops-service/lib/{hash_ext.rb => core_ext/hash.rb}        | 0
 .../lib/{nil_class_ext.rb => core_ext/nil_class.rb}         | 0
 devops-service/lib/{string_ext.rb => core_ext/string.rb}    | 0
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename devops-service/lib/{hash_ext.rb => core_ext/hash.rb} (100%)
 rename devops-service/lib/{nil_class_ext.rb => core_ext/nil_class.rb} (100%)
 rename devops-service/lib/{string_ext.rb => core_ext/string.rb} (100%)

diff --git a/devops-service/core/devops-service.rb b/devops-service/core/devops-service.rb
index cd33364..7bf32f1 100644
--- a/devops-service/core/devops-service.rb
+++ b/devops-service/core/devops-service.rb
@@ -1,7 +1,7 @@
 require "wisper"
-require "lib/hash_ext"
-require "lib/nil_class_ext"
-require "lib/string_ext"
+require "lib/core_ext/hash"
+require "lib/core_ext/nil_class"
+require "lib/core_ext/string"
 
 require_relative "devops-loader"
 require_relative "devops-application"
diff --git a/devops-service/lib/hash_ext.rb b/devops-service/lib/core_ext/hash.rb
similarity index 100%
rename from devops-service/lib/hash_ext.rb
rename to devops-service/lib/core_ext/hash.rb
diff --git a/devops-service/lib/nil_class_ext.rb b/devops-service/lib/core_ext/nil_class.rb
similarity index 100%
rename from devops-service/lib/nil_class_ext.rb
rename to devops-service/lib/core_ext/nil_class.rb
diff --git a/devops-service/lib/string_ext.rb b/devops-service/lib/core_ext/string.rb
similarity index 100%
rename from devops-service/lib/string_ext.rb
rename to devops-service/lib/core_ext/string.rb

From ee9605fc4d6a6b175b2d994feb81f9ce78c67d40 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 20 Jan 2016 01:43:34 +0300
Subject: [PATCH 19/35] introduced ExpirationScheduler

---
 .../lib/executors/expiration_scheduler.rb     | 35 ++++++++++++++
 .../lib/executors/server_executor.rb          | 33 +++----------
 .../executors/expiration_scheduler_spec.rb    | 47 +++++++++++++++++++
 .../spec/executors/server_executor_spec.rb    |  2 +-
 .../workers/delete_server_worker.rb           |  1 +
 5 files changed, 90 insertions(+), 28 deletions(-)
 create mode 100644 devops-service/lib/executors/expiration_scheduler.rb
 create mode 100644 devops-service/spec/executors/expiration_scheduler_spec.rb

diff --git a/devops-service/lib/executors/expiration_scheduler.rb b/devops-service/lib/executors/expiration_scheduler.rb
new file mode 100644
index 0000000..5eee52f
--- /dev/null
+++ b/devops-service/lib/executors/expiration_scheduler.rb
@@ -0,0 +1,35 @@
+require "workers/delete_server_worker"
+
+module Devops
+  module Executor
+    class ExpirationScheduler
+      def initialize(expires, server)
+        @expires, @server = expires, server
+      end
+
+      def schedule_expiration!
+        return unless @expires
+        DeleteServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name)
+      end
+
+      def interval_in_seconds
+        interval = @expires.to_i
+        measure_unit = @expires.chars.last
+        case measure_unit
+        when 's'
+          interval
+        when 'm'
+          interval * 60
+        when 'h'
+          interval * 60 * 60
+        when 'd'
+          interval * 60 * 60 * 24
+        when 'w'
+          interval * 60 * 60 * 24 * 7
+        else
+          raise 'Wrong interval format'
+        end
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index ef0b9c9..16b53da 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -1,6 +1,5 @@
 require "lib/knife/knife_factory"
-require "workers/worker"
-require "workers/delete_server_worker"
+require "lib/executors/expiration_scheduler"
 require "hooks"
 
 module Devops
@@ -108,7 +107,7 @@ module Devops
           @out.flush
           DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
 
-          schedule_expiration(@server)
+          schedule_expiration()
 
           unless options["without_bootstrap"]
             bootstrap_options = {
@@ -491,30 +490,10 @@ module Devops
 
       private
 
-      def schedule_expiration(server)
-        expires = @deploy_env.expires
-        return unless expires
-        interval = interval_in_seconds(expires)
-        @out <<  "Planning expiration in #{expires}"
-        DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name)
-      end
-
-      def interval_in_seconds(interval_as_string)
-        interval = interval_as_string.to_i
-        measure_unit = interval_as_string.chars.last
-        case measure_unit
-        when 's'
-          interval
-        when 'm'
-          interval * 60
-        when 'h'
-          interval * 60 * 60
-        when 'd'
-          interval * 60 * 60 * 24
-        when 'w'
-          interval * 60 * 60 * 24 * 7
-        else
-          raise 'Wrong interval format'
+      def schedule_expiration
+        if @deploy_env.expires
+          @out <<  "Planning expiration in #{@deploy_env.expires}"
+          ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration!
         end
       end
 
diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/expiration_scheduler_spec.rb
new file mode 100644
index 0000000..6964e5d
--- /dev/null
+++ b/devops-service/spec/executors/expiration_scheduler_spec.rb
@@ -0,0 +1,47 @@
+require 'lib/executors/expiration_scheduler'
+
+RSpec.describe Devops::Executor::ExpirationScheduler do
+  let(:server) { build(:server) }
+
+  describe '#schedule_expiration!' do
+    it 'schedules server deleting at given time' do
+      expect(DeleteServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name')
+      described_class.new('2m', server).schedule_expiration!
+    end
+
+    it "doesn't schedule job if expires is nil" do
+      expect(DeleteServerWorker).not_to receive(:perform_in)
+      described_class.new(nil, server).schedule_expiration!
+    end
+  end
+
+  describe '#interval_in_seconds' do
+    def interval_in_seconds(expires)
+      described_class.new(expires, server).interval_in_seconds
+    end
+
+    it 'recognizes seconds' do
+      expect(interval_in_seconds('2s')).to eq 2
+    end
+
+    it 'recognizes minutes' do
+      expect(interval_in_seconds('3m')).to eq 180
+    end
+
+    it 'recognizes hours' do
+      expect(interval_in_seconds('1h')).to eq 3600
+    end
+
+    it 'recognizes days' do
+      expect(interval_in_seconds('1d')).to eq 86400
+    end
+
+    it 'recognizes weeks' do
+      expect(interval_in_seconds('1w')).to eq 604800
+    end
+
+    it 'raises on wrong format' do
+      expect { interval_in_seconds('wrong') }.to raise_error(StandardError)
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index f3e8ff1..6e8db02 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -464,7 +464,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
 
-  describe '#deploy_server', stubbed_knife: true, stubbed_connector: true do
+  describe '#deploy_server', stubbed_knife: true do
     let(:deploy_info) { @deploy_info }
     let(:json_file_name) { 'json.json' }
     let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) }
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb
index c8fe476..c4e02eb 100644
--- a/devops-service/workers/delete_server_worker.rb
+++ b/devops-service/workers/delete_server_worker.rb
@@ -1,6 +1,7 @@
 require "db/mongo/models/server"
 require "db/mongo/models/report"
 require "lib/executors/server_executor"
+require "workers/worker"
 
 class DeleteServerWorker < Worker
 

From f4d13ce19f3d5accd550d2ea7cf77467d7c20250 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 20 Jan 2016 10:23:34 +0300
Subject: [PATCH 20/35] add specs for ServerExecutor#unbootstrap

---
 .../lib/executors/server_executor.rb          | 20 ++-----
 .../spec/executors/server_executor_spec.rb    | 56 ++++++++++++++++++-
 2 files changed, 57 insertions(+), 19 deletions(-)

diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 16b53da..7fb58fa 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -177,7 +177,7 @@ module Devops
             DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
             return error_code(:server_bootstrap_fail)
           end
-          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless connected_successfully?
+          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless last_command_successful?
         rescue ArgumentError => e
           @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
           @out.flush
@@ -299,7 +299,6 @@ module Devops
         knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
-      # there were changes in unbootstrap in other branches; leave it for now
       def unbootstrap
         k = Devops::Db.connector.key(@server.key)
         cert_path = k.path
@@ -307,14 +306,13 @@ module Devops
         res = delete_from_chef_server(@server.chef_node_name)
         begin
           new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}"
-#          r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
           cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} \"/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name}; else echo not found; fi'\""
           DevopsLogger.logger.info("Trying to run command '#{cmd}'")
-          r = `#{cmd}`.strip
+          r = execute_system_command(cmd).strip
           if r == 'not found'
             res[:server] = "Directory '/etc/chef' does not exists"
           else
-            raise(r) unless $?.success?
+            raise(r) unless last_command_successful?
             res[:server] = "'/etc/chef' renamed to '#{new_name}'"
           end
         rescue => e
@@ -412,16 +410,6 @@ module Devops
         }
       end
 
-=begin
-      def delete_etc_chef s, cert_path
-        cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
-        cmd += " sudo " unless s.remote_user == "root"
-        cmd += "rm -Rf /etc/chef"
-        r = `#{cmd}`
-        raise(r) unless $?.success?
-      end
-=end
-
       def delete_server
         mongo = ::Devops::Db.connector
         if @server.static?
@@ -514,7 +502,7 @@ module Devops
         `#{cmd}`
       end
 
-      def connected_successfully?
+      def last_command_successful?
         $?.success?
       end
 
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 6e8db02..106837d 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -212,7 +212,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
     before do
       allow(executor).to receive(:sleep)
-      allow(executor).to receive(:connected_successfully?).and_return(true)
+      allow(executor).to receive(:last_command_successful?).and_return(true)
       allow(executor).to receive(:execute_system_command)
       allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node')
       allow(stubbed_connector).to receive(:key).and_return(image)
@@ -239,7 +239,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     end
 
     context "couldn't ssh to server" do
-      before { allow(executor).to receive(:connected_successfully?) { false } }
+      before { allow(executor).to receive(:last_command_successful?) { false } }
 
       it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do
         max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT
@@ -254,7 +254,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
     context 'after successful ssh check' do
-      before { allow(executor).to receive(:connected_successfully?).and_return(false, true) }
+      before { allow(executor).to receive(:last_command_successful?).and_return(false, true) }
 
       it "sets default chef node name if it's nil" do
         executor.server.chef_node_name = nil
@@ -407,6 +407,56 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   end
 
 
+  describe '#unbootstrap', stubbed_knife: true do
+    before do
+      allow(stubbed_connector).to receive_message_chain('key.path') { 'path_to_key' }
+      allow(stubbed_knife).to receive(:chef_node_delete)
+      allow(stubbed_knife).to receive(:chef_client_delete)
+      allow(executor).to receive(:execute_system_command) { '' }
+      allow(executor).to receive(:last_command_successful?) { true }
+      allow(executor).to receive(:sleep)
+    end
+
+    it 'deletes node from chef server' do
+      allow(executor).to receive(:delete_from_chef_server).and_call_original
+      expect(executor).to receive(:delete_from_chef_server)
+      executor.unbootstrap
+    end
+
+    it "uses server's key" do
+      expect(executor).to receive(:execute_system_command).with(%r(ssh -i path_to_key))
+      executor.unbootstrap
+    end
+
+    it 'backups /etc/chef' do
+      expect(executor).to receive(:execute_system_command).with(%r(mv /etc/chef ))
+      executor.unbootstrap
+    end
+
+    it 'returns a hash with :chef_node, :chef_client and :server keys' do
+      expect(executor.unbootstrap).to be_a(Hash).and include(:chef_node, :chef_client, :server)
+    end
+
+    it "writes successful message into result's :server key" do
+      expect(executor.unbootstrap[:server]).to include('renamed')
+    end
+
+    context "when command returned 'not found'" do
+      it "writes error message into result's :server key" do
+        allow(executor).to receive(:execute_system_command) { 'not found' }
+        expect(executor.unbootstrap[:server]).to eq("Directory '/etc/chef' does not exists")
+      end
+    end
+
+    context "if command wasn't successful" do
+      it 'returns hash with error after 5.retries' do
+        allow(executor).to receive(:last_command_successful?) { false }
+        expect(executor).to receive(:sleep).with(1).exactly(5).times
+        expect(executor.unbootstrap).to be_a(Hash).and include(:error)
+      end
+    end
+  end
+
 
   describe '#deploy_server_with_tags', stubbed_knife: true do
     let(:current_tags) { @current_tags }

From a96ea689223a6364b6b097da30d45973e1ffed54 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 20 Jan 2016 10:32:57 +0300
Subject: [PATCH 21/35] change subject with named let)

---
 .../spec/executors/server_executor_spec.rb    | 135 +++++++++---------
 1 file changed, 66 insertions(+), 69 deletions(-)

diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 106837d..7426437 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -78,12 +78,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
   describe '#create_server' do
-    let!(:without_bootstrap) { @without_bootstrap = true }
-    let!(:run_list) { @run_list = %w(role[asd]) }
-    let!(:key) { @key = 'key' }
-    let!(:image) { double('Image instance', remote_user: 'remote_user') }
-
-    subject {
+    let(:image) { double('Image instance', remote_user: 'remote_user') }
+    let(:create_server) {
       executor.create_server(
         'created_by' => 'user',
         'run_list' => @run_list,
@@ -97,10 +93,13 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       allow(provider).to receive(:create_server) { true }
       allow(stubbed_connector).to receive(:image) { image }
       allow(stubbed_connector).to receive(:server_insert)
+      @without_bootstrap = true
+      @run_list = %w(role[asd])
+      @key = 'key'
     end
 
     it 'builds server model from given options' do
-      subject
+      create_server
       expect(executor.server.created_by).to eq 'user'
       expect(executor.server.chef_node_name).to eq 'node_name'
       expect(executor.server.key).to eq @key
@@ -109,46 +108,46 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
     it 'sets run list to an empty array by default' do
       @run_list = nil
-      subject
+      create_server
       expect(executor.server.run_list).to eq []
     end
 
     it 'sets key to default provider ssh key by default' do
       @key = nil
       allow(provider).to receive(:ssh_key) { 'default_key' }
-      subject
+      create_server
       expect(executor.server.key).to eq 'default_key'
     end
 
     it 'runs hooks' do
       expect(executor).to receive(:run_hook).with(:before_create).ordered
       expect(executor).to receive(:run_hook).with(:after_create).ordered
-      subject
+      create_server
     end
 
     it 'creates server in cloud' do
       expect(provider).to receive(:create_server).with(
         an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output
       )
-      subject
+      create_server
     end
 
     it 'inserts built server into mongo' do
       expect(stubbed_connector).to receive(:server_insert)
-      subject
+      create_server
     end
 
     it 'schedules expiration for server' do
       deploy_env.expires = '2m'
       allow(DeleteServerWorker).to receive(:perform_in)
       expect(DeleteServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
-      subject
+      create_server
     end
 
     it "doesn't schedule expiration if deploy_env.expires is nil" do
       deploy_env.expires = nil
       expect(DeleteServerWorker).not_to receive(:perform_in)
-      subject
+      create_server
     end
 
     context 'without_bootstrap option is false' do
@@ -157,7 +156,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         allow(image).to receive(:bootstrap_template) { 'template' }
         allow(executor).to receive(:two_phase_bootstrap)
         expect(executor).to receive(:two_phase_bootstrap)
-        subject
+        create_server
       end
     end
 
@@ -167,7 +166,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         allow(image).to receive(:bootstrap_template) { 'template' }
         allow(executor).to receive(:two_phase_bootstrap)
         expect(executor).to receive(:two_phase_bootstrap)
-        subject
+        create_server
       end
     end
 
@@ -175,7 +174,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it "doesn't launch bootstrap" do
         @without_bootstrap = true
         expect(executor).not_to receive(:two_phase_bootstrap)
-        subject
+        create_server
       end
     end
 
@@ -187,19 +186,19 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
       it 'rollbacks server creating' do
         expect(executor).to receive(:roll_back)
-        subject
+        create_server
       end
 
       it 'deletes server from mongo' do
         expect(stubbed_connector).to receive(:server_delete)
-        subject
+        create_server
       end
     end
 
     context "if creating server in cloud wasn't successful" do
       it 'returns creating_server_in_cloud_failed error code' do
         allow(provider).to receive(:create_server) { false }
-        expect(subject).to eq 10
+        expect(create_server).to eq 10
       end
     end
   end
@@ -207,8 +206,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
   describe '#bootstrap', stubbed_knife: true do
-    subject { executor.bootstrap({}) }
     let(:image) { double('Key instance', path: 'path') }
+    let(:bootstrap) { executor.bootstrap({}) }
 
     before do
       allow(executor).to receive(:sleep)
@@ -223,19 +222,19 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     it 'run before hook' do
       expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered
       expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered
-      subject
+      bootstrap
     end
 
     context "when server's private ip is unset" do
       it 'returns server_bootstrap_private_ip_unset error code' do
         server.private_ip = nil
-        expect(subject).to eq 4
+        expect(bootstrap).to eq 4
       end
     end
 
     it 'tries to ssh to server' do
       expect(executor).to receive(:execute_system_command).with(/ssh/)
-      subject
+      bootstrap
     end
 
     context "couldn't ssh to server" do
@@ -244,11 +243,11 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do
         max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT
         expect(executor).to receive(:execute_system_command).exactly(max_retries).times
-        subject
+        bootstrap
       end
 
       it 'returns server_bootstrap_fail error code' do
-        expect(subject).to eq 2
+        expect(bootstrap).to eq 2
       end
     end
 
@@ -258,24 +257,24 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
       it "sets default chef node name if it's nil" do
         executor.server.chef_node_name = nil
-        expect {subject}.to change {executor.server.chef_node_name}.to 'chef_node'
+        expect {bootstrap}.to change {executor.server.chef_node_name}.to 'chef_node'
       end
 
       it 'executes knife bootstrap' do
         expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array))
-        subject
+        bootstrap
       end
 
       it "bootstraps to public ip if it's set" do
         server.public_ip = '8.8.8.8'
         expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array))
-        subject
+        bootstrap
       end
 
       context 'after successful bootstrap' do
         it "updates server's chef node name in db" do
           expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server))
-          subject
+          bootstrap
         end
       end
 
@@ -283,12 +282,12 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) }
 
         it 'returns :server_bootstrap_fail code' do
-          expect(subject).to eq 2
+          expect(bootstrap).to eq 2
         end
 
         it "doesn't run after hook" do
           expect(executor).to receive(:run_hook).with(:before_bootstrap, output)
-          subject
+          bootstrap
         end
       end
     end
@@ -297,7 +296,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
   describe '#two_phase_bootstrap', stubbed_knife: true do
-    subject { executor.two_phase_bootstrap({}) }
+    let(:two_phase_bootstrap) { executor.two_phase_bootstrap({}) }
 
     before do
       allow(provider).to receive(:run_list) {[]}
@@ -319,37 +318,37 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
         it 'builds run list' do
           expect(executor).to receive(:compute_run_list)
-          subject
+          two_phase_bootstrap
         end
 
         it 'sets run list to chef node' do
           expect(stubbed_knife).to receive(:set_run_list)
-          subject
+          two_phase_bootstrap
         end
 
         it 'deploys server' do
           expect(executor).to receive(:deploy_server)
-          subject
+          two_phase_bootstrap
         end
 
         context 'if deploy was successful' do
           it 'returns 0' do
             allow(executor).to receive(:deploy_server) { 0 }
-            expect(subject).to eq 0
+            expect(two_phase_bootstrap).to eq 0
           end
         end
 
         context "if deploy wasn't successful" do
           it 'returns :deploy_failed code' do
             allow(executor).to receive(:deploy_server) { 1 }
-            expect(subject).to eq 8
+            expect(two_phase_bootstrap).to eq 8
           end
         end
 
         context 'when an error occured during deploy' do
           it 'returns :deploy_unknown_error code' do
             allow(executor).to receive(:deploy_server) { raise }
-            expect(subject).to eq 6
+            expect(two_phase_bootstrap).to eq 6
           end
         end
       end
@@ -361,7 +360,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
           allow(stubbed_connector).to receive(:server_delete)
           expect(executor).to receive(:roll_back).ordered
           expect(stubbed_connector).to receive(:server_delete).ordered
-          subject
+          two_phase_bootstrap
         end
       end
     end
@@ -369,21 +368,19 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context "when bootstrap wasn't successful" do
       it 'returns :server_bootstrap_fail error code' do
         allow(executor).to receive(:bootstrap) { 1 }
-        expect(subject).to eq 2
+        expect(two_phase_bootstrap).to eq 2
       end
     end
 
     context 'when an error occured during bootstrap' do
       it 'returns :server_bootstrap_unknown_error error code' do
         allow(executor).to receive(:bootstrap) { raise }
-        expect(subject).to eq 7
+        expect(two_phase_bootstrap).to eq 7
       end
     end
   end
 
   describe '#check_server_on_chef_server', stubbed_knife: true do
-    subject { executor.check_server_on_chef_server }
-
     before do
       server.chef_node_name = 'a'
       allow(stubbed_knife).to receive(:chef_node_list) { @node_list }
@@ -392,17 +389,17 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
     it 'returns true when node_name in node list and in client list' do
       @node_list = %w(a); @client_list = %w(a)
-      expect(subject).to be true
+      expect(executor.check_server_on_chef_server).to be true
     end
 
     it "returns false if node name isn't in node list" do
       @node_list = []; @client_list = %w(a)
-      expect(subject).to be false
+      expect(executor.check_server_on_chef_server).to be false
     end
 
     it "returns false if node name isn't in node list" do
       @node_list = %w(a); @client_list = []
-      expect(subject).to be false
+      expect(executor.check_server_on_chef_server).to be false
     end
   end
 
@@ -464,7 +461,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     let(:joined_initial_tags) { initial_tags.join(' ') }
     let(:given_tags) { %w(c d) }
     let(:joined_given_tags) { given_tags.join(' ') }
-    subject { executor.deploy_server_with_tags(given_tags, {}) }
+    let(:deploy_server_with_tags) { executor.deploy_server_with_tags(given_tags, {}) }
 
     before do
       @current_tags = initial_tags.dup
@@ -490,7 +487,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_initial_tags, joined_given_tags).ordered
         expect(executor).to receive(:deploy_server).ordered
         expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_given_tags, joined_initial_tags).ordered
-        subject
+        deploy_server_with_tags
       end
     end
 
@@ -498,7 +495,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it 'restores tags anyway' do
         allow(executor).to receive(:deploy_server) { raise }
         expect {
-          subject
+          deploy_server_with_tags
         }.to raise_error StandardError
         expect(current_tags).to eq initial_tags
       end
@@ -507,7 +504,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     context 'if cannot add tags to server' do
       it 'returns :server_cannot_update_tags code' do
         allow(stubbed_knife).to receive(:swap_tags) { false }
-        expect(subject).to eq 3
+        expect(deploy_server_with_tags).to eq 3
       end
     end
   end
@@ -519,7 +516,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     let(:json_file_name) { 'json.json' }
     let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) }
 
-    subject { executor.deploy_server(deploy_info) }
+    let(:deploy_server) { executor.deploy_server(deploy_info) }
 
     before do
       allow(executor).to receive(:run_hook).with(:before_deploy, any_args)
@@ -534,7 +531,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
     it 'runs before_deploy and after_deploy hooks' do
       expect(executor).to receive(:run_hook).with(:before_deploy, any_args).ordered
       expect(executor).to receive(:run_hook).with(:after_deploy, any_args).ordered
-      subject
+      deploy_server
     end
 
     context 'when uses json file' do
@@ -560,13 +557,13 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
 
       it 'writes deploy_info to json file if it not exists' do
-        expect { subject }.to change { Dir.entries(SpecSupport.tmp_dir)}
+        expect { deploy_server }.to change { Dir.entries(SpecSupport.tmp_dir)}
       end
 
       it "writes deploy_info to given json file name if it doesn't exist" do
         FileUtils.rm(json_file_path) if File.exists?(json_file_path)
         deploy_info['json_file'] = json_file_name
-        expect { subject }.to change {
+        expect { deploy_server }.to change {
           Dir.entries(SpecSupport.tmp_dir)
         }
         FileUtils.rm(json_file_path)
@@ -575,7 +572,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it 'reads json from file if it exists' do
         deploy_info['json_file'] = json_file_name
         File.open(json_file_path, 'w') { |file| file.puts '{"foo": "bar"'}
-        expect { subject }.not_to change {
+        expect { deploy_server }.not_to change {
           Dir.entries(SpecSupport.tmp_dir)
         }
         FileUtils.rm(json_file_path)
@@ -585,7 +582,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         deploy_info['json_file'] = json_file_name
         regexp = %r(-j http://host.com:8080/api/v2.0/deploy/data/#{json_file_name})
         expect(stubbed_knife).to receive(:ssh_stream).with(anything, regexp, any_args)
-        subject
+        deploy_server
       end
     end
 
@@ -598,41 +595,41 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it "adds run list to command if server's stack is set" do
         server.stack = 'stack'
         expect(stubbed_knife).to receive(:ssh_stream).with(anything, %r(-r foo,bar), any_args)
-        subject
+        deploy_server
       end
 
       it "doesn't add run list to command if server's stack is unset" do
         expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args)
-        subject
+        deploy_server
       end
     end
 
     it "uses server's key" do
       expect(stubbed_connector).to receive(:key).with('key_id')
       expect(stubbed_knife).to receive(:ssh_stream).with(any_args, 'path_to_key')
-      subject
+      deploy_server
     end
 
     it "uses public ip if it's set" do
       server.public_ip = '127.0.0.1'
       expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, '127.0.0.1', any_args)
-      subject
+      deploy_server
     end
 
     it "uses private_ip if public_ip isn't set" do
       expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, server.private_ip, any_args)
-      subject
+      deploy_server
     end
 
     context 'if deploy was successful' do
       it "updates server's last operation" do
         expect(server).to receive(:set_last_operation).with('deploy', anything)
         expect(stubbed_connector).to receive(:server_update).with(server)
-        subject
+        deploy_server
       end
 
       it 'returns 0' do
-        expect(subject).to eq 0
+        expect(deploy_server).to eq 0
       end
     end
 
@@ -641,26 +638,26 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       it "doesn't run after_deploy hook" do
         expect(executor).to receive(:run_hook).with(:before_deploy, any_args)
         expect(executor).not_to receive(:run_hook).with(:after_deploy, any_args)
-        subject
+        deploy_server
       end
 
       it 'returns 1' do
-        expect(subject).to eq 1
+        expect(deploy_server).to eq 1
       end
     end
   end
 
 
   describe '#delete_from_chef_server', stubbed_knife: true do
-    subject { executor.delete_from_chef_server('foo') }
+    let(:delete_from_chef_server) { executor.delete_from_chef_server('foo') }
     before do
       allow(stubbed_knife).to receive(:chef_client_delete)
       allow(stubbed_knife).to receive(:chef_node_delete)
-      subject
+      delete_from_chef_server
     end
 
     it 'returns hash with :chef_node and :chef_client keys' do
-      expect(subject).to be_a(Hash).and include(:chef_node, :chef_client)
+      expect(delete_from_chef_server).to be_a(Hash).and include(:chef_node, :chef_client)
     end
 
     it 'calls to :chef_node_delete and :chef_client_delete' do

From 80f39e4e546d974b5f46e65bc798053c9002468c Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 20 Jan 2016 13:20:10 +0300
Subject: [PATCH 22/35] remove unused methods

---
 devops-service/db/mongo/models/image.rb | 4 ----
 devops-service/db/mongo/models/key.rb   | 4 ----
 devops-service/db/mongo/models/user.rb  | 7 +------
 3 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/devops-service/db/mongo/models/image.rb b/devops-service/db/mongo/models/image.rb
index dace22e..2eb249a 100644
--- a/devops-service/db/mongo/models/image.rb
+++ b/devops-service/db/mongo/models/image.rb
@@ -73,10 +73,6 @@ module Devops
         }
       end
 
-      def self.create_from_json! json
-        Image.new( JSON.parse(json) )
-      end
-
     end
   end
 end
diff --git a/devops-service/db/mongo/models/key.rb b/devops-service/db/mongo/models/key.rb
index 32332ff..edaac58 100644
--- a/devops-service/db/mongo/models/key.rb
+++ b/devops-service/db/mongo/models/key.rb
@@ -30,10 +30,6 @@ module Devops
         key
       end
 
-      def self.create_from_json json
-        Key.new( JSON.parse(json) )
-      end
-
       def filename
         File.basename(self.path)
       end
diff --git a/devops-service/db/mongo/models/user.rb b/devops-service/db/mongo/models/user.rb
index c24247e..c96b3dd 100644
--- a/devops-service/db/mongo/models/user.rb
+++ b/devops-service/db/mongo/models/user.rb
@@ -75,17 +75,12 @@ module Devops
         user
       end
 
-      def self.create_from_json json
-        User.new( JSON.parse(json) )
-      end
-
       def to_hash_without_id
-        o = {
+        {
           "email" => self.email,
           "password" => self.password,
           "privileges" => self.privileges
         }
-        o
       end
 
       def check_privileges cmd, required_privelege

From df44b5b26996aac1e12a7f95eb2283350c937b84 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 20 Jan 2016 14:49:30 +0300
Subject: [PATCH 23/35] add missing specs

---
 devops-service/db/mongo/models/project.rb       |  1 +
 .../models/stack_template/stack_template_ec2.rb |  4 ----
 devops-service/spec/models/project_spec.rb      | 17 +++++++++++++++++
 devops-service/spec/models/user_spec.rb         | 14 +++++++++++++-
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb
index c4edc04..0bb61ac 100644
--- a/devops-service/db/mongo/models/project.rb
+++ b/devops-service/db/mongo/models/project.rb
@@ -22,6 +22,7 @@ module Devops
       #define_hook :after_add_deploy_env
 
       attr_accessor :id, :deploy_envs, :type, :archived, :description, :run_list
+      attr_accessor :components
 
       MULTI_TYPE = "multi"
 
diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
index 89f7a0d..7a2636e 100644
--- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
+++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
@@ -16,10 +16,6 @@ module Devops
         super.merge(template_url: template_url)
       end
 
-      def delete_template_file_from_storage
-        raise 'Implement me'
-      end
-
       class << self
 
         def create(attrs)
diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb
index dd892b1..fdb8b50 100644
--- a/devops-service/spec/models/project_spec.rb
+++ b/devops-service/spec/models/project_spec.rb
@@ -23,6 +23,23 @@ RSpec.describe Devops::Model::Project, type: :model do
       project = build(:project, with_deploy_env_identifiers: ['foo', nil])
       expect(project).not_to be_valid
     end
+
+    describe 'components validation' do
+      it 'is valid with components with filenames' do
+        project.components = {'foo' => {'filename' => 'bar'}}
+        expect{project.validate_components}.not_to raise_error
+      end
+
+      it "isn't valid if components isn't a hash" do
+        project.components = []
+        expect{project.validate_components}.to raise_error InvalidRecord
+      end
+
+      it "raises InvalidRecord if one of componentsц hasn't filename" do
+        project.components = {'foo' => {}}
+        expect{project.validate_components}.to raise_error InvalidRecord
+      end
+    end
   end
 
   describe '.fields' do
diff --git a/devops-service/spec/models/user_spec.rb b/devops-service/spec/models/user_spec.rb
index c060ff4..ad9bc69 100644
--- a/devops-service/spec/models/user_spec.rb
+++ b/devops-service/spec/models/user_spec.rb
@@ -79,13 +79,25 @@ RSpec.describe Devops::Model::User, type: :model do
     end
   end
 
+  describe '.build_from_bson' do
+    it 'builds User model from given hash and assigns id' do
+      model = described_class.build_from_bson('_id' => 'foo', 'username' => 'not shown', 'email' => 'baz')
+      expect(model.id).to eq 'foo'
+      expect(model.email).to eq 'baz'
+    end
+  end
+
   describe '#check_privileges' do
     it "raises InvalidPrivileges if user hasn't specified privilege" do
       expect { user.check_privileges('key', 'w') }.to raise_error(InvalidPrivileges)
     end
 
     it 'does nothing is user has specified privilege' do
-      user.check_privileges('key', 'r')
+      expect{user.check_privileges('key', 'r')}.not_to raise_error
+    end
+
+    it 'raises InvalidPrivileges if given privelege is wrong' do
+      expect{user.check_privileges('key', 't')}.to raise_error InvalidPrivileges
     end
   end
 

From 4bab2579708a2ddb41d50cf7f2699630949f7b35 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <achuchkalov@ggasoftware.com>
Date: Wed, 20 Jan 2016 18:47:07 +0400
Subject: [PATCH 24/35] simplify stubbing in validators

---
 .../db/mongo/models/deploy_env/deploy_env_ec2.rb |  2 +-
 devops-service/db/validators/base.rb             |  2 ++
 .../db/validators/deploy_env/flavor.rb           |  8 +-------
 .../db/validators/deploy_env/groups.rb           | 10 +++-------
 devops-service/db/validators/deploy_env/image.rb |  8 +-------
 .../db/validators/deploy_env/stack_template.rb   | 12 ++----------
 .../db/validators/field_validators/flavor.rb     |  8 +-------
 .../db/validators/field_validators/image.rb      |  8 +-------
 devops-service/db/validators/helpers/users.rb    |  7 +------
 .../models/deploy_env/deploy_env_ec2_spec.rb     | 16 +++++++++++++++-
 .../deploy_env/deploy_env_openstack_spec.rb      | 16 +++++++++++++++-
 .../models/deploy_env/deploy_env_static_spec.rb  |  4 ++--
 devops-service/spec/models/project_spec.rb       | 16 +++++++++++++++-
 .../shared_contexts/stubbed_env_validators.rb    |  5 +++--
 14 files changed, 63 insertions(+), 59 deletions(-)

diff --git a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
index 49cdfad..f4259cf 100644
--- a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
+++ b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
@@ -53,7 +53,7 @@ module Devops
       def subnets_filter
         networks = provider_instance.networks
 
-        unless self.subnets.empty?
+        if subnets && !subnets.empty?
           network = networks.detect {|n| n["name"] == self.subnets[0]}
           if network
             {"vpc-id" => network["vpcId"] }
diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb
index 6e3b488..26c623f 100644
--- a/devops-service/db/validators/base.rb
+++ b/devops-service/db/validators/base.rb
@@ -12,6 +12,7 @@ module Validators
       raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}")
     end
 
+    # :nocov:
     def valid?
       raise 'override me'
     end
@@ -19,6 +20,7 @@ module Validators
     def message
       raise 'override me'
     end
+    # :nocov:
 
     class << self
       private
diff --git a/devops-service/db/validators/deploy_env/flavor.rb b/devops-service/db/validators/deploy_env/flavor.rb
index e0ff359..b763af5 100644
--- a/devops-service/db/validators/deploy_env/flavor.rb
+++ b/devops-service/db/validators/deploy_env/flavor.rb
@@ -3,7 +3,7 @@ module Validators
 
     def valid?
       return true unless @model.flavor
-      available_flavors.detect do |flavor|
+      @model.provider_instance.flavors.detect do |flavor|
         flavor['id'] == @model.flavor
       end
     end
@@ -11,11 +11,5 @@ module Validators
     def message
       "Invalid flavor '#{@model.flavor}'."
     end
-
-    private
-
-    def available_flavors
-      @model.provider_instance.flavors
-    end
   end
 end
\ No newline at end of file
diff --git a/devops-service/db/validators/deploy_env/groups.rb b/devops-service/db/validators/deploy_env/groups.rb
index 43f7027..8e3db24 100644
--- a/devops-service/db/validators/deploy_env/groups.rb
+++ b/devops-service/db/validators/deploy_env/groups.rb
@@ -3,6 +3,9 @@ module Validators
 
     def valid?
       return true if @model.groups.nil?
+      subnets_filter = @model.subnets_filter
+      available_groups = @model.provider_instance.groups(subnets_filter).keys
+
       @invalid_groups = @model.groups - available_groups
       @invalid_groups.empty?
     end
@@ -10,12 +13,5 @@ module Validators
     def message
       "Invalid groups '#{@invalid_groups.join("', '")}'."
     end
-
-    private
-
-    def available_groups
-      subnets_filter = @model.subnets_filter
-      @model.provider_instance.groups(subnets_filter).keys
-    end
   end
 end
\ No newline at end of file
diff --git a/devops-service/db/validators/deploy_env/image.rb b/devops-service/db/validators/deploy_env/image.rb
index 6056d88..ee69561 100644
--- a/devops-service/db/validators/deploy_env/image.rb
+++ b/devops-service/db/validators/deploy_env/image.rb
@@ -6,7 +6,7 @@ module Validators
 
     def valid?
       return true unless @model.image
-      available_images.detect do |image|
+      get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
         image["id"] == @model.image
       end
     end
@@ -14,11 +14,5 @@ module Validators
     def message
       "Invalid image '#{@model.image}'."
     end
-
-    private
-
-    def available_images
-      get_available_provider_images(::Devops::Db.connector, @model.provider)
-    end
   end
 end
diff --git a/devops-service/db/validators/deploy_env/stack_template.rb b/devops-service/db/validators/deploy_env/stack_template.rb
index 9b20d6d..eee09de 100644
--- a/devops-service/db/validators/deploy_env/stack_template.rb
+++ b/devops-service/db/validators/deploy_env/stack_template.rb
@@ -4,21 +4,13 @@ module Validators
     def valid?
       return true unless @model.stack_template
 
-      available_stack_templates.detect do |template|
-        template['id'] == @model.stack_template
+      Devops::Db.connector.stack_templates.detect do |template|
+        template.id == @model.stack_template
       end
     end
 
     def message
       "Invalid stack template '#{@model.stack_template}'."
     end
-
-    private
-
-    def available_stack_templates
-      # map to hash to simplify mocks. Later replace this method with something more suitable
-      Devops::Db.connector.stack_templates.map(&:to_hash)
-    end
-
   end
 end
diff --git a/devops-service/db/validators/field_validators/flavor.rb b/devops-service/db/validators/field_validators/flavor.rb
index 4f86df4..a06d104 100644
--- a/devops-service/db/validators/field_validators/flavor.rb
+++ b/devops-service/db/validators/field_validators/flavor.rb
@@ -4,7 +4,7 @@ module Validators
     class Flavor < Base
 
       def valid?
-        available_flavors.detect do |flavor|
+        @model.provider_instance.flavors.detect do |flavor|
           flavor['id'] == @value
         end
       end
@@ -12,12 +12,6 @@ module Validators
       def message
         "Invalid flavor '#{@value}'."
       end
-
-      private
-
-      def available_flavors
-        @model.provider_instance.flavors
-      end
     end
   end
 end
diff --git a/devops-service/db/validators/field_validators/image.rb b/devops-service/db/validators/field_validators/image.rb
index 3dd727b..d8df841 100644
--- a/devops-service/db/validators/field_validators/image.rb
+++ b/devops-service/db/validators/field_validators/image.rb
@@ -7,7 +7,7 @@ module Validators
       include ::ImageCommands
 
       def valid?
-        available_images.detect do |image|
+        get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
           image["id"] == @value
         end
       end
@@ -15,12 +15,6 @@ module Validators
       def message
         "Invalid image '#{@value}'."
       end
-
-      private
-
-      def available_images
-        get_available_provider_images(::Devops::Db.connector, @model.provider)
-      end
     end
   end
 end
diff --git a/devops-service/db/validators/helpers/users.rb b/devops-service/db/validators/helpers/users.rb
index 7b534bf..430a7bc 100644
--- a/devops-service/db/validators/helpers/users.rb
+++ b/devops-service/db/validators/helpers/users.rb
@@ -2,6 +2,7 @@ module Validators
   class Helpers::Users < Base
 
     def valid?
+      available_users = ::Devops::Db.connector.users_names(@model)
       @nonexistent_users = (@model || []) - available_users
       @nonexistent_users.empty?
     end
@@ -9,11 +10,5 @@ module Validators
     def message
       Devops::Messages.t("project.deploy_env.validation.users.not_exist", users: @nonexistent_users.join("', '"))
     end
-
-    private
-
-    def available_users
-      ::Devops::Db.connector.users_names(@model)
-    end
   end
 end
diff --git a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
index 8bbf1d0..a80bcef 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
@@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvEc2, type: :model do
   let(:env) { build(:deploy_env_ec2) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_env_validators: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true do
+    before do
+      provider_double = instance_double('Provider::Ec2',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
+    end
+
     it_behaves_like 'deploy env'
     it_behaves_like 'cloud deploy env'
   end
diff --git a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
index e1a54de..c7ba3a8 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
@@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvOpenstack, type: :model do
   let(:env) { build(:deploy_env_openstack) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_env_validators: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true do
+    before do
+      provider_double = instance_double('Provider::Openstack',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(openstack) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_openstack, id: 'template')] }
+    end
+
     it_behaves_like 'deploy env'
     it_behaves_like 'cloud deploy env'
   end
diff --git a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
index 45ad446..58f170f 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
@@ -5,10 +5,10 @@ RSpec.describe Devops::Model::DeployEnvStatic, type: :model do
   let(:env) { build(:deploy_env_static) }
 
 
-  describe 'it inherits from deploy env', stubbed_logger: true do
+  describe 'it inherits from deploy env', stubbed_logger: true, stubbed_connector: true do
     before do
       allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(static))
-      allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
     end
 
     it_behaves_like 'deploy env'
diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb
index fdb8b50..f264e2a 100644
--- a/devops-service/spec/models/project_spec.rb
+++ b/devops-service/spec/models/project_spec.rb
@@ -3,7 +3,21 @@ require 'db/mongo/models/project'
 RSpec.describe Devops::Model::Project, type: :model do
   let(:project) { build(:project) }
 
-  describe 'validation rules:', stubbed_env_validators: true do
+  describe 'validation rules:', stubbed_connector: true do
+    before do
+      provider_double = instance_double('Provider::Ec2',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
+    end
+
     include_examples 'field type validation', :id, :not_nil, :non_empty_string
     include_examples 'field type validation', :deploy_envs, :not_nil, :non_empty_array
     include_examples 'field type validation', :description, :maybe_nil, :maybe_empty_string
diff --git a/devops-service/spec/shared_contexts/stubbed_env_validators.rb b/devops-service/spec/shared_contexts/stubbed_env_validators.rb
index 69f081c..998f27d 100644
--- a/devops-service/spec/shared_contexts/stubbed_env_validators.rb
+++ b/devops-service/spec/shared_contexts/stubbed_env_validators.rb
@@ -1,9 +1,10 @@
 RSpec.shared_context 'stubbed calls to connector in env validators', stubbed_env_validators: true do
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2 openstack))
+    allow_any_instance_of(env_class).to receive_message_chain('provider_instance.flavors').and_return [{'id' => 'flavor'}]
     allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
-    allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
-    allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
+    # allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
+    # allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
     allow_any_instance_of(Validators::DeployEnv::Groups).to receive(:available_groups).and_return(['default'])
     allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])
     allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])

From 3b8fc97861f6901a5baf157dcff94bd03369ea37 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 21 Jan 2016 02:26:27 +0300
Subject: [PATCH 25/35] remove unused condition

---
 .../db/validators/image/bootstrap_template.rb        | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/devops-service/db/validators/image/bootstrap_template.rb b/devops-service/db/validators/image/bootstrap_template.rb
index 2ced224..67bc070 100644
--- a/devops-service/db/validators/image/bootstrap_template.rb
+++ b/devops-service/db/validators/image/bootstrap_template.rb
@@ -8,22 +8,12 @@ module Validators
       include BootstrapTemplatesCommands
 
       def valid?
-        if @model.bootstrap_template
-          available_templates.include?(@model.bootstrap_template)
-        else
-          true
-        end
+        get_templates.include?(@model.bootstrap_template)
       end
 
       def message
         "Invalid bootstrap template '#{@model.bootstrap_template}' for image '#{@model.id}'"
       end
-
-      private
-
-      def available_templates
-        get_templates
-      end
     end
   end
 end

From f016727f37beb52501f853707d9e14c6f8304cdd Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 21 Jan 2016 20:46:15 +0300
Subject: [PATCH 26/35] stub logger in various places

---
 devops-service/db/validators/base.rb          |  2 +-
 .../lib/executors/server_executor.rb          |  1 +
 .../spec/executors/server_executor_spec.rb    | 41 ++++---------------
 .../models/deploy_env/deploy_env_ec2_spec.rb  |  2 +-
 .../deploy_env/deploy_env_openstack_spec.rb   |  2 +-
 devops-service/spec/models/image_spec.rb      |  2 +-
 devops-service/spec/models/key_spec.rb        |  2 +-
 devops-service/spec/models/project_spec.rb    |  2 +-
 .../openstack_provider_account_spec.rb        |  2 +-
 devops-service/spec/models/server_spec.rb     |  2 +-
 .../spec/models/stack/stack_ec2_spec.rb       |  2 +-
 11 files changed, 17 insertions(+), 43 deletions(-)

diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb
index c7d9250..bec8722 100644
--- a/devops-service/db/validators/base.rb
+++ b/devops-service/db/validators/base.rb
@@ -9,7 +9,7 @@ module Validators
     def validate!
       raise InvalidRecord.new(message) unless valid?
     rescue StandardError => e
-      puts [e.message, e.backtrace].join("\n")
+      DevopsLogger.logger.error [e.message, e.backtrace].join("\n")
       raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}")
     end
 
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 091bf1b..06d8425 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -300,6 +300,7 @@ module Devops
         knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
+      # returns a hash with :chef_node, :chef_client and :server keys
       def unbootstrap
         k = Devops::Db.connector.key(@server.key)
         cert_path = k.path
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 7426437..de70722 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -412,45 +412,18 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
       allow(executor).to receive(:execute_system_command) { '' }
       allow(executor).to receive(:last_command_successful?) { true }
       allow(executor).to receive(:sleep)
+      allow(Net::SSH).to receive(:start)
     end
 
-    it 'deletes node from chef server' do
-      allow(executor).to receive(:delete_from_chef_server).and_call_original
-      expect(executor).to receive(:delete_from_chef_server)
+    it 'connects by ssh' do
+      expect(Net::SSH).to receive(:start)
       executor.unbootstrap
     end
 
-    it "uses server's key" do
-      expect(executor).to receive(:execute_system_command).with(%r(ssh -i path_to_key))
-      executor.unbootstrap
-    end
-
-    it 'backups /etc/chef' do
-      expect(executor).to receive(:execute_system_command).with(%r(mv /etc/chef ))
-      executor.unbootstrap
-    end
-
-    it 'returns a hash with :chef_node, :chef_client and :server keys' do
-      expect(executor.unbootstrap).to be_a(Hash).and include(:chef_node, :chef_client, :server)
-    end
-
-    it "writes successful message into result's :server key" do
-      expect(executor.unbootstrap[:server]).to include('renamed')
-    end
-
-    context "when command returned 'not found'" do
-      it "writes error message into result's :server key" do
-        allow(executor).to receive(:execute_system_command) { 'not found' }
-        expect(executor.unbootstrap[:server]).to eq("Directory '/etc/chef' does not exists")
-      end
-    end
-
-    context "if command wasn't successful" do
-      it 'returns hash with error after 5.retries' do
-        allow(executor).to receive(:last_command_successful?) { false }
-        expect(executor).to receive(:sleep).with(1).exactly(5).times
-        expect(executor.unbootstrap).to be_a(Hash).and include(:error)
-      end
+    it 'returns hash with error after 5 unsuccessful retries' do
+      allow(Net::SSH).to receive(:start) { raise }
+      expect(Net::SSH).to receive(:start).exactly(5).times
+      expect(executor.unbootstrap).to be_a(Hash).and include(:error)
     end
   end
 
diff --git a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
index a80bcef..dd3c181 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
@@ -5,7 +5,7 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvEc2, type: :model do
   let(:env) { build(:deploy_env_ec2) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_connector: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
     before do
       provider_double = instance_double('Provider::Ec2',
         flavors: [{'id' => 'flavor'}],
diff --git a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
index c7ba3a8..a7f180b 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
@@ -5,7 +5,7 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvOpenstack, type: :model do
   let(:env) { build(:deploy_env_openstack) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_connector: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
     before do
       provider_double = instance_double('Provider::Openstack',
         flavors: [{'id' => 'flavor'}],
diff --git a/devops-service/spec/models/image_spec.rb b/devops-service/spec/models/image_spec.rb
index 769e04a..9d78424 100644
--- a/devops-service/spec/models/image_spec.rb
+++ b/devops-service/spec/models/image_spec.rb
@@ -14,7 +14,7 @@ RSpec.describe Devops::Model::Image, type: :model do
     expect(image).to be_valid
   end
 
-  describe 'validation' do
+  describe 'validation', stubbed_logger: true do
     include_examples 'field type validation', :id, :not_nil, :non_empty_string, :field_validator
     include_examples 'field type validation', :remote_user, :not_nil, :non_empty_string, :field_validator
     include_examples 'field type validation', :name, :not_nil, :non_empty_string, :field_validator
diff --git a/devops-service/spec/models/key_spec.rb b/devops-service/spec/models/key_spec.rb
index 8a2108c..bed5d79 100644
--- a/devops-service/spec/models/key_spec.rb
+++ b/devops-service/spec/models/key_spec.rb
@@ -7,7 +7,7 @@ RSpec.describe Devops::Model::Key, type: :model do
     expect(key).to be_valid
   end
 
-  describe 'validations' do
+  describe 'validations', stubbed_logger: true do
     it 'key file should exist in file system' do
       expect(build(:key, path: './not_exist')).not_to be_valid
     end
diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb
index f264e2a..5fc0521 100644
--- a/devops-service/spec/models/project_spec.rb
+++ b/devops-service/spec/models/project_spec.rb
@@ -3,7 +3,7 @@ require 'db/mongo/models/project'
 RSpec.describe Devops::Model::Project, type: :model do
   let(:project) { build(:project) }
 
-  describe 'validation rules:', stubbed_connector: true do
+  describe 'validation rules:', stubbed_connector: true, stubbed_logger: true do
     before do
       provider_double = instance_double('Provider::Ec2',
         flavors: [{'id' => 'flavor'}],
diff --git a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
index 97aabf6..9efdebd 100644
--- a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
+++ b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
@@ -3,7 +3,7 @@ require 'spec_helper'
 RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do
   let(:provider_account) { build(:openstack_provider_account) }
 
-  it "should not validate access_key_id" do
+  it "should not validate access_key_id", stubbed_logger: true do
     fields = described_class.field_validators.values.flatten.flatten.map{|t| t[:field]}
     expect(fields).not_to include(:access_key_id, :secret_access_key)
   end
diff --git a/devops-service/spec/models/server_spec.rb b/devops-service/spec/models/server_spec.rb
index 961da4e..0beb489 100644
--- a/devops-service/spec/models/server_spec.rb
+++ b/devops-service/spec/models/server_spec.rb
@@ -11,7 +11,7 @@ RSpec.describe Devops::Model::Server, type: :model do
     expect(server).to be_valid
   end
 
-  describe 'validation rules:' do
+  describe 'validation rules:', stubbed_logger: true do
     include_examples 'field type validation', :id, :not_nil, :non_empty_string
     include_examples 'field type validation', :provider, :not_nil, :non_empty_string
     include_examples 'field type validation', :remote_user, :not_nil, :non_empty_string
diff --git a/devops-service/spec/models/stack/stack_ec2_spec.rb b/devops-service/spec/models/stack/stack_ec2_spec.rb
index 17fa499..1e02927 100644
--- a/devops-service/spec/models/stack/stack_ec2_spec.rb
+++ b/devops-service/spec/models/stack/stack_ec2_spec.rb
@@ -11,7 +11,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
     expect(build(:stack_ec2)).to be_valid
   end
 
-  describe 'validation rules:' do
+  describe 'validation rules:', stubbed_logger: true do
     include_examples 'field type validation', :id, :not_nil, :non_empty_string, :field_validator
     include_examples 'field type validation', :project, :not_nil, :non_empty_string, :field_validator
     include_examples 'field type validation', :deploy_env, :not_nil, :non_empty_string, :field_validator

From 89a36ea27645c58ba3338d5ad3b453d3d73cd582 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 21 Jan 2016 21:21:27 +0300
Subject: [PATCH 27/35] add support for rcov formatter

---
 devops-service/Gemfile             | 1 +
 devops-service/Gemfile.lock        | 5 ++++-
 devops-service/spec/spec_helper.rb | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/devops-service/Gemfile b/devops-service/Gemfile
index d6628eb..9a51237 100644
--- a/devops-service/Gemfile
+++ b/devops-service/Gemfile
@@ -37,4 +37,5 @@ group :devepoment do
   gem 'byebug'
   gem 'guard-rspec', require: false
   gem 'simplecov', require: false
+  gem 'simplecov-rcov', require: false
 end
diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock
index d9ca5a3..7ee4373 100644
--- a/devops-service/Gemfile.lock
+++ b/devops-service/Gemfile.lock
@@ -297,6 +297,8 @@ GEM
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
+    simplecov-rcov (0.2.3)
+      simplecov (>= 0.4.1)
     sinatra (1.4.5)
       rack (~> 1.4)
       rack-protection (~> 1.4)
@@ -359,6 +361,7 @@ DEPENDENCIES
   rspec_junit_formatter
   sidekiq (= 3.2.6)
   simplecov
+  simplecov-rcov
   sinatra (= 1.4.5)
   sinatra-contrib
   sinatra-websocket
@@ -367,4 +370,4 @@ DEPENDENCIES
   wisper
 
 BUNDLED WITH
-   1.10.6
+   1.11.2
diff --git a/devops-service/spec/spec_helper.rb b/devops-service/spec/spec_helper.rb
index 5dd1d0a..10f8ebf 100644
--- a/devops-service/spec/spec_helper.rb
+++ b/devops-service/spec/spec_helper.rb
@@ -16,6 +16,10 @@ end
 
 def check_coverage
   require 'simplecov'
+  if ENV['JENKINS']
+    require 'simplecov-rcov'
+    SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+  end
   SimpleCov.start do
     add_filter { |src| src.filename =~ /spec\// }
   end

From 930a5a28b8a613dc34438e6e780c98da5074e2f8 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 21 Jan 2016 21:31:40 +0300
Subject: [PATCH 28/35] use specific tmp folder

---
 .../models/provider_account/openstack_provider_account_spec.rb | 1 +
 devops-service/spec/support/spec_support.rb                    | 3 ++-
 devops-service/spec/support/tmp/.gitkeep                       | 0
 3 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 devops-service/spec/support/tmp/.gitkeep

diff --git a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
index 9efdebd..0c17dce 100644
--- a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
+++ b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
@@ -4,6 +4,7 @@ RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do
   let(:provider_account) { build(:openstack_provider_account) }
 
   it "should not validate access_key_id", stubbed_logger: true do
+    pending "waiting for mongoid support to be finished"
     fields = described_class.field_validators.values.flatten.flatten.map{|t| t[:field]}
     expect(fields).not_to include(:access_key_id, :secret_access_key)
   end
diff --git a/devops-service/spec/support/spec_support.rb b/devops-service/spec/support/spec_support.rb
index 6b96ed8..69bd0e4 100644
--- a/devops-service/spec/support/spec_support.rb
+++ b/devops-service/spec/support/spec_support.rb
@@ -7,8 +7,9 @@ module SpecSupport
     File.join(ROOT, 'spec/support/templates/blank_file')
   end
 
+  # for specs which write files
   def self.tmp_dir
-    File.join(ROOT, 'tmp/')
+    File.join(ROOT, 'spec/support/tmp/')
   end
 
   def self.db_params
diff --git a/devops-service/spec/support/tmp/.gitkeep b/devops-service/spec/support/tmp/.gitkeep
new file mode 100644
index 0000000..e69de29

From 65de9531f8441d45ae40c7f620c80076d32e8931 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 27 Jan 2016 11:51:54 +0300
Subject: [PATCH 29/35] fix requiring StackBootstrapWorker error

---
 devops-service/app/api2/handlers/stack.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb
index 6f5a910..17531e2 100644
--- a/devops-service/app/api2/handlers/stack.rb
+++ b/devops-service/app/api2/handlers/stack.rb
@@ -1,6 +1,7 @@
 require "lib/executors/server_executor"
 require 'db/mongo/models/stack/stack_factory'
 require "app/api2/parsers/stack"
+require 'workers/stack_bootstrap_worker'
 require_relative "request_handler"
 
 module Devops

From ee23d88d7070274195d0137526a5a05c1ee41f23 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Thu, 28 Jan 2016 14:54:12 +0300
Subject: [PATCH 30/35] update stack template specs

---
 .../stack_template/stack_template_ec2_spec.rb    | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
index 3386445..b94506f 100644
--- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
+++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
@@ -6,21 +6,23 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do
 
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
-    allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true }
-    allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} }
+    provider_double = instance_double('Provider::Ec2',
+      validate_stack_template: true,
+      store_stack_template: {'url' => 'template_url'}
+    )
+    allow(Provider::ProviderFactory).to receive(:get) { provider_double }
   end
 
   it_behaves_like 'stack template'
 
   it 'uploads file to S3' do
-    expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template')
-    params = {
-      'id' => 'foo',
+    result = described_class.create('id' => 'foo',
       'template_body' => '{}',
       'owner' => 'root',
       'provider' => 'ec2'
-    }
-    expect(described_class.create(params)).to be_an_instance_of(described_class)
+    )
+    expect(result).to be_an_instance_of(described_class)
+    expect(result.template_url).to eq 'template_url'
   end
 
 end
\ No newline at end of file

From 8d01ff77b98cb3380a34d8a495edb4c34bd40139 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 2 Feb 2016 14:34:17 +0700
Subject: [PATCH 31/35] rename delete_server_worker to
 delete_expired_server_worker

---
 devops-service/db/mongo/models/report.rb                    | 1 +
 devops-service/lib/executors/expiration_scheduler.rb        | 4 ++--
 devops-service/spec/executors/expiration_scheduler_spec.rb  | 4 ++--
 devops-service/spec/executors/server_executor_spec.rb       | 6 +++---
 ...ete_server_worker.rb => delete_expired_server_worker.rb} | 6 +++---
 5 files changed, 11 insertions(+), 10 deletions(-)
 rename devops-service/workers/{delete_server_worker.rb => delete_expired_server_worker.rb} (83%)

diff --git a/devops-service/db/mongo/models/report.rb b/devops-service/db/mongo/models/report.rb
index c557b22..e0d03db 100644
--- a/devops-service/db/mongo/models/report.rb
+++ b/devops-service/db/mongo/models/report.rb
@@ -11,6 +11,7 @@ module Devops
       STACK_TYPE = 5
       DEPLOY_STACK_TYPE = 6
       DELETE_SERVER_TYPE = 7
+      EXPIRE_SERVER_TYPE = 8
 
       attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
 
diff --git a/devops-service/lib/executors/expiration_scheduler.rb b/devops-service/lib/executors/expiration_scheduler.rb
index 5eee52f..d84702d 100644
--- a/devops-service/lib/executors/expiration_scheduler.rb
+++ b/devops-service/lib/executors/expiration_scheduler.rb
@@ -1,4 +1,4 @@
-require "workers/delete_server_worker"
+require "workers/delete_expired_server_worker"
 
 module Devops
   module Executor
@@ -9,7 +9,7 @@ module Devops
 
       def schedule_expiration!
         return unless @expires
-        DeleteServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name)
+        DeleteExpiredServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name)
       end
 
       def interval_in_seconds
diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/expiration_scheduler_spec.rb
index 6964e5d..f8009f8 100644
--- a/devops-service/spec/executors/expiration_scheduler_spec.rb
+++ b/devops-service/spec/executors/expiration_scheduler_spec.rb
@@ -5,12 +5,12 @@ RSpec.describe Devops::Executor::ExpirationScheduler do
 
   describe '#schedule_expiration!' do
     it 'schedules server deleting at given time' do
-      expect(DeleteServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name')
+      expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name')
       described_class.new('2m', server).schedule_expiration!
     end
 
     it "doesn't schedule job if expires is nil" do
-      expect(DeleteServerWorker).not_to receive(:perform_in)
+      expect(DeleteExpiredServerWorker).not_to receive(:perform_in)
       described_class.new(nil, server).schedule_expiration!
     end
   end
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index de70722..142f193 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -139,14 +139,14 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
 
     it 'schedules expiration for server' do
       deploy_env.expires = '2m'
-      allow(DeleteServerWorker).to receive(:perform_in)
-      expect(DeleteServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
+      allow(DeleteExpiredServerWorker).to receive(:perform_in)
+      expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
       create_server
     end
 
     it "doesn't schedule expiration if deploy_env.expires is nil" do
       deploy_env.expires = nil
-      expect(DeleteServerWorker).not_to receive(:perform_in)
+      expect(DeleteExpiredServerWorker).not_to receive(:perform_in)
       create_server
     end
 
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_expired_server_worker.rb
similarity index 83%
rename from devops-service/workers/delete_server_worker.rb
rename to devops-service/workers/delete_expired_server_worker.rb
index c4e02eb..a749e73 100644
--- a/devops-service/workers/delete_server_worker.rb
+++ b/devops-service/workers/delete_expired_server_worker.rb
@@ -3,13 +3,13 @@ require "db/mongo/models/report"
 require "lib/executors/server_executor"
 require "workers/worker"
 
-class DeleteServerWorker < Worker
+class DeleteExpiredServerWorker < Worker
 
   def perform(options)
     chef_node_name = options.fetch('server_chef_node_name')
-    puts "Expire server '#{chef_node_name}'."
 
     call() do |out, file|
+      out.puts "Expire server '#{chef_node_name}'."
       server = mongo.server_by_chef_node_name(chef_node_name)
       report = save_report(file, server)
 
@@ -28,7 +28,7 @@ class DeleteServerWorker < Worker
       "created_by" => 'SYSTEM',
       "project" => server.project,
       "deploy_env" => server.deploy_env,
-      "type" => Devops::Model::Report::DELETE_SERVER_TYPE
+      "type" => Devops::Model::Report::EXPIRE_SERVER_TYPE
     )
     mongo.save_report(report)
     report

From 775c9758cae801fa3f4a3fa140da0e49dadb8495 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Tue, 2 Feb 2016 15:43:13 +0700
Subject: [PATCH 32/35] move servers deleting to background

---
 .../lib/devops-client/handler/server.rb       |  5 +--
 .../devops-client/options/server_options.rb   |  2 +-
 .../lib/devops-client/output/project.rb       | 10 ++---
 devops-service/app/api2/handlers/project.rb   | 13 ++-----
 devops-service/app/api2/handlers/server.rb    |  9 +++--
 devops-service/app/api2/routes/server.rb      |  3 +-
 .../lib/executors/server_executor.rb          | 29 ++++++++------
 .../spec/executors/server_executor_spec.rb    | 10 ++---
 .../workers/delete_server_worker.rb           | 39 +++++++++++++++++++
 devops-service/workers/run_workers.rb         |  2 +
 10 files changed, 78 insertions(+), 44 deletions(-)
 create mode 100644 devops-service/workers/delete_server_worker.rb

diff --git a/devops-client/lib/devops-client/handler/server.rb b/devops-client/lib/devops-client/handler/server.rb
index 1d85e02..55fbdcf 100644
--- a/devops-client/lib/devops-client/handler/server.rb
+++ b/devops-client/lib/devops-client/handler/server.rb
@@ -87,10 +87,7 @@ class Server < Handler
       end
       if question(I18n.t("handler.server.question.delete", :name => name))
         puts "Server '#{name}', deleting..."
-        o = delete("/server/#{name}", options)
-        ["server", "chef_node", "chef_client", "message"].each do |k|
-          puts o[k] unless o[k].nil?
-        end
+        puts delete("/server/#{name}", options)
       end
     end
     ""
diff --git a/devops-client/lib/devops-client/options/server_options.rb b/devops-client/lib/devops-client/options/server_options.rb
index 7a44bc0..662db24 100644
--- a/devops-client/lib/devops-client/options/server_options.rb
+++ b/devops-client/lib/devops-client/options/server_options.rb
@@ -126,7 +126,7 @@ class ServerOptions < CommonOptions
         options[:groups] = groups.split(",")
       end
 
-      parser.recognize_option_value(:private_ip, 'server', short: '-N', i18n_scope: 'create')
+      parser.recognize_option_value(:private_ip, short: '-N', i18n_scope: 'create')
 
       # it was disabled somewhy
       # parser.on('--public-ip', "Associate public IP with server") do
diff --git a/devops-client/lib/devops-client/output/project.rb b/devops-client/lib/devops-client/output/project.rb
index b137aad..c080e45 100644
--- a/devops-client/lib/devops-client/output/project.rb
+++ b/devops-client/lib/devops-client/output/project.rb
@@ -154,17 +154,13 @@ module Output
     def delete_servers_output
       output = ''
 
-      if @data['deleted'].empty?
+      if @data['reports'].empty?
         output << 'There are no deleted servers.'
       else
-        output << "Deleted servers:\n----\n"
-        output << @data['deleted'].join("\n")
+        output << "Reports for servers deleting:\n----\n"
+        output << @data['reports'].join("\n")
       end
 
-      if !@data['failed'].empty?
-        output << "\nThere were errors during deleting these servers:\n----\n"
-        output << @data['failed'].join("\n")
-      end
       output
     end
 
diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb
index 3e36bb9..7193e8b 100644
--- a/devops-service/app/api2/handlers/project.rb
+++ b/devops-service/app/api2/handlers/project.rb
@@ -4,6 +4,7 @@ require "workers/project_test_worker"
 require "app/api2/parsers/project"
 require "lib/project/type/types_factory"
 require "lib/executors/server_executor"
+require "workers/delete_server_worker"
 
 require_relative "../helpers/version_2.rb"
 require_relative "request_handler"
@@ -314,16 +315,10 @@ module Devops
         private
 
         def delete_chosen_servers!(servers)
-          deleted, failed = [], []
-          servers.each do |server|
-            begin
-              Devops::Executor::ServerExecutor.new(server, '').delete_server
-              deleted << server.id
-            rescue
-              failed << server.id
-            end
+          reports = servers.map do |server|
+            Worker.start_async(DeleteServerWorker, 'server_id' => server.id)
           end
-          {deleted: deleted, failed: failed}
+          {reports: reports}
         end
 
       end
diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb
index 6b35d72..f6a462c 100644
--- a/devops-service/app/api2/handlers/server.rb
+++ b/devops-service/app/api2/handlers/server.rb
@@ -10,6 +10,7 @@ require "db/mongo/models/server"
 
 require "workers/create_server_worker"
 require "workers/bootstrap_worker"
+require "workers/delete_server_worker"
 require "app/api2/parsers/server"
 require_relative "request_handler"
 
@@ -47,10 +48,10 @@ module Devops
         end
 
         def delete id
-          s = get_server_by_key(id, parser.instance_key)
-          ### Authorization
-          Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
-          Devops::Executor::ServerExecutor.new(s, "").delete_server
+          server = get_server_by_key(id, parser.instance_key)
+          Devops::Db.connector.check_project_auth server.project, server.deploy_env, parser.current_user
+          jid = Worker.start_async(DeleteServerWorker, 'server_id' => id)
+          [jid]
         end
 
         def create_server_stream out
diff --git a/devops-service/app/api2/routes/server.rb b/devops-service/app/api2/routes/server.rb
index 69bd54e..b411fbc 100644
--- a/devops-service/app/api2/routes/server.rb
+++ b/devops-service/app/api2/routes/server.rb
@@ -127,8 +127,7 @@ module Devops
           #   200 - Deleted
           hash["DELETE"] = lambda {|id|
             check_privileges("server", "w")
-            info, r = Devops::API2_0::Handler::Server.new(request).delete(id)
-            create_response(info, r)
+            json Devops::API2_0::Handler::Server.new(request).delete(id)
           }
           app.multi_routes "/server/:id", {:headers => [:accept, :content_type]}, hash
 
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index a8aebc6..e861220 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -432,23 +432,25 @@ module Devops
             unbootstrap
           end
           mongo.server_delete @server.id
-          msg = "Static server '#{@server.id}' is removed"
-          DevopsLogger.logger.info msg
-          return msg, nil
+          puts_and_flush "Static server '#{@server.id}' is removed"
+          return 0
         end
-        r = delete_from_chef_server(@server.chef_node_name)
+
+        puts_and_flush "Deleting from chef server:"
+        delete_from_chef_server(@server.chef_node_name).each do |key, result|
+          @out.puts "#{key} - #{result}"
+        end
+
+        puts_and_flush "Deleting from cloud:"
         provider = @server.provider_instance
         begin
-          r[:server] = provider.delete_server @server
+          puts_and_flush provider.delete_server @server
         rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
-          r[:server] = "Server with id '#{@server.id}' not found in '#{provider.name}' servers"
-          DevopsLogger.logger.warn r[:server]
+          puts_and_flush "Server with id '#{@server.id}' not found among '#{provider.name}' servers"
         end
         mongo.server_delete @server.id
-        info = "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed"
-        DevopsLogger.logger.info info
-        r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
-        return info, r
+        puts_and_flush "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed."
+        0
       end
 
       def roll_back
@@ -493,6 +495,11 @@ module Devops
 
       private
 
+      def puts_and_flush(message)
+        @out.puts message
+        @out.flush
+      end
+
       def schedule_expiration
         if @deploy_env.expires
           @out <<  "Planning expiration in #{@deploy_env.expires}"
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index 142f193..bc19b58 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -660,9 +660,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         delete_server
       end
 
-      it 'returns message and nil' do
-        expect(delete_server.first).to be_a(String)
-        expect(delete_server.last).to be nil
+      it 'returns 0' do
+        expect(delete_server).to eq 0
       end
 
       it "doesn't try to remove it from cloud" do
@@ -707,9 +706,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
         delete_server
       end
 
-      it 'returns message and hash with :chef_node, :chef_client and :server keys' do
-        expect(delete_server.first).to be_a(String)
-        expect(delete_server.last).to be_a(Hash).and include(:chef_client, :chef_node, :server)
+      it 'returns 0' do
+        expect(delete_server).to eq 0
       end
     end
   end
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb
new file mode 100644
index 0000000..4354e16
--- /dev/null
+++ b/devops-service/workers/delete_server_worker.rb
@@ -0,0 +1,39 @@
+require "db/mongo/models/server"
+require "db/mongo/models/report"
+require "lib/executors/server_executor"
+require "workers/worker"
+require 'byebug'
+
+class DeleteServerWorker < Worker
+
+  # options should contain 'server_id'
+  def perform(options)
+    server_id = options.fetch('server_id')
+
+    call() do |out, file|
+      out.puts "Deleting server with id #{server_id}" and out.flush
+      @server = mongo.server_by_instance_id(server_id)
+      report = save_report(file)
+
+      e = Devops::Executor::ServerExecutor.new(@server, out)
+      e.report = report
+      e.delete_server
+    end
+  end
+
+  private
+
+  def save_report(file)
+    report = Devops::Model::Report.new(
+      "file" => file,
+      "_id" => jid,
+      "created_by" => 'SYSTEM',
+      "project" => @server.project,
+      "deploy_env" => @server.deploy_env,
+      "type" => Devops::Model::Report::DELETE_SERVER_TYPE
+    )
+    mongo.save_report(report)
+    report
+  end
+
+end
diff --git a/devops-service/workers/run_workers.rb b/devops-service/workers/run_workers.rb
index 7dac55e..6e25e39 100644
--- a/devops-service/workers/run_workers.rb
+++ b/devops-service/workers/run_workers.rb
@@ -5,6 +5,8 @@ require File.join(root, "deploy_worker")
 require File.join(root, "bootstrap_worker")
 require File.join(root, "project_test_worker")
 require File.join(root, "stack_bootstrap_worker")
+require File.join(root, "delete_server_worker")
+require File.join(root, "delete_expired_server_worker")
 
 config = {}
 #require File.join(root, "../proxy")

From 92598d86ed5acb2701dc8b0d72910b9fe1d6a1dd Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Sun, 31 Jan 2016 21:52:11 +0700
Subject: [PATCH 33/35] add possibility to set chef_node_name mask for stacks;
 cover StackBootstrapWorker with specs

---
 devops-service/Guardfile                      |   1 +
 devops-service/commands/stack.rb              |  62 ------
 devops-service/db/validators/base.rb          |   2 -
 .../lib/executors/server_executor.rb          |  17 +-
 devops-service/lib/puts_and_flush.rb          |   9 +
 .../spec/executors/server_executor_spec.rb    |   6 +-
 .../spec/shared_contexts/stubbed_logger.rb    |   6 +-
 .../workers/chef_node_name_builder_spec.rb    |  45 ++++
 .../workers/stack_bootstrap_worker_spec.rb    | 128 +++++++++++
 .../stack_servers_bootstrapper_spec.rb        |  59 +++++
 .../workers/stack_servers_persister_spec.rb   | 116 ++++++++++
 .../spec/workers/stack_synchronizer_spec.rb   |  82 +++++++
 .../stack_bootstrap/chef_node_name_builder.rb |  15 ++
 .../workers/stack_bootstrap/errors.rb         |   4 +
 .../stack_servers_bootstrapper.rb             |  78 +++++++
 .../stack_servers_persister.rb                |  69 ++++++
 .../stack_bootstrap/stack_synchronizer.rb     |  71 +++++++
 .../workers/stack_bootstrap_worker.rb         | 201 +++++-------------
 devops-service/workers/worker.rb              |   3 +-
 19 files changed, 743 insertions(+), 231 deletions(-)
 delete mode 100644 devops-service/commands/stack.rb
 create mode 100644 devops-service/lib/puts_and_flush.rb
 create mode 100644 devops-service/spec/workers/chef_node_name_builder_spec.rb
 create mode 100644 devops-service/spec/workers/stack_bootstrap_worker_spec.rb
 create mode 100644 devops-service/spec/workers/stack_servers_bootstrapper_spec.rb
 create mode 100644 devops-service/spec/workers/stack_servers_persister_spec.rb
 create mode 100644 devops-service/spec/workers/stack_synchronizer_spec.rb
 create mode 100644 devops-service/workers/stack_bootstrap/chef_node_name_builder.rb
 create mode 100644 devops-service/workers/stack_bootstrap/errors.rb
 create mode 100644 devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb
 create mode 100644 devops-service/workers/stack_bootstrap/stack_servers_persister.rb
 create mode 100644 devops-service/workers/stack_bootstrap/stack_synchronizer.rb

diff --git a/devops-service/Guardfile b/devops-service/Guardfile
index a66e388..82943a3 100644
--- a/devops-service/Guardfile
+++ b/devops-service/Guardfile
@@ -43,4 +43,5 @@ guard :rspec, cmd: "rspec" do
   # Devops files
   watch(%r{db/.+\.rb}) { rspec.spec_dir }
   watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
+  watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" }
 end
diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb
deleted file mode 100644
index ba98ded..0000000
--- a/devops-service/commands/stack.rb
+++ /dev/null
@@ -1,62 +0,0 @@
-module StackCommands
-  extend self
-
-  RESULT_CODES = {
-    stack_rolled_back: 1,
-    unkown_status: 2,
-    timeout: 3,
-    error: 5
-  }
-
-  def self.result_codes
-    RESULT_CODES
-  end
-
-  def self.result_code(code)
-    result_codes.fetch(code)
-  end
-
-  def sync_stack_proc
-    lambda do |out, stack, mongo|
-      # 5 tries each 5 seconds, then 200 tries each 10 seconds
-      sleep_times = [5]*5 + [10]*200
-
-      begin
-        out << "Syncing stack '#{stack.id}'...\n"
-        events_keys = []
-        sleep_times.each do |sleep_time|
-          sleep sleep_time
-          stack.sync_details!
-          stack.events.each do |event|
-            unless events_keys.include?(event["event_id"])
-              events_keys << event["event_id"]
-              out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
-            end
-          end
-          case stack.stack_status
-          when 'CREATE_IN_PROGRESS'
-            out.flush
-          when 'CREATE_COMPLETE'
-            mongo.stack_update(stack)
-            out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n"
-            out.flush
-            return 0
-          when 'ROLLBACK_COMPLETE'
-            out << "\nStack '#{stack.id}' status is rolled back\n"
-            return StackCommands.result_code(:stack_rolled_back)
-          else
-            out.puts "\nUnknown stack status: '#{stack.stack_status}'"
-            return StackCommands.result_code(:unkown_status)
-          end
-        end
-        out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds."
-        return StackCommands.result_code(:timeout)
-      rescue StandardError => e
-        logger.error e.message
-        out << "Error: #{e.message}\n"
-        return StackCommands.result_code(:error)
-      end
-    end
-  end
-
-end
diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb
index bec8722..677e9f2 100644
--- a/devops-service/db/validators/base.rb
+++ b/devops-service/db/validators/base.rb
@@ -13,7 +13,6 @@ module Validators
       raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}")
     end
 
-    # :nocov:
     def valid?
       raise 'override me'
     end
@@ -21,7 +20,6 @@ module Validators
     def message
       raise 'override me'
     end
-    # :nocov:
 
     class << self
       private
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index a8aebc6..ff4c9f6 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -53,16 +53,20 @@ module Devops
         @current_user = options[:current_user]
       end
 
-      def self.error_code(symbolic_code)
-        ERROR_CODES.fetch(symbolic_code)
+      def self.error_code(reason)
+        ERROR_CODES.fetch(reason)
       end
 
-      def self.symbolic_error_code(integer_code)
+      def self.reason_from_error_code(integer_code)
         ERROR_CODES.key(integer_code) || :unknown_error
       end
 
-      def error_code(symbolic_code)
-        self.class.error_code(symbolic_code)
+      def self.bootstrap_errors_reasons
+        [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error]
+      end
+
+      def error_code(reason)
+        self.class.error_code(reason)
       end
 
       def create_server_object options
@@ -511,8 +515,6 @@ module Devops
         cmd
       end
 
-      # to simplify testing
-      # :nocov:
       def execute_system_command(cmd)
         `#{cmd}`
       end
@@ -524,7 +526,6 @@ module Devops
       def knife_instance
         @knife_instance ||= KnifeFactory.instance
       end
-      # :nocov:
 
     end
   end
diff --git a/devops-service/lib/puts_and_flush.rb b/devops-service/lib/puts_and_flush.rb
new file mode 100644
index 0000000..a0af609
--- /dev/null
+++ b/devops-service/lib/puts_and_flush.rb
@@ -0,0 +1,9 @@
+module PutsAndFlush
+  private
+
+  # out stream should be defined
+  def puts_and_flush(message)
+    out.puts message
+    out.flush
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
index de70722..3ba6eb9 100644
--- a/devops-service/spec/executors/server_executor_spec.rb
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -53,13 +53,13 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec
   end
 
 
-  describe '.symbolic_error_code' do
+  describe '.reason_from_error_code' do
     it 'returns symbol given an integer' do
-      expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail
+      expect(described_class.reason_from_error_code(2)).to eq :server_bootstrap_fail
     end
 
     it "returns :unknown_error if can't recognize error code" do
-      expect(described_class.symbolic_error_code(123)).to eq :unknown_error
+      expect(described_class.reason_from_error_code(123)).to eq :unknown_error
     end
   end
 
diff --git a/devops-service/spec/shared_contexts/stubbed_logger.rb b/devops-service/spec/shared_contexts/stubbed_logger.rb
index 486caa1..6105101 100644
--- a/devops-service/spec/shared_contexts/stubbed_logger.rb
+++ b/devops-service/spec/shared_contexts/stubbed_logger.rb
@@ -1,8 +1,6 @@
 RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do
   before do
-    allow(DevopsLogger).to receive_message_chain('logger.debug')
-    allow(DevopsLogger).to receive_message_chain('logger.info')
-    allow(DevopsLogger).to receive_message_chain('logger.error')
-    allow(DevopsLogger).to receive_message_chain('logger.warn')
+    logger = double('logger', debug: nil, info: nil, error: nil, warn: nil)
+    allow(DevopsLogger).to receive(:logger) { logger }
   end
 end
\ No newline at end of file
diff --git a/devops-service/spec/workers/chef_node_name_builder_spec.rb b/devops-service/spec/workers/chef_node_name_builder_spec.rb
new file mode 100644
index 0000000..246dcab
--- /dev/null
+++ b/devops-service/spec/workers/chef_node_name_builder_spec.rb
@@ -0,0 +1,45 @@
+require 'workers/stack_bootstrap/chef_node_name_builder'
+RSpec.describe ChefNodeNameBuilder do
+  let(:server_info) do
+    {
+      'id' => 'server1',
+      'name' => 'server_name',
+      'key_name' => 'key',
+      'private_ip' => '127.0.0.1',
+      'public_ip' => '127.0.0.2',
+      'tags' => {
+        'cid:priority' => '3'
+      }
+    }
+  end
+  let(:project) { build(:project, id: 'proj', with_deploy_env_identifiers: %w(dev)) }
+  let(:env) { project.deploy_env('dev') }
+  let(:name_builder) { described_class.new(server_info, project, env) }
+  let(:build_name) { name_builder.build_node_name }
+
+  def set_mask(mask)
+    server_info['tags']['cid:node-name-mask'] = mask
+  end
+
+  describe '#build_node_name' do
+    it 'uses default mask ("$project-$nodename-$env")' do
+      expect(build_name).to eq 'proj-server1-dev'
+    end
+
+    it 'substitutes project, env and nodename' do
+      set_mask('$project/$env/$nodename')
+      expect(build_name).to eq 'proj/dev/server1'
+    end
+
+    it 'substitutes $time' do
+      set_mask('$nodename-$time')
+      expect(build_name).to match /server1-\d+/
+    end
+
+    it 'substitutes underscores to dashes' do
+      server_info['id'] = 'server_1'
+      expect(build_name).to match 'proj-server-1-dev'
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb
new file mode 100644
index 0000000..b2ede82
--- /dev/null
+++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb
@@ -0,0 +1,128 @@
+require 'workers/stack_bootstrap_worker'
+
+RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:file) { 'temp.txt' }
+
+  let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
+  let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
+  let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
+  let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) }
+  let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
+  let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
+  let(:worker) { described_class.new }
+
+  before do
+    allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
+    allow(stubbed_connector).to receive(:save_report)
+    allow(stubbed_connector).to receive(:stack_insert)
+
+    allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
+    allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
+    allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
+    allow(worker).to receive(:call).and_yield(out, file)
+    allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
+  end
+
+
+  it 'requires "stack_attributes" in options' do
+    expect{
+      worker.perform({})
+    }.to raise_error KeyError
+  end
+
+  it 'saves report about operation' do
+    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report))
+    perform_without_bootstrap
+  end
+
+  it 'saves report about operation, creates stack and persists stack servers' do
+    allow(worker).to receive(:create_stack).and_call_original
+    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)).ordered
+    expect(worker).to receive(:create_stack).ordered
+    expect(stack_servers_persister).to receive(:persist).ordered
+    perform_without_bootstrap
+  end
+
+  context 'if without_bootstrap is true' do
+    it "doesn't bootstrap servers" do
+      expect(stack_servers_bootstrapper).not_to receive(:bootstrap)
+      perform_without_bootstrap
+    end
+
+    it 'returns 0' do
+      expect(perform_without_bootstrap).to eq 0
+    end
+  end
+
+  context 'if without_bootstrap is false or not set' do
+    it 'bootstraps servers in order by priorities, separately' do
+      first_servers = build_list(:server, 2)
+      last_servers = build_list(:server, 3)
+      allow(stack_servers_persister).to receive(:persist) {
+        {3 => first_servers, 1 => last_servers}
+      }
+      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered
+      expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered
+      perform_with_bootstrap
+    end
+
+    context 'when bootstraping servers was successful' do
+      it 'returns 0' do
+        expect(perform_with_bootstrap).to eq 0
+      end
+    end
+
+    context 'when a known error occured during servers bootstrap' do
+      before do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError }
+      end
+
+      it 'rollbacks stack and returns 2' do
+        expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
+        expect(stubbed_connector).to receive(:stack_servers_delete)
+        expect(stubbed_connector).to receive(:stack_delete)
+        perform_with_bootstrap
+      end
+
+      it 'returns 2' do
+        allow(worker).to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 2
+      end
+    end
+
+    context 'when a known error occured during servers deploy' do
+      it "doesn't rollback stack and returns 3" do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError }
+        expect(worker).not_to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 3
+      end
+    end
+
+    context "when a servers bootstrap & deploy haven't been finished due to timeout" do
+      it "doesn't rollback stack and returns 3" do
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout }
+        expect(worker).not_to receive(:rollback_stack!)
+        expect(perform_with_bootstrap).to eq 4
+      end
+    end
+
+    context 'when an unknown error occured during servers bootsrap and deploy' do
+      it 'rollbacks stack and reraises that error' do
+        error = StandardError.new
+        allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error }
+        allow(worker).to receive(:rollback_stack!)
+        expect(worker).to receive(:rollback_stack!)
+        expect{perform_with_bootstrap}.to raise_error(error)
+      end
+    end
+  end
+
+  context "when stack creation wasn't successful" do
+    it 'returns 1' do
+      allow(stack_synchronizer).to receive(:sync) { 5 }
+      allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
+      expect(perform_without_bootstrap).to eq 1
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb b/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb
new file mode 100644
index 0000000..374f57c
--- /dev/null
+++ b/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb
@@ -0,0 +1,59 @@
+require 'workers/stack_bootstrap_worker'
+
+RSpec.describe StackServersBootstrapper, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:jid) { 1000 }
+  let(:bootstrapper) { described_class.new(out, jid) }
+  let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
+  let(:bootstrap_job_ids) { %w(100 200) }
+  let(:subreport1) { build(:report, id: bootstrap_job_ids.first) }
+  let(:subreport2) { build(:report, id: bootstrap_job_ids.last) }
+
+  describe '#bootstrap' do
+    let(:bootstrap!) { bootstrapper.bootstrap(servers) }
+
+    before do
+      allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
+      allow(stubbed_connector).to receive(:add_report_subreports)
+      allow(stubbed_connector).to receive(:report) do |subreport_id|
+        subreport_id == '100' ? subreport1 : subreport2
+      end
+      allow(bootstrapper).to receive(:sleep)
+    end
+
+    it 'start bootstrap workers' do
+      expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
+      bootstrap!
+    end
+
+    it 'add subreports' do
+      expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
+      bootstrap!
+    end
+
+    it 'waits for job to end' do
+      allow(subreport1).to receive(:status).and_return('running', 'running', 'running', 'completed')
+      allow(subreport2).to receive(:status).and_return('running', 'running', 'running', 'completed')
+      expect(bootstrapper).to receive(:sleep).exactly(2*4).times
+      bootstrap!
+    end
+
+    it 'raises StackServerBootstrapError if an error occured during bootstrap' do
+      allow(subreport1).to receive(:status) {'failed'}
+      allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:server_bootstrap_fail) }
+      expect { bootstrap! }.to raise_error StackServerBootstrapError
+    end
+
+    it 'raises StackServerDeployError if an error occured during deploy' do
+      allow(subreport1).to receive(:status) {'failed'}
+      allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:deploy_failed) }
+      expect { bootstrap! }.to raise_error StackServerDeployError
+    end
+
+    it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do
+      allow(subreport1).to receive(:status) {'running'}
+      expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/devops-service/spec/workers/stack_servers_persister_spec.rb b/devops-service/spec/workers/stack_servers_persister_spec.rb
new file mode 100644
index 0000000..c24ba8b
--- /dev/null
+++ b/devops-service/spec/workers/stack_servers_persister_spec.rb
@@ -0,0 +1,116 @@
+require 'workers/stack_bootstrap/stack_servers_persister'
+
+RSpec.describe StackServersPersister, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:run_list) { ['role[asd]'] }
+  let(:stack) { build(:stack, deploy_env: 'foo', run_list: run_list) }
+  let(:project) { build(:project, id: 'name') }
+  let(:persister) { described_class.new(stack, out) }
+  let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
+  let(:server_info_hash) do
+    {
+      'id' => 'server1',
+      'name' => 'server_name',
+      'key_name' => 'key',
+      'private_ip' => '127.0.0.1',
+      'public_ip' => '127.0.0.2',
+      'tags' => {
+        'cid:priority' => '3'
+      }
+    }
+  end
+
+  before do
+    allow(stubbed_connector).to receive(:project) { project }
+    allow(stubbed_connector).to receive(:image) {
+      instance_double(Devops::Model::Image, remote_user: 'user')
+    }
+    allow(stubbed_connector).to receive(:server_insert)
+    allow(stack).to receive(:provider_instance) { provider }
+    allow(provider).to receive(:stack_servers) {[server_info_hash]}
+  end
+
+  describe '#persist' do
+    it 'fetches stack servers info' do
+      expect(provider).to receive(:stack_servers).with(stack)
+      persister.persist
+    end
+
+    it "doesn't raise error if cid:priority tag is absent" do
+      server_info_hash['tags'].delete('cid:priority')
+      expect {persister.persist}.not_to raise_error
+    end
+
+    it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
+      result = persister.persist
+      expect(result).to be_a(Hash)
+      expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
+    end
+
+    it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.id).to eq 'server1'
+        expect(server.key).to eq 'key'
+        expect(server.private_ip).to eq '127.0.0.1'
+        expect(server.public_ip).to eq '127.0.0.2'
+      end
+      persister.persist
+    end
+
+    it 'takes created_by, run_list and stack attrs from stack' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.created_by).to eq 'root'
+        expect(server.run_list).to eq run_list
+        expect(server.stack).to eq 'iamstack'
+      end
+      persister.persist
+    end
+
+    it 'takes remote_user from image user' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.remote_user).to eq 'user'
+      end
+      persister.persist
+    end
+
+    it "takes deploy_env from project's deploy_env identifier" do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.deploy_env).to eq 'foo'
+      end
+      persister.persist
+    end
+
+    it "takes default provider's ssh key if info doesn't contain it" do
+      allow(provider).to receive(:ssh_key) { 'default_key' }
+      server_info_hash.delete('key_name')
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.key).to eq 'default_key'
+      end
+      persister.persist
+    end
+
+    it "sets server's run list to empty array if stack's run_list is nil" do
+      stack.run_list = nil
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.run_list).to eq []
+      end
+      persister.persist
+    end
+
+    it 'build chef_node_name with default mask "$project-$nodename-$env"' do
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.chef_node_name).to eq 'name-server1-foo'
+      end
+      persister.persist
+    end
+
+    it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
+      server_info_hash['tags']['cid:node-name-mask'] = '$project-$nodename-123'
+      expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(server.chef_node_name).to eq 'name-server1-123'
+      end
+      persister.persist
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/devops-service/spec/workers/stack_synchronizer_spec.rb b/devops-service/spec/workers/stack_synchronizer_spec.rb
new file mode 100644
index 0000000..83a43cf
--- /dev/null
+++ b/devops-service/spec/workers/stack_synchronizer_spec.rb
@@ -0,0 +1,82 @@
+require 'workers/stack_bootstrap/stack_synchronizer'
+RSpec.describe StackSynchronizer, stubbed_connector: true do
+  let(:out) { double(:out, puts: nil, flush: nil) }
+  let(:stack) { build(:stack) }
+  let(:syncer) { described_class.new(stack, out) }
+
+  before do
+    allow(stack).to receive(:sync_details!)
+    allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
+    allow(syncer).to receive(:sleep)
+    allow(stubbed_connector).to receive(:stack_update)
+
+    lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
+    allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
+  end
+
+  describe '#sync' do
+    it 'waits for stack creating to be finished' do
+      expect(syncer).to receive(:sleep).at_least(10).times
+      expect(stack).to receive(:sync_details!).at_least(10).times
+      syncer.sync
+    end
+
+    it 'prints each message only once' do
+      event1 = {'event_id' => 1, 'timestamp' => 't1'}
+      event2 = {'event_id' => 2, 'timestamp' => 't2'}
+      event3 = {'event_id' => 3, 'timestamp' => 't3'}
+
+      allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
+      syncer.sync
+      expect(out).to have_received(:puts).with(/t1/).once.ordered
+      expect(out).to have_received(:puts).with(/t2/).once.ordered
+      expect(out).to have_received(:puts).with(/t3/).once.ordered
+    end
+
+    context 'when stack creating was successful' do
+      it 'updates stack in DB when stack creating is finished and returns 0' do
+        expect(stubbed_connector).to receive(:stack_update).with(stack)
+        expect(syncer.sync).to eq 0
+      end
+    end
+
+    context 'when stack was rollbacked' do
+      it 'returns 1 (:stack_rolled_back)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE')
+        expect(syncer.sync).to eq 1
+      end
+    end
+
+    context 'when unkown stack status was found' do
+      it 'returns 2 (:unkown_status)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown')
+        expect(syncer.sync).to eq 2
+      end
+    end
+
+    context "when stack hasn't been synced in an hour" do
+      it 'returns 3 (:timeout)' do
+        allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
+        expect(syncer.sync).to eq 3
+      end
+    end
+
+    context 'when an error occured during syncing', stubbed_logger: true do
+      it 'returns 5 (:error)' do
+        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+        allow(stubbed_connector).to receive(:stack_update) { raise }
+        expect(syncer.sync).to eq 5
+      end
+    end
+  end
+
+  describe '#reason_from_error_code' do
+    it 'returns reason as symbol for integer error_code' do
+      expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
+      expect(syncer.reason_from_error_code(2)).to eq :unkown_status
+      expect(syncer.reason_from_error_code(3)).to eq :timeout
+      expect(syncer.reason_from_error_code(5)).to eq :error
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb b/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb
new file mode 100644
index 0000000..660b23a
--- /dev/null
+++ b/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb
@@ -0,0 +1,15 @@
+class ChefNodeNameBuilder
+  def initialize(server_info, project, env)
+    @server_info, @project, @env = server_info, project, env
+    @mask = server_info['tags']['cid:node-name-mask'] || '$project-$nodename-$env'
+  end
+
+  def build_node_name
+    @mask.gsub!('$project', @project.id)
+    @mask.gsub!('$env', @env.identifier)
+    @mask.gsub!('$nodename', @server_info['id'])
+    @mask.gsub!('$time', Time.now.to_i.to_s)
+    @mask.gsub!('_', '-')
+    @mask
+  end
+end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap/errors.rb b/devops-service/workers/stack_bootstrap/errors.rb
new file mode 100644
index 0000000..e9d6035
--- /dev/null
+++ b/devops-service/workers/stack_bootstrap/errors.rb
@@ -0,0 +1,4 @@
+class StackCreatingError < StandardError; end
+class StackServerBootstrapError < StandardError; end
+class StackServerDeployError < StandardError; end
+class StackServerBootstrapDeployTimeout < StandardError; end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb
new file mode 100644
index 0000000..77baab6
--- /dev/null
+++ b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb
@@ -0,0 +1,78 @@
+require 'workers/bootstrap_worker'
+require "workers/stack_bootstrap/errors"
+
+class StackServersBootstrapper
+  include PutsAndFlush
+  attr_reader :out
+
+  def initialize(out, jid)
+    @out, @jid = out, jid
+  end
+
+  def bootstrap(servers)
+    @servers = servers
+    puts_and_flush "\nStart bootstraping stack servers"
+
+    servers_jobs_ids = start_workers
+    ::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values)
+
+    out.puts
+    servers_jobs_ids.each do |server_id, subreport_id|
+      job_result_code = wait_for_job(server_id, subreport_id)
+      check_job_result!(server_id, job_result_code)
+    end
+    puts_and_flush "Stack servers have been bootstraped"
+  end
+
+  private
+
+  def check_job_result!(server_id, job_result_code)
+    return if job_result_code == 0
+
+    reason = Devops::Executor::ServerExecutor.reason_from_error_code(job_result_code)
+    puts_and_flush "Operation result for #{server_id}: #{reason}"
+
+    if error_occured_during_bootstrap?(reason)
+      raise StackServerBootstrapError # will cause rollback of a stack
+    else
+      raise StackServerDeployError #will not cause rollback of a stack
+    end
+  end
+
+  def error_occured_during_bootstrap?(reason)
+    Devops::Executor::ServerExecutor.bootstrap_errors_reasons.include?(reason)
+  end
+
+  def wait_for_job(server_id, subreport_id)
+    1000.times do
+      sleep(5)
+      subreport = ::Devops::Db.connector.report(subreport_id)
+      case subreport.status
+      when Worker::STATUS::COMPLETED
+        puts_and_flush "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
+        return 0
+      when Worker::STATUS::FAILED
+        puts_and_flush "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
+        return subreport.job_result_code
+      end
+    end
+    puts_and_flush "Waiting for job #{subreport_id} halted: timeout reached."
+    raise StackServerBootstrapDeployTimeout
+  end
+
+  # returns hash: {server_id => worker_job_id}
+  def start_workers
+    servers_jobs_ids = {}
+    @servers.each do |server|
+      job_id = Worker.start_async(::BootstrapWorker,
+        server_attrs: server.to_mongo_hash,
+        bootstrap_template: 'omnibus',
+        owner: server.created_by
+      )
+      @out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}"
+      servers_jobs_ids[server.id] = job_id
+    end
+    puts_and_flush "\n"
+    servers_jobs_ids
+  end
+end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb
new file mode 100644
index 0000000..3a30eab
--- /dev/null
+++ b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb
@@ -0,0 +1,69 @@
+require 'workers/stack_bootstrap/chef_node_name_builder'
+
+class StackServersPersister
+  include PutsAndFlush
+  attr_reader :stack, :out
+
+  def initialize(stack, out)
+    @stack, @out = stack, out
+    @project = mongo.project(stack.project)
+    @deploy_env = @project.deploy_env(stack.deploy_env)
+    @provider = stack.provider_instance
+  end
+
+  # returns: {  priority_as_integer => [Servers] }
+  def persist
+    puts_and_flush 'Start syncing stack servers with CID'
+
+    stack_servers_with_priority = {}
+    stack_servers_info.each do |priority, info_array|
+      stack_servers_with_priority[priority] = info_array.map do |info_hash|
+        out.puts "Instance '#{info_hash["id"]}' has been launched with stack."
+        persist_stack_server(info_hash)
+      end
+    end
+    puts_and_flush "Stack servers have been synced with CID"
+    stack_servers_with_priority.each do |priority, servers|
+      out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
+    end
+    out.flush
+    stack_servers_with_priority
+  end
+
+  private
+
+  # returns: {priority_as_integer => array_of_sersvers_info}
+  def stack_servers_info
+    stack_servers = @provider.stack_servers(stack)
+    stack_servers.each do |info|
+      info['tags']['cid:priority'] = info['tags']['cid:priority'].to_i
+    end
+    stack_servers.group_by{|info| info['tags']['cid:priority']}
+  end
+
+  # takes a hash, returns Server model
+  def persist_stack_server(info_hash)
+    server_attrs = {
+      '_id' => info_hash['id'],
+      'chef_node_name' => ChefNodeNameBuilder.new(info_hash, @project, @deploy_env).build_node_name,
+      'created_by' => stack.owner,
+      'deploy_env' => @deploy_env.identifier,
+      'key' => info_hash['key_name'] || @provider.ssh_key,
+      'project' => @project.id,
+      'provider' => @provider.name,
+      'remote_user' => mongo.image(@deploy_env.image).remote_user,
+      'private_ip' => info_hash['private_ip'],
+      'public_ip' => info_hash['public_ip'],
+      'run_list' => stack.run_list || [],
+      'stack' => stack.name
+    }
+
+    server = ::Devops::Model::Server.new(server_attrs)
+    mongo.server_insert(server)
+    server
+  end
+
+  def mongo
+    Devops::Db.connector
+  end
+end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb
new file mode 100644
index 0000000..7144d9f
--- /dev/null
+++ b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb
@@ -0,0 +1,71 @@
+class StackSynchronizer
+  include PutsAndFlush
+  attr_reader :out, :stack
+
+  def initialize(stack, out)
+    @stack, @out = stack, out
+    @printed_events = []
+  end
+
+  def sync
+    puts_and_flush "Syncing stack '#{stack.id}'..."
+
+    # 5 tries each 5 seconds, then 200 tries each 10 seconds
+    sleep_times = [5]*5 + [10]*400
+
+    sleep_times.each do |sleep_time|
+      sleep sleep_time
+      stack.sync_details!
+      print_new_events
+      case stack.stack_status
+      when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS'
+      when 'CREATE_COMPLETE'
+        ::Devops::Db.connector.stack_update(stack)
+        puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
+        return 0
+      when 'ROLLBACK_COMPLETE'
+        puts_and_flush "Stack '#{stack.id}' status is rolled back"
+        return error_code(:stack_rolled_back)
+      else
+        puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
+        return error_code(:unkown_status)
+      end
+    end
+    puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
+    error_code(:timeout)
+  rescue StandardError => e
+    DevopsLogger.logger.error e.message
+    puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}"
+    error_code(:error)
+  end
+
+  def reason_from_error_code(code)
+    error_codes.key(code)
+  end
+
+  private
+
+  def error_code(reason)
+    error_codes.fetch(reason)
+  end
+
+  def error_codes
+    {
+      stack_rolled_back: 1,
+      unkown_status: 2,
+      timeout: 3,
+      error: 5
+    }
+  end
+
+
+  def print_new_events
+    stack.events.each do |event|
+      unless @printed_events.include?(event["event_id"])
+        @printed_events << event["event_id"]
+        out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
+      end
+    end
+    out.flush
+  end
+end
\ No newline at end of file
diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb
index da7f712..51e8b0a 100644
--- a/devops-service/workers/stack_bootstrap_worker.rb
+++ b/devops-service/workers/stack_bootstrap_worker.rb
@@ -1,14 +1,13 @@
-require "commands/stack"
 require "db/mongo/models/stack/stack_factory"
 require "db/mongo/models/project"
 require "db/mongo/models/report"
+require "workers/stack_bootstrap/stack_synchronizer"
+require "workers/stack_bootstrap/stack_servers_bootstrapper"
+require "workers/stack_bootstrap/stack_servers_persister"
+require "workers/stack_bootstrap/errors"
 
-class StackCreatingError < StandardError; end
-class BootstrapingStackServerError < StandardError; end
-class DeployingStackServerError < StandardError; end
 
 class StackBootstrapWorker < Worker
-  include StackCommands
 
   def perform(options)
     stack_attrs = options.fetch('stack_attributes')
@@ -18,38 +17,33 @@ class StackBootstrapWorker < Worker
       without_bootstrap = stack_attrs.delete('without_bootstrap')
       @out.puts "Received 'without_bootstrap' option" if without_bootstrap
 
-      report = save_report(file, stack_attrs)
+      save_report(file, stack_attrs)
 
       begin
-        stack = create_stack(stack_attrs)
+        @stack = create_stack(stack_attrs)
 
-        #TODO: errors
         begin
-          servers_with_priority = persist_stack_servers!(stack)
-          unless without_bootstrap
-            sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
-            sorted_keys.each do |key|
-              @out.puts "Servers with priority '#{key}':"
-              bootstrap_servers!(servers_with_priority[key], report)
-            end
-          end
-          @out.puts "Done."
+          @servers_with_priority = stack_servers_persister.persist
+          bootstrap_in_priority_order unless without_bootstrap
           0
-        rescue BootstrapingStackServerError
-          @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
-          rollback_stack!(stack)
+        rescue StackServerBootstrapError
+          puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
+          rollback_stack!(@stack)
           2
-        rescue DeployingStackServerError => e
-          @out.puts "\nStack was launched, but an error occured during deploying stack servers."
-          @out.puts "You can redeploy stack after fixing the error."
+        rescue StackServerDeployError => e
+          out.puts "\nStack was launched, but an error occured during deploying stack servers."
+          puts_and_flush "You can redeploy stack after fixing the error."
           3
+        rescue StackServerBootstrapDeployTimeout
+          puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
+          4
         rescue StandardError => e
-          @out.puts "\nAn error occured. Initiating stack rollback."
-          rollback_stack!(stack)
+          puts_and_flush "\nAn error occured. Initiating stack rollback."
+          rollback_stack!(@stack)
           raise e
         end
       rescue StackCreatingError
-        @out.puts "Stack creating error"
+        puts_and_flush "Stack creating error"
         1
       end
     end
@@ -57,97 +51,52 @@ class StackBootstrapWorker < Worker
 
   private
 
-  def rollback_stack!(stack)
-    @out.puts "\nStart rollback of a stack"
-    stack.delete_stack_in_cloud!
-    Devops::Db.connector.stack_servers_delete(stack.name)
-    Devops::Db.connector.stack_delete(stack.id)
-    @out.puts "Rollback has been completed"
+  def stack_synchronizer(stack)
+    StackSynchronizer.new(stack, out)
   end
 
+  def stack_servers_persister
+    @stack_servers_persister ||= StackServersPersister.new(@stack, out)
+  end
+
+  def stack_servers_bootstrapper
+    @stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid)
+  end
+
+  # builds and persist stack model, initiate stack creating in cloud
   def create_stack(stack_attrs)
     stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
     mongo.stack_insert(stack)
-    operation_result = sync_stack_proc.call(@out, stack, mongo)
+    synchronizer = stack_synchronizer(stack)
+    operation_result = synchronizer.sync
 
     if operation_result == 0
-      @out.puts "\nStack '#{stack.name}' has been created"
-      @out.flush
+      puts_and_flush "\nStack '#{stack.name}' has been created"
       stack
     else
-      human_readable_code = StackCommands.result_codes.key(operation_result)
-      @out.puts "An error ocurred during stack creating"
-      @out.puts "Stack creating operation result was #{human_readable_code}"
+      human_readable_code = synchronizer.reason_from_error_code(operation_result)
+      out.puts "An error ocurred during stack creating"
+      puts_and_flush "Stack creating operation result was #{human_readable_code}"
       raise StackCreatingError
     end
   end
 
-  def bootstrap_servers!(servers, report)
-    @out << "\nStart bootstraping stack servers\n"
-
-    subreports = []
-    data = {}
-    servers.each do |server|
-      sjid = Worker.start_async(BootstrapWorker,
-        server_attrs: server.to_mongo_hash,
-        bootstrap_template: 'omnibus',
-        owner: server.created_by
-      )
-      subreports << sjid
-      @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
-      data[server.id] = sjid
+  # Bootstrap servers with high priorities first
+  def bootstrap_in_priority_order
+    sorted_priorities = @servers_with_priority.keys.sort.reverse
+    sorted_priorities.each do |priority|
+      @out.puts "Servers with priority '#{priority}':"
+      stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority])
     end
-    @out.puts
-    @out.flush
-    mongo.add_report_subreports(jid, subreports)
-    results = []
-    data.each do |server_id, subreport_id|
-      begin
-        sleep(5)
-        subreport = mongo.report(subreport_id)
-        status = subreport.status
-        if status == Worker::STATUS::COMPLETED
-          @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
-          break
-        elsif status == Worker::STATUS::FAILED
-          results << subreport.job_result_code
-          @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
-          break
-        end
-      end while(true)
-    end
-    @out.flush
-    results.empty? ? 0 : -5
+    puts_and_flush "Done."
   end
 
-  def check_bootstrap_results!(results)
-    if results.values.all?(&:zero?)
-      # everything is OK
-      @out.puts "Stack servers have been bootstraped"
-      @out.flush
-      return 0
-    end
-
-    @out.puts
-    results.each do |chef_node_name, code|
-      human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code)
-      @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
-    end
-
-    if errors_in_bootstrapping_present?(results.values)
-      raise BootstrapingStackServerError # will cause rollback of a stack
-    else
-      raise DeployingStackServerError #will not cause rollback of a stack
-    end
-  end
-
-  def errors_in_bootstrapping_present?(result_codes)
-    bootstrap_error_codes = []
-    [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
-      bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code)
-    end
-
-    (bootstrap_error_codes & result_codes).size > 0
+  def rollback_stack!(stack)
+    puts_and_flush "\nStart rollback of a stack"
+    stack.delete_stack_in_cloud!
+    Devops::Db.connector.stack_servers_delete(stack.name)
+    Devops::Db.connector.stack_delete(stack.id)
+    puts_and_flush "Rollback has been completed"
   end
 
   def save_report(file, stack_attrs)
@@ -164,54 +113,4 @@ class StackBootstrapWorker < Worker
     mongo.save_report(report)
     report
   end
-
-  # returns
-  # {
-  #   "priority" => [Servers]
-  # }
-  def persist_stack_servers!(stack)
-    @out.puts "Start syncing stack servers with CID"
-    @out.flush
-    project = mongo.project(stack.project)
-    deploy_env = project.deploy_env(stack.deploy_env)
-    provider = stack.provider_instance
-
-    stack_servers = provider.stack_servers(stack)
-    stack_servers.each do |info|
-      info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
-    end
-    stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
-    stack_servers_with_priority = {}
-    stack_servers_info.each do |priority, info_array|
-      stack_servers_with_priority[priority] = info_array.map do |extended_info|
-        @out.puts "Instance '#{extended_info["id"]}' has been launched with stack."
-        server_attrs = {
-          'provider' => provider.name,
-          'project' => project.id,
-          'deploy_env' => deploy_env.identifier,
-          'remote_user' => mongo.image(deploy_env.image).remote_user,
-          'key' => extended_info["key_name"] || provider.ssh_key,
-          '_id' => extended_info["id"],
-          'chef_node_name' => extended_info["name"],
-          'private_ip' => extended_info["private_ip"],
-          'public_ip' => extended_info["public_ip"],
-          'created_by' => stack.owner,
-          'run_list' => stack.run_list || [],
-          'stack' => stack.name
-        }
-
-        server = ::Devops::Model::Server.new(server_attrs)
-        mongo.server_insert(server)
-        # server.chef_node_name =  provider.create_default_chef_node_name(server)
-        server
-      end
-    end
-    @out.puts "Stack servers have been synced with CID"
-    stack_servers_with_priority.each do |priority, servers|
-      @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
-    end
-    @out.flush
-    stack_servers_with_priority
-  end
-
 end
diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb
index eb5af3e..c7daada 100644
--- a/devops-service/workers/worker.rb
+++ b/devops-service/workers/worker.rb
@@ -12,11 +12,12 @@ require "core/devops-logger"
 require "core/devops-db"
 require "providers/provider_factory"
 require "lib/knife/knife_factory"
-
+require "lib/puts_and_flush"
 
 # All options keys MUST be a symbol!!!
 class Worker
   include Sidekiq::Worker
+  include PutsAndFlush
 
   attr_accessor :out
 

From 724349b5783e479093c25fcf7d181ac535603907 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 10 Feb 2016 17:05:35 +0300
Subject: [PATCH 34/35] fix deleting by node name; set owner properly

---
 devops-service/app/api2/handlers/project.rb    | 3 ++-
 devops-service/app/api2/handlers/server.rb     | 5 +++--
 devops-service/workers/delete_server_worker.rb | 8 ++++----
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb
index 7193e8b..5f8f918 100644
--- a/devops-service/app/api2/handlers/project.rb
+++ b/devops-service/app/api2/handlers/project.rb
@@ -315,8 +315,9 @@ module Devops
         private
 
         def delete_chosen_servers!(servers)
+          current_user = parser.current_user
           reports = servers.map do |server|
-            Worker.start_async(DeleteServerWorker, 'server_id' => server.id)
+            Worker.start_async(DeleteServerWorker, 'server_id' => server.id, 'current_user' => current_user)
           end
           {reports: reports}
         end
diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb
index f6a462c..aa77939 100644
--- a/devops-service/app/api2/handlers/server.rb
+++ b/devops-service/app/api2/handlers/server.rb
@@ -49,8 +49,9 @@ module Devops
 
         def delete id
           server = get_server_by_key(id, parser.instance_key)
-          Devops::Db.connector.check_project_auth server.project, server.deploy_env, parser.current_user
-          jid = Worker.start_async(DeleteServerWorker, 'server_id' => id)
+          current_user = parser.current_user
+          Devops::Db.connector.check_project_auth server.project, server.deploy_env, current_user
+          jid = Worker.start_async(DeleteServerWorker, 'server_id' => server.id, 'current_user' => current_user)
           [jid]
         end
 
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb
index 4354e16..77f3e5c 100644
--- a/devops-service/workers/delete_server_worker.rb
+++ b/devops-service/workers/delete_server_worker.rb
@@ -2,18 +2,18 @@ require "db/mongo/models/server"
 require "db/mongo/models/report"
 require "lib/executors/server_executor"
 require "workers/worker"
-require 'byebug'
 
 class DeleteServerWorker < Worker
 
   # options should contain 'server_id'
   def perform(options)
     server_id = options.fetch('server_id')
+    current_user = options.fetch('current_user')
 
     call() do |out, file|
       out.puts "Deleting server with id #{server_id}" and out.flush
       @server = mongo.server_by_instance_id(server_id)
-      report = save_report(file)
+      report = save_report(file, current_user)
 
       e = Devops::Executor::ServerExecutor.new(@server, out)
       e.report = report
@@ -23,11 +23,11 @@ class DeleteServerWorker < Worker
 
   private
 
-  def save_report(file)
+  def save_report(file, current_user)
     report = Devops::Model::Report.new(
       "file" => file,
       "_id" => jid,
-      "created_by" => 'SYSTEM',
+      "created_by" => current_user,
       "project" => @server.project,
       "deploy_env" => @server.deploy_env,
       "type" => Devops::Model::Report::DELETE_SERVER_TYPE

From 7e6c5c2f723dad846361c4a48e35b74418746350 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 10 Feb 2016 20:39:49 +0300
Subject: [PATCH 35/35] create report before executing job's body

---
 devops-service/db/mongo/connectors/report.rb  |  1 +
 devops-service/db/mongo/mongo_connector.rb    |  2 +-
 .../workers/stack_bootstrap_worker_spec.rb    | 12 +++---
 devops-service/workers/bootstrap_worker.rb    | 37 ++++++++-----------
 .../workers/create_server_worker.rb           | 20 ++++------
 .../workers/delete_server_worker.rb           | 17 +++------
 devops-service/workers/deploy_worker.rb       | 22 +++++------
 devops-service/workers/project_test_worker.rb | 21 ++++-------
 .../workers/stack_bootstrap_worker.rb         | 16 +++-----
 devops-service/workers/worker.rb              | 20 ++++++----
 10 files changed, 75 insertions(+), 93 deletions(-)

diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb
index 0a8d449..0f9d5c0 100644
--- a/devops-service/db/mongo/connectors/report.rb
+++ b/devops-service/db/mongo/connectors/report.rb
@@ -3,6 +3,7 @@ require "date"
 module Connectors
   class Report < Base
     include Helpers::ShowCommand,
+      Helpers::UpdateCommand,
       Helpers::ListCommand
 
     def initialize(db)
diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb
index 60deaa7..cc44133 100644
--- a/devops-service/db/mongo/mongo_connector.rb
+++ b/devops-service/db/mongo/mongo_connector.rb
@@ -31,7 +31,7 @@ class MongoConnector
     [:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
     :user_update, :create_root_user, :check_user_privileges] => :users_connector,
     [:keys, :key, :key_insert, :key_delete] => :keys_connector,
-    [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
+    [:save_report, :report_update, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
     [:insert_statistic, :search_statistic] => :statistics_connector,
     [:provider_accounts, :provider_account_insert, :provider_account_delete, :provider_account] => :provider_accounts_connector
   )
diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb
index b2ede82..ff5cefe 100644
--- a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb
+++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb
@@ -15,12 +15,14 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
     allow(stubbed_connector).to receive(:save_report)
+    allow(stubbed_connector).to receive(:report_update)
     allow(stubbed_connector).to receive(:stack_insert)
 
     allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
     allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
     allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
-    allow(worker).to receive(:call).and_yield(out, file)
+    allow(worker).to receive(:call).and_yield
+    worker.out = out
     allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
   end
 
@@ -31,14 +33,14 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
     }.to raise_error KeyError
   end
 
-  it 'saves report about operation' do
-    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report))
+  it 'updates report about operation' do
+    expect(stubbed_connector).to receive(:report_update).with(instance_of(Devops::Model::Report))
     perform_without_bootstrap
   end
 
-  it 'saves report about operation, creates stack and persists stack servers' do
+  it 'updates report about operation, creates stack and persists stack servers' do
     allow(worker).to receive(:create_stack).and_call_original
-    expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)).ordered
+    expect(stubbed_connector).to receive(:report_update).with(instance_of(Devops::Model::Report)).ordered
     expect(worker).to receive(:create_stack).ordered
     expect(stack_servers_persister).to receive(:persist).ordered
     perform_without_bootstrap
diff --git a/devops-service/workers/bootstrap_worker.rb b/devops-service/workers/bootstrap_worker.rb
index 001f0d2..1e40e05 100644
--- a/devops-service/workers/bootstrap_worker.rb
+++ b/devops-service/workers/bootstrap_worker.rb
@@ -6,24 +6,18 @@ require "db/mongo/models/report"
 
 class BootstrapWorker < Worker
 
+  # options must contain 'server_attrs', 'owner'
   def perform(options)
-    server_attrs = options.fetch('server_attrs')
-#    bootstrap_template = options.fetch('bootstrap_template')
-    owner = options.fetch('owner')
-    options = convert_config(options)
+    call do
+      owner = options.fetch('owner')
+      converted_options = convert_config(options)
 
-    call() do |out, file|
-      server = Devops::Model::Server.new(server_attrs)
-      report = save_report(file, owner, server)
+      server = Devops::Model::Server.new(options.fetch('server_attrs'))
+      report = save_report(owner, server)
 
-=begin
-      options = {
-        bootstrap_template: bootstrap_template
-      }
-=end
       executor = Devops::Executor::ServerExecutor.new(server, out, current_user: owner)
       executor.report = report
-      status = executor.two_phase_bootstrap(options)
+      status = executor.two_phase_bootstrap(converted_options)
       mongo.set_report_server_data(jid, server.chef_node_name, server.public_ip || server.private_ip)
       status
     end
@@ -31,18 +25,19 @@ class BootstrapWorker < Worker
 
   private
 
-  def save_report(file, owner, server)
-    report_data = {
-      "file" => file,
-      "_id" => jid,
+  def save_report(owner, server)
+    update_report(
       "created_by" => owner,
       "project" => server.project,
       "deploy_env" => server.deploy_env,
       "type" => Devops::Model::Report::BOOTSTRAP_TYPE
-    }
-    report = Devops::Model::Report.new(report_data)
-    mongo.save_report(report)
-    report
+    )
+  end
+
+  def convert_config conf
+    config = {}
+    conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}
+    config
   end
 end
 
diff --git a/devops-service/workers/create_server_worker.rb b/devops-service/workers/create_server_worker.rb
index bc2655a..78c0e59 100644
--- a/devops-service/workers/create_server_worker.rb
+++ b/devops-service/workers/create_server_worker.rb
@@ -6,14 +6,15 @@ require "lib/executors/server_executor"
 
 class CreateServerWorker < Worker
 
+  # options must contain 'server_attrs', 'owner'
   def perform(options)
-    server_attrs = options.fetch('server_attrs')
-    owner = options.fetch('owner')
+    call do
+      server_attrs = options.fetch('server_attrs')
+      owner = options.fetch('owner')
 
-    call() do |out, file|
       project = mongo.project(server_attrs["project"])
       env = project.deploy_env(server_attrs["deploy_env"])
-      report = save_report(file, project, env, owner)
+      report = save_report(project, env, owner)
 
       e = Devops::Executor::ServerExecutor.new(nil, out)
       e.project = project
@@ -26,18 +27,13 @@ class CreateServerWorker < Worker
 
   private
 
-  def save_report(file, project, env, owner)
-    report_data = {
-      "file" => file,
-      "_id" => jid,
+  def save_report(project, env, owner)
+    update_report(
       "created_by" => owner,
       "project" => project.id,
       "deploy_env" => env.identifier,
       "type" => Devops::Model::Report::SERVER_TYPE
-    }
-    report = Devops::Model::Report.new(report_data)
-    mongo.save_report(report)
-    report
+    )
   end
 
 end
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb
index 77f3e5c..937f00f 100644
--- a/devops-service/workers/delete_server_worker.rb
+++ b/devops-service/workers/delete_server_worker.rb
@@ -5,15 +5,14 @@ require "workers/worker"
 
 class DeleteServerWorker < Worker
 
-  # options should contain 'server_id'
+  # options must contain 'server_id', 'current_user'
   def perform(options)
-    server_id = options.fetch('server_id')
-    current_user = options.fetch('current_user')
+    call do
+      server_id = options.fetch('server_id')
 
-    call() do |out, file|
       out.puts "Deleting server with id #{server_id}" and out.flush
       @server = mongo.server_by_instance_id(server_id)
-      report = save_report(file, current_user)
+      report = save_report(options.fetch('current_user'))
 
       e = Devops::Executor::ServerExecutor.new(@server, out)
       e.report = report
@@ -23,17 +22,13 @@ class DeleteServerWorker < Worker
 
   private
 
-  def save_report(file, current_user)
-    report = Devops::Model::Report.new(
-      "file" => file,
-      "_id" => jid,
+  def save_report(current_user)
+    update_report(
       "created_by" => current_user,
       "project" => @server.project,
       "deploy_env" => @server.deploy_env,
       "type" => Devops::Model::Report::DELETE_SERVER_TYPE
     )
-    mongo.save_report(report)
-    report
   end
 
 end
diff --git a/devops-service/workers/deploy_worker.rb b/devops-service/workers/deploy_worker.rb
index b6e48d0..791c86d 100644
--- a/devops-service/workers/deploy_worker.rb
+++ b/devops-service/workers/deploy_worker.rb
@@ -6,15 +6,16 @@ require "db/mongo/models/report"
 
 class DeployWorker < Worker
 
+  # options must contain 'server_attrs', 'owner', 'tags', 'deploy_info'
   def perform(options)
-    server_attrs = options.fetch('server_attrs')
-    owner = options.fetch('owner')
-    tags = options.fetch('tags')
-    deploy_info = options.fetch('deploy_info')
+    call do
+      server_attrs = options.fetch('server_attrs')
+      owner = options.fetch('owner')
+      tags = options.fetch('tags')
+      deploy_info = options.fetch('deploy_info')
 
-    call() do |out, file|
       server = Devops::Model::Server.new(server_attrs)
-      report = save_report(file, owner, server)
+      report = save_report(owner, server)
 
       executor = Devops::Executor::ServerExecutor.new(server, out, current_user: owner)
       executor.report = report
@@ -24,14 +25,11 @@ class DeployWorker < Worker
 
   private
 
-  def save_report(file, owner, server)
+  def save_report(owner, server)
     report_data = {
-      "file" => file,
-      "_id" => jid,
       "created_by" => owner,
       "project" => server.project,
       "deploy_env" => server.deploy_env,
-      "status" => STATUS::RUNNING,
       "chef_node_name" => server.chef_node_name,
       "host" => server.public_ip || server.private_ip
     }
@@ -41,9 +39,7 @@ class DeployWorker < Worker
       report_data["type"] = Devops::Model::Report::DEPLOY_STACK_TYPE
       report_data["stack"] = server.stack
     end
-    report = Devops::Model::Report.new(report_data)
-    mongo.save_report(report)
-    report
+    update_report(report_data)
   end
 
 end
diff --git a/devops-service/workers/project_test_worker.rb b/devops-service/workers/project_test_worker.rb
index 760b06c..2d604a7 100644
--- a/devops-service/workers/project_test_worker.rb
+++ b/devops-service/workers/project_test_worker.rb
@@ -11,15 +11,15 @@ class ProjectTestWorker < Worker
   include StatusCommands
 
   def perform(params)
-    user = params.fetch('user')
-    project_name = params.fetch('project')
-    deploy_env_name = params.fetch('deploy_env')
+    call do
+      user = params.fetch('user')
+      project_name = params.fetch('project')
+      deploy_env_name = params.fetch('deploy_env')
 
-    call() do |out, file|
       DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})"
       project = mongo.project(project_name)
       env = project.deploy_env(deploy_env_name)
-      report = save_report(file, user, project_name, deploy_env_name)
+      report = save_report(user, project_name, deploy_env_name)
 
       executor = Devops::Executor::ServerExecutor.new(nil, out)
       executor.project = project
@@ -102,17 +102,12 @@ class ProjectTestWorker < Worker
 
   private
 
-  def save_report(file, user, project_name, deploy_env_name)
-    report_data = {
-      "file" => file,
-      "_id" => jid,
+  def save_report(user, project_name, deploy_env_name)
+    update_report(
       "created_by" => user,
       "project" => project_name,
       "deploy_env" => deploy_env_name,
       "type" => Devops::Model::Report::PROJECT_TEST_TYPE
-    }
-    report = Devops::Model::Report.new(report_data)
-    mongo.save_report(report)
-    report
+    )
   end
 end
diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb
index 51e8b0a..3d1f3fb 100644
--- a/devops-service/workers/stack_bootstrap_worker.rb
+++ b/devops-service/workers/stack_bootstrap_worker.rb
@@ -9,15 +9,15 @@ require "workers/stack_bootstrap/errors"
 
 class StackBootstrapWorker < Worker
 
+  # options must contain 'stack_attributes'
   def perform(options)
-    stack_attrs = options.fetch('stack_attributes')
+    call do
+      stack_attrs = options.fetch('stack_attributes')
 
-    call() do |out, file|
-      @out = out
       without_bootstrap = stack_attrs.delete('without_bootstrap')
       @out.puts "Received 'without_bootstrap' option" if without_bootstrap
 
-      save_report(file, stack_attrs)
+      save_report(stack_attrs)
 
       begin
         @stack = create_stack(stack_attrs)
@@ -99,10 +99,8 @@ class StackBootstrapWorker < Worker
     puts_and_flush "Rollback has been completed"
   end
 
-  def save_report(file, stack_attrs)
-    report = ::Devops::Model::Report.new(
-      "file" => file,
-      "_id" => jid,
+  def save_report(stack_attrs)
+    update_report(
       "created_by" => stack_attrs['owner'],
       "project" => stack_attrs["project"],
       "deploy_env" => stack_attrs["deploy_env"],
@@ -110,7 +108,5 @@ class StackBootstrapWorker < Worker
       "subreports" => [],
       "stack" => stack_attrs['name']
     )
-    mongo.save_report(report)
-    report
   end
 end
diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb
index c7daada..6a6ec95 100644
--- a/devops-service/workers/worker.rb
+++ b/devops-service/workers/worker.rb
@@ -83,15 +83,16 @@ class Worker
   def call_async()
     dir = DevopsConfig[:report_dir_v2]
     # directory is created on server start in config.ru
-    file = File.join(dir, jid)
+    @file = File.join(dir, jid)
 
+    create_report
     update_job_status(STATUS::INIT, nil)
-    File.open(file, "w") do |out|
+    File.open(@file, "w") do |out|
       begin
         update_job_status(STATUS::RUNNING, nil)
         self.out = out
 
-        job_result = yield(out, file)
+        job_result = yield
         canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED)
         update_job_status(canonical_status, job_result)
       rescue StandardError, RecordNotFound => e
@@ -125,10 +126,15 @@ class Worker
     status
   end
 
-  def convert_config conf
-    config = {}
-    conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}
-    config
+  def create_report
+    report = Devops::Model::Report.new('_id' => jid, 'file' => @file)
+    mongo.save_report(report)
   end
 
+  def update_report(additional_report_attrs)
+    report_attrs = additional_report_attrs.merge('_id' => jid, 'file' => @file)
+    report = Devops::Model::Report.new(report_attrs)
+    mongo.report_update(report)
+    report
+  end
 end