diff --git a/.gitignore b/.gitignore
index 49cc974..c742b09 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 devops-service/tests/features/support/config.yml
 .devops_files/
 devops-service/plugins
-devops-service/spec/examples.txt
\ No newline at end of file
+devops-service/spec/examples.txt
+devops-service/coverage
+devops-service/tmp
\ No newline at end of file
diff --git a/devops-service/Gemfile b/devops-service/Gemfile
index 13b88c3..9a51237 100644
--- a/devops-service/Gemfile
+++ b/devops-service/Gemfile
@@ -36,4 +36,6 @@ end
 group :devepoment do
   gem 'byebug'
   gem 'guard-rspec', require: false
+  gem 'simplecov', require: false
+  gem 'simplecov-rcov', require: false
 end
diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock
index 31da4a0..7ee4373 100644
--- a/devops-service/Gemfile.lock
+++ b/devops-service/Gemfile.lock
@@ -60,6 +60,7 @@ GEM
       gherkin (~> 2.12.0)
     daemons (1.2.3)
     diff-lcs (1.2.5)
+    docile (1.1.5)
     em-websocket (0.3.8)
       addressable (>= 2.1.1)
       eventmachine (>= 0.12.9)
@@ -291,6 +292,13 @@ GEM
       json
       redis (>= 3.0.6)
       redis-namespace (>= 1.3.1)
+    simplecov (0.11.1)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    simplecov-rcov (0.2.3)
+      simplecov (>= 0.4.1)
     sinatra (1.4.5)
       rack (~> 1.4)
       rack-protection (~> 1.4)
@@ -352,9 +360,14 @@ DEPENDENCIES
   rspec (~> 3.3)
   rspec_junit_formatter
   sidekiq (= 3.2.6)
+  simplecov
+  simplecov-rcov
   sinatra (= 1.4.5)
   sinatra-contrib
   sinatra-websocket
   test-unit
   thin (~> 1.5.1)
   wisper
+
+BUNDLED WITH
+   1.11.2
diff --git a/devops-service/Guardfile b/devops-service/Guardfile
index 13d5834..a66e388 100644
--- a/devops-service/Guardfile
+++ b/devops-service/Guardfile
@@ -42,4 +42,5 @@ guard :rspec, cmd: "rspec" do
 
   # Devops files
   watch(%r{db/.+\.rb}) { rspec.spec_dir }
+  watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
 end
diff --git a/devops-service/app/api2/handlers/provider.rb b/devops-service/app/api2/handlers/provider.rb
index 8e5445d..7e74f27 100644
--- a/devops-service/app/api2/handlers/provider.rb
+++ b/devops-service/app/api2/handlers/provider.rb
@@ -27,14 +27,14 @@ module Devops
         def add_account provider
           account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account)
           account.validate_fields!
-          Devops::Db.connector.provider_accounts_insert(account)
+          Devops::Db.connector.provider_account_insert(account)
           ::Provider::ProviderFactory.add_account(provider, account)
           account.to_hash
         end
 
         def delete_account name, provider
           account = Devops::Db.connector.provider_account(provider, name)
-          Devops::Db.connector.provider_accounts_delete(name)
+          Devops::Db.connector.provider_account_delete(name)
           ::Provider::ProviderFactory.delete_account(provider, account)
           account.to_hash
         end
diff --git a/devops-service/commands/knife_commands.rb b/devops-service/commands/knife_commands.rb
index 45c4835..73c41bf 100644
--- a/devops-service/commands/knife_commands.rb
+++ b/devops-service/commands/knife_commands.rb
@@ -38,6 +38,13 @@ class KnifeCommands
     knife("tag delete #{name} #{tagsStr}")
   end
 
+  # extracted from server_executor.rb
+  def swap_tags(name, from, to)
+    tags_delete(name, from)
+    result = tags_create(name, to)
+    !!result[1]
+  end
+
   def create_role role_name, project, env
     file = "/tmp/new_role.json"
     File.open(file, "w") do |f|
diff --git a/devops-service/core/devops-service.rb b/devops-service/core/devops-service.rb
index cd33364..7bf32f1 100644
--- a/devops-service/core/devops-service.rb
+++ b/devops-service/core/devops-service.rb
@@ -1,7 +1,7 @@
 require "wisper"
-require "lib/hash_ext"
-require "lib/nil_class_ext"
-require "lib/string_ext"
+require "lib/core_ext/hash"
+require "lib/core_ext/nil_class"
+require "lib/core_ext/string"
 
 require_relative "devops-loader"
 require_relative "devops-application"
diff --git a/devops-service/db/mongo/connectors/helpers/delete_command.rb b/devops-service/db/mongo/connectors/helpers/delete_command.rb
index b66be9c..8841aaa 100644
--- a/devops-service/db/mongo/connectors/helpers/delete_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/delete_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module DeleteCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
+        resource_name = base.to_s.underscore_class
         method_name = "#{resource_name}_delete".to_sym
         alias_method method_name, :delete
       end
diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb
index be380a9..446ca5f 100644
--- a/devops-service/db/mongo/connectors/helpers/insert_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module InsertCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
+        resource_name = base.to_s.underscore_class
         method_name = "#{resource_name}_insert".to_sym
         alias_method method_name, :insert
       end
@@ -24,7 +22,7 @@ module Connectors
         rescue Mongo::OperationFailure => e
           # exception's message doesn't always start from error code
           if e.message =~ /11000/
-            resource_name = StringHelper.underscore_class(record.class)
+            resource_name = record.class.to_s.underscore_class
             raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
           end
         end
diff --git a/devops-service/db/mongo/connectors/helpers/list_command.rb b/devops-service/db/mongo/connectors/helpers/list_command.rb
index aaab3b8..a53dcb0 100644
--- a/devops-service/db/mongo/connectors/helpers/list_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/list_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module ListCommand
@@ -9,9 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base).to_sym
-        method_name = StringHelper.pluralize(resource_name)
-        alias_method method_name, :list
+        alias_method base.to_s.underscore_class.pluralize, :list
       end
 
       # query options is needed, for example, for fields limiting
diff --git a/devops-service/db/mongo/connectors/helpers/show_command.rb b/devops-service/db/mongo/connectors/helpers/show_command.rb
index cb06d74..2889d72 100644
--- a/devops-service/db/mongo/connectors/helpers/show_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/show_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module ShowCommand
@@ -9,7 +7,7 @@ module Connectors
       # We need this alias to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        method_name = StringHelper.underscore_class(base).to_sym
+        method_name = base.to_s.underscore_class.to_sym
         alias_method method_name, :show
       end
 
diff --git a/devops-service/db/mongo/connectors/helpers/update_command.rb b/devops-service/db/mongo/connectors/helpers/update_command.rb
index 674eb60..a8f88b6 100644
--- a/devops-service/db/mongo/connectors/helpers/update_command.rb
+++ b/devops-service/db/mongo/connectors/helpers/update_command.rb
@@ -1,5 +1,3 @@
-require 'lib/string_helper'
-
 module Connectors
   module Helpers
     module UpdateCommand
@@ -8,8 +6,7 @@ module Connectors
       # We need second method name to forward methods from MongoConnector to resources connectors.
 
       def self.included(base)
-        resource_name = StringHelper.underscore_class(base)
-        method_name = "#{resource_name}_update".to_sym
+        method_name = "#{base.to_s.underscore_class}_update".to_sym
         alias_method method_name, :update
       end
 
diff --git a/devops-service/db/mongo/connectors/provider_accounts.rb b/devops-service/db/mongo/connectors/provider_account.rb
similarity index 90%
rename from devops-service/db/mongo/connectors/provider_accounts.rb
rename to devops-service/db/mongo/connectors/provider_account.rb
index 4bd235a..931f99f 100644
--- a/devops-service/db/mongo/connectors/provider_accounts.rb
+++ b/devops-service/db/mongo/connectors/provider_account.rb
@@ -1,5 +1,5 @@
 module Connectors
-  class ProviderAccounts < Base
+  class ProviderAccount < Base
 
     include Helpers::InsertCommand,
             Helpers::DeleteCommand
@@ -16,7 +16,7 @@ module Connectors
     def provider_account provider, account
       c = Provider::ProviderFactory.get_account_class(provider)
       bson = collection.find({provider: provider, _id: account}).to_a.first
-      raise RecordNotFound.new("'Account #{account}' for provider '#{provider}' not found") unless bson
+      raise RecordNotFound.new("Account '#{account}' for provider '#{provider}' not found") unless bson
       c.build_from_bson(bson)
     end
 
diff --git a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
index 49cdfad..f4259cf 100644
--- a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
+++ b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
@@ -53,7 +53,7 @@ module Devops
       def subnets_filter
         networks = provider_instance.networks
 
-        unless self.subnets.empty?
+        if subnets && !subnets.empty?
           network = networks.detect {|n| n["name"] == self.subnets[0]}
           if network
             {"vpc-id" => network["vpcId"] }
diff --git a/devops-service/db/mongo/models/image.rb b/devops-service/db/mongo/models/image.rb
index 0263562..8ccfc59 100644
--- a/devops-service/db/mongo/models/image.rb
+++ b/devops-service/db/mongo/models/image.rb
@@ -73,10 +73,6 @@ module Devops
         }
       end
 
-      def self.create_from_json! json
-        Image.new( JSON.parse(json) )
-      end
-
     end
   end
 end
diff --git a/devops-service/db/mongo/models/key.rb b/devops-service/db/mongo/models/key.rb
index 32332ff..edaac58 100644
--- a/devops-service/db/mongo/models/key.rb
+++ b/devops-service/db/mongo/models/key.rb
@@ -30,10 +30,6 @@ module Devops
         key
       end
 
-      def self.create_from_json json
-        Key.new( JSON.parse(json) )
-      end
-
       def filename
         File.basename(self.path)
       end
diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb
index c4edc04..0bb61ac 100644
--- a/devops-service/db/mongo/models/project.rb
+++ b/devops-service/db/mongo/models/project.rb
@@ -22,6 +22,7 @@ module Devops
       #define_hook :after_add_deploy_env
 
       attr_accessor :id, :deploy_envs, :type, :archived, :description, :run_list
+      attr_accessor :components
 
       MULTI_TYPE = "multi"
 
diff --git a/devops-service/db/mongo/models/provider_accounts/provider_account.rb b/devops-service/db/mongo/models/provider_accounts/provider_account.rb
index d3c9c42..efc6dd1 100644
--- a/devops-service/db/mongo/models/provider_accounts/provider_account.rb
+++ b/devops-service/db/mongo/models/provider_accounts/provider_account.rb
@@ -61,6 +61,12 @@ module Devops
         }
       end
 
+      # absent of "id" attribute can cause some inconviniences.
+      # for example, we have "record.id" call in InsertCommand
+      def id
+        account_name
+      end
+
     end
   end
 end
diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
index 20b3d03..4cd52f3 100644
--- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
+++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
@@ -16,10 +16,6 @@ module Devops
         super.merge(template_url: template_url)
       end
 
-      def delete_template_file_from_storage
-        raise 'Implement me'
-      end
-
       def update_template_url
         self.template_url = generate_template_file_and_upload_to_storage(id, template_body)
       end
diff --git a/devops-service/db/mongo/models/user.rb b/devops-service/db/mongo/models/user.rb
index c24247e..c96b3dd 100644
--- a/devops-service/db/mongo/models/user.rb
+++ b/devops-service/db/mongo/models/user.rb
@@ -75,17 +75,12 @@ module Devops
         user
       end
 
-      def self.create_from_json json
-        User.new( JSON.parse(json) )
-      end
-
       def to_hash_without_id
-        o = {
+        {
           "email" => self.email,
           "password" => self.password,
           "privileges" => self.privileges
         }
-        o
       end
 
       def check_privileges cmd, required_privelege
diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb
index baa47ff..60deaa7 100644
--- a/devops-service/db/mongo/mongo_connector.rb
+++ b/devops-service/db/mongo/mongo_connector.rb
@@ -33,7 +33,7 @@ class MongoConnector
     [:keys, :key, :key_insert, :key_delete] => :keys_connector,
     [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
     [:insert_statistic, :search_statistic] => :statistics_connector,
-    [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_account] => :provider_accounts_connector
+    [:provider_accounts, :provider_account_insert, :provider_account_delete, :provider_account] => :provider_accounts_connector
   )
 
   def initialize(db, host, port=27017, user=nil, password=nil)
@@ -48,7 +48,7 @@ class MongoConnector
   private
 
   def provider_accounts_connector
-    @provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db)
+    @provider_accounts_connector ||= Connectors::ProviderAccount.new(@db)
   end
 
   def images_connector
diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb
index 677e9f2..bec8722 100644
--- a/devops-service/db/validators/base.rb
+++ b/devops-service/db/validators/base.rb
@@ -13,6 +13,7 @@ module Validators
       raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}")
     end
 
+    # :nocov:
     def valid?
       raise 'override me'
     end
@@ -20,6 +21,7 @@ module Validators
     def message
       raise 'override me'
     end
+    # :nocov:
 
     class << self
       private
diff --git a/devops-service/db/validators/deploy_env/flavor.rb b/devops-service/db/validators/deploy_env/flavor.rb
index e0ff359..b763af5 100644
--- a/devops-service/db/validators/deploy_env/flavor.rb
+++ b/devops-service/db/validators/deploy_env/flavor.rb
@@ -3,7 +3,7 @@ module Validators
 
     def valid?
       return true unless @model.flavor
-      available_flavors.detect do |flavor|
+      @model.provider_instance.flavors.detect do |flavor|
         flavor['id'] == @model.flavor
       end
     end
@@ -11,11 +11,5 @@ module Validators
     def message
       "Invalid flavor '#{@model.flavor}'."
     end
-
-    private
-
-    def available_flavors
-      @model.provider_instance.flavors
-    end
   end
 end
\ No newline at end of file
diff --git a/devops-service/db/validators/deploy_env/groups.rb b/devops-service/db/validators/deploy_env/groups.rb
index 43f7027..8e3db24 100644
--- a/devops-service/db/validators/deploy_env/groups.rb
+++ b/devops-service/db/validators/deploy_env/groups.rb
@@ -3,6 +3,9 @@ module Validators
 
     def valid?
       return true if @model.groups.nil?
+      subnets_filter = @model.subnets_filter
+      available_groups = @model.provider_instance.groups(subnets_filter).keys
+
       @invalid_groups = @model.groups - available_groups
       @invalid_groups.empty?
     end
@@ -10,12 +13,5 @@ module Validators
     def message
       "Invalid groups '#{@invalid_groups.join("', '")}'."
     end
-
-    private
-
-    def available_groups
-      subnets_filter = @model.subnets_filter
-      @model.provider_instance.groups(subnets_filter).keys
-    end
   end
 end
\ No newline at end of file
diff --git a/devops-service/db/validators/deploy_env/image.rb b/devops-service/db/validators/deploy_env/image.rb
index 6056d88..ee69561 100644
--- a/devops-service/db/validators/deploy_env/image.rb
+++ b/devops-service/db/validators/deploy_env/image.rb
@@ -6,7 +6,7 @@ module Validators
 
     def valid?
       return true unless @model.image
-      available_images.detect do |image|
+      get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
         image["id"] == @model.image
       end
     end
@@ -14,11 +14,5 @@ module Validators
     def message
       "Invalid image '#{@model.image}'."
     end
-
-    private
-
-    def available_images
-      get_available_provider_images(::Devops::Db.connector, @model.provider)
-    end
   end
 end
diff --git a/devops-service/db/validators/deploy_env/stack_template.rb b/devops-service/db/validators/deploy_env/stack_template.rb
index 9b20d6d..eee09de 100644
--- a/devops-service/db/validators/deploy_env/stack_template.rb
+++ b/devops-service/db/validators/deploy_env/stack_template.rb
@@ -4,21 +4,13 @@ module Validators
     def valid?
       return true unless @model.stack_template
 
-      available_stack_templates.detect do |template|
-        template['id'] == @model.stack_template
+      Devops::Db.connector.stack_templates.detect do |template|
+        template.id == @model.stack_template
       end
     end
 
     def message
       "Invalid stack template '#{@model.stack_template}'."
     end
-
-    private
-
-    def available_stack_templates
-      # map to hash to simplify mocks. Later replace this method with something more suitable
-      Devops::Db.connector.stack_templates.map(&:to_hash)
-    end
-
   end
 end
diff --git a/devops-service/db/validators/field_validators/flavor.rb b/devops-service/db/validators/field_validators/flavor.rb
index 4f86df4..a06d104 100644
--- a/devops-service/db/validators/field_validators/flavor.rb
+++ b/devops-service/db/validators/field_validators/flavor.rb
@@ -4,7 +4,7 @@ module Validators
     class Flavor < Base
 
       def valid?
-        available_flavors.detect do |flavor|
+        @model.provider_instance.flavors.detect do |flavor|
           flavor['id'] == @value
         end
       end
@@ -12,12 +12,6 @@ module Validators
       def message
         "Invalid flavor '#{@value}'."
       end
-
-      private
-
-      def available_flavors
-        @model.provider_instance.flavors
-      end
     end
   end
 end
diff --git a/devops-service/db/validators/field_validators/image.rb b/devops-service/db/validators/field_validators/image.rb
index 3dd727b..d8df841 100644
--- a/devops-service/db/validators/field_validators/image.rb
+++ b/devops-service/db/validators/field_validators/image.rb
@@ -7,7 +7,7 @@ module Validators
       include ::ImageCommands
 
       def valid?
-        available_images.detect do |image|
+        get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
           image["id"] == @value
         end
       end
@@ -15,12 +15,6 @@ module Validators
       def message
         "Invalid image '#{@value}'."
       end
-
-      private
-
-      def available_images
-        get_available_provider_images(::Devops::Db.connector, @model.provider)
-      end
     end
   end
 end
diff --git a/devops-service/db/validators/helpers/users.rb b/devops-service/db/validators/helpers/users.rb
index 7b534bf..430a7bc 100644
--- a/devops-service/db/validators/helpers/users.rb
+++ b/devops-service/db/validators/helpers/users.rb
@@ -2,6 +2,7 @@ module Validators
   class Helpers::Users < Base
 
     def valid?
+      available_users = ::Devops::Db.connector.users_names(@model)
       @nonexistent_users = (@model || []) - available_users
       @nonexistent_users.empty?
     end
@@ -9,11 +10,5 @@ module Validators
     def message
       Devops::Messages.t("project.deploy_env.validation.users.not_exist", users: @nonexistent_users.join("', '"))
     end
-
-    private
-
-    def available_users
-      ::Devops::Db.connector.users_names(@model)
-    end
   end
 end
diff --git a/devops-service/db/validators/image/bootstrap_template.rb b/devops-service/db/validators/image/bootstrap_template.rb
index 2ced224..67bc070 100644
--- a/devops-service/db/validators/image/bootstrap_template.rb
+++ b/devops-service/db/validators/image/bootstrap_template.rb
@@ -8,22 +8,12 @@ module Validators
       include BootstrapTemplatesCommands
 
       def valid?
-        if @model.bootstrap_template
-          available_templates.include?(@model.bootstrap_template)
-        else
-          true
-        end
+        get_templates.include?(@model.bootstrap_template)
       end
 
       def message
         "Invalid bootstrap template '#{@model.bootstrap_template}' for image '#{@model.id}'"
       end
-
-      private
-
-      def available_templates
-        get_templates
-      end
     end
   end
 end
diff --git a/devops-service/lib/hash_ext.rb b/devops-service/lib/core_ext/hash.rb
similarity index 100%
rename from devops-service/lib/hash_ext.rb
rename to devops-service/lib/core_ext/hash.rb
diff --git a/devops-service/lib/nil_class_ext.rb b/devops-service/lib/core_ext/nil_class.rb
similarity index 100%
rename from devops-service/lib/nil_class_ext.rb
rename to devops-service/lib/core_ext/nil_class.rb
diff --git a/devops-service/lib/core_ext/string.rb b/devops-service/lib/core_ext/string.rb
new file mode 100644
index 0000000..54475e3
--- /dev/null
+++ b/devops-service/lib/core_ext/string.rb
@@ -0,0 +1,27 @@
+class String
+  def present?
+    !empty?
+  end
+
+  def blank?
+    empty?
+  end
+
+  # from ActiveSupport
+  def underscore
+    gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    tr("-", "_").
+    downcase
+  end
+
+  def underscore_class
+    split('::').last.underscore
+  end
+
+  # rough simplification
+  def pluralize
+    "#{self}s"
+  end
+end
\ No newline at end of file
diff --git a/devops-service/lib/executors/expiration_scheduler.rb b/devops-service/lib/executors/expiration_scheduler.rb
new file mode 100644
index 0000000..5eee52f
--- /dev/null
+++ b/devops-service/lib/executors/expiration_scheduler.rb
@@ -0,0 +1,35 @@
+require "workers/delete_server_worker"
+
+module Devops
+  module Executor
+    class ExpirationScheduler
+      def initialize(expires, server)
+        @expires, @server = expires, server
+      end
+
+      def schedule_expiration!
+        return unless @expires
+        DeleteServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name)
+      end
+
+      def interval_in_seconds
+        interval = @expires.to_i
+        measure_unit = @expires.chars.last
+        case measure_unit
+        when 's'
+          interval
+        when 'm'
+          interval * 60
+        when 'h'
+          interval * 60 * 60
+        when 'd'
+          interval * 60 * 60 * 24
+        when 'w'
+          interval * 60 * 60 * 24 * 7
+        else
+          raise 'Wrong interval format'
+        end
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb
index 88ac412..a8aebc6 100644
--- a/devops-service/lib/executors/server_executor.rb
+++ b/devops-service/lib/executors/server_executor.rb
@@ -1,6 +1,5 @@
 require "lib/knife/knife_factory"
-require "workers/worker"
-require "workers/delete_server_worker"
+require "lib/executors/expiration_scheduler"
 require "hooks"
 require 'net/ssh'
 
@@ -9,12 +8,16 @@ module Devops
     class ServerExecutor
       include Hooks
 
-      RESULT_CODES = {
+      ERROR_CODES = {
         server_bootstrap_fail: 2,
+        server_cannot_update_tags: 3,
+        server_bootstrap_private_ip_unset: 4,
         server_not_in_chef_nodes: 5,
         server_bootstrap_unknown_error: 7,
         deploy_unknown_error: 6,
-        deploy_failed: 8
+        deploy_failed: 8,
+        creating_server_unknown_error: 9,
+        creating_server_in_cloud_failed: 10
       }
 
       # waiting for 5*60 seconds (5 min)
@@ -34,46 +37,32 @@ module Devops
       define_hook :before_bootstrap
       define_hook :after_bootstrap
 
-      before_deploy :create_run_list
+      before_deploy :add_run_list_to_deploy_info
+
+
+      attr_accessor :server, :deploy_env, :report, :project
 
       def initialize server, out, options={}
         if server
           @project = Devops::Db.connector.project(server.project)
           @deploy_env = @project.deploy_env(server.deploy_env)
         end
-        @knife_instance = KnifeFactory.instance
         @server = server
         @out = out
         @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
         @current_user = options[:current_user]
       end
 
-      def self.result_code(symbolic_code)
-        RESULT_CODES.fetch(symbolic_code)
+      def self.error_code(symbolic_code)
+        ERROR_CODES.fetch(symbolic_code)
       end
 
-      def self.symbolic_result_code(integer_code)
-        RESULT_CODES.key(integer_code) || :unknown_error
+      def self.symbolic_error_code(integer_code)
+        ERROR_CODES.key(integer_code) || :unknown_error
       end
 
-      def result_code(symbolic_code)
-        self.class.result_code(symbolic_code)
-      end
-
-      def report= r
-        @report = r
-      end
-
-      def project= p
-        @project = p
-      end
-
-      def deploy_env= e
-        @deploy_env = e
-      end
-
-      def server
-        @server
+      def error_code(symbolic_code)
+        self.class.error_code(symbolic_code)
       end
 
       def create_server_object options
@@ -109,7 +98,9 @@ module Devops
           res[:before] = self.run_hook :before_create
           @out << "Done\n"
 
-          return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
+          unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
+            return error_code(:creating_server_in_cloud_failed)
+          end
           mongo.server_insert @server
 
           @out << "\nAfter create hooks...\n"
@@ -118,7 +109,7 @@ module Devops
           @out.flush
           DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
 
-          schedule_expiration(@server)
+          schedule_expiration()
 
           unless options["without_bootstrap"]
             bootstrap_options = {
@@ -135,11 +126,15 @@ module Devops
           DevopsLogger.logger.error e.message
           roll_back
           mongo.server_delete @server.id
-          # return 5
-          return result_code(:server_not_in_chef_nodes)
+          error_code(:creating_server_unknown_error)
         end
       end
 
+      # options:
+      #   :run_list (optional)
+      #   :bootstrap_template (optional)
+      #   :chef_environment (optional)
+      #   :config (optional)
       def bootstrap options
         @out << "\n\nBootstrap...\n"
         @out.flush
@@ -151,7 +146,7 @@ module Devops
         @out << "Done\n"
         if @server.private_ip.nil?
           @out << "Error: Private IP is null"
-          return false
+          return error_code(:server_bootstrap_private_ip_unset)
         end
         ja = {
           :provider => @server.provider,
@@ -166,13 +161,7 @@ module Devops
 
         address = "#{@server.remote_user}@#{ip}"
 
-        cmd = 'ssh '
-        cmd << "-i #{cert_path} "
-        cmd << '-q '
-        cmd << '-o StrictHostKeyChecking=no '
-        cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
-        cmd << "#{address} 'exit'"
-        cmd << " 2>&1"
+        cmd = check_ssh_command(cert_path, address)
 
         @out << "\nWaiting for SSH..."
         @out << "\nTest command: '#{cmd}'\n"
@@ -181,16 +170,16 @@ module Devops
         retries_amount = 0
         begin
           sleep(5)
-          res = `#{cmd}`
+          res = execute_system_command(cmd)
           retries_amount += 1
-          if retries_amount > MAX_SSH_RETRIES_AMOUNT
+          if retries_amount >= MAX_SSH_RETRIES_AMOUNT
             @out.puts "Can not connect to #{address}"
             @out.puts res
             @out.flush
             DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
-            return result_code(:server_bootstrap_fail)
+            return error_code(:server_bootstrap_fail)
           end
-          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
+          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless last_command_successful?
         rescue ArgumentError => e
           @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
           @out.flush
@@ -200,7 +189,7 @@ module Devops
         provider = @server.provider_instance
         @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
 
-        r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
+        r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
 
         if r == 0
           @out << "Chef node name: #{@server.chef_node_name}\n"
@@ -217,10 +206,16 @@ module Devops
         else
           @out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
           @out.flush
-          result_code(:server_bootstrap_fail)
+          error_code(:server_bootstrap_fail)
         end
       end
 
+      # options:
+      #   :cert_path (required)
+      #   :run_list (optional)
+      #   :bootstrap_template (optional)
+      #   :chef_environment (optional)
+      #   :config (optional)
       def bootstrap_options attributes, options
         bootstrap_options = [
           "-x #{@server.remote_user}",
@@ -242,6 +237,7 @@ module Devops
         @out << "Done\n"
       end
 
+      # essentially, it just bootstrap and then deploy
       def two_phase_bootstrap options
         prepare_two_phase_bootstrap(options)
         # bootstrap phase
@@ -252,14 +248,14 @@ module Devops
           bootstrap_status = bootstrap(options)
 
           if bootstrap_status == 0
-            if check_server
+            if check_server_on_chef_server
               @out << "Server #{@server.chef_node_name} is created"
             else
               @out.puts "Can not find client or node on chef-server"
               roll_back
               @out.flush
               mongo.server_delete @server.id
-              return result_code(:server_not_in_chef_nodes)
+              return error_code(:server_not_in_chef_nodes)
             end
           else
             # @out << roll_back
@@ -269,22 +265,20 @@ module Devops
             DevopsLogger.logger.error msg
             @out.puts msg
             @out.flush
-            return result_code(:server_bootstrap_fail)
+            return error_code(:server_bootstrap_fail)
           end
         rescue => e
           @out << "\nError: #{e.message}\n"
           @out.flush
-          return result_code(:server_bootstrap_unknown_error)
+          return error_code(:server_bootstrap_unknown_error)
         end
 
         # deploy phase. Assume that all servers are bootstraped successfully here.
         begin
-          #raise "hello"
-          @out << "\n"
           run_list = compute_run_list
-          @out << "\nComputed run list: #{run_list.join(", ")}"
+          @out << "\n\nComputed run list: #{run_list.join(", ")}"
           @out.flush
-          @knife_instance.set_run_list(@server.chef_node_name, run_list)
+          knife_instance.set_run_list(@server.chef_node_name, run_list)
           deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
           deploy_status = deploy_server(deploy_info)
           if deploy_status == 0
@@ -294,19 +288,20 @@ module Devops
             msg << "\nDeploing server operation status was #{deploy_status}"
             DevopsLogger.logger.error msg
             @out << "\n" + msg + "\n"
-            result_code(:deploy_failed)
+            error_code(:deploy_failed)
           end
         rescue => e
           @out << "\nError: #{e.message}\n"
           DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
-          result_code(:deploy_unknown_error)
+          error_code(:deploy_unknown_error)
         end
       end
 
-      def check_server
-        @knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
+      def check_server_on_chef_server
+        knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
       end
 
+      # returns a hash with :chef_node, :chef_client and :server keys
       def unbootstrap
         k = Devops::Db.connector.key(@server.key)
         cert_path = k.path
@@ -346,35 +341,28 @@ module Devops
       end
 
       def deploy_server_with_tags tags, deploy_info
-        old_tags_str = nil
-        new_tags_str = nil
-        unless tags.empty?
-          old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
-          @out << "Server tags: #{old_tags_str}\n"
-          @knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
+        return deploy_server(deploy_info) if tags.empty?
 
-          new_tags_str = tags.join(" ")
-          @out << "Server new tags: #{new_tags_str}\n"
-          cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
-          unless cmd[1]
-            m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
-            DevopsLogger.logger.error(m)
-            @out << m + "\n"
-            return 3
-          end
-          DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
+        old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
+        new_tags_str = tags.join(" ")
+
+        @out.puts "Temporarily changing tags (#{old_tags_str}) to (#{new_tags_str})"
+        unless knife_instance.swap_tags(@server.chef_node_name, old_tags_str, new_tags_str)
+          m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
+          DevopsLogger.logger.error(m)
+          @out.puts m
+          return error_code(:server_cannot_update_tags)
         end
+        DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
 
-        r = deploy_server deploy_info
-
-        unless tags.empty?
-          @out << "Restore tags\n"
-          cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
-          DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
-          cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
-          DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
+        begin
+          deploy_result = deploy_server deploy_info
+        ensure
+          @out.puts "Restoring tags"
+          knife_instance.swap_tags(@server.chef_node_name, new_tags_str, old_tags_str)
+          DevopsLogger.logger.info("Restoring tags for #{@server.chef_node_name}: from #{new_tags_str} back to (#{old_tags_str})")
         end
-        return r
+        deploy_result
       end
 
       def deploy_server deploy_info
@@ -397,7 +385,7 @@ module Devops
               f.write json
             end
           end
-          @out << "Deploy Input Parameters:\n"
+          @out.puts "Deploy Input Parameters:"
           @out.puts json
           @out.flush
           cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
@@ -412,7 +400,7 @@ module Devops
         end
         @out.flush
         k = Devops::Db.connector.key(@server.key)
-        lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
+        lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
         r = /Chef\sClient\sfinished/i
 
         if lline && lline[r]
@@ -432,21 +420,11 @@ module Devops
 
       def delete_from_chef_server node_name
         {
-          :chef_node => @knife_instance.chef_node_delete(node_name),
-          :chef_client => @knife_instance.chef_client_delete(node_name)
+          :chef_node => knife_instance.chef_node_delete(node_name),
+          :chef_client => knife_instance.chef_client_delete(node_name)
         }
       end
 
-=begin
-      def delete_etc_chef s, cert_path
-        cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
-        cmd += " sudo " unless s.remote_user == "root"
-        cmd += "rm -Rf /etc/chef"
-        r = `#{cmd}`
-        raise(r) unless $?.success?
-      end
-=end
-
       def delete_server
         mongo = ::Devops::Db.connector
         if @server.static?
@@ -487,7 +465,7 @@ module Devops
         end
       end
 
-      def create_run_list out, deploy_info
+      def add_run_list_to_deploy_info out, deploy_info
         out << "\nGenerate run list hook...\n"
         if deploy_info["run_list"]
           out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
@@ -496,14 +474,6 @@ module Devops
         out << "Project run list: #{@project.run_list.join(", ")}\n"
         out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
         out << "Server run list: #{@server.run_list.join(", ")}\n"
-=begin
-        rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
-        if @server.stack
-          stack = Devops::Db.connector.stack(@server.stack)
-          out << "Stack run list: #{stack.run_list.join(", ")}\n"
-          rlist.merge(stack.run_list)
-        end
-=end
         deploy_info["run_list"] = compute_run_list
         out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
       end
@@ -511,47 +481,51 @@ module Devops
       def compute_run_list
         rlist = []
         [@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
-          rlist += sub_run_list if sub_run_list.is_a?(Array)
+          rlist += sub_run_list if sub_run_list
         end
-        rlist = Set.new(rlist)
         if @server.stack
           stack = Devops::Db.connector.stack(@server.stack)
-#          out << "Stack run list: #{stack.run_list.join(", ")}\n"
           srl = stack.run_list
-          rlist.merge(srl) if srl.is_a?(Array)
+          rlist += srl if srl
         end
-        rlist.to_a
+        rlist.uniq
       end
 
       private
 
-      def schedule_expiration(server)
-        expires = @deploy_env.expires
-        return unless expires
-        interval = interval_in_seconds(expires)
-        @out <<  "Planning expiration in #{expires}"
-        DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name)
-      end
-
-      def interval_in_seconds(interval_as_string)
-        interval = interval_as_string.to_i
-        measure_unit = interval_as_string.chars.last
-        case measure_unit
-        when 's'
-          interval
-        when 'm'
-          interval * 60
-        when 'h'
-          interval * 60 * 60
-        when 'd'
-          interval * 60 * 60 * 24
-        when 'w'
-          interval * 60 * 60 * 24 * 7
-        else
-          raise 'Wrong interval format'
+      def schedule_expiration
+        if @deploy_env.expires
+          @out <<  "Planning expiration in #{@deploy_env.expires}"
+          ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration!
         end
       end
 
+      def check_ssh_command(cert_path, address)
+        cmd = 'ssh '
+        cmd << "-i #{cert_path} "
+        cmd << '-q '
+        cmd << '-o StrictHostKeyChecking=no '
+        cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
+        cmd << "#{address} 'exit'"
+        cmd << " 2>&1"
+        cmd
+      end
+
+      # to simplify testing
+      # :nocov:
+      def execute_system_command(cmd)
+        `#{cmd}`
+      end
+
+      def last_command_successful?
+        $?.success?
+      end
+
+      def knife_instance
+        @knife_instance ||= KnifeFactory.instance
+      end
+      # :nocov:
+
     end
   end
 end
diff --git a/devops-service/lib/string_ext.rb b/devops-service/lib/string_ext.rb
deleted file mode 100644
index ce2e9ca..0000000
--- a/devops-service/lib/string_ext.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-class String
-  def present?
-    !empty?
-  end
-
-  def blank?
-    empty?
-  end
-end
\ No newline at end of file
diff --git a/devops-service/lib/string_helper.rb b/devops-service/lib/string_helper.rb
deleted file mode 100644
index 74a6727..0000000
--- a/devops-service/lib/string_helper.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-module StringHelper
-  extend self
-
-  # from Rails' ActiveSupport
-  def underscore(string)
-    string.gsub(/::/, '/').
-    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-    gsub(/([a-z\d])([A-Z])/,'\1_\2').
-    tr("-", "_").
-    downcase
-  end
-
-  def underscore_class(klass, without_ancestors=true)
-    class_name = if without_ancestors
-      klass.to_s.split('::').last
-    else
-      klass.to_s
-    end
-    StringHelper.underscore(class_name)
-  end
-
-  # from Rails' ActiveSupport
-  def camelize(term)
-    string = term.to_s
-    string = string.sub(/^[a-z\d]*/) { $&.capitalize }
-    string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
-    string.gsub!(/\//, '::')
-    string
-  end
-
-  # rough simplification
-  def pluralize(string)
-    "#{string}s"
-  end
-end
diff --git a/devops-service/lib/users_permissions_updater.rb b/devops-service/migrations/users_permissions_updater.rb
similarity index 100%
rename from devops-service/lib/users_permissions_updater.rb
rename to devops-service/migrations/users_permissions_updater.rb
diff --git a/devops-service/spec/connectors/provider_accounts_connector_spec.rb b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
new file mode 100644
index 0000000..2500179
--- /dev/null
+++ b/devops-service/spec/connectors/provider_accounts_connector_spec.rb
@@ -0,0 +1,58 @@
+require 'db/mongo/models/provider_accounts/ec2_provider_account'
+require 'db/mongo/models/provider_accounts/openstack_provider_account'
+require 'db/mongo/connectors/provider_account'
+require 'spec/connectors/tester_connector/provider_account'
+
+RSpec.describe Connectors::ProviderAccount, type: :connector do
+  set_tester_connector TesterConnector::ProviderAccount
+
+  include_examples 'mongo connector', {
+    model_name: :provider_account,
+    factory_name: :ec2_provider_account,
+    only: [:insert, :delete],
+    field_to_update: :description
+  }
+
+  describe '#provider_accounts', cleanup_after: :all do
+    before(:all) do
+      @tester_connector.create(id: 'foo', provider: 'ec2')
+      @tester_connector.create(id: 'bar', provider: 'openstack')
+    end
+
+    it 'returns array of Ec2ProviderAccount if @provider is ec2' do
+      expect(
+        @connector.provider_accounts('ec2')
+      ).to be_an_array_of(Devops::Model::Ec2ProviderAccount).and have_size(1)
+    end
+
+    it 'returns array of Ec2ProviderAccount if @provider is openstack' do
+      expect(
+        @connector.provider_accounts('openstack')
+      ).to be_an_array_of(Devops::Model::OpenstackProviderAccount).and have_size(1)
+    end
+  end
+
+  describe '#provider_account', cleanup_after: :all do
+    before(:all) do
+      @tester_connector.create(id: 'foo', provider: 'ec2')
+      @tester_connector.create(id: 'bar', provider: 'openstack')
+    end
+
+    it 'returns ec2 provider account' do
+      acc = @connector.provider_account('ec2', 'foo')
+      expect(acc).to be_a(Devops::Model::Ec2ProviderAccount)
+      expect(acc.account_name).to eq 'foo'
+    end
+
+    it 'returns openstack provider account' do
+      acc = @connector.provider_account('openstack', 'bar')
+      expect(acc).to be_a(Devops::Model::OpenstackProviderAccount)
+      expect(acc.account_name).to eq 'bar'
+    end
+
+    it 'raises error if account is missing' do
+      expect{@connector.provider_account('ec2', 'missing')}.to raise_error(RecordNotFound)
+    end
+  end
+
+end
diff --git a/devops-service/spec/connectors/tester_connector/provider_account.rb b/devops-service/spec/connectors/tester_connector/provider_account.rb
new file mode 100644
index 0000000..38a8feb
--- /dev/null
+++ b/devops-service/spec/connectors/tester_connector/provider_account.rb
@@ -0,0 +1,6 @@
+require_relative 'base'
+
+module TesterConnector
+  class ProviderAccount < Base
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/expiration_scheduler_spec.rb
new file mode 100644
index 0000000..6964e5d
--- /dev/null
+++ b/devops-service/spec/executors/expiration_scheduler_spec.rb
@@ -0,0 +1,47 @@
+require 'lib/executors/expiration_scheduler'
+
+RSpec.describe Devops::Executor::ExpirationScheduler do
+  let(:server) { build(:server) }
+
+  describe '#schedule_expiration!' do
+    it 'schedules server deleting at given time' do
+      expect(DeleteServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name')
+      described_class.new('2m', server).schedule_expiration!
+    end
+
+    it "doesn't schedule job if expires is nil" do
+      expect(DeleteServerWorker).not_to receive(:perform_in)
+      described_class.new(nil, server).schedule_expiration!
+    end
+  end
+
+  describe '#interval_in_seconds' do
+    def interval_in_seconds(expires)
+      described_class.new(expires, server).interval_in_seconds
+    end
+
+    it 'recognizes seconds' do
+      expect(interval_in_seconds('2s')).to eq 2
+    end
+
+    it 'recognizes minutes' do
+      expect(interval_in_seconds('3m')).to eq 180
+    end
+
+    it 'recognizes hours' do
+      expect(interval_in_seconds('1h')).to eq 3600
+    end
+
+    it 'recognizes days' do
+      expect(interval_in_seconds('1d')).to eq 86400
+    end
+
+    it 'recognizes weeks' do
+      expect(interval_in_seconds('1w')).to eq 604800
+    end
+
+    it 'raises on wrong format' do
+      expect { interval_in_seconds('wrong') }.to raise_error(StandardError)
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb
new file mode 100644
index 0000000..de70722
--- /dev/null
+++ b/devops-service/spec/executors/server_executor_spec.rb
@@ -0,0 +1,792 @@
+require 'lib/executors/server_executor'
+
+RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do
+  let(:project) { build(:project) }
+  let(:deploy_env) { project.deploy_env('foo') }
+  let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
+  let(:output) { File.open(File::NULL, "w") }
+  let(:provider) { double('Provider instance') }
+  let(:executor) { described_class.new(server, output) }
+
+
+  before do
+    allow(stubbed_connector).to receive(:project) { project }
+    allow(executor.deploy_env).to receive(:provider_instance) { provider }
+    allow(server).to receive(:provider_instance) { provider }
+  end
+
+  describe '#initialize' do
+    it 'sets server, project, deploy_env, out instance variables' do
+      expect(executor.server).to eq server
+      expect(executor.deploy_env).to eq deploy_env
+      expect(executor).to have_instance_variable_value(:project, project)
+      expect(executor).to have_instance_variable_value(:out, output)
+    end
+
+    it 'defines :flush method on @out if it is absent' do
+      out = Class.new.new
+      expect(out).not_to respond_to(:flush)
+      described_class.new(server, out)
+      expect(out).to respond_to(:flush)
+    end
+
+    it 'sets current_user from options' do
+      user = double
+      executor = described_class.new(server, '', {current_user: user})
+      expect(executor).to have_instance_variable_value(:current_user, user)
+    end
+  end
+
+
+  describe '#report=' do
+    it 'sets report instance variable' do
+      executor.report= 'foo'
+      expect(executor).to have_instance_variable_value(:report, 'foo')
+    end
+  end
+
+  describe '#project=' do
+    it 'sets project instance variable' do
+      executor.project= 'foo'
+      expect(executor).to have_instance_variable_value(:project, 'foo')
+    end
+  end
+
+
+  describe '.symbolic_error_code' do
+    it 'returns symbol given an integer' do
+      expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail
+    end
+
+    it "returns :unknown_error if can't recognize error code" do
+      expect(described_class.symbolic_error_code(123)).to eq :unknown_error
+    end
+  end
+
+
+
+  describe '#create_server_object' do
+    it 'builds Server object' do
+      server = executor.create_server_object('created_by' => 'me')
+      expect(server).to be_a(Devops::Model::Server)
+      expect(server.project).to eq 'my_project'
+      expect(server.deploy_env).to eq 'foo'
+      expect(server.created_by).to eq 'me'
+    end
+  end
+
+
+
+  describe '#create_server' do
+    let(:image) { double('Image instance', remote_user: 'remote_user') }
+    let(:create_server) {
+      executor.create_server(
+        'created_by' => 'user',
+        'run_list' => @run_list,
+        'name' => 'node_name',
+        'key' => @key,
+        'without_bootstrap' => @without_bootstrap
+      )
+    }
+
+    before do
+      allow(provider).to receive(:create_server) { true }
+      allow(stubbed_connector).to receive(:image) { image }
+      allow(stubbed_connector).to receive(:server_insert)
+      @without_bootstrap = true
+      @run_list = %w(role[asd])
+      @key = 'key'
+    end
+
+    it 'builds server model from given options' do
+      create_server
+      expect(executor.server.created_by).to eq 'user'
+      expect(executor.server.chef_node_name).to eq 'node_name'
+      expect(executor.server.key).to eq @key
+      expect(executor.server.run_list).to eq @run_list
+    end
+
+    it 'sets run list to an empty array by default' do
+      @run_list = nil
+      create_server
+      expect(executor.server.run_list).to eq []
+    end
+
+    it 'sets key to default provider ssh key by default' do
+      @key = nil
+      allow(provider).to receive(:ssh_key) { 'default_key' }
+      create_server
+      expect(executor.server.key).to eq 'default_key'
+    end
+
+    it 'runs hooks' do
+      expect(executor).to receive(:run_hook).with(:before_create).ordered
+      expect(executor).to receive(:run_hook).with(:after_create).ordered
+      create_server
+    end
+
+    it 'creates server in cloud' do
+      expect(provider).to receive(:create_server).with(
+        an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output
+      )
+      create_server
+    end
+
+    it 'inserts built server into mongo' do
+      expect(stubbed_connector).to receive(:server_insert)
+      create_server
+    end
+
+    it 'schedules expiration for server' do
+      deploy_env.expires = '2m'
+      allow(DeleteServerWorker).to receive(:perform_in)
+      expect(DeleteServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
+      create_server
+    end
+
+    it "doesn't schedule expiration if deploy_env.expires is nil" do
+      deploy_env.expires = nil
+      expect(DeleteServerWorker).not_to receive(:perform_in)
+      create_server
+    end
+
+    context 'without_bootstrap option is false' do
+      it 'launches bootstrap' do
+        @without_bootstrap = false
+        allow(image).to receive(:bootstrap_template) { 'template' }
+        allow(executor).to receive(:two_phase_bootstrap)
+        expect(executor).to receive(:two_phase_bootstrap)
+        create_server
+      end
+    end
+
+    context 'without_bootstrap option is nil' do
+      it 'launches bootstrap' do
+        @without_bootstrap = nil
+        allow(image).to receive(:bootstrap_template) { 'template' }
+        allow(executor).to receive(:two_phase_bootstrap)
+        expect(executor).to receive(:two_phase_bootstrap)
+        create_server
+      end
+    end
+
+    context 'without_bootstrap option is true' do
+      it "doesn't launch bootstrap" do
+        @without_bootstrap = true
+        expect(executor).not_to receive(:two_phase_bootstrap)
+        create_server
+      end
+    end
+
+    context 'if error has been raised during execution' do
+      before do
+        allow(stubbed_connector).to receive(:server_delete)
+        allow(provider).to receive(:create_server) { raise }
+      end
+
+      it 'rollbacks server creating' do
+        expect(executor).to receive(:roll_back)
+        create_server
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete)
+        create_server
+      end
+    end
+
+    context "if creating server in cloud wasn't successful" do
+      it 'returns creating_server_in_cloud_failed error code' do
+        allow(provider).to receive(:create_server) { false }
+        expect(create_server).to eq 10
+      end
+    end
+  end
+
+
+
+  describe '#bootstrap', stubbed_knife: true do
+    let(:image) { double('Key instance', path: 'path') }
+    let(:bootstrap) { executor.bootstrap({}) }
+
+    before do
+      allow(executor).to receive(:sleep)
+      allow(executor).to receive(:last_command_successful?).and_return(true)
+      allow(executor).to receive(:execute_system_command)
+      allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node')
+      allow(stubbed_connector).to receive(:key).and_return(image)
+      allow(stubbed_connector).to receive(:server_set_chef_node_name)
+      allow(stubbed_knife).to receive(:knife_bootstrap).and_return(0)
+    end
+
+    it 'run before hook' do
+      expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered
+      expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered
+      bootstrap
+    end
+
+    context "when server's private ip is unset" do
+      it 'returns server_bootstrap_private_ip_unset error code' do
+        server.private_ip = nil
+        expect(bootstrap).to eq 4
+      end
+    end
+
+    it 'tries to ssh to server' do
+      expect(executor).to receive(:execute_system_command).with(/ssh/)
+      bootstrap
+    end
+
+    context "couldn't ssh to server" do
+      before { allow(executor).to receive(:last_command_successful?) { false } }
+
+      it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do
+        max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT
+        expect(executor).to receive(:execute_system_command).exactly(max_retries).times
+        bootstrap
+      end
+
+      it 'returns server_bootstrap_fail error code' do
+        expect(bootstrap).to eq 2
+      end
+    end
+
+
+    context 'after successful ssh check' do
+      before { allow(executor).to receive(:last_command_successful?).and_return(false, true) }
+
+      it "sets default chef node name if it's nil" do
+        executor.server.chef_node_name = nil
+        expect {bootstrap}.to change {executor.server.chef_node_name}.to 'chef_node'
+      end
+
+      it 'executes knife bootstrap' do
+        expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array))
+        bootstrap
+      end
+
+      it "bootstraps to public ip if it's set" do
+        server.public_ip = '8.8.8.8'
+        expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array))
+        bootstrap
+      end
+
+      context 'after successful bootstrap' do
+        it "updates server's chef node name in db" do
+          expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server))
+          bootstrap
+        end
+      end
+
+      context "if bootstraping wasn't successful" do
+        before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) }
+
+        it 'returns :server_bootstrap_fail code' do
+          expect(bootstrap).to eq 2
+        end
+
+        it "doesn't run after hook" do
+          expect(executor).to receive(:run_hook).with(:before_bootstrap, output)
+          bootstrap
+        end
+      end
+    end
+  end
+
+
+
+  describe '#two_phase_bootstrap', stubbed_knife: true do
+    let(:two_phase_bootstrap) { executor.two_phase_bootstrap({}) }
+
+    before do
+      allow(provider).to receive(:run_list) {[]}
+      allow(stubbed_connector).to receive(:server_delete)
+    end
+
+    context 'when bootstrap was successful' do
+      before do
+        allow(executor).to receive(:bootstrap) { 0 }
+        allow(executor).to receive(:check_server_on_chef_server) { false }
+      end
+
+      context 'if node presents on chef server' do
+        before do
+          allow(executor).to receive(:check_server_on_chef_server) { true }
+          allow(executor).to receive(:deploy_server)
+          allow(stubbed_knife).to receive(:set_run_list)
+        end
+
+        it 'builds run list' do
+          expect(executor).to receive(:compute_run_list)
+          two_phase_bootstrap
+        end
+
+        it 'sets run list to chef node' do
+          expect(stubbed_knife).to receive(:set_run_list)
+          two_phase_bootstrap
+        end
+
+        it 'deploys server' do
+          expect(executor).to receive(:deploy_server)
+          two_phase_bootstrap
+        end
+
+        context 'if deploy was successful' do
+          it 'returns 0' do
+            allow(executor).to receive(:deploy_server) { 0 }
+            expect(two_phase_bootstrap).to eq 0
+          end
+        end
+
+        context "if deploy wasn't successful" do
+          it 'returns :deploy_failed code' do
+            allow(executor).to receive(:deploy_server) { 1 }
+            expect(two_phase_bootstrap).to eq 8
+          end
+        end
+
+        context 'when an error occured during deploy' do
+          it 'returns :deploy_unknown_error code' do
+            allow(executor).to receive(:deploy_server) { raise }
+            expect(two_phase_bootstrap).to eq 6
+          end
+        end
+      end
+
+      context "if node doesn't present on chef server" do
+        it 'roll backs and then deletes server from mongo' do
+          allow(executor).to receive(:check_server_on_chef_server) { false }
+          allow(executor).to receive(:roll_back)
+          allow(stubbed_connector).to receive(:server_delete)
+          expect(executor).to receive(:roll_back).ordered
+          expect(stubbed_connector).to receive(:server_delete).ordered
+          two_phase_bootstrap
+        end
+      end
+    end
+
+    context "when bootstrap wasn't successful" do
+      it 'returns :server_bootstrap_fail error code' do
+        allow(executor).to receive(:bootstrap) { 1 }
+        expect(two_phase_bootstrap).to eq 2
+      end
+    end
+
+    context 'when an error occured during bootstrap' do
+      it 'returns :server_bootstrap_unknown_error error code' do
+        allow(executor).to receive(:bootstrap) { raise }
+        expect(two_phase_bootstrap).to eq 7
+      end
+    end
+  end
+
+  describe '#check_server_on_chef_server', stubbed_knife: true do
+    before do
+      server.chef_node_name = 'a'
+      allow(stubbed_knife).to receive(:chef_node_list) { @node_list }
+      allow(stubbed_knife).to receive(:chef_client_list) { @client_list }
+    end
+
+    it 'returns true when node_name in node list and in client list' do
+      @node_list = %w(a); @client_list = %w(a)
+      expect(executor.check_server_on_chef_server).to be true
+    end
+
+    it "returns false if node name isn't in node list" do
+      @node_list = []; @client_list = %w(a)
+      expect(executor.check_server_on_chef_server).to be false
+    end
+
+    it "returns false if node name isn't in node list" do
+      @node_list = %w(a); @client_list = []
+      expect(executor.check_server_on_chef_server).to be false
+    end
+  end
+
+
+  describe '#unbootstrap', stubbed_knife: true do
+    before do
+      allow(stubbed_connector).to receive_message_chain('key.path') { 'path_to_key' }
+      allow(stubbed_knife).to receive(:chef_node_delete)
+      allow(stubbed_knife).to receive(:chef_client_delete)
+      allow(executor).to receive(:execute_system_command) { '' }
+      allow(executor).to receive(:last_command_successful?) { true }
+      allow(executor).to receive(:sleep)
+      allow(Net::SSH).to receive(:start)
+    end
+
+    it 'connects by ssh' do
+      expect(Net::SSH).to receive(:start)
+      executor.unbootstrap
+    end
+
+    it 'returns hash with error after 5 unsuccessful retries' do
+      allow(Net::SSH).to receive(:start) { raise }
+      expect(Net::SSH).to receive(:start).exactly(5).times
+      expect(executor.unbootstrap).to be_a(Hash).and include(:error)
+    end
+  end
+
+
+  describe '#deploy_server_with_tags', stubbed_knife: true do
+    let(:current_tags) { @current_tags }
+    let(:initial_tags) { %w(a b) }
+    let(:joined_initial_tags) { initial_tags.join(' ') }
+    let(:given_tags) { %w(c d) }
+    let(:joined_given_tags) { given_tags.join(' ') }
+    let(:deploy_server_with_tags) { executor.deploy_server_with_tags(given_tags, {}) }
+
+    before do
+      @current_tags = initial_tags.dup
+      allow(stubbed_knife).to receive(:tags_list) { @current_tags }
+      allow(stubbed_knife).to receive(:swap_tags) do |_, tags_to_delete, tags_to_add|
+        @current_tags -= tags_to_delete.split
+        @current_tags += tags_to_add.split
+      end
+      allow(executor).to receive(:deploy_server)
+    end
+
+    context 'when tags are empty' do
+      it 'just deploys server' do
+        expect(executor).to receive(:deploy_server)
+        expect(stubbed_knife).not_to receive(:swap_tags)
+        executor.deploy_server_with_tags([], {})
+      end
+    end
+
+    context 'when tags are not empty' do
+      it 'temporarily swaps current_tags with given ones, deploys server and then restores tags' do
+        expect(stubbed_knife).to receive(:tags_list).ordered
+        expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_initial_tags, joined_given_tags).ordered
+        expect(executor).to receive(:deploy_server).ordered
+        expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_given_tags, joined_initial_tags).ordered
+        deploy_server_with_tags
+      end
+    end
+
+    context 'if error occures during deploy' do
+      it 'restores tags anyway' do
+        allow(executor).to receive(:deploy_server) { raise }
+        expect {
+          deploy_server_with_tags
+        }.to raise_error StandardError
+        expect(current_tags).to eq initial_tags
+      end
+    end
+
+    context 'if cannot add tags to server' do
+      it 'returns :server_cannot_update_tags code' do
+        allow(stubbed_knife).to receive(:swap_tags) { false }
+        expect(deploy_server_with_tags).to eq 3
+      end
+    end
+  end
+
+
+
+  describe '#deploy_server', stubbed_knife: true do
+    let(:deploy_info) { @deploy_info }
+    let(:json_file_name) { 'json.json' }
+    let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) }
+
+    let(:deploy_server) { executor.deploy_server(deploy_info) }
+
+    before do
+      allow(executor).to receive(:run_hook).with(:before_deploy, any_args)
+      allow(executor).to receive(:run_hook).with(:after_deploy, any_args)
+      allow(stubbed_knife).to receive(:ssh_stream) { 'Chef Client finished'}
+      allow(stubbed_connector).to receive(:key) { double('Key', path: 'path_to_key') }
+      allow(stubbed_connector).to receive(:server_update)
+      @deploy_info = {}
+    end
+
+
+    it 'runs before_deploy and after_deploy hooks' do
+      expect(executor).to receive(:run_hook).with(:before_deploy, any_args).ordered
+      expect(executor).to receive(:run_hook).with(:after_deploy, any_args).ordered
+      deploy_server
+    end
+
+    context 'when uses json file' do
+      before(:all) do
+        @tmp_files_at_start = Dir.entries(SpecSupport.tmp_dir)
+      end
+      before do
+        allow(DevopsConfig).to receive(:config).and_return({
+          project_info_dir: SpecSupport.tmp_dir,
+          address: 'host.com',
+          port: '8080',
+          url_prefix: 'api'
+        })
+        deploy_info['use_json_file'] = true
+      end
+
+      after(:all) do
+        diff = Dir.entries(SpecSupport.tmp_dir) - @tmp_files_at_start
+        diff.each do |file|
+          FileUtils.rm(File.join(SpecSupport.tmp_dir, file))
+        end
+      end
+
+
+      it 'writes deploy_info to json file if it not exists' do
+        expect { deploy_server }.to change { Dir.entries(SpecSupport.tmp_dir)}
+      end
+
+      it "writes deploy_info to given json file name if it doesn't exist" do
+        FileUtils.rm(json_file_path) if File.exists?(json_file_path)
+        deploy_info['json_file'] = json_file_name
+        expect { deploy_server }.to change {
+          Dir.entries(SpecSupport.tmp_dir)
+        }
+        FileUtils.rm(json_file_path)
+      end
+
+      it 'reads json from file if it exists' do
+        deploy_info['json_file'] = json_file_name
+        File.open(json_file_path, 'w') { |file| file.puts '{"foo": "bar"'}
+        expect { deploy_server }.not_to change {
+          Dir.entries(SpecSupport.tmp_dir)
+        }
+        FileUtils.rm(json_file_path)
+      end
+
+      it 'adds link to json to deploy command' do
+        deploy_info['json_file'] = json_file_name
+        regexp = %r(-j http://host.com:8080/api/v2.0/deploy/data/#{json_file_name})
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, regexp, any_args)
+        deploy_server
+      end
+    end
+
+    context "doesn't use json file" do
+      before do
+        deploy_info['use_json_file'] = false
+        deploy_info['run_list'] = %w(foo bar)
+      end
+
+      it "adds run list to command if server's stack is set" do
+        server.stack = 'stack'
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, %r(-r foo,bar), any_args)
+        deploy_server
+      end
+
+      it "doesn't add run list to command if server's stack is unset" do
+        expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args)
+        deploy_server
+      end
+    end
+
+    it "uses server's key" do
+      expect(stubbed_connector).to receive(:key).with('key_id')
+      expect(stubbed_knife).to receive(:ssh_stream).with(any_args, 'path_to_key')
+      deploy_server
+    end
+
+    it "uses public ip if it's set" do
+      server.public_ip = '127.0.0.1'
+      expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, '127.0.0.1', any_args)
+      deploy_server
+    end
+
+    it "uses private_ip if public_ip isn't set" do
+      expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, server.private_ip, any_args)
+      deploy_server
+    end
+
+    context 'if deploy was successful' do
+      it "updates server's last operation" do
+        expect(server).to receive(:set_last_operation).with('deploy', anything)
+        expect(stubbed_connector).to receive(:server_update).with(server)
+        deploy_server
+      end
+
+      it 'returns 0' do
+        expect(deploy_server).to eq 0
+      end
+    end
+
+    context "when deploy wasn't successful" do
+      before { allow(stubbed_knife).to receive(:ssh_stream) { 'fail'} }
+      it "doesn't run after_deploy hook" do
+        expect(executor).to receive(:run_hook).with(:before_deploy, any_args)
+        expect(executor).not_to receive(:run_hook).with(:after_deploy, any_args)
+        deploy_server
+      end
+
+      it 'returns 1' do
+        expect(deploy_server).to eq 1
+      end
+    end
+  end
+
+
+  describe '#delete_from_chef_server', stubbed_knife: true do
+    let(:delete_from_chef_server) { executor.delete_from_chef_server('foo') }
+    before do
+      allow(stubbed_knife).to receive(:chef_client_delete)
+      allow(stubbed_knife).to receive(:chef_node_delete)
+      delete_from_chef_server
+    end
+
+    it 'returns hash with :chef_node and :chef_client keys' do
+      expect(delete_from_chef_server).to be_a(Hash).and include(:chef_node, :chef_client)
+    end
+
+    it 'calls to :chef_node_delete and :chef_client_delete' do
+      expect(stubbed_knife).to have_received(:chef_client_delete)
+      expect(stubbed_knife).to have_received(:chef_node_delete)
+    end
+  end
+
+
+  describe '#delete_server' do
+    let(:delete_server) { executor.delete_server }
+
+    context 'when server is static' do
+      before do
+        server.provider = 'static'
+        allow(stubbed_connector).to receive(:server_delete).with(server.id)
+        allow(executor).to receive(:unbootstrap)
+      end
+
+      it 'performs unbootstrap' do
+        expect(executor).to receive(:unbootstrap)
+        delete_server
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete).with(server.id)
+        delete_server
+      end
+
+      it 'returns message and nil' do
+        expect(delete_server.first).to be_a(String)
+        expect(delete_server.last).to be nil
+      end
+
+      it "doesn't try to remove it from cloud" do
+        expect{delete_server}.not_to raise_error
+      end
+    end
+
+    context "when server isn't static", stubbed_knife: true do
+      before do
+        allow(server).to receive_message_chain('provider_instance.delete_server')
+        allow(stubbed_connector).to receive(:server_delete).with(server.id)
+        allow(stubbed_knife).to receive(:chef_node_delete)
+        allow(stubbed_knife).to receive(:chef_client_delete)
+      end
+
+      it 'deletes from info about note chef server' do
+        allow(executor).to receive(:delete_from_chef_server).and_call_original
+        expect(executor).to receive(:delete_from_chef_server)
+        delete_server
+      end
+
+      it "doesn't unbootstrap server" do
+        expect(executor).not_to receive(:unbootstrap)
+        delete_server
+      end
+
+      it 'deletes server from cloud' do
+        expect(server).to receive_message_chain('provider_instance.delete_server').with(server)
+        delete_server
+      end
+
+      it "doesn't raise error if server wasn't found in cloud" do
+        allow(server).to receive_message_chain('provider_instance.name')
+        allow(server).to receive_message_chain('provider_instance.delete_server') {
+          raise Fog::Compute::OpenStack::NotFound
+        }
+        expect { delete_server }.not_to raise_error
+      end
+
+      it 'deletes server from mongo' do
+        expect(stubbed_connector).to receive(:server_delete).with(server.id)
+        delete_server
+      end
+
+      it 'returns message and hash with :chef_node, :chef_client and :server keys' do
+        expect(delete_server.first).to be_a(String)
+        expect(delete_server.last).to be_a(Hash).and include(:chef_client, :chef_node, :server)
+      end
+    end
+  end
+
+
+  describe '#rollback' do
+    before do
+      allow(executor).to receive(:delete_from_chef_server) { {} }
+      allow(server).to receive_message_chain('provider_instance.delete_server')
+    end
+
+    it "does nothing if server.id is nil" do
+      server.id = nil
+      expect(executor).not_to receive(:delete_from_chef_server)
+      expect(server).not_to receive(:provider_instance)
+      executor.roll_back
+    end
+
+    it 'deletes node from chef server and instance from cloud' do
+      expect(executor).to receive(:delete_from_chef_server)
+      expect(server).to receive_message_chain('provider_instance.delete_server')
+      executor.roll_back
+    end
+
+    it "doesn't raise if deleting server in cloud raises an error" do
+      allow(server).to receive_message_chain('provider_instance.delete_server') { raise }
+      expect { executor.roll_back }.not_to raise_error
+    end
+  end
+
+
+  describe '#add_run_list_to_deploy_info' do
+    it "doesn't change deploy info if it already includes run list" do
+      deploy_info = {'run_list' => %w(foo)}
+      expect {
+        executor.add_run_list_to_deploy_info(output, deploy_info)
+      }.not_to change { deploy_info }
+    end
+
+    it 'computes and adds run_list to deploy_info' do
+      deploy_info = {}
+      allow(executor).to receive(:compute_run_list) { %w(foo) }
+      expect(executor).to receive(:compute_run_list)
+      executor.add_run_list_to_deploy_info(output, deploy_info)
+      expect(deploy_info['run_list']).to eq %w(foo)
+    end
+  end
+
+  describe '#compute_run_list' do
+    before do
+      allow(deploy_env).to receive_message_chain('provider_instance.run_list') { %w(a) }
+      project.run_list = %w(b)
+      deploy_env.run_list = %w(c)
+      server.run_list = %w(d)
+    end
+
+    it "returns array with run list merged from provider's, project's, env's and server's run lists" do
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d))
+    end
+
+    it "includes stack's run list if stack is set", stubbed_connector: true do
+      server.stack = 'stack'
+      allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: %w(e)) }
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d e))
+    end
+
+    it "doesn't contain nils" do
+      server.run_list = nil
+      server.stack = 'stack'
+      allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: nil) }
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c))
+    end
+
+    it 'returns uniq elements' do
+      project.run_list = %w(a)
+      deploy_env.run_list = %w(a)
+      expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a d))
+    end
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/factories/key.rb b/devops-service/spec/factories/key.rb
index 06fb96e..ccc0085 100644
--- a/devops-service/spec/factories/key.rb
+++ b/devops-service/spec/factories/key.rb
@@ -3,7 +3,7 @@ require 'db/mongo/models/key'
 FactoryGirl.define do
   factory :key, class: Devops::Model::Key do
     id 'user_key'
-    path SpecSupport::BLANK_FILE
+    path SpecSupport.blank_file
     scope 'user'
   end
 end
\ No newline at end of file
diff --git a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
index 1886a1c..dd3c181 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
@@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvEc2, type: :model do
   let(:env) { build(:deploy_env_ec2) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
+    before do
+      provider_double = instance_double('Provider::Ec2',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
+    end
+
     it_behaves_like 'deploy env'
     it_behaves_like 'cloud deploy env'
   end
diff --git a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
index 96fdfd7..a7f180b 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
@@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
 RSpec.describe Devops::Model::DeployEnvOpenstack, type: :model do
   let(:env) { build(:deploy_env_openstack) }
 
-  describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do
+  describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
+    before do
+      provider_double = instance_double('Provider::Openstack',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(openstack) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_openstack, id: 'template')] }
+    end
+
     it_behaves_like 'deploy env'
     it_behaves_like 'cloud deploy env'
   end
diff --git a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
index 45ad446..58f170f 100644
--- a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
+++ b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
@@ -5,10 +5,10 @@ RSpec.describe Devops::Model::DeployEnvStatic, type: :model do
   let(:env) { build(:deploy_env_static) }
 
 
-  describe 'it inherits from deploy env', stubbed_logger: true do
+  describe 'it inherits from deploy env', stubbed_logger: true, stubbed_connector: true do
     before do
       allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(static))
-      allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
     end
 
     it_behaves_like 'deploy env'
diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb
index 6457ebb..5fc0521 100644
--- a/devops-service/spec/models/project_spec.rb
+++ b/devops-service/spec/models/project_spec.rb
@@ -3,7 +3,21 @@ require 'db/mongo/models/project'
 RSpec.describe Devops::Model::Project, type: :model do
   let(:project) { build(:project) }
 
-  describe 'validation rules:', stubbed_env_validators: true, stubbed_logger: true do
+  describe 'validation rules:', stubbed_connector: true, stubbed_logger: true do
+    before do
+      provider_double = instance_double('Provider::Ec2',
+        flavors: [{'id' => 'flavor'}],
+        networks: [{'default' => {'vpcId' => 'foo'}}],
+        groups: {'default' => nil},
+        images: [{'id' => 'image'}]
+      )
+      allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
+      allow(Provider::ProviderFactory).to receive(:get) { provider_double }
+      allow(stubbed_connector).to receive(:users_names) { %w(root) }
+      allow(stubbed_connector).to receive(:available_images) { %w(image) }
+      allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
+    end
+
     include_examples 'field type validation', :id, :not_nil, :non_empty_string
     include_examples 'field type validation', :deploy_envs, :not_nil, :non_empty_array
     include_examples 'field type validation', :description, :maybe_nil, :maybe_empty_string
@@ -23,6 +37,23 @@ RSpec.describe Devops::Model::Project, type: :model do
       project = build(:project, with_deploy_env_identifiers: ['foo', nil])
       expect(project).not_to be_valid
     end
+
+    describe 'components validation' do
+      it 'is valid with components with filenames' do
+        project.components = {'foo' => {'filename' => 'bar'}}
+        expect{project.validate_components}.not_to raise_error
+      end
+
+      it "isn't valid if components isn't a hash" do
+        project.components = []
+        expect{project.validate_components}.to raise_error InvalidRecord
+      end
+
+      it "raises InvalidRecord if one of componentsц hasn't filename" do
+        project.components = {'foo' => {}}
+        expect{project.validate_components}.to raise_error InvalidRecord
+      end
+    end
   end
 
   describe '.fields' do
diff --git a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
index 4e31025..0c17dce 100644
--- a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
+++ b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
@@ -1,6 +1,5 @@
 require 'spec_helper'
 
-# не пытайся выделить в shared_specs, фигня выйдет
 RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do
   let(:provider_account) { build(:openstack_provider_account) }
 
diff --git a/devops-service/spec/models/server_spec.rb b/devops-service/spec/models/server_spec.rb
index c744618..0beb489 100644
--- a/devops-service/spec/models/server_spec.rb
+++ b/devops-service/spec/models/server_spec.rb
@@ -44,6 +44,14 @@ RSpec.describe Devops::Model::Server, type: :model do
     end
   end
 
+  describe '.build_from_bson' do
+    it 'takes a hash and returns instance of Server model' do
+      model = described_class.build_from_bson('id' => 'foo')
+      expect(model).to be_an_instance_of(described_class)
+      expect(model.id).to eq 'foo'
+    end
+  end
+
   it '#to_hash_without_id returns not nil fields' do
     server = described_class.new('run_list' => [], 'project' =>  'asd')
     expect(server.to_hash_without_id.keys).to match_array(%w(run_list project))
diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
index 3386445..b94506f 100644
--- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
+++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
@@ -6,21 +6,23 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do
 
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
-    allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true }
-    allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} }
+    provider_double = instance_double('Provider::Ec2',
+      validate_stack_template: true,
+      store_stack_template: {'url' => 'template_url'}
+    )
+    allow(Provider::ProviderFactory).to receive(:get) { provider_double }
   end
 
   it_behaves_like 'stack template'
 
   it 'uploads file to S3' do
-    expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template')
-    params = {
-      'id' => 'foo',
+    result = described_class.create('id' => 'foo',
       'template_body' => '{}',
       'owner' => 'root',
       'provider' => 'ec2'
-    }
-    expect(described_class.create(params)).to be_an_instance_of(described_class)
+    )
+    expect(result).to be_an_instance_of(described_class)
+    expect(result.template_url).to eq 'template_url'
   end
 
 end
\ No newline at end of file
diff --git a/devops-service/spec/models/user_spec.rb b/devops-service/spec/models/user_spec.rb
index c060ff4..ad9bc69 100644
--- a/devops-service/spec/models/user_spec.rb
+++ b/devops-service/spec/models/user_spec.rb
@@ -79,13 +79,25 @@ RSpec.describe Devops::Model::User, type: :model do
     end
   end
 
+  describe '.build_from_bson' do
+    it 'builds User model from given hash and assigns id' do
+      model = described_class.build_from_bson('_id' => 'foo', 'username' => 'not shown', 'email' => 'baz')
+      expect(model.id).to eq 'foo'
+      expect(model.email).to eq 'baz'
+    end
+  end
+
   describe '#check_privileges' do
     it "raises InvalidPrivileges if user hasn't specified privilege" do
       expect { user.check_privileges('key', 'w') }.to raise_error(InvalidPrivileges)
     end
 
     it 'does nothing is user has specified privilege' do
-      user.check_privileges('key', 'r')
+      expect{user.check_privileges('key', 'r')}.not_to raise_error
+    end
+
+    it 'raises InvalidPrivileges if given privelege is wrong' do
+      expect{user.check_privileges('key', 't')}.to raise_error InvalidPrivileges
     end
   end
 
diff --git a/devops-service/spec/shared_contexts/stubbed_connector.rb b/devops-service/spec/shared_contexts/stubbed_connector.rb
index 7d6dabb..49b202a 100644
--- a/devops-service/spec/shared_contexts/stubbed_connector.rb
+++ b/devops-service/spec/shared_contexts/stubbed_connector.rb
@@ -1,5 +1,5 @@
 RSpec.shared_context 'stubbed calls to connector', stubbed_connector: true do
-  let(:stubbed_connector) { double() }
+  let(:stubbed_connector) { instance_double(MongoConnector) }
   before do
     allow(Devops::Db).to receive(:connector) { stubbed_connector }
   end
diff --git a/devops-service/spec/shared_contexts/stubbed_env_validators.rb b/devops-service/spec/shared_contexts/stubbed_env_validators.rb
index 69f081c..998f27d 100644
--- a/devops-service/spec/shared_contexts/stubbed_env_validators.rb
+++ b/devops-service/spec/shared_contexts/stubbed_env_validators.rb
@@ -1,9 +1,10 @@
 RSpec.shared_context 'stubbed calls to connector in env validators', stubbed_env_validators: true do
   before do
     allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2 openstack))
+    allow_any_instance_of(env_class).to receive_message_chain('provider_instance.flavors').and_return [{'id' => 'flavor'}]
     allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
-    allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
-    allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
+    # allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
+    # allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
     allow_any_instance_of(Validators::DeployEnv::Groups).to receive(:available_groups).and_return(['default'])
     allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])
     allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])
diff --git a/devops-service/spec/shared_contexts/stubbed_knife.rb b/devops-service/spec/shared_contexts/stubbed_knife.rb
new file mode 100644
index 0000000..042bcc7
--- /dev/null
+++ b/devops-service/spec/shared_contexts/stubbed_knife.rb
@@ -0,0 +1,6 @@
+RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do
+  let(:stubbed_knife) { instance_double(KnifeCommands) }
+  before do
+    allow(KnifeFactory).to receive(:instance) { stubbed_knife }
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/shared_contexts/stubbed_logger.rb b/devops-service/spec/shared_contexts/stubbed_logger.rb
index f2d79e9..486caa1 100644
--- a/devops-service/spec/shared_contexts/stubbed_logger.rb
+++ b/devops-service/spec/shared_contexts/stubbed_logger.rb
@@ -3,5 +3,6 @@ RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do
     allow(DevopsLogger).to receive_message_chain('logger.debug')
     allow(DevopsLogger).to receive_message_chain('logger.info')
     allow(DevopsLogger).to receive_message_chain('logger.error')
+    allow(DevopsLogger).to receive_message_chain('logger.warn')
   end
 end
\ No newline at end of file
diff --git a/devops-service/spec/spec_helper.rb b/devops-service/spec/spec_helper.rb
index 9f2500f..10f8ebf 100644
--- a/devops-service/spec/spec_helper.rb
+++ b/devops-service/spec/spec_helper.rb
@@ -4,32 +4,53 @@ require 'factory_girl'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/inflector'
 
-# setup load_path and require support files
-root = File.join(File.dirname(__FILE__), "..")
-$LOAD_PATH.push root unless $LOAD_PATH.include? root
-
-# suppress output
-original_stdout = $stdout
-$stdout = File.open(File::NULL, "w")
-
-Dir[("./spec/support/**/*.rb")].each { |f| require f }
-Dir[("./spec/shared_contexts/**/*.rb")].each { |f| require f }
-
-# Factory girl configuration
-FactoryGirl.define do
-  # do not try to persist, but raise validation errors
-  to_create { |model| model.validate! }
+def suppress_output!
+  original_stdout = $stdout
+  $stdout = File.open(File::NULL, "w")
+  RSpec.configure do |config|
+    config.after(:all) do
+      $stdout = original_stdout
+    end
+  end
 end
-FactoryGirl.find_definitions
+
+def check_coverage
+  require 'simplecov'
+  if ENV['JENKINS']
+    require 'simplecov-rcov'
+    SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+  end
+  SimpleCov.start do
+    add_filter { |src| src.filename =~ /spec\// }
+  end
+end
+
+def require_support_files
+  root = File.join(File.dirname(__FILE__), "..")
+  $LOAD_PATH.push root unless $LOAD_PATH.include?(root)
+  Dir[("#{root}/spec/support/**/*.rb")].each { |f| require f }
+  Dir[("#{root}/spec/shared_contexts/**/*.rb")].each { |f| require f }
+end
+
+def setup_factory_girl
+  FactoryGirl.define do
+    # do not persist, but raise validation errors
+    to_create { |model| model.validate! }
+  end
+  FactoryGirl.find_definitions
+  RSpec.configure { |config| config.include FactoryGirl::Syntax::Methods }
+end
+
+
+# extra configuration
+suppress_output!
+check_coverage if ENV['COVERAGE']
+require_support_files
+setup_factory_girl
+
 
 # RSpec configuration
 RSpec.configure do |config|
-  config.include FactoryGirl::Syntax::Methods
-
-  config.after(:all) do
-    $stdout = original_stdout
-  end
-
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest
   # assertions if you prefer.
diff --git a/devops-service/spec/support/instance_variable_matcher.rb b/devops-service/spec/support/instance_variable_matcher.rb
new file mode 100644
index 0000000..51f5097
--- /dev/null
+++ b/devops-service/spec/support/instance_variable_matcher.rb
@@ -0,0 +1,6 @@
+RSpec::Matchers.define :have_instance_variable_value do |name, value|
+  match do |actual|
+    actual.instance_variable_defined?("@#{name}") &&
+      actual.instance_variable_get("@#{name}") == value
+  end
+end
\ No newline at end of file
diff --git a/devops-service/spec/support/spec_support.rb b/devops-service/spec/support/spec_support.rb
index ab47bc6..69bd0e4 100644
--- a/devops-service/spec/support/spec_support.rb
+++ b/devops-service/spec/support/spec_support.rb
@@ -2,7 +2,15 @@ require 'yaml'
 
 module SpecSupport
   ROOT = File.join(__dir__, '../../')
-  BLANK_FILE = File.join(ROOT, 'spec/support/blank_file')
+
+  def self.blank_file
+    File.join(ROOT, 'spec/support/templates/blank_file')
+  end
+
+  # for specs which write files
+  def self.tmp_dir
+    File.join(ROOT, 'spec/support/tmp/')
+  end
 
   def self.db_params
     @db_params ||= begin
@@ -32,4 +40,8 @@ module SpecSupport
       end
     end
   end
+
+  def self.root
+    File.join(__dir__, '../../')
+  end
 end
\ No newline at end of file
diff --git a/devops-service/spec/support/blank_file b/devops-service/spec/support/templates/blank_file
similarity index 100%
rename from devops-service/spec/support/blank_file
rename to devops-service/spec/support/templates/blank_file
diff --git a/devops-service/spec/support/tmp/.gitkeep b/devops-service/spec/support/tmp/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb
index c8fe476..c4e02eb 100644
--- a/devops-service/workers/delete_server_worker.rb
+++ b/devops-service/workers/delete_server_worker.rb
@@ -1,6 +1,7 @@
 require "db/mongo/models/server"
 require "db/mongo/models/report"
 require "lib/executors/server_executor"
+require "workers/worker"
 
 class DeleteServerWorker < Worker
 
diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb
index 6ad25d7..da7f712 100644
--- a/devops-service/workers/stack_bootstrap_worker.rb
+++ b/devops-service/workers/stack_bootstrap_worker.rb
@@ -130,7 +130,7 @@ class StackBootstrapWorker < Worker
 
     @out.puts
     results.each do |chef_node_name, code|
-      human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code)
+      human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code)
       @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
     end
 
@@ -144,7 +144,7 @@ class StackBootstrapWorker < Worker
   def errors_in_bootstrapping_present?(result_codes)
     bootstrap_error_codes = []
     [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
-      bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
+      bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code)
     end
 
     (bootstrap_error_codes & result_codes).size > 0