CID-472: sync groups

devops-client/lib/devops-client/handler/stack.rb

@@ -41,30 +41,31 @@ class Stack < Handler
   end
 
   def create_handler
-    q = {}
+    attrs = {}
-
+    attrs[:project] = options[:project] || resources_selector.select_available_project
-    q[:without_bootstrap] = options[:without_bootstrap]
+    attrs[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(attrs[:project])
-    # q[:provider] = options[:provider] || resources_selector.select_available_provider
+    env = fetcher.fetch_project(attrs[:project])['deploy_envs'].detect {|env| env['identifier'] == attrs[:deploy_env]}
-    q[:project] = options[:project] || resources_selector.select_available_project
+    attrs[:provider] = env['provider']
-    q[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(q[:project])
-    env = fetcher.fetch_project(q[:project])['deploy_envs'].detect {|env| env['identifier'] == q[:deploy_env]}
-    q[:provider] = env['provider']
 
     params_filepath = options[:parameters_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.parameters_file'))
     if params_filepath.empty?
-      q[:parameters] = {}
+      attrs[:parameters] = {}
     else
-      q[:parameters] = JSON.parse(File.read(params_filepath))
+      attrs[:parameters] = JSON.parse(File.read(params_filepath))
     end
 
     tags_filepath = options[:tags_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.tags_file'))
     if tags_filepath.empty?
-      q[:tags] = {}
+      attrs[:tags] = {}
     else
-      q[:tags] = JSON.parse(File.read(tags_filepath))
+      attrs[:tags] = JSON.parse(File.read(tags_filepath))
     end
 
-    json = JSON.pretty_generate(q)
+    json = JSON.pretty_generate(
+      without_bootstrap: options[:without_bootstrap] || false,
+      skip_rollback: options[:skip_rollback] || false,
+      stack_attributes: attrs
+    )
     if question(I18n.t("handler.stack.question.create")) {puts json}
       job_ids = post_body "/stack", json
       reports_urls(job_ids)

devops-client/lib/devops-client/options/stack_options.rb

@@ -31,6 +31,7 @@ class StackOptions < CommonOptions
       parser.recognize_option_value(:tags_file)
       parser.recognize_option_value(:run_list)
       parser.recognize_option_value(:without_bootstrap, type: :switch, switch_value: true)
+      parser.recognize_option_value(:skip_rollback, type: :switch, switch_value: true)
     end
   end
 

devops-service/app/api2/handlers/provider_notification.rb

@@ -0,0 +1,22 @@
+require 'workers/stack_sync_worker'
+require_relative "request_handler"
+
+module Devops
+  module API2_0
+    module Handler
+      class ProviderNotification < RequestHandler
+
+        def autoscaling_groups_change(group_id, provider_account)
+          provider = ::Provider::ProviderFactory.get('ec2', provider_account)
+          stack_id = provider.stack_id_of_autoscaling_group(group_id)
+          stack = ::Devops::Db.connector.stack_by_id(stack_id)
+          jid = Worker.start_async(StackSyncWorker, stack_name: stack.name)
+          puts jid
+          jid
+        end
+
+      end
+    end
+  end
+end
+

devops-service/app/api2/handlers/stack.rb

@@ -21,16 +21,16 @@ module Devops
 
         def create_stack
           object = parser.create
-          project = Devops::Db.connector.project(object["project"])
+          stack_attrs = object['stack_attributes']
-          env = project.deploy_env(object["deploy_env"])
+          project = Devops::Db.connector.project(stack_attrs['project'])
+          env = project.deploy_env(stack_attrs['deploy_env'])
           raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil?
-          object["stack_template"] = env.stack_template
+          add_stack_attributes(stack_attrs, env, parser)
-          object["owner"] = parser.current_user
-          object["provider"] = env.provider
-          object["provider_account"] = env.provider_account
 
           jid = Worker.start_async(StackBootstrapWorker,
-            stack_attributes: object
+            stack_attributes: stack_attrs,
+            without_bootstrap: object['without_bootstrap'],
+            skip_rollback: object['skip_rollback']
           )
           [jid]
         end
@@ -48,7 +48,7 @@ module Devops
 
         def sync id
           stack = self.stack(id)
-          stack.sync!
+          stack.sync_status_and_events!
           Devops::Db.connector.stack_update(stack)
 
           stack
@@ -201,6 +201,15 @@ module Devops
           stack.change_stack_template!(stack_template_id)
         end
 
+        private
+
+        def add_stack_attributes(stack_attrs, env, parser)
+          stack_attrs['stack_template'] = env.stack_template
+          stack_attrs['owner'] = parser.current_user
+          stack_attrs['provider'] = env.provider
+          stack_attrs['provider_account'] = env.provider_account
+        end
+
       end
     end
   end

devops-service/app/api2/parsers/stack.rb

@@ -7,10 +7,11 @@ module Devops
 
         def create
           @body ||= create_object_from_json_body
-          project_name = check_string(@body["project"], "Parameter 'project' must be a not empty string")
+          stack_attributes = @body.fetch('stack_attributes')
-          env_name = check_string(@body["deploy_env"], "Parameter 'deploy_env' must be a not empty string")
+          project_name = check_string(stack_attributes["project"], "Parameter 'project' must be a not empty string")
-          check_string(@body["name"], "Parameter 'name' must be a not empty string", true, false)
+          env_name = check_string(stack_attributes["deploy_env"], "Parameter 'deploy_env' must be a not empty string")
-          list = check_array(@body["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true)
+          check_string(stack_attributes["name"], "Parameter 'name' must be a not empty string", true, false)
+          list = check_array(stack_attributes["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true)
           Validators::Helpers::RunList.new(list).validate! unless list.nil?
           @body
         end

devops-service/app/api2/routes/provider_notification.rb

@@ -0,0 +1,24 @@
+module Devops
+  module API2_0
+    module Routes
+      module ProviderNotificationRoutes
+
+        def self.registered(app)
+
+          # * *Request*
+          #   - method : POST
+          #   Checks if given autoscaling group is launched within stack that is handled by CID.
+          #   If so, starts syncing that stack. Otherwise returns 404 error.
+          #
+          # * *Returns* :
+          #   report_id
+          app.post_with_headers '/provider_notifications/aws/:provider_account/autoscaling_groups/:id/changes', :headers => [:accept] do |provider_account, group_id|
+            check_privileges("stack", "r")
+            json Devops::API2_0::Handler::ProviderNotification.new(request).autoscaling_groups_change(group_id, provider_account)
+          end
+
+        end
+      end
+    end
+  end
+end

devops-service/app/api2/routes/stack.rb

@@ -30,7 +30,7 @@ module Devops
           #     - Content-Type: application/json
           #   - body :
           #     {
-          #       "without_bootstrap": null,
+          #       "stack_attributes": {
           #         "project": "project_name",
           #         "deploy_env": "test",
           #         "provider": "ec2",
@@ -41,6 +41,9 @@ module Devops
           #           "KeyName": "Value"
           #         }
           #       }
+          #       "without_bootstrap": false,
+          #       "skip_rollback": false
+          #     }
           #
           # * *Returns* :
           #   [report_id]

devops-service/app/devops-api2.rb

@@ -19,9 +19,9 @@ module Devops
       require_relative "api2/handlers/server"
       require_relative "api2/handlers/image"
       require_relative "api2/handlers/project"
-
       require_relative "api2/handlers/stack"
       require_relative "api2/handlers/stack_template"
+      require_relative "api2/handlers/provider_notification"
 
       require 'lib/stubber'
     end
@@ -70,6 +70,7 @@ module Devops
       require_relative "api2/routes/stack_template"
       require_relative "api2/routes/statistic"
       require_relative "api2/routes/report"
+      require_relative "api2/routes/provider_notification"
 
       routes = Devops::API2_0::Routes.constants.collect{|s| Devops::API2_0::Routes.const_get(s)}.select {|const| const.class == Module}
       routes.each do |r|

devops-service/db/mongo/connectors/stack.rb

@@ -21,6 +21,14 @@ module Connectors
       collection.update({"_id" => id}, {"$set" => {"run_list" => run_list}})
     end
 
+    def lock_persisting_stack(id)
+      collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => true}})
+    end
+
+    def unlock_persisting_stack(id)
+      collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => false}})
+    end
+
     def stack(name, options={})
       query = {'name' => name}.merge(options)
       bson = collection.find(query).to_a.first
@@ -28,6 +36,13 @@ module Connectors
       model_from_bson(bson)
     end
 
+    def stack_by_id(id)
+      query = {'_id' => id}
+      bson = collection.find(query).to_a.first
+      raise RecordNotFound.new("'#{id}' not found") unless bson
+      model_from_bson(bson)
+    end
+
     private
 
     def model_from_bson(bson)

devops-service/db/mongo/models/report.rb

@@ -12,6 +12,10 @@ module Devops
       DEPLOY_STACK_TYPE = 6
       DELETE_SERVER_TYPE = 7
       EXPIRE_SERVER_TYPE = 8
+      SYNC_STACK_TYPE = 9
+
+
+      SYSTEM_OWNER = 'SYSTEM'
 
       attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
 

devops-service/db/mongo/models/stack/stack_base.rb

@@ -7,7 +7,7 @@ module Devops
 
       include ModelWithProvider
 
-      attr_accessor :id, :name, :project, :deploy_env, :stack_template, :parameters, :events, :owner, :run_list, :stack_status
+      attr_accessor :parameters, :events, :stack_status, :persisting_is_locked
 
       set_field_validators :id, [::Validators::FieldValidator::NotNil,
                                       ::Validators::FieldValidator::FieldType::String,
@@ -56,6 +56,7 @@ module Devops
         self.run_list = attrs['run_list'] || []
         self.tags = attrs['tags'] || {}
         self.stack_status = attrs['stack_status']
+        self.persisting_is_locked = attrs['persisting_is_locked']
         self
       end
 
@@ -70,7 +71,8 @@ module Devops
           stack_status: stack_status,
           owner: owner,
           run_list: run_list,
-          tags: tags
+          tags: tags,
+          persisting_is_locked: persisting_is_locked
         }.merge(provider_hash)
       end
 
@@ -87,7 +89,7 @@ module Devops
         provider_instance.delete_stack(self)
       end
 
-      def sync!
+      def sync_status_and_events!
         self.stack_status = provider_instance.stack_details(self)[:stack_status]
         self.events = provider_instance.stack_events(self)
       rescue Fog::AWS::CloudFormation::NotFound
@@ -111,6 +113,16 @@ module Devops
         Devops::Db.connector.stack_template(stack_template)
       end
 
+      def lock_persisting!
+        self.persisting_is_locked = true
+        Devops::Db.connector.lock_persisting_stack(id)
+      end
+
+      def unlock_persisting!
+        self.persisting_is_locked = false
+        Devops::Db.connector.unlock_persisting_stack(id)
+      end
+
       class << self
 
         # attrs should include:
@@ -122,7 +134,7 @@ module Devops
         def create(attrs, out)
           model = new(attrs)
           model.create_stack_in_cloud!(out)
-          model.sync!
+          model.sync_status_and_events!
           model
         end
 

devops-service/db/mongo/mongo_connector.rb

@@ -17,7 +17,8 @@ class MongoConnector
   delegate(
     [:images, :image, :image_insert, :image_delete, :image_update] => :images_connector,
     [:stack_templates, :stack_template, :stack_template_insert, :stack_template_delete, :stack_template_update] => :stack_templates_connector,
-    [:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list] => :stacks_connector,
+    [:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list,
+      :stack_by_id, :lock_persisting_stack, :unlock_persisting_stack] => :stacks_connector,
     [:available_images, :add_available_images, :delete_available_images] => :filters_connector,
     [:project, :projects_all, :projects, :project_names_with_envs,
     :projects_by_image, :projects_by_user, :project_insert, :project_update,

devops-service/lib/executors/server_executor.rb

@@ -131,7 +131,7 @@ module Devops
         res = self.run_hook(:before_bootstrap, @out)
         @out << "Done\n"
         if @server.private_ip.nil?
-          @out << "Error: Private IP is null"
+          @out.puts "Error: Private IP is null"
           return error_code(:server_bootstrap_private_ip_unset)
         end
         ja = {
@@ -238,13 +238,15 @@ module Devops
               @out << "Server #{@server.chef_node_name} is created"
             else
               @out.puts "Can not find client or node on chef-server"
-              roll_back
+              @out.puts "Skip rollback because :skip_rollback option is set"
+              roll_back unless options[:skip_rollback]
               @out.flush
               mongo.server_delete @server.id
               return error_code(:server_not_in_chef_nodes)
             end
           else
-            roll_back
+            @out.puts "Skip rollback because :skip_rollback option is set"
+            roll_back unless options[:skip_rollback]
             mongo.server_delete @server.id
             msg = "Failed while bootstraping server with id '#{@server.id}'\n"
             msg << "Bootstraping operation result was #{bootstrap_status}"

devops-service/lib/executors/stack_executor.rb

@@ -4,6 +4,7 @@ module Devops; module Executor; class StackExecutor; end; end; end # predeclare
 require_relative "stack_executor/stack_creation_waiter"
 require_relative "stack_executor/stack_servers_persister"
 require_relative "stack_executor/prioritized_groups_bootstrapper"
+require_relative "stack_executor/stack_servers_fetcher"
 
 module Devops
   module Executor
@@ -14,38 +15,52 @@ module Devops
       def initialize(options)
         @out = options.fetch(:out)
         @stack = options[:stack]
-      end
+        self.just_persisted_by_priority = {}
-
-      def wait_till_stack_is_created
-        wait_result = StackCreationWaiter.new(stack, out).sync
-
-        if wait_result.ok?
-          puts_and_flush "\nStack '#{stack.name}' has been created"
-          true
-        else
-          puts_and_flush "An error ocurred during stack creation: #{wait_result.reason}"
-          false
-        end
       end
 
       def create_stack(stack_attrs)
+        stack_attrs.merge!('persisting_is_locked' => true)
         @stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, out)
         mongo.stack_insert(@stack)
       end
 
-      def persist_stack_servers
+      def wait_till_stack_is_created
-        puts_and_flush 'Start saving stack servers into CID database.'
+        wait_result = waiter.wait
-        persister = StackServersPersister.new(stack, out)
+        stack.unlock_persisting!
-        persister.persist_new_servers
+        wait_result.ok?
-        puts_and_flush "Stack servers have been saved."
-        {
-          just_persisted_by_priority: persister.just_persisted_by_priority,
-          deleted: persister.deleted
-        }
       end
 
-      def bootstrap_servers_by_priority(servers_by_priorities, jid)
+      def persist_new_servers
-        PrioritizedGroupsBootstrapper.new(out, jid, servers_by_priorities).bootstrap_servers_by_priority
+        reload_stack
+        raise 'It seems that stack is synchronizing at the moment' if stack.persisting_is_locked
+        begin
+          stack.lock_persisting!
+          fetcher.new_servers_by_priorities.each do |priority, provider_infos|
+            servers = provider_infos.map {|info| persister.persist(info) }
+            just_persisted_by_priority[priority] = servers
+          end
+
+          just_persisted_by_priority.values.flatten.each do |server|
+            puts_and_flush "\n\nPersisted server #{server.id}: #{JSON.pretty_generate(server.to_hash)}"
+          end
+        ensure
+          stack.unlock_persisting!
+        end
+      end
+
+      def bootstrap_just_persisted(jid)
+        puts_and_flush "Bootstrapping just persisted servers" if just_persisted_by_priority.values.flatten.any?
+        just_persisted_by_priority.each do |priority, servers|
+          puts_and_flush "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
+        end
+        PrioritizedGroupsBootstrapper.new(out, jid, just_persisted_by_priority).bootstrap_servers_by_priority
+      end
+
+      def delete_stale_servers
+        fetcher.stale_servers.each do |server|
+          server_executor = Devops::Executor::ServerExecutor.new(server, out)
+          server_executor.delete_server
+        end
       end
 
       def delete_stack
@@ -56,10 +71,28 @@ module Devops
 
       private
 
+      attr_accessor :just_persisted_by_priority
+
       def mongo
         Devops::Db.connector
       end
 
+      def reload_stack
+        @stack = mongo.stack(@stack.name)
+      end
+
+      def fetcher
+        @fetcher ||= StackServersFetcher.new(stack, out)
+      end
+
+      def persister
+        @persister ||= StackServersPersister.new(@stack)
+      end
+
+      def waiter
+        StackCreationWaiter.new(stack, out)
+      end
+
     end
   end
 end

devops-service/lib/executors/stack_executor/chef_node_name_builder.rb

@@ -25,7 +25,8 @@ class Devops::Executor::StackExecutor
     def initialize(attrs)
       @server_info = attrs[:provider_server_info]
       @project, @env = attrs[:project_id], attrs[:env_id]
-      @mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK
+      @mask = @server_info['tags']['cid:node-name-mask'] if @server_info['tags']
+      @mask ||= DEFAULT_MASK
     end
 
 

devops-service/lib/executors/stack_executor/servers_bootstrapper.rb

@@ -28,7 +28,7 @@ class Devops::Executor::StackExecutor
 
       @server_bootstrap_jobs.map do |server_id, job_id|
         result = wait_for_bootstrap_job(job_id)
-        puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
+        puts_and_flush Devops::Messages.t("stack_executor.servers_bootstrapper.result.#{result.reason}", server_id: server_id, job_id: job_id)
         result
       end
     end
@@ -41,7 +41,8 @@ class Devops::Executor::StackExecutor
         job_id = Worker.start_async(::BootstrapWorker,
           server_attrs: server.to_mongo_hash,
           bootstrap_template: 'omnibus',
-          owner: server.created_by
+          owner: server.created_by,
+          skip_rollback: true
         )
         @out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
         @server_bootstrap_jobs[server.id] = job_id

devops-service/lib/executors/stack_executor/stack_creation_waiter.rb

@@ -22,12 +22,12 @@ class Devops::Executor::StackExecutor
       @sync_result = nil
     end
 
-    def sync
+    def wait
       puts_and_flush "Syncing stack '#{stack.id}'..."
 
       sleep_times.detect do |sleep_time|
         sleep sleep_time
-        stack.sync!
+        stack.sync_status_and_events!
         print_new_events
 
         update_stack_status if stack_status_changed?
@@ -75,7 +75,7 @@ class Devops::Executor::StackExecutor
     end
 
     def print_result_message
-      puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}",
+      puts_and_flush Devops::Messages.t("stack_executor.stack_creation_waiter.result.#{@sync_result.reason}",
         stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
     end
 

devops-service/lib/executors/stack_executor/stack_servers_fetcher.rb

@@ -0,0 +1,97 @@
+class Devops::Executor::StackExecutor
+  class StackServersFetcher
+    include PutsAndFlush
+    attr_reader :out
+
+    NEW = :new
+    PERSISTED = :persisted
+    STALE = :stale
+
+    def initialize(stack, out)
+      @stack, @out = stack, out
+      @by_state = {NEW => [], PERSISTED => [], STALE => []}
+      fetch
+    end
+
+
+    def new_servers_by_priorities
+      servers_with_priority = {}
+      @by_state[NEW].each do |provider_info|
+        priority = priority_from_info(provider_info)
+        servers_with_priority[priority] ||= []
+        servers_with_priority[priority] << provider_info
+      end
+      servers_with_priority
+    end
+
+    def servers_to_persist
+      @by_state[NEW]
+    end
+
+    def already_persisted_servers
+      @by_state[PERSISTED]
+    end
+
+    def stale_servers
+      @by_state[STALE]
+    end
+
+    private
+
+    def fetch
+      @servers_info = @stack.provider_instance.stack_servers(@stack)
+      divide_into_states
+      output_fetched_servers
+    end
+
+    def priority_from_info(provider_info)
+      if provider_info['tags']
+        priority = provider_info['tags']['cid:priority'].to_i
+      else
+        0
+      end
+    end
+
+    def divide_into_states
+      persisted = Devops::Db.connector.stack_servers(@stack.name)
+      persisted_ids = persisted.map(&:id)
+      in_cloud_ids = @servers_info.map {|info| info['id']}
+      new_ids = in_cloud_ids - persisted_ids
+      deleted_ids = persisted_ids - in_cloud_ids
+
+      @servers_info.each do |info|
+        if new_ids.include?(info['id'])
+          @by_state[NEW] << info
+        else
+          @by_state[PERSISTED] << info
+        end
+      end
+
+      @by_state[STALE] = persisted.select { |server| deleted_ids.include?(server.id) }
+    end
+
+    # Do not move it to stack executor till set events handling properly.
+    # For now there may be already persisted servers on creation because of too early events from aws.
+    def output_fetched_servers
+      if servers_to_persist.any?
+        out.puts 'Servers to persist:'
+        servers_to_persist.each { |info| out.puts JSON.pretty_generate(info) }
+      else
+        out.puts 'There are no servers to persist.'
+      end
+
+      if already_persisted_servers.any?
+        out.puts "\nAlready persisted servers:"
+        already_persisted_servers.each { |info| out.puts JSON.pretty_generate(info) }
+      end
+
+      if stale_servers.any?
+        out.puts "\nStale servers:"
+        stale_servers.each { |server| out.puts JSON.pretty_generate(server.to_hash) }
+      else
+        out.puts "\nThere are no stale servers."
+      end
+      out.flush
+    end
+  end
+end

devops-service/lib/executors/stack_executor/stack_servers_persister.rb

@@ -1,102 +1,43 @@
 require_relative 'chef_node_name_builder'
 
-# Fetches info about stack servers from provider and then persist them in mongo.
 class Devops::Executor::StackExecutor
   class StackServersPersister
-    include PutsAndFlush
+    attr_reader :stack
-    attr_reader :stack, :out, :deleted, :servers_info
 
-    NEW = 'new'
+    def initialize(stack)
-    DELETED = 'deleted'
+      @stack = stack
-    PERSISTED = 'persisted'
-    JUST_PERSISTED = 'just_persisted'
-
-    def initialize(stack, out)
-      @stack, @out = stack, out
       @project = mongo.project(stack.project)
       @deploy_env = @project.deploy_env(stack.deploy_env)
-      fetch_provider_servers_info
-      set_servers_states
     end
 
-    def persist_new_servers
+    def persist(provider_info)
-      with_state(NEW).each do |info|
-        info[:server] = persist_stack_server(info[:provider_info])
-        info[:state] = JUST_PERSISTED
-        puts_and_flush "Persisted server with id '#{info[:id]}' and name '#{info[:server].chef_node_name}'"
-      end
-    end
-
-    # returns: { priority_as_integer => [Servers] }
-    def just_persisted_by_priority
-      stack_servers_with_priority = {}
-      with_state(JUST_PERSISTED).each do |info|
-        stack_servers_with_priority[info[:priority]] ||= []
-        stack_servers_with_priority[info[:priority]] << info[:server]
-      end
-      stack_servers_with_priority
-    end
-
-    private
-
-    def fetch_provider_servers_info
-      @servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info|
-        {
-          id: provider_info['id'],
-          provider_info: provider_info,
-          priority: provider_info['tags']['cid:priority'].to_i
-        }
-      end
-    end
-
-    def set_servers_states
-      persisted = Devops::Db.connector.stack_servers(stack.id)
-      persisted_ids = persisted.map(&:id)
-      in_cloud_ids = @servers_info.map {|info| info[:id]}
-      new_ids = in_cloud_ids - persisted_ids
-      deleted_ids = persisted_ids - in_cloud_ids
-
-      @servers_info.each do |info|
-        if new_ids.include?(info[:id])
-          info[:state] = NEW
-        else
-          info[:state] = PERSISTED
-        end
-      end
-
-      @deleted = persisted.select { |server| deleted_ids.include?(server.id) }
-    end
-
-    def with_state(state)
-      @servers_info.select { |info| info[:state] == state }
-    end
-
-    # takes a hash, returns Server model
-    def persist_stack_server(server_info)
       server_attrs = {
-        '_id' => server_info['id'],
+        '_id' => provider_info['id'],
-        'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values),
+        'chef_node_name' => get_name_builder(provider_info).build_node_name!(incrementers_values),
         'created_by' => stack.owner,
         'deploy_env' => @deploy_env.identifier,
-        'key' => server_info['key_name'] || stack.provider_instance.ssh_key,
+        'key' => provider_info['key_name'] || stack.provider_instance.ssh_key,
         'project' => @project.id,
         'provider' => @stack.provider,
         'provider_account' => @stack.provider_account,
         'remote_user' => mongo.image(@deploy_env.image).remote_user,
-        'private_ip' => server_info['private_ip'],
+        'private_ip' => provider_info['private_ip'],
-        'public_ip' => server_info['public_ip'],
+        'public_ip' => provider_info['public_ip'],
         'run_list' => stack.run_list || [],
         'stack' => stack.name
       }
 
       server = ::Devops::Model::Server.new(server_attrs)
       mongo.server_insert(server)
+      # here custom insert method is used and it doesn't return server model
       server
     end
 
-    def get_name_builder(server_info)
+    private
+
+    def get_name_builder(provider_info)
       ChefNodeNameBuilder.new(
-        provider_server_info: server_info,
+        provider_server_info: provider_info,
         project_id: @project.id,
         env_id: @deploy_env.identifier,
         owner: stack.owner

devops-service/lib/sync_stack_lambda.py

@@ -8,10 +8,11 @@ import urllib2
 def lambda_handler(event, context):
   print("Received event: " + json.dumps(event, indent=2))
 
-  stack_id = event['Records'][0]['Sns']['Message']
+  host = 'http://CHANGE_ME'
-  url = 'http://example.com/v2.0/stack/%s/sync' % stack_id
+  group_name = event['detail']['AutoScalingGroupName']
-  username = 'root'
+  url = '%s/v2.0/provider_notifications/aws/first/autoscaling_groups/%s/changes' % (host, group_name)
-  password = 'pass'
+  username = 'CHANGE_ME'
+  password = 'CHANGE_ME'
 
   request = urllib2.Request(url, '')
   base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')

devops-service/messages/en.yml

@@ -13,15 +13,24 @@ en:
           Stack was launched, but an error occured during deploying stack servers.
           You can redeploy stack after fixing the error.
         timeout_reached: Bootstrap or deploy wasn't completed due to timeout.
+    stack_sync:
+      bootstrap_result:
+        ok: "Stack has been successfully synced."
+        bootstrap_error: An error occured during new stack servers bootstrapp.
+        deploy_error: An error occured during new stack servers deploy.
+        timeout_reached: Bootstrap or deploy of new servers wasn't completed due to timeout.
+  stack_executor:
     servers_bootstrapper:
-      bootstrap_servers:
+      result:
         ok: "Server '%{server_id}' has been bootstraped (job %{job_id})."
         timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
         bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
         deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."
     stack_creation_waiter:
       result:
-      ok: "Stack '%{stack_id}' status is now %{status}"
+        ok: |
+          Stack '%{stack_id}' status is now %{status}
+          Stack has been created
         stack_rolled_back: "Stack '%{stack_id}' status is now %{status}"
         stack_deleted:  "Stack '%{stack_id}' status is now %{status}"
         stack_not_found: "Stack '%{stack_id}' status is now %{status}"

devops-service/providers/ec2.rb

@@ -211,10 +211,6 @@ module Provider
       connection_compute(connection_options)
     end
 
-    def cloud_formation
-      @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options)
-    end
-
     def create_stack(stack, out)
       begin
         out << "Creating stack for project '#{stack.project}' and environment '#{stack.deploy_env}'...\n"
@@ -291,26 +287,20 @@ module Provider
       cloud_formation.describe_stack_events(stack.name).body['StackEvents'].map{|se| {"timestamp" => se["Timestamp"], "stack_name" => se["StackName"], "stack_id" => se["StackId"], "event_id" => se["EventId"], "reason" => se["ResourceStatusReason"], "status" => se["ResourceStatus"]}}.sort{|se1, se2| se1["timestamp"] <=> se2["timestamp"]}
     end
 
-    # не работает, не используется
-    # def stack_resource(stack, resource_id)
-    #   physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id
-    #   compute.servers.get(physical_id)
-    # end
-
     def stack_servers(stack)
-      # orchestration.describe_stack_resources возвращает мало информации
       resources = compute.describe_instances(
         'tag-key' => 'aws:cloudformation:stack-id',
         'tag-value' => stack.id
       ).body["reservationSet"]
 
-      # В ресурсах могут лежать не только конкретные инстансы, но и MasterNodesGroup, которые управляют
+
-      # несколькими инстансами. Обрабатываем эту ситуацию.
+      # There may be not only instances but autoscaling groups with several instances.
+      # Handle such situations.
       instances = resources.map { |resource| resource["instancesSet"] }.flatten
+      instances.delete_if {|i| i['instanceState']['name'] == 'terminated'}
 
       instances.map do |instance|
         {
-          # 'name' => instance["tagSet"]["Name"],
           'name' => [stack.name, instance_name(instance)].join('-'),
           'id' => instance["instanceId"],
           'key_name' => instance["keyName"],
@@ -325,6 +315,18 @@ module Provider
       "stack-#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}".gsub('_', '-')
     end
 
+    def stack_id_of_autoscaling_group(id)
+      response = auto_scaling.describe_auto_scaling_groups('AutoScalingGroupNames' => [id])
+      tags = response.body['DescribeAutoScalingGroupsResult']['AutoScalingGroups'].first['Tags']
+      stack_id_tag = tags.find {|t| t['Key'] == 'aws:cloudformation:stack-id'}
+      if stack_id_tag
+        stack_id_tag['Value']
+      end
+    rescue StandardError => e
+      puts "An error occured while analyzing group '#{id}': #{[e.name, e.message, e.backtrace].join("\n")}"
+      nil
+    end
+
     def describe_vpcs
       self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } }
     end
@@ -391,14 +393,18 @@ module Provider
       r
     end
 
-    def orchestration
-      @orchestration ||= Fog::AWS::CloudFormation.new(connection_options)
-    end
-
     def storage
       @storage ||= Fog::Storage.new(connection_options)
     end
 
+    def cloud_formation
+      @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options)
+    end
+
+    def auto_scaling
+      @auto_scaling ||= Fog::AWS::AutoScaling.new(connection_options)
+    end
+
     def stack_templates_bucket
       bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplatesnibrdev'
       bucket = storage.directories.get(bucket_name)

devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb

@@ -7,7 +7,7 @@ class Devops::Executor::StackExecutor
     let(:syncer) { described_class.new(stack, out) }
 
     before do
-      allow(stack).to receive(:sync!)
+      allow(stack).to receive(:sync_status_and_events!)
       allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
       allow(syncer).to receive(:sleep)
       allow(stubbed_connector).to receive(:stack_update)
@@ -15,17 +15,17 @@ class Devops::Executor::StackExecutor
 
     def setup_statuses(statuses_array)
       statuses = statuses_array.to_enum
-      allow(stack).to receive(:sync!) {
+      allow(stack).to receive(:sync_status_and_events!) {
         stack.stack_status = statuses.next
       }
     end
 
-    describe '#sync', stubbed_logger: true do
+    describe '#wait', stubbed_logger: true do
       it 'waits for stack creating to be finished' do
         setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
         expect(syncer).to receive(:sleep).at_least(10).times
-        expect(stack).to receive(:sync!).at_least(10).times
+        expect(stack).to receive(:sync_status_and_events!).at_least(10).times
-        syncer.sync
+        syncer.wait
       end
 
       it 'prints each message only once' do
@@ -39,13 +39,13 @@ class Devops::Executor::StackExecutor
         expect(out).to receive(:puts).with(/t1/).once.ordered
         expect(out).to receive(:puts).with(/t2/).once.ordered
         expect(out).to receive(:puts).with(/t3/).once.ordered
-        syncer.sync
+        syncer.wait
       end
 
       it 'updates stack when status is changed' do
         setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
         expect(stubbed_connector).to receive(:stack_update).exactly(3).times
-        syncer.sync
+        syncer.wait
       end
 
       context 'when stack creating was successful' do
@@ -53,7 +53,7 @@ class Devops::Executor::StackExecutor
           setup_statuses(['CREATE_COMPLETE'])
           expect(stubbed_connector).to receive(:stack_update).with(stack)
           expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
-          expect(syncer.sync).to be_ok
+          expect(syncer.wait).to be_ok
         end
       end
 
@@ -61,7 +61,7 @@ class Devops::Executor::StackExecutor
         it 'returns 1 (:stack_rolled_back)' do
           setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
           expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
-          expect(syncer.sync).to be_stack_rolled_back
+          expect(syncer.wait).to be_stack_rolled_back
         end
       end
 
@@ -69,7 +69,7 @@ class Devops::Executor::StackExecutor
         it 'returns 2 (:unkown_status)' do
           setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
           expect(out).to receive(:puts).with(/unknown/)
-          expect(syncer.sync).to be_unkown_status
+          expect(syncer.wait).to be_unkown_status
         end
       end
 
@@ -77,7 +77,7 @@ class Devops::Executor::StackExecutor
         it 'returns 3 (:timeout)' do
           allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
           expect(out).to receive(:puts).with(/hasn't been synced/)
-          expect(syncer.sync).to be_timeout
+          expect(syncer.wait).to be_timeout
         end
       end
 
@@ -85,7 +85,7 @@ class Devops::Executor::StackExecutor
         it 'returns 5 (:error)' do
           setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
           allow(stubbed_connector).to receive(:stack_update) { raise }
-          expect(syncer.sync.code).to eq 5
+          expect(syncer.wait.code).to eq 5
         end
       end
     end

devops-service/spec/executors/stack_executor/stack_servers_fetcher_spec.rb

@@ -0,0 +1,55 @@
+require 'lib/executors/stack_executor/stack_servers_fetcher'
+
+class Devops::Executor::StackExecutor
+  RSpec.describe StackServersFetcher, stubbed_connector: true do
+    def info_hash_for_id(id)
+      {
+        'id' => id,
+        'name' => 'server_name',
+        'key_name' => 'key',
+        'private_ip' => '127.0.0.1',
+        'public_ip' => '127.0.0.2',
+        'tags' => {
+          'cid:priority' => '3',
+          'Name' => 'server1'
+        }
+      }
+    end
+
+    let(:out) { double('out', puts: nil, flush: nil) }
+    let(:stack) { build(:stack_ec2) }
+    let(:fetcher) { described_class.new(stack, out) }
+    let(:new_server_info) { info_hash_for_id('i-new') }
+    let(:persisted_server_info) { info_hash_for_id('i-persisted') }
+    let(:stale_server) { build(:server, id: 'i-deleted') }
+
+    before do
+      allow(stubbed_connector).to receive(:stack_servers) { [stale_server, build(:server, id: 'i-persisted')] }
+      provider = instance_double(Provider::Ec2, name: 'ec2')
+      allow(stack).to receive(:provider_instance) { provider }
+      allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
+    end
+
+    describe '#new_servers_by_priorities' do
+      let(:new_servers_by_priorities) { fetcher.new_servers_by_priorities }
+
+      it 'returns new servers info by :new key divided by priorities' do
+        expect(new_servers_by_priorities).to be_a(Hash)
+        expect(new_servers_by_priorities.length).to eq 1
+        expect(new_servers_by_priorities[3].length).to eq 1
+        expect(new_servers_by_priorities[3].first).to eq new_server_info
+      end
+
+      it "sets priority to 0 if it's absent" do
+        new_server_info['tags'].delete('cid:priority')
+        expect(new_servers_by_priorities[0].first).to eq new_server_info
+      end
+    end
+
+    describe '#stale_servers' do
+      it 'returns array of stale servers' do
+        expect(fetcher.stale_servers).to match_array [stale_server]
+      end
+    end
+  end
+end

devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb

@@ -2,9 +2,9 @@ require 'lib/executors/stack_executor/stack_servers_persister'
 
 class Devops::Executor::StackExecutor
   RSpec.describe StackServersPersister, stubbed_connector: true do
-    def info_hash_for_id(id)
+    let(:provider_info) {
       {
-        'id' => id,
+        'id' => 'i-new',
         'name' => 'server_name',
         'key_name' => 'key',
         'private_ip' => '127.0.0.1',
@@ -14,153 +14,87 @@ class Devops::Executor::StackExecutor
           'Name' => 'server1'
         }
       }
-    end
+    }
 
-    let(:out) { double(:out, puts: nil, flush: nil) }
     let(:run_list) { ['role[asd]'] }
+    let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
     let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
     let(:project) { build(:project, id: 'name') }
-    let(:persister) { described_class.new(stack, out) }
+    let(:persister) { described_class.new(stack) }
-    let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
-    let(:new_server_info) { info_hash_for_id('i-new') }
-    let(:deleted_server_info) { info_hash_for_id('i-deleted') }
-    let(:persisted_server_info) { info_hash_for_id('i-persisted') }
 
     before do
       allow(stubbed_connector).to receive(:project) { project }
       allow(stubbed_connector).to receive(:image) {
         instance_double(Devops::Model::Image, remote_user: 'user')
       }
-      allow(stubbed_connector).to receive(:server_insert)
+      allow(stubbed_connector).to receive(:server_insert) {|server| server}
-      allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
       allow(stack).to receive(:provider_instance) { provider }
-      allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
     end
 
-    describe '#initialize' do
+    describe '#persist' do
-      it 'fetches stack servers info' do
+      let(:persist) { persister.persist(provider_info) }
-        expect(provider).to receive(:stack_servers).with(stack)
+
-        described_class.new(stack, out)
+      it 'persist record to mongo' do
+        expect(stubbed_connector).to receive(:server_insert)
+        persist
       end
 
-      it "doesn't raise error if cid:priority tag is absent" do
-        new_server_info['tags'].delete('cid:priority')
-        expect { described_class.new(stack, out) }.not_to raise_error
-      end
-
-      it 'sets proper statuses' do
-        infos = described_class.new(stack, out).servers_info
-        expect(infos.length).to eq 2
-        expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
-        expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
-      end
-
-      it 'sets deleted servers' do
-        deleted = described_class.new(stack, out).deleted
-        expect(deleted.length).to eq 1
-        expect(deleted.first.id).to eq 'i-deleted'
-      end
-    end
-
-    describe '#persist_new_servers' do
       it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.to_hash).to include(
-          expect(server.id).to eq 'i-new'
+          'id' => 'i-new',
-          expect(server.key).to eq 'key'
+          'key' => 'key',
-          expect(server.private_ip).to eq '127.0.0.1'
+          'private_ip' => '127.0.0.1',
-          expect(server.public_ip).to eq '127.0.0.2'
+          'public_ip' => '127.0.0.2'
-        end
+        )
-        persister.persist_new_servers
       end
 
       it 'takes created_by, run_list and stack attrs from stack' do
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.to_hash).to include(
-          expect(server.created_by).to eq 'root'
+          'created_by' => 'root',
-          expect(server.run_list).to eq run_list
+          'run_list' => run_list,
-          expect(server.stack).to eq 'iamstack'
+          'stack' => 'iamstack'
-        end
+        )
-        persister.persist_new_servers
       end
 
       it 'takes remote_user from image user' do
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.remote_user).to eq 'user'
-          expect(server.remote_user).to eq 'user'
-        end
-        persister.persist_new_servers
       end
 
       it "takes deploy_env from project's deploy_env identifier" do
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.deploy_env).to eq 'foo'
-          expect(server.deploy_env).to eq 'foo'
-        end
-        persister.persist_new_servers
       end
 
       it "takes default provider's ssh key if info doesn't contain it" do
         allow(provider).to receive(:ssh_key) { 'default_key' }
-        new_server_info.delete('key_name')
+        provider_info.delete('key_name')
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.key).to eq 'default_key'
-          expect(server.key).to eq 'default_key'
-        end
-        persister.persist_new_servers
       end
 
       it "sets server's run list to empty array if stack's run_list is nil" do
         stack.run_list = nil
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.run_list).to eq []
-          expect(server.run_list).to eq []
-        end
-        persister.persist_new_servers
       end
 
       it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.chef_node_name).to eq 'name-foo-i-new'
-          expect(server.chef_node_name).to eq 'name-foo-i-new'
-        end
-        persister.persist_new_servers
       end
 
       it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
-        new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
+        provider_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.chef_node_name).to eq 'name-server1-123'
-          expect(server.chef_node_name).to eq 'name-server1-123'
-        end
-        persister.persist_new_servers
       end
 
       it "sets provider and provider account from stack" do
         stack.provider_account = 'foo'
-        expect(stubbed_connector).to receive(:server_insert) do |server|
+        expect(persist.provider_account).to eq 'foo'
-          expect(server.provider).to eq 'ec2'
-          expect(server.provider_account).to eq 'foo'
-        end
-        persister.persist_new_servers
       end
 
       describe 'incremented variables' do
         it 'substitutes :increment-groupid: with incrementing numbers' do
-          allow(provider).to receive(:stack_servers) {[
+          provider_info['tags']['cid:node-name-mask'] = 'node-:increment-group1:-dev'
-            {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
+          expect(persister.persist(provider_info).chef_node_name).to eq 'node-01-dev'
-            {'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
+          expect(persister.persist(provider_info).chef_node_name).to eq 'node-02-dev'
-          ]}
+        end
-          expect(stubbed_connector).to receive(:server_insert) do |server|
-            expect(server.chef_node_name).to eq 'node-01-dev'
-          end.ordered
-          expect(stubbed_connector).to receive(:server_insert) do |server|
-            expect(server.chef_node_name).to eq 'node-02-dev'
-          end
-          persister.persist_new_servers
-        end
-      end
-    end
-
-    describe '#just_persisted_by_priority' do
-      it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
-        persister.persist_new_servers
-        result = persister.just_persisted_by_priority
-        expect(result).to be_a(Hash)
-        expect(result.size).to eq 1
-        expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
       end
     end
   end

devops-service/spec/executors/stack_executor_spec.rb

@@ -6,51 +6,119 @@ class Devops::Executor::StackExecutor
     let(:stack) { build(:stack) }
     let(:executor_without_stack) { described_class.new(out: out) }
     let(:executor_with_stack) { described_class.new(out: out, stack: stack) }
+    let(:new_servers_by_priorities) {
+      {
+        0 => [{id: 1}],
+        2 => [{id: 2}]
+      }
+    }
+    let(:just_persisted_by_priority) {
+      {
+        0 => [double('info 1', id: 1)],
+        2 => [double('info 2', id: 2)]
+      }
+    }
+    let(:fetcher) {
+      instance_double(StackServersFetcher,
+        new_servers_by_priorities: new_servers_by_priorities,
+        stale_servers: build_list(:server, 2),
+        fetch: nil
+      )
+    }
+
+    before do
+      allow(executor_with_stack).to receive(:fetcher) { fetcher }
+    end
+
 
     describe '#wait_till_stack_is_created' do
+      let(:waiter) { instance_double(StackCreationWaiter, wait: double("creation_result", ok?: true)) }
+
+      before do
+        allow(executor_with_stack).to receive(:waiter) { waiter }
+        allow(stack).to receive(:unlock_persisting!)
+      end
+
+      it 'waites till stack is created, then fetches stack servers and unlocks stack persisting' do
+        expect(waiter).to receive(:wait).ordered
+        expect(fetcher).to receive(:fetch).ordered
+        expect(stack).to receive(:unlock_persisting!).ordered
+        executor_with_stack.wait_till_stack_is_created
+      end
+
       it "return true if syncer returns ok" do
-        allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) }
         expect(executor_with_stack.wait_till_stack_is_created).to be true
       end
 
       it "return false if syncer returns not ok" do
-        allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') }
+        allow(waiter).to receive(:wait) { double("creation_result", ok?: false, reason: '') }
         expect(executor_with_stack.wait_till_stack_is_created).to be false
       end
     end
 
-    describe '#create_stack', stubbed_connector: true do
+    describe '#create_stack' do
+      before { expect(stubbed_connector).to receive(:stack_insert) }
+
       it 'initiate creation in cloud and persists stack' do
         expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out)
-        expect(stubbed_connector).to receive(:stack_insert)
+        executor_with_stack.create_stack({'provider' => 'ec2'})
+      end
+
+      it 'locks persisting on create' do
+        expect(Devops::Model::StackFactory).to receive(:create) do |_, params|
+          expect(params).to include('persisting_is_locked' => true)
+        end
         executor_with_stack.create_stack({'provider' => 'ec2'})
       end
     end
 
-    describe '#persist_stack_servers' do
+    describe '#persist_new_servers' do
-      let(:persister) {
+      let(:persister) { instance_double(StackServersPersister, persist: build(:server)) }
-        instance_double(StackServersPersister,
-          persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil
-        )
-      }
-      before { allow(StackServersPersister).to receive(:new).and_return(persister) }
 
-      it 'calls StackServersPersister#persist_new_servers' do
+      before do
-        expect(persister).to receive(:persist_new_servers)
+        allow(executor_with_stack).to receive(:persister) { persister }
-        executor_with_stack.persist_stack_servers
+        allow(stack).to receive(:lock_persisting!)
+        allow(stack).to receive(:unlock_persisting!)
+        allow(stubbed_connector).to receive(:stack) { stack }
       end
 
-      it 'returns hash with :just_persisted_by_priority and :deleted keys' do
+      it 'calls StackServersPersister#persist for each server' do
-        expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil)
+        expect(persister).to receive(:persist).exactly(2).times
+        executor_with_stack.persist_new_servers
+      end
+
+      it 'locks persisting of a stack before start and unlocks after finish' do
+        expect(stack).to receive(:lock_persisting!).ordered
+        expect(persister).to receive(:persist).ordered
+        expect(stack).to receive(:unlock_persisting!).ordered
+        executor_with_stack.persist_new_servers
+      end
+
+      it 'unlocks persisting even in case of failures' do
+        allow(persister).to receive(:persist) { raise }
+        expect(stack).to receive(:unlock_persisting!)
+        expect { executor_with_stack.persist_new_servers }.to raise_error StandardError
       end
     end
 
-    describe '#bootstrap_servers_by_priority' do
+    describe '#bootstrap_just_persisted' do
       it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do
         result = double('bootstrap_result')
+        allow(executor_with_stack).to receive(:just_persisted_by_priority) { just_persisted_by_priority }
         allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result }
         expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority)
-        expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result
+        expect(executor_with_stack.bootstrap_just_persisted(1000)).to eq result
+      end
+    end
+
+    describe '#delete_stale_servers' do
+      it 'builds server executor per stale server and properly delete them' do
+        executor1 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil)
+        executor2 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil)
+        allow(Devops::Executor::ServerExecutor).to receive(:new).and_return(executor1, executor2)
+        expect(executor1).to receive(:delete_server).ordered
+        expect(executor2).to receive(:delete_server).ordered
+        executor_with_stack.delete_stale_servers
       end
     end
 
@@ -61,7 +129,6 @@ class Devops::Executor::StackExecutor
         expect(stubbed_connector).to receive(:stack_delete).ordered
         executor_with_stack.delete_stack
       end
-
     end
   end
 end

devops-service/spec/models/stack/stack_ec2_spec.rb

@@ -80,7 +80,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
   end
 
 
-  describe '#sync!' do
+  describe '#sync_status_and_events!' do
     let(:fresh_events) { double('fresh_events') }
     let(:provider) {
       instance_double('Provider::Ec2',
@@ -95,12 +95,12 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
 
     it "get fresh stack details and updates stack status" do
       expect(provider).to receive(:stack_details)
-      expect {stack.sync!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE')
+      expect {stack.sync_status_and_events!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE')
     end
 
     it "get fresh stack events and stores it in @events" do
       expect(stack).to receive_message_chain('provider_instance.stack_events')
-      expect {stack.sync!}.to change {stack.events}.from(nil).to(fresh_events)
+      expect {stack.sync_status_and_events!}.to change {stack.events}.from(nil).to(fresh_events)
     end
   end
 
@@ -140,7 +140,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
 
     before do
       allow_any_instance_of(described_class).to receive(:create_stack_in_cloud!)
-      allow_any_instance_of(described_class).to receive(:sync!)
+      allow_any_instance_of(described_class).to receive(:sync_status_and_events!)
     end
 
     it "returns instance of #{described_class.name}" do
@@ -153,7 +153,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
     end
 
     it 'synchronizes details' do
-      expect_any_instance_of(described_class).to receive(:sync!)
+      expect_any_instance_of(described_class).to receive(:sync_status_and_events!)
       subject
     end
   end

devops-service/spec/workers/stack_bootstrap_worker_spec.rb

@@ -3,15 +3,16 @@ require 'lib/executors/stack_executor'
 
 RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do
   let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
-  let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
-  let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
   let(:worker) { described_class.new }
+  let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
+  let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs, 'without_bootstrap' => true) }
   let(:executor) {
     instance_double(Devops::Executor::StackExecutor,
       wait_till_stack_is_created: true,
       create_stack: Devops::Model::StackEc2.new(stack_attrs),
-      persist_stack_servers: nil,
+      persist_new_servers: nil,
-      delete_stack: nil
+      delete_stack: nil,
+      bootstrap_just_persisted: bootstrap_result(:ok)
     )
   }
 
@@ -22,8 +23,6 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
   before do
     allow(worker).to receive(:update_report)
     allow(worker).to receive(:executor) { executor }
-    allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} }
-    allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:ok) }
   end
 
 
@@ -39,7 +38,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
   it 'updates report about operation, creates stack and persists stack servers' do
     expect(worker).to receive(:update_report).ordered
     expect(executor).to receive(:create_stack).ordered
-    expect(worker).to receive(:persist_stack_servers).ordered
+    expect(executor).to receive(:persist_new_servers).ordered
     perform_without_bootstrap
   end
 
@@ -55,7 +54,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
 
   context 'if without_bootstrap is true' do
     it "doesn't bootstrap servers" do
-      expect(worker).not_to receive(:bootstrap_servers_by_priority)
+      expect(worker).not_to receive(:bootstrap_or_rollback_if_failed)
       perform_without_bootstrap
     end
 
@@ -70,27 +69,27 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
     end
 
     it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do
-      allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:bootstrap_error) }
+      allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:bootstrap_error) }
       expect(executor).to receive(:delete_stack)
       perform_with_bootstrap
       expect(perform_with_bootstrap).to eq 2
     end
 
     it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do
-      allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:deploy_error) }
+      allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:deploy_error) }
       expect(worker).not_to receive(:rollback_stack!)
       expect(perform_with_bootstrap).to eq 3
     end
 
     it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do
-      allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:timeout_reached) }
+      allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:timeout_reached) }
       expect(worker).not_to receive(:rollback_stack!)
       expect(perform_with_bootstrap).to eq 4
     end
 
     it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do
       error = StandardError.new
-      allow(worker).to receive(:bootstrap_servers_by_priority) { raise error }
+      allow(executor).to receive(:bootstrap_just_persisted) { raise error }
       expect(worker).to receive(:rollback_stack!)
       expect{perform_with_bootstrap}.to raise_error(error)
     end

devops-service/workers/bootstrap_worker.rb

@@ -6,10 +6,19 @@ require "db/mongo/models/report"
 
 class BootstrapWorker < Worker
 
-  # options must contain 'server_attrs', 'owner'
+  # @options
+  # 'server_attrs':       required
+  # 'owner':              required
+  # 'skip_rollback':      optional
+  # 'deploy_info':        optional
+  # 'deployers':          optional
+  # 'bootstrap_template': optional
+  # 'chef_environment':   optional
+  # 'config':             optional, whatever this parameter really means
   def perform(options)
     call do
       owner = options.fetch('owner')
+      skip_rollback = options['skip_rollback']
       converted_options = convert_config(options)
 
       server = Devops::Model::Server.new(options.fetch('server_attrs'))

devops-service/workers/delete_expired_server_worker.rb

@@ -22,7 +22,7 @@ class DeleteExpiredServerWorker < Worker
 
   def save_report(server)
     update_report(
-      "created_by" => 'SYSTEM',
+      "created_by" => Devops::Model::Report::SYSTEM_OWNER,
       "project" => server.project,
       "deploy_env" => server.deploy_env,
       "type" => Devops::Model::Report::EXPIRE_SERVER_TYPE

devops-service/workers/run_workers.rb

@@ -7,6 +7,7 @@ require File.join(root, "project_test_worker")
 require File.join(root, "stack_bootstrap_worker")
 require File.join(root, "delete_server_worker")
 require File.join(root, "delete_expired_server_worker")
-
+require File.join(root, "stack_sync_worker")
+require 'byebug'
 config = {}
 #require File.join(root, "../proxy")

devops-service/workers/stack_bootstrap_worker.rb

@@ -2,14 +2,16 @@ require 'lib/executors/stack_executor'
 
 class StackBootstrapWorker < Worker
 
-  # options must contain 'stack_attributes'
+  # @options:
+  #   'stack_attributes', required
+  #   'without_bootstrap', optional. false by default
+  #   'skip_rollback', optional. false by default
   def perform(options)
     call do
+      puts_and_flush JSON.pretty_generate(options)
       stack_attrs = options.fetch('stack_attributes')
-
+      without_bootstrap = options['without_bootstrap'] || false
-      without_bootstrap = stack_attrs.delete('without_bootstrap')
+      skip_rollback = options['skip_rollback'] || false
-      skip_rollback = false # take it from options in future
-      @out.puts "Received 'without_bootstrap' option" if without_bootstrap
 
       save_report(stack_attrs)
 
@@ -20,7 +22,7 @@ class StackBootstrapWorker < Worker
       end
 
       begin
-        persist_stack_servers
+        executor.persist_new_servers
         if without_bootstrap
           0
         else
@@ -40,17 +42,13 @@ class StackBootstrapWorker < Worker
     @executor ||= Devops::Executor::StackExecutor.new(out: out)
   end
 
+  # let it stay inside method to improve readability of #perform method
   def create_stack(stack_attrs)
     @stack = executor.create_stack(stack_attrs)
   end
 
-  def persist_stack_servers
-    @servers_by_priorities = executor.persist_stack_servers[:just_persisted_by_priority]
-  end
-
-  # options should contain :skip_rollback
   def bootstrap_or_rollback_if_failed(options)
-    bootstrap_result = bootstrap_servers_by_priority
+    bootstrap_result = executor.bootstrap_just_persisted(jid)
     puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}")
     if bootstrap_result.bootstrap_error? && !options[:skip_rollback]
       rollback_stack!
@@ -58,14 +56,6 @@ class StackBootstrapWorker < Worker
     bootstrap_result.code
   end
 
-  def bootstrap_servers_by_priority
-    out.puts "Bootstrapping just persisted servers"
-    @servers_by_priorities.each do |priority, servers|
-      out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
-    end
-    out.flush
-    executor.bootstrap_servers_by_priority(@servers_by_priorities, jid)
-  end
 
   def rollback_stack!
     puts_and_flush "\nStart rollback of a stack"

devops-service/workers/stack_sync_worker.rb

@@ -0,0 +1,62 @@
+require 'lib/executors/stack_executor'
+
+class StackSyncWorker < Worker
+
+  MAX_LOCK_WAITING_TIME = 15000
+
+  # @options:
+  #   'stack_name'    required
+  #   'created_by'  optional
+  def perform(options)
+    call do
+      stack_name = options.fetch('stack_name')
+      created_by = options['created_by'] || ::Devops::Model::Report::SYSTEM_OWNER
+      @stack = mongo.stack(stack_name)
+
+      save_report(created_by)
+      wait_until_stack_is_unlocked
+      puts_and_flush 'Persisting new servers.'
+      executor.persist_new_servers
+
+      puts_and_flush "\n\nDeleting stale servers."
+      executor.delete_stale_servers
+
+      puts_and_flush "\n\nBootstrapping just persisted servers."
+      bootstrap_result = executor.bootstrap_just_persisted(jid)
+      puts_and_flush Devops::Messages.t("worker.stack_sync.bootstrap_result.#{bootstrap_result.reason}")
+      bootstrap_result.code
+    end
+  end
+
+  private
+
+  def executor
+    @executor ||= Devops::Executor::StackExecutor.new(out: out, stack: @stack)
+  end
+
+  def wait_until_stack_is_unlocked
+    return unless @stack.persisting_is_locked
+    puts_and_flush 'Stack is locked, waiting...'
+    waiting_time = 0
+    sleep_time = 10
+    loop do
+      sleep(sleep_time)
+      @stack = mongo.stack(@stack.name)
+      return unless @stack.persisting_is_locked
+
+      waiting_time += sleep_time
+      raise "Stack has been locked for too long" if waiting_time > MAX_LOCK_WAITING_TIME
+    end
+  end
+
+  def save_report(created_by)
+    update_report(
+      "created_by" => created_by,
+      "project" => @stack.project,
+      "deploy_env" => @stack.deploy_env,
+      "type" => ::Devops::Model::Report::SYNC_STACK_TYPE,
+      "subreports" => [],
+      "stack" => @stack.name
+    )
+  end
+end