#730: Detailed stack creating logs

devops-service/app/api2/handlers/project.rb

@@ -79,6 +79,7 @@ module Devops
         def add_or_update_deploy_env id, deploy_env
           project = Devops::Db.connector.project(id)
           env = parser.add_or_update_deploy_env
+          env.identifier = deploy_env if env.identifier.nil?
           begin
             db_env = project.deploy_env(deploy_env)
             unless env.identifier == deploy_env
@@ -90,6 +91,7 @@ module Devops
               raise InvalidRecord.new("Can not change environment '#{deploy_env}' to '#{env.identifier}', environment '#{env.identifier}' already exist") unless deploy_env == env.identifier
             rescue RecordNotFound => e
             end
+            env.validate!
             project.delete_deploy_env(deploy_env)
             project.add_deploy_env(env)
             "Deploy environment '#{deploy_env}' has been updated in project '#{project.id}'"
@@ -165,42 +167,56 @@ module Devops
         end
 
         def deploy_project_stream out, id
+          # check if project exist
+          project = Devops::Db.connector.project(id)
           deploy_env, servers = parser.deploy
           keys = {}
-          dbserver = servers(id, deploy_env, servers)
+          dbserver = Devops::Db.connector.servers(id, deploy_env, servers, true)
           out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n")
           status = []
-          servers.each do |s|
+          deploy_info_buf = {}
-            project = begin
+          dbservers.each do |s|
+            begin
               Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
             rescue InvalidPrivileges, RecordNotFound  => e
               out << e.message + "\n"
               status.push 2
               next
             end
-            deploy_info = project.deploy_info(s.deploy_env, nil)
+            deploy_env_model = project.deploy_env(s.deploy_env)
+            deploy_info = if deploy_info_buf[s.deploy_env]
+              deploy_info_buf[s.deploy_env]
+            else
+              # мы не можем указать один build_number для всех окружений, поэтому nil
+              deploy_info_buf[s.deploy_env] = project.deploy_info(deploy_env_model, nil)
+            end
             status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info))
           end
           status
         end
 
         def deploy_project id
+          # check if project exist
+          project_model = Devops::Db.connector.project(id)
           deploy_env, servers = parser.deploy
           files = []
-          servers(id, deploy_env, servers).each do |s|
+          dbservers = Devops::Db.connector.servers(id, deploy_env, servers, true)
+          #out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n")
+          deploy_info_buf = {}
+          dbservers.each do |s|
             begin
               Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
             rescue InvalidPrivileges, RecordNotFound  => e
               next
             end
 
-            project_model = project(s.project)
+            deploy_env_model = project_model.deploy_env(s.deploy_env)
-            deploy_env_model = project_model.deploy_env(deploy_env)
+            deploy_info = if deploy_info_buf[s.deploy_env]
-
+              deploy_info_buf[s.deploy_env]
-            puts '!!! WARNING !!!'
+            else
-            puts 'build_number is set to empty string'
+              # мы не можем указать один build_number для всех окружений, поэтому nil
-            build_number = ''
+              deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil)
-            deploy_info = project_model.deploy_info(deploy_env_model, build_number)
+            end
 
             uri = Worker.start_async(DeployWorker, @request,
               server_attrs: s.to_hash,
@@ -213,11 +229,6 @@ module Devops
           files
         end
 
-        def servers project_id, deploy_env, servers
-          project = Devops::Db.connector.project(project_id)
-          dbservers = Devops::Db.connector.servers(project_id, deploy_env, servers, true)
-        end
-
         def archive_project id
           project = Devops::Db.connector.project(id)
           Devops::Db.connector.archive_project(id)

devops-service/app/api2/parsers/project.rb

@@ -52,7 +52,6 @@ module Devops
 
         def update
           body = create_object_from_json_body
-          check_string(body["name"], "Parameter 'name' must be a not empty string", true, false)
           check_string(body["description"], "Parameter 'description' must be a string", true, true)
           #check_array(body["deploy_envs"], "Parameter 'deploy_envs' must be a not empty array of objects", Hash)
           rl = check_array(body["run_list"], "Parameter 'run_list' must be an array of string", String, true, true)

devops-service/commands/stack.rb

@@ -1,48 +1,54 @@
 module StackCommands
   extend self
 
+  def self.result_codes
+    {
+      stack_rolled_back: 1,
+      unkown_status: 2,
+      timeout: 3,
+      error: 5
+    }
+  end
+
+  def self.result_code(code)
+    result_codes.fetch(code)
+  end
+
   def sync_stack_proc
     lambda do |out, stack, mongo|
-      # two tries each 4 seconds, then 5 tries each 10 seconds, then 10 tries each 30 seconds.
+      # 5 tries each 5 seconds, then 200 tries each 10 seconds
-      sleep_times = [4]*2 + [10]*5 + [30]*10
+      sleep_times = [5]*5 + [10]*200
 
       begin
         out << "Syncing stack '#{stack.id}'...\n"
         sleep_times.each do |sleep_time|
           sleep sleep_time
           stack.sync_details!
-	  case stack.stack_status
+          case stack.stack_status
-	  when 'CREATE_IN_PROGRESS'
+          when 'CREATE_IN_PROGRESS'
             out << "."
-	    out.flush
+            out.flush
-	  when 'CREATE_COMPLETE'
+          when 'CREATE_COMPLETE'
             mongo.stack_update(stack)
             out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n"
             out.flush
             return 0
-	  when 'ROLLBACK_COMPLETE'
+          when 'ROLLBACK_COMPLETE'
             out << "\nStack '#{stack.id}' status is rolled back\n"
-	    return 1
+            return StackCommands.result_code(:stack_rolled_back)
-	  else
+          else
-	    out << "\nUnknown status: '#{stack.stack_status}'"
+            out.puts "\nUnknown stack status: '#{stack.stack_status}'"
-	    return 2
+            return StackCommands.result_code(:unkown_status)
-	  end
+          end
         end
+        out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds."
+        return StackCommands.result_code(:timeout)
       rescue StandardError => e
         logger.error e.message
         out << "Error: #{e.message}\n"
-        return 5
+        return StackCommands.result_code(:error)
       end
     end
   end
 
-=begin
-  def bootstrap_stack_servers_proc(out, stack, mongo, provider, logger)
-    mongo.stack_servers(stack.id).each do |server|
-      key = mongo.key(server.key)
-      two_phase_bootstrap(server, out, provider, mongo, key.path, logger)
-    end
-  end
-=end
-
 end

devops-service/db/mongo/connectors/base.rb

@@ -6,10 +6,24 @@ require "exceptions/invalid_privileges"
 module Connectors
   class Base
 
+    # если хотим создавать индексы при старте приложения, нужно сначала создать коллекцию
+    def initialize db
+      names = db.collection_names
+      unless names.include?(self.collection_name)
+        db.create_collection(self.collection_name)
+        puts "Collection '#{self.collection_name}' has been created"
+      end
+      self.collection = db.collection(self.collection_name)
+    end
+
     def create_indexes
 
     end
 
+    def collection_name
+      raise "owerride me"
+    end
+
     # Yes, we can implement connectors without attr_accessor, storing collection directly
     # in instance variable like
     # @collection = db.collection('users')

devops-service/db/mongo/connectors/filter.rb

@@ -2,7 +2,11 @@ module Connectors
   class Filter < Base
 
     def initialize(db)
-      self.collection = db.collection('filters')
+      super(db)
+    end
+
+    def collection_name
+      'filters'
     end
 
     def available_images provider

devops-service/db/mongo/connectors/image.rb

@@ -9,7 +9,11 @@ module Connectors
       Helpers::UpdateCommand
 
     def initialize(db)
-      self.collection = db.collection('images')
+      super(db)
+    end
+
+    def collection_name
+      'images'
     end
 
     def images(provider=nil)

devops-service/db/mongo/connectors/key.rb

@@ -7,7 +7,11 @@ module Connectors
       Helpers::DeleteCommand
 
     def initialize(db)
-      self.collection = db.collection('keys')
+      super(db)
+    end
+
+    def collection_name
+      'keys'
     end
 
     def key(id, scope=nil)

devops-service/db/mongo/connectors/project.rb

@@ -8,7 +8,11 @@ module Connectors
 
 
     def initialize(db)
-      @collection = db.collection('projects')
+      super(db)
+    end
+
+    def collection_name
+      'projects'
     end
 
     def is_project_exists?(project)
@@ -104,7 +108,6 @@ module Connectors
     end
 
     def add_deploy_env_to_project id, env
-      env.validate!
       @collection.update({"_id" => id}, {'$push' => {deploy_envs: env.to_hash} })
     end
 
@@ -119,7 +122,7 @@ module Connectors
     end
 
     def project_update id, params
-      raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"]
+      #raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"]
       keys = %w(run_list description)
       params.delete_if{|k,v| !keys.include?(k)}
       @collection.update({"_id" => id}, {'$set' => params })

devops-service/db/mongo/connectors/project_template.rb

@@ -8,7 +8,11 @@ module Connectors
 
 
     def initialize(db)
-      @collection = db.collection('project_templates')
+      super(db)
+    end
+
+    def collection_name
+      'project_templates'
     end
   end
 end

devops-service/db/mongo/connectors/report.rb

@@ -6,7 +6,11 @@ module Connectors
       Helpers::ListCommand
 
     def initialize(db)
-      self.collection = db.collection('reports')
+      super(db)
+    end
+
+    def collection_name
+      'reports'
     end
 
     def save_report r

devops-service/db/mongo/connectors/server.rb

@@ -5,7 +5,11 @@ module Connectors
 
 
     def initialize(db)
-      self.collection = db.collection('servers')
+      super(db)
+    end
+
+    def collection_name
+      'servers'
     end
 
     def servers_find(query, fields=nil)

devops-service/db/mongo/connectors/stack.rb

@@ -7,7 +7,11 @@ module Connectors
       Helpers::UpdateCommand
 
     def initialize(db)
-      self.collection = db.collection('stacks')
+      super(db)
+    end
+
+    def collection_name
+      'stacks'
     end
 
     def stacks(options={})

devops-service/db/mongo/connectors/stack_template.rb

@@ -8,7 +8,11 @@ module Connectors
       Helpers::UpdateCommand
 
     def initialize(db)
-      self.collection = db.collection('stack_templates')
+      super(db)
+    end
+
+    def collection_name
+      "stack_templates"
     end
 
     def stack_templates(provider=nil)

devops-service/db/mongo/connectors/statictic.rb

@@ -2,7 +2,11 @@ module Connectors
   class Statistic < Base
 
     def initialize(db)
-      self.collection = db.collection('statistic')
+      super(db)
+    end
+
+    def collection_name
+      'statistic'
     end
 
     def insert_statistic(user, path, method, body, response_code)
@@ -45,7 +49,8 @@ module Connectors
     private
 
       def mongo_sort_order(order)
-        raise "Wrong sort order" unless %w(asc desc).include?(order)
+        # asc by default and if order id invalid value, it is not a reason for response with code 500
+        #raise "Wrong sort order" unless %w(asc desc).include?(order)
         order == 'asc' ? 1 : -1
       end
 

devops-service/db/mongo/connectors/user.rb

@@ -8,7 +8,11 @@ module Connectors
       Helpers::UpdateCommand
 
     def initialize(db)
-      self.collection = db.collection('users')
+      super(db)
+    end
+
+    def collection_name
+      'users'
     end
 
     def user_auth user, password

devops-service/lib/executors/server_executor.rb

@@ -35,6 +35,25 @@ module Devops
         @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
       end
 
+      def result_codes
+        self.class.result_codes
+      end
+
+      def self.result_codes
+        {
+          server_bootstrap_fail: 2,
+          server_not_in_chef_nodes: 5
+        }
+      end
+
+      def self.result_code(code)
+        result_codes.fetch(code)
+      end
+
+      def result_code(code)
+        self.class.result_code(code)
+      end
+
       def report= r
         @report = r
       end
@@ -95,7 +114,8 @@ module Devops
           DevopsLogger.logger.error e.message
           roll_back
           mongo.server_delete @server.id
-          return 5
+          # return 5
+          return result_code(:server_not_in_chef_nodes)
         end
       end
 
@@ -127,12 +147,13 @@ module Devops
         cmd = 'ssh '
         cmd << "-i #{cert_path} "
         cmd << '-q '
+        cmd << '-o StrictHostKeyChecking=no '
         cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
         cmd << "#{address} 'exit'"
         cmd << " 2>&1"
 
-        @out << "\nWaiting for SSH..."
+        @out.puts "\nWaiting for SSH..."
-        @out << "Test command: '#{cmd}'\n"
+        @out.puts "Test command: '#{cmd}'"
         @out.flush
 
         retries_amount = 0
@@ -140,14 +161,17 @@ module Devops
           sleep(5)
           res = `#{cmd}`
           retries_amount += 1
-          if retries_amount == max_retries_amount
+          if retries_amount > max_retries_amount
-            @out << "\nCan not connect to #{address}"
+            @out.puts "Can not connect to #{address}"
-            @out << "\n" + res
+            @out.puts res
+            @out.flush
             DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
-            return false
+            return result_code(:server_bootstrap_fail)
           end
           raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
         rescue ArgumentError => e
+          @out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})"
+          @out.flush
           retry
         end
 
@@ -195,7 +219,8 @@ module Devops
           else
             @out << roll_back
             mongo.server_delete @server.id
-            return 5
+            # return 5
+            return result_code(:server_not_in_chef_nodes)
           end
           @out << "\n"
           @out.flush
@@ -284,7 +309,7 @@ module Devops
         cmd = "chef-client --no-color"
         if deploy_info["use_json_file"]
           deploy_info.delete("use_json_file")
-          @out << "Build information:\n"
+          @out << "Deploy Input Parameters:\n"
           json = JSON.pretty_generate(deploy_info)
           @out << json
           @out << "\n"
@@ -394,7 +419,7 @@ module Devops
       private
 
       def max_retries_amount
-        120
+        20
       end
 
       def schedule_expiration(server)

devops-service/providers/ec2.rb

@@ -103,7 +103,14 @@ module Provider
       options = {
         "InstanceType" => flavor,
 #        "Placement.AvailabilityZone" => s.options[:availability_zone],
-        "KeyName" => self.ssh_key
+        "KeyName" => self.ssh_key,
+        "Tags" => {
+          "Name" => s.chef_node_name,
+          "cid:project" => s.project,
+          "cid:deployEnv" => s.deploy_env,
+          "cid:user" => s.created_by,
+          "cid:remoteUser" => s.remote_user
+        }
       }
       vpcId = nil
       unless subnets.empty?
@@ -150,7 +157,7 @@ module Provider
       end
       s.public_ip = details["ipAddress"]
       s.private_ip = details["privateIpAddress"]
-      set_server_tags(s)
+#      set_server_tags(s)
       out << "\nDone\n\n"
       out << s.info
 
@@ -159,11 +166,6 @@ module Provider
 
     def set_server_tags s
       tags = {
-        "Name" => s.chef_node_name,
-        "cid:project" => s.project,
-        "cid:deployEnv" => s.deploy_env,
-        "cid:user" => s.created_by,
-        "cid:remoteUser" => s.remote_user
       }
       compute.create_tags(s.id, tags)
     end
@@ -226,12 +228,18 @@ module Provider
           {
             'TemplateBody' => stack.template_body,
             'Parameters' => stack.parameters || {},
-            'Capabilities' => ['CAPABILITY_IAM']
+            'Capabilities' => ['CAPABILITY_IAM'],
+            'Tags' => {
+              "StackName" => stack.name,
+              "cid:project" => stack.project,
+              "cid:deployEnv" => stack.deploy_env,
+              "cid:user" => stack.owner
+            }
           }
         )
         stack.id = response.body['StackId']
         out << "Stack id: #{stack.id}\n"
-        set_stack_tags(stack, out)
+        #set_stack_tags(stack, out)
         out.flush
       rescue Excon::Errors::Conflict => e
         raise ProviderErrors::NameConflict
@@ -250,11 +258,7 @@ module Provider
     end
 
     def set_stack_tags stack, out=""
-       tags = {
+      tags = {
-        "StackName" => stack.name,
-        "cid:project" => stack.project,
-        "cid:deployEnv" => stack.deploy_env,
-        "cid:user" => stack.owner
 #        "cid:remoteUser" => s.remote_user
       }
       #ids = stack_resources(stack).map {|resource| resource['PhysicalResourceId']}
@@ -279,7 +283,11 @@ module Provider
     end
 
     def stack_details(stack)
-      cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'][0]
+      response = cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks']
+      # somewhy it sometimes returns blank results
+      response.detect do |t|
+        t.has_key?('StackName')
+      end
     end
 
     def stack_resources(stack)

devops-service/workers/stack_bootstrap_worker.rb

@@ -4,6 +4,10 @@ require "db/mongo/models/stack/stack_factory"
 require "db/mongo/models/project"
 require "db/mongo/models/report"
 
+class StackCreatingError < StandardError; end
+class BootstrapingStackServerError < StandardError; end
+class DeployingStackServerError < StandardError; end
+
 class StackBootstrapWorker < Worker
   include StackCommands
 
@@ -12,31 +16,98 @@ class StackBootstrapWorker < Worker
     stack_attrs = options.fetch('stack_attributes')
 
     call(provider_name) do |provider, out, file|
+      @out = out
       without_bootstrap = stack_attrs.delete('without_bootstrap')
       report = save_report(file, stack_attrs)
 
-      stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, out)
+      begin
-      mongo.stack_insert(stack)
+        stack = create_stack(provider_name, stack_attrs)
-
+      rescue StackCreatingError
-      operation_result = sync_stack_proc.call(out, stack, mongo)
+        return 1
-      if operation_result == 0
+      end
-        out << "\nStack '#{stack.name}' has been created\n"
+
-        out.flush
+      begin
-        servers = persist_stack_servers!(stack, provider)
+        servers = persist_stack_servers!(stack, provider)
-        bootstrap_servers(servers, out, report) unless without_bootstrap
+        bootstrap_servers!(servers, report) unless without_bootstrap
+      rescue BootstrapingStackServerError
+        @out.puts "An error occured during bootstraping stack servers. Initiating stack rollback.\n\n"
+        rollback_stack!(stack)
+        1
+      rescue DeployingStackServerError => e
+        @out.puts "Stack was launched, but an error occured during deploying stack servers."
+        @out.puts "You can redeploy stack after fixing the error.\n\n"
+        1
+      rescue StandardError => e
+        @out.puts "An error occured. Initiating stack rollback.\n\n"
+        rollback_stack!(stack)
+        raise e
       end
-      operation_result
     end
   end
 
   private
 
-  def bootstrap_servers(servers, out, report)
+  def rollback_stack!(stack)
-    out << "\n"
+    @out.puts "\nStart rollback of a stack"
-    servers.map do |server|
+    stack.delete_stack_in_cloud!
-      executor = Devops::Executor::ServerExecutor.new(server, out)
+    Devops::Db.connector.stack_servers_delete(stack.name)
+    Devops::Db.connector.stack_delete(stack.id)
+    @out.puts "\nRollback has been completed"
+  end
+
+  def create_stack(provider_name, stack_attrs)
+    stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out)
+    mongo.stack_insert(stack)
+    operation_result = sync_stack_proc.call(@out, stack, mongo)
+
+    if operation_result == 0
+      @out.puts "\nStack '#{stack.name}' has been created"
+      @out.flush
+      stack
+    else
+      human_readable_code = StackCommands.result_codes.key(operation_result)
+      @out.puts "An error ocurred during stack creating"
+      @out.puts "Stack creating operation result was #{human_readable_code}"
+      raise StackCreatingError
+    end
+  end
+
+  def bootstrap_servers!(servers, report)
+    @out << "\nStart bootstraping stack servers\n"
+    bootstraping_results = {}
+    servers.each do |server|
+      executor = Devops::Executor::ServerExecutor.new(server, @out)
       executor.report = report
-      executor.two_phase_bootstrap({})
+      bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({})
+    end
+    check_bootstrap_results!(bootstraping_results)
+  end
+
+  def check_bootstrap_results!(results)
+    if results.values.all?(&:zero?)
+      # everything is OK
+      @out.puts "Stack servers have been bootstraped"
+      @out.flush
+      return 0
+    end
+
+    results.each do |chef_node_name, code|
+      human_readable_code = Devops::Executor::ServerExecutor.result_codes.key(code)
+      @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
+    end
+
+    if errors_in_bootstrapping_present?(results)
+      # An error occured during servers bootsraping, so rollback stack
+      raise BootstrapingStackServerError
+    else
+      # An error occured during servers deploying, rollback isn't needed
+      raise DeployingStackServerError
+    end
+  end
+
+  def errors_in_bootstrapping_present?(results)
+    results.values.any? do |result|
+      result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail)
     end
   end
 
@@ -54,10 +125,12 @@ class StackBootstrapWorker < Worker
   end
 
   def persist_stack_servers!(stack, provider)
+    @out.puts "Start syncing stack servers with CID"
+    @out.flush
     project = mongo.project(stack.project)
     deploy_env = project.deploy_env(stack.deploy_env)
 
-    provider.stack_servers(stack).map do |extended_info|
+    servers = provider.stack_servers(stack).map do |extended_info|
       server_attrs = {
         'provider' => provider.name,
         'project' => project.id,
@@ -78,6 +151,9 @@ class StackBootstrapWorker < Worker
       # server.chef_node_name =  provider.create_default_chef_node_name(server)
       server
     end
+    @out.puts "Stack servers have been synced with CID"
+    @out.flush
+    servers
   end
 
 end