Merge branch 'devops_3' of http://git.stu.neva.ru/cloudtechlab/devops-service into devops_3

devops-service/lib/executors/server_executor.rb

@@ -10,9 +10,14 @@ module Devops
 
       RESULT_CODES = {
         server_bootstrap_fail: 2,
-        server_not_in_chef_nodes: 5
+        server_not_in_chef_nodes: 5,
+        server_bootstrap_unknown_error: 7,
+        deploy_unknown_error: 6,
+        deploy_failed: 8
       }
 
+      MAX_SSH_RETRIES_AMOUNT = 20
+
       #params:
       #  out - container for output data
       #  deploy_info - hash with deploy data
@@ -45,7 +50,7 @@ module Devops
       end
 
       def self.symbolic_result_code(integer_code)
-        RESULT_CODES.key(integer_code) || :deploy_error
+        RESULT_CODES.key(integer_code) || :unknown_error
       end
 
       def result_code(symbolic_code)
@@ -159,7 +164,7 @@ module Devops
           sleep(5)
           res = `#{cmd}`
           retries_amount += 1
-          if retries_amount > max_retries_amount
+          if retries_amount > MAX_SSH_RETRIES_AMOUNT
             @out.puts "Can not connect to #{address}"
             @out.puts res
             @out.flush
@@ -168,13 +173,14 @@ module Devops
           end
           raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
         rescue ArgumentError => e
-          @out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})"
+          @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
           @out.flush
           retry
         end
 
         provider = @server.provider_instance
         @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
+
         r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
 
         if r == 0
@@ -210,43 +216,61 @@ module Devops
       end
 
       def two_phase_bootstrap options
+        # bootstrap phase
+        begin
           provider = @server.provider_instance
           mongo = ::Devops::Db.connector
           options[:run_list] = provider.run_list
-        status = bootstrap(options)
+          bootstrap_status = bootstrap(options)
-        if status == 0
+
+          if bootstrap_status == 0
             if check_server
               @out << "Server #{@server.chef_node_name} is created"
             else
               @out << roll_back
+              @out.flush
               mongo.server_delete @server.id
-            # return 5
               return result_code(:server_not_in_chef_nodes)
             end
+          else
+            # @out << roll_back
+            # mongo.server_delete @server.id
+            msg = "Failed while bootstraping server with id '#{@server.id}'\n"
+            msg << "Bootstraping operation result was #{bootstrap_status}"
+            DevopsLogger.logger.error msg
+            @out.puts msg
+            @out.flush
+            return result_code(:server_bootstrap_fail)
+          end
+        rescue => e
+          @out << "\nError: #{e.message}\n"
+          @out.flush
+          return result_code(:server_bootstrap_unknown_error)
+        end
+
+        # deploy phase. Assume that all servers are bootstraped successfully here.
+        begin
+          raise "hello"
           @out << "\n"
           @out.flush
-
           # run_list = @server.run_list + provider.run_list
           # @out << "\nRun list: #{run_list.inspect}"
           # s.options[:run_list] += run_list
           # @knife_instance.set_run_list(@server.chef_node_name, run_list)
-          status = deploy_server(@project.deploy_info(@deploy_env))
+          deploy_status = deploy_server(@project.deploy_info(@deploy_env))
-          if status != 0
+          if deploy_status == 0
-            msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
+            0
-            DevopsLogger.logger.error msg
-            @out << "\n" + msg + "\n"
-          end
           else
-          msg = "Failed while bootstraping server with id '#{@server.id}'"
+            msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
+            msg << "\nDeploing server operation status was #{deploy_status}"
             DevopsLogger.logger.error msg
             @out << "\n" + msg + "\n"
-#          @out << roll_back
+            result_code(:deploy_failed)
-#          mongo.server_delete @server.id
           end
-        status
         rescue => e
           @out << "\nError: #{e.message}\n"
-        return 6
+          result_code(:deploy_unknown_error)
+        end
       end
 
       def check_server
@@ -419,10 +443,6 @@ module Devops
 
       private
 
-      def max_retries_amount
-        20
-      end
-
       def schedule_expiration(server)
         expires = @deploy_env.expires
         return unless expires

devops-service/workers/stack_bootstrap_worker.rb

@@ -81,6 +81,7 @@ class StackBootstrapWorker < Worker
       executor = Devops::Executor::ServerExecutor.new(server, @out)
       executor.report = report
       bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({})
+      @out.flush
     end
     check_bootstrap_results!(bootstraping_results)
   end
@@ -99,17 +100,20 @@ class StackBootstrapWorker < Worker
       @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
     end
 
-    if errors_in_bootstrapping_present?(results)
+    if errors_in_bootstrapping_present?(results.values)
       raise BootstrapingStackServerError # will cause rollback of a stack
     else
       raise DeployingStackServerError #will not cause rollback of a stack
     end
   end
 
-  def errors_in_bootstrapping_present?(results)
+  def errors_in_bootstrapping_present?(result_codes)
-    results.values.any? do |result|
+    bootstrap_error_codes = []
-      result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail)
+    [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
+      bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
     end
+
+    (bootstrap_error_codes & result_codes).size > 0
   end
 
   def save_report(file, stack_attrs)