Merge branch '#855' into features

devops-service/app/api2/handlers/deploy.rb

@@ -19,10 +19,8 @@ module Devops
           names = body["names"]
           tags = body["tags"] || []
           run_list = body["run_list"]
-          dir = DevopsConfig.config[:report_dir_v2]
           files = []
           jid = nil
-          uri = URI.parse(parser.request.url)
           owner = parser.current_user
           @deploy_info_buf = {}
           servers(names).each do |s|
@@ -36,7 +34,7 @@ module Devops
               deploy_info = create_deploy_info(s, project, body["build_number"])
               deploy_info["run_list"] = run_list if run_list
 
-              uri = Worker.start_async(DeployWorker, @request,
+              jid = Worker.start_async(DeployWorker,
                 server_attrs: s.to_hash,
                 owner: owner,
                 tags: tags,
@@ -63,7 +61,7 @@ module Devops
               }
               Devops::Db.connector.save_report(Report.new(o))
             end
-            files.push(uri)
+            files.push(jid)
           end
           files
         end

devops-service/app/api2/handlers/project.rb

@@ -251,13 +251,13 @@ module Devops
               deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil)
             end
 
-            uri = Worker.start_async(DeployWorker, @request,
+            jid = Worker.start_async(DeployWorker,
               server_attrs: s.to_hash,
               owner: parser.current_user,
               tags: [],
               deploy_info: deploy_info
             )
-            files.push uri
+            files.push jid
           end
           files
         end
@@ -288,14 +288,14 @@ module Devops
             raise InvalidRecord.new(msg)
           end
 
-          uri = Worker.start_async(ProjectTestWorker, @request,
+          jid = Worker.start_async(ProjectTestWorker,
             project: project.id,
             deploy_env: env.identifier,
             user: @request.env['REMOTE_USER']
           )
 
           sleep 1
-          return [uri]
+          return [jid]
         end
 
       end

devops-service/app/api2/handlers/server.rb

@@ -71,11 +71,11 @@ module Devops
           user = parser.current_user
 
           check_if_server_attrs_are_valid(body, user)
-          uri = Worker.start_async(CreateServerWorker, @request,
+          jid = Worker.start_async(CreateServerWorker,
             server_attrs: body,
             owner: user
           )
-          [uri]
+          [jid]
         end
 
         def deploy node_name
@@ -185,19 +185,16 @@ module Devops
         end
 
         def bootstrap_server
-          options = prepare_bootstrap_server
+          server, rl, bootstrap_template = prepare_bootstrap_server
-          s = options.delete(:server)
-          options[:provider_name] = s.provider
-          options[:server_attrs] = s.to_mongo_hash
-          options[:owner] = parser.current_user
-#          dir = DevopsConfig[:report_dir_v2]
-#          files = []
-#          uri = URI.parse(@request.url)
 
-          uri = Worker.start_async(BootstrapWorker, @request, options)
+          jid = Worker.start_async(BootstrapWorker,
+            server_attrs: server.to_mongo_hash,
+            bootstrap_template: bootstrap_template,
+            owner: parser.current_user
+          )
 
           sleep 1
-          [uri]
+          [jid]
         end
 
         def prepare_bootstrap_server

devops-service/app/api2/handlers/stack.rb

@@ -28,13 +28,10 @@ module Devops
           object["provider"] = env.provider
           object["provider_account"] = env.provider_account
 
-          # TODO: without provider_name
+          jid = Worker.start_async(StackBootstrapWorker,
-          uri = Worker.start_async(StackBootstrapWorker, @request,
-            provider_name: env.provider,
             stack_attributes: object
           )
-          puts "Syncing report is located here: #{uri}"
+          [jid]
-          [uri]
         end
 
         def stack id
@@ -102,7 +99,6 @@ module Devops
         def deploy id
           stack = self.stack(id)
           owner = parser.current_user
-          status = []
           project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner)
           deploy_env_model = project.deploy_env(stack.deploy_env)
           body = parser.deploy
@@ -114,7 +110,7 @@ module Devops
           servers.each do |s|
             begin
               deploy_info = project.deploy_info(deploy_env_model, nil)
-              uri = Worker.start_async(DeployWorker, @request,
+              jid = Worker.start_async(DeployWorker,
                 server_attrs: s.to_hash,
                 owner: owner,
                 tags: tags,
@@ -140,7 +136,7 @@ module Devops
               }
               Devops::Db.connector.save_report(Report.new(o))
             end
-            files.push uri
+            files.push jid
           end
           files
         end

devops-service/db/mongo/connectors/report.rb

@@ -51,8 +51,18 @@ module Connectors
       collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)}
     end
 
-    def set_report_status(jid, status)
+    def add_report_subreports(jid, subreports)
-      collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}})
+      collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}})
+    end
+
+    def set_report_status(jid, status, job_result_code)
+      set = {"status" => status, "updated_at" => Time.new}
+      set["job_result_code"] = job_result_code unless job_result_code.nil?
+      collection.update({"_id" => jid}, {"$set" => set})
+    end
+
+    def set_report_status(jid, status, job_result_code)
+      collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}})
     end
 
     def set_report_server_data id, chef_node_name, host

devops-service/db/mongo/models/report.rb

@@ -12,7 +12,7 @@ module Devops
       DEPLOY_STACK_TYPE = 6
       DELETE_SERVER_TYPE = 7
 
-      attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack
+      attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
 
       def initialize r
         self.id = r["_id"]
@@ -21,13 +21,15 @@ module Devops
         self.project = r["project"]
         self.deploy_env = r["deploy_env"]
         self.type = r["type"]
-        self.created_at = r["created_at"]
+#        self.created_at = r["created_at"]
         self.chef_node_name = r["chef_node_name"]
         self.host = r["host"]
         self.stack = r["stack"]
         self.created_at = r["created_at"].localtime unless r["created_at"].nil?
         self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil?
         self.status = r["status"]
+        self.subreports = r["subreports"]
+        self.job_result_code = r["job_result_code"]
       end
 
       def to_hash_without_id
@@ -42,7 +44,9 @@ module Devops
           "chef_node_name" => self.chef_node_name,
           "host" => self.host,
           "status" => self.status,
-          "stack" => self.stack
+          "stack" => self.stack,
+          "subreports" => self.subreports,
+          "job_result_code" => self.job_result_code
         }
       end
 

devops-service/db/mongo/mongo_connector.rb

@@ -31,7 +31,7 @@ class MongoConnector
     [:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
     :user_update, :create_root_user, :check_user_privileges] => :users_connector,
     [:keys, :key, :key_insert, :key_delete] => :keys_connector,
-    [:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector,
+    [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
     [:insert_statistic, :search_statistic] => :statistics_connector,
     [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector
   )

devops-service/providers/ec2.rb

@@ -319,6 +319,7 @@ module Provider
           'key_name' => instance["keyName"],
           'private_ip' => instance["privateIpAddress"],
           'public_ip' => instance["ipAddress"],
+          'tags' => instance["tagSet"]
         }
       end
     end

devops-service/workers/bootstrap_worker.rb

@@ -1,20 +1,18 @@
 require File.join(File.dirname(__FILE__), "worker")
 
 require "lib/executors/server_executor"
-require "providers/provider_factory"
 require "db/mongo/models/server"
 require "db/mongo/models/report"
 
 class BootstrapWorker < Worker
 
   def perform(options)
-    provider_name = options.fetch('provider_name')
     server_attrs = options.fetch('server_attrs')
 #    bootstrap_template = options.fetch('bootstrap_template')
     owner = options.fetch('owner')
     options = convert_config(options)
 
-    call(provider_name) do |provider, out, file|
+    call() do |out, file|
       server = Devops::Model::Server.new(server_attrs)
       report = save_report(file, owner, server)
 

devops-service/workers/create_server_worker.rb

@@ -1,6 +1,5 @@
 require File.join(File.dirname(__FILE__), "worker")
 
-require "providers/provider_factory"
 require "db/mongo/models/server"
 require "db/mongo/models/report"
 require "lib/executors/server_executor"
@@ -11,7 +10,7 @@ class CreateServerWorker < Worker
     server_attrs = options.fetch('server_attrs')
     owner = options.fetch('owner')
 
-    call(nil) do |not_used, out, file|
+    call() do |out, file|
       project = mongo.project(server_attrs["project"])
       env = project.deploy_env(server_attrs["deploy_env"])
       report = save_report(file, project, env, owner)

devops-service/workers/delete_server_worker.rb

@@ -8,7 +8,7 @@ class DeleteServerWorker < Worker
     chef_node_name = options.fetch('server_chef_node_name')
     puts "Expire server '#{chef_node_name}'."
 
-    call(nil) do |not_used, out, file|
+    call() do |out, file|
       server = mongo.server_by_chef_node_name(chef_node_name)
       report = save_report(file, server)
 

devops-service/workers/deploy_worker.rb

@@ -12,7 +12,7 @@ class DeployWorker < Worker
     tags = options.fetch('tags')
     deploy_info = options.fetch('deploy_info')
 
-    call(nil) do |not_used, out, file|
+    call() do |out, file|
       server = Devops::Model::Server.new(server_attrs)
       report = save_report(file, owner, server)
 

devops-service/workers/project_test_worker.rb

@@ -15,7 +15,7 @@ class ProjectTestWorker < Worker
     project_name = params.fetch('project')
     deploy_env_name = params.fetch('deploy_env')
 
-    call(nil) do |not_used, out, file|
+    call() do |out, file|
       DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})"
       project = mongo.project(project_name)
       env = project.deploy_env(deploy_env_name)

devops-service/workers/stack_bootstrap_worker.rb

@@ -1,4 +1,3 @@
-require "providers/provider_factory"
 require "commands/stack"
 require "db/mongo/models/stack/stack_factory"
 require "db/mongo/models/project"
@@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker
   include StackCommands
 
   def perform(options)
-    provider_name = options.fetch('provider_name')
     stack_attrs = options.fetch('stack_attributes')
 
-    call(provider_name) do |provider, out, file|
+    call() do |out, file|
       @out = out
       without_bootstrap = stack_attrs.delete('without_bootstrap')
       @out.puts "Received 'without_bootstrap' option" if without_bootstrap
@@ -23,14 +21,23 @@ class StackBootstrapWorker < Worker
       report = save_report(file, stack_attrs)
 
       begin
-        stack = create_stack(provider_name, stack_attrs)
+        stack = create_stack(stack_attrs)
       rescue StackCreatingError
         return 1
       end
 
+      #TODO: errors
       begin
-        servers = persist_stack_servers!(stack, provider)
+        servers_with_priority = persist_stack_servers!(stack)
-        bootstrap_servers!(servers, report) unless without_bootstrap
+        unless without_bootstrap
+          sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
+          sorted_keys.each do |key|
+            @out.puts "Servers with priority '#{key}':"
+            bootstrap_servers!(servers_with_priority[key], report)
+          end
+        end
+        @out.puts "Done."
+        0
       rescue BootstrapingStackServerError
         @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
         rollback_stack!(stack)
@@ -57,8 +64,8 @@ class StackBootstrapWorker < Worker
     @out.puts "Rollback has been completed"
   end
 
-  def create_stack(provider_name, stack_attrs)
+  def create_stack(stack_attrs)
-    stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out)
+    stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
     mongo.stack_insert(stack)
     operation_result = sync_stack_proc.call(@out, stack, mongo)
 
@@ -76,15 +83,40 @@ class StackBootstrapWorker < Worker
 
   def bootstrap_servers!(servers, report)
     @out << "\nStart bootstraping stack servers\n"
-    bootstraping_results = {}
+
+    subreports = []
+    data = {}
     servers.each do |server|
-      executor = Devops::Executor::ServerExecutor.new(server, @out)
+      sjid = Worker.start_async(BootstrapWorker,
-      executor.report = report
+        server_attrs: server.to_mongo_hash,
-      #TODO: can stack choose bootstrap template?
+        bootstrap_template: 'omnibus',
-      bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'})
+        owner: server.created_by
-      @out.flush
+      )
+      subreports << sjid
+      @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
+      data[server.id] = sjid
     end
-    check_bootstrap_results!(bootstraping_results)
+    @out.puts
+    @out.flush
+    mongo.add_report_subreports(jid, subreports)
+    results = []
+    data.each do |server_id, subreport_id|
+      begin
+        sleep(5)
+        subreport = mongo.report(subreport_id)
+        status = subreport.status
+        if status == Worker::STATUS::COMPLETED
+          @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
+          break
+        elsif status == Worker::STATUS::FAILED
+          results << subreport.job_result_code
+          @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
+          break
+        end
+      end while(true)
+    end
+    @out.flush
+    results.empty? ? 0 : -5
   end
 
   def check_bootstrap_results!(results)
@@ -124,42 +156,60 @@ class StackBootstrapWorker < Worker
       "created_by" => stack_attrs['owner'],
       "project" => stack_attrs["project"],
       "deploy_env" => stack_attrs["deploy_env"],
-      "type" => ::Devops::Model::Report::STACK_TYPE
+      "type" => ::Devops::Model::Report::STACK_TYPE,
+      "subreports" => [],
+      "stack" => stack_attrs['name']
     )
     mongo.save_report(report)
     report
   end
 
-  def persist_stack_servers!(stack, provider)
+  # returns
+  # {
+  #   "priority" => [Servers]
+  # }
+  def persist_stack_servers!(stack)
     @out.puts "Start syncing stack servers with CID"
     @out.flush
     project = mongo.project(stack.project)
     deploy_env = project.deploy_env(stack.deploy_env)
+    provider = stack.provider_instance
 
-    servers = provider.stack_servers(stack).map do |extended_info|
+    stack_servers = provider.stack_servers(stack)
-      server_attrs = {
+    stack_servers.each do |info|
-        'provider' => provider.name,
+      info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
-        'project' => project.id,
+    end
-        'deploy_env' => deploy_env.identifier,
+    stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
-        'remote_user' => mongo.image(deploy_env.image).remote_user,
+    stack_servers_with_priority = {}
-        'key' => extended_info["key_name"] || provider.ssh_key,
+    stack_servers_info.each do |priority, info_array|
-        '_id' => extended_info["id"],
+      stack_servers_with_priority[priority] = info_array.map do |extended_info|
-        'chef_node_name' => extended_info["name"],
+        server_attrs = {
-        'private_ip' => extended_info["private_ip"],
+          'provider' => provider.name,
-        'public_ip' => extended_info["public_ip"],
+          'project' => project.id,
-        'created_by' => stack.owner,
+          'deploy_env' => deploy_env.identifier,
-        'run_list' => stack.run_list || [],
+          'remote_user' => mongo.image(deploy_env.image).remote_user,
-        'stack' => stack.name
+          'key' => extended_info["key_name"] || provider.ssh_key,
-      }
+          '_id' => extended_info["id"],
+          'chef_node_name' => extended_info["name"],
+          'private_ip' => extended_info["private_ip"],
+          'public_ip' => extended_info["public_ip"],
+          'created_by' => stack.owner,
+          'run_list' => stack.run_list || [],
+          'stack' => stack.name
+        }
 
-      server = ::Devops::Model::Server.new(server_attrs)
+        server = ::Devops::Model::Server.new(server_attrs)
-      mongo.server_insert(server)
+        mongo.server_insert(server)
-      # server.chef_node_name =  provider.create_default_chef_node_name(server)
+        # server.chef_node_name =  provider.create_default_chef_node_name(server)
-      server
+        server
+      end
     end
     @out.puts "Stack servers have been synced with CID"
+    stack_servers_with_priority.each do |priority, servers|
+      @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
+    end
     @out.flush
-    servers
+    stack_servers_with_priority
   end
 
 end

devops-service/workers/worker.rb

@@ -28,14 +28,16 @@ class Worker
     IN_QUEUE = "queued"
   end
 
-  def self.start_async(worker_class, request, job_options)
+  def self.start_async(worker_class, job_options)
-    jid = worker_class.perform_async(job_options)
+    jid = worker_class.perform_async(job_options.dup)
     Worker.set_status jid, Worker::STATUS::IN_QUEUE
     DevopsLogger.logger.info "Job '#{jid}' has been queued"
-
+    jid
+=begin
     uri = URI.parse(request.url)
     uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}"
     uri.to_s
+=end
   end
 
   def self.start_sync(worker_class, request, job_options, out)
@@ -54,14 +56,13 @@ class Worker
     Sidekiq.redis {|con| con.hset "devops", id, status}
   end
 
-  def call provider_name, &block
+  def call &block
     begin
-      initialize_devops(provider_name)
+      initialize_devops()
-      provider = ::Provider::ProviderFactory.get(provider_name) if provider_name
       if jid
-        call_async(provider, &block)
+        call_async(&block)
       else
-        call_sync(provider, &block)
+        call_sync(&block)
       end
     rescue StandardError => e
       DevopsLogger.logger.error "#{e.message}:\n#{e.backtrace.join("\n")}"
@@ -70,7 +71,7 @@ class Worker
 
   private
 
-  def initialize_devops(provider_name)
+  def initialize_devops()
     DevopsLogger.logger = logger
     DevopsConfig.read
     DevopsService.init
@@ -78,33 +79,33 @@ class Worker
   end
 
   # outputs to file
-  def call_async(provider)
+  def call_async()
     dir = DevopsConfig[:report_dir_v2]
     # directory is created on server start in config.ru
     file = File.join(dir, jid)
 
-    update_job_status(STATUS::INIT)
+    update_job_status(STATUS::INIT, nil)
     File.open(file, "w") do |out|
       begin
-        update_job_status(STATUS::RUNNING)
+        update_job_status(STATUS::RUNNING, nil)
         self.out = out
 
-        job_result = yield(provider, out, file)
+        job_result = yield(out, file)
         canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED)
-        update_job_status(canonical_status)
+        update_job_status(canonical_status, job_result)
       rescue StandardError, RecordNotFound => e
         out << "\n #{e.class}\n #{e.message}\n"
         out << e.backtrace.join("\n")
-        update_job_status(STATUS::FAILED)
+        update_job_status(STATUS::FAILED, -100)
       end
     end
   end
 
   # outputs to STDOUT
-  def call_sync(provider)
+  def call_sync()
     out = STDOUT
     begin
-      yield(provider, out, '')
+      yield(out, '')
     rescue StandardError, RecordNotFound => e
       out << "\n"
       out << e.message
@@ -117,17 +118,12 @@ class Worker
     ::Devops::Db.connector
   end
 
-  def update_job_status(status)
+  def update_job_status(status, job_result_code)
     set_status(jid, status)
-    mongo.set_report_status(jid, status)
+    mongo.set_report_status(jid, status, job_result_code)
     status
   end
 
-  def init_provider(provider_name)
-    ::Provider::ProviderFactory.init(DevopsConfig.config)
-    ::Provider::ProviderFactory.get(provider_name) if provider_name
-  end
-
   def convert_config conf
     config = {}
     conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}