workers

devops-service/db/mongo/mongo_connector.rb

@@ -410,6 +410,10 @@ class MongoConnector
     Report.new(r)
   end
 
+  def set_report_status id, status
+    @reports.update({"_id" => id}, {"$set" => {"status" => status, "updated_at" => Time.new}})
+  end
+
 private
   def find_server params
     s = @servers.find(create_query(params)).to_a[0]

devops-service/workers/bootstrap_worker.rb

@@ -12,44 +12,28 @@ class BootstrapWorker < Worker
   include ServerCommands
 
   def perform(dir, e_provider, server, owner, conf)
-    set_status jid, "init"
+    call(conf, e_provider, dir) do |mongo, provider, out, file|
-    config = convert_config(conf)
+      s = Server.new(server)
-    file = File.join(dir, jid)
+      s.options = convert_config(server["options"])
-    File.open(file, "w") do |out|
+      o = {
-      begin
+        "file" => file,
-        set_status jid, "running"
+        "_id" => jid,
-        mongo = mongo_connector(config)
+        "created_by" => owner,
-        ::Provider::ProviderFactory.init(config)
+        "project" => s.project,
-        provider = ::Provider::ProviderFactory.get(e_provider)
+        "deploy_env" => s.deploy_env,
-        logger.debug "Provider: #{provider.inspect}"
+        "type" => Report::BOOTSTRAP_TYPE
-        s = Server.new(server)
+      }
-        s.options = convert_config(server["options"])
+      mongo.save_report(Report.new(o))
-        o = {
-          "file" => file,
-          "_id" => jid,
-          "created_by" => owner,
-          "project" => s.project,
-          "deploy_env" => s.deploy_env,
-          "type" => Report::BOOTSTRAP_TYPE
-        }
-        mongo.save_report(Report.new(o))
 
-        key = mongo.key(s.key)
+      key = mongo.key(s.key)
-        out << "\nBootstrap with run list: #{s.options[:run_list].inspect}"
+      out << "\nBootstrap with run list: #{s.options[:run_list].inspect}"
-        status = bootstrap(s, out, key.path, logger)
+      status = bootstrap(s, out, key.path, logger)
-        if status == 0
+      if status == 0
-          out << "Chef node name: #{s.chef_node_name}\n"
+        out << "Chef node name: #{s.chef_node_name}\n"
-          mongo.server_set_chef_node_name s
+        mongo.server_set_chef_node_name s
-          out << "Chef node name has been updated\n"
+        out << "Chef node name has been updated\n"
-        end
-        set_status jid, (status == 0 ? "completed" : "failed")
-      rescue Exception => e
-        out << "\n"
-        out << e.message
-        out << "\n"
-        out << e.backtrace.join("\n")
-        set_status jid, "failed"
       end
+      status
     end
   end
 end

devops-service/workers/create_server_worker.rb

@@ -12,36 +12,21 @@ class CreateServerWorker < Worker
   include ServerCommands
 
   def perform(dir, e_provider, server, owner, conf)
-    set_status jid, "init"
+    call(conf, e_provider, dir) do |mongo, provider, out, file|
-    config = convert_config(conf)
+      s = Server.new(server)
-    file = File.join(dir, jid)
+      s.options = convert_config(server["options"])
-    File.open(file, "w") do |out|
+      o = {
-      begin
+        "file" => file,
-        set_status jid, "running"
+        "_id" => jid,
-        mongo = mongo_connector(config)
+        "created_by" => owner,
-        ::Provider::ProviderFactory.init(config)
+        "project" => s.project,
-        provider = ::Provider::ProviderFactory.get(e_provider)
+        "deploy_env" => s.deploy_env,
-        s = Server.new(server)
+        "type" => Report::SERVER_TYPE
-        s.options = convert_config(server["options"])
+      }
-        o = {
+      mongo.save_report(Report.new(o))
-          "file" => file,
-          "_id" => jid,
-          "created_by" => owner,
-          "project" => s.project,
-          "deploy_env" => s.deploy_env,
-          "type" => Report::SERVER_TYPE
-        }
-        mongo.save_report(Report.new(o))
 
-        status = create_server_proc.call(out, s, provider, mongo)
+      status = create_server_proc.call(out, s, provider, mongo)
-        set_status jid, (status == 0 ? "completed" : "failed")
+      status
-      rescue Exception => e
-        out << "\n"
-        out << e.message
-        out << "\n"
-        out << e.backtrace.join("\n")
-        set_status jid, "failed"
-      end
     end
   end
 end

devops-service/workers/deploy_worker.rb

@@ -11,33 +11,20 @@ class DeployWorker < Worker
   include DeployCommands
 
   def perform(dir, server, tags, owner, conf)
-    set_status jid, "init"
+    call(conf, nil, dir) do |mongo, provider, out, file|
-    config = convert_config(conf)
+      s = Server.new(server)
-    file = File.join(dir, jid)
+      o = {
-    File.open(file, "w") do |out|
+        "file" => file,
-      begin
+        "_id" => jid,
-        set_status jid, "running"
+        "created_by" => owner,
-        mongo = mongo_connector(config)
+        "project" => s.project,
-        s = Server.new(server)
+        "deploy_env" => s.deploy_env,
-        o = {
+        "type" => Report::DEPLOY_TYPE
-          "file" => file,
+      }
-          "_id" => jid,
+      mongo.save_report(Report.new(o))
-          "created_by" => owner,
-          "project" => s.project,
-          "deploy_env" => s.deploy_env,
-          "type" => Report::DEPLOY_TYPE
-        }
-        mongo.save_report(Report.new(o))
 
-        r = deploy_server_proc.call(out, s, mongo, tags)
+      status = deploy_server_proc.call(out, s, mongo, tags)
-        set_status jid, (r == 0 ? "completed" : "failed")
+      status
-      rescue Exception => e
-        out << "\n"
-        out << e.message
-        out << "\n"
-        out << e.backtrace.join("\n")
-        set_status jid, "failed"
-      end
     end
   end
 end

devops-service/workers/project_test_worker.rb

@@ -13,113 +13,97 @@ class ProjectTestWorker < Worker
   include StatusCommands
 
   def perform(dir, params, conf)
-    FileUtils.mkdir_p(dir) unless File.exists?(dir)
+    call(conf, nil, dir) do |mongo, provider, out, file|
-    set_status jid, "init"
+      logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{params["user"]})"
-    config = convert_config(conf)
+      project = mongo.project(params["project"])
-    file = File.join(dir, jid)
+      env = project.deploy_env(params["env"])
-    File.open(file, "w") do |out|
+      user = params["user"]
-      begin
+      o = {
-        set_status jid, "running"
+        "file" => file,
-        logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{params["user"]})"
+        "_id" => jid,
-        mongo = mongo_connector(config)
+        "created_by" => user,
-        ::Provider::ProviderFactory.init(config)
+        "project" => params["project"],
-        project = mongo.project(params["project"])
+        "deploy_env" => params["env"],
-        env = project.deploy_env(params["env"])
+        "type" => Report::PROJECT_TEST_TYPE
-        user = params["user"]
+      }
-        o = {
+      mongo.save_report(Report.new(o))
-          "file" => file,
-          "_id" => jid,
-          "created_by" => user,
-          "project" => params["project"],
-          "deploy_env" => params["env"],
-          "type" => Report::PROJECT_TEST_TYPE
-        }
-        mongo.save_report(Report.new(o))
 
-        provider = ::Provider::ProviderFactory.get(env.provider)
+      provider = ::Provider::ProviderFactory.get(env.provider)
-        servers = extract_servers(provider, project, env, {}, user, mongo)
+      servers = extract_servers(provider, project, env, {}, user, mongo)
-        result = {:servers => []}
+      result = {:servers => []}
-        project.deploy_envs = [ env ]
+      project.deploy_envs = [ env ]
-        result[:project] = project.to_hash
+      result[:project] = project.to_hash
-        status = 0
+      status = 0
-        servers.each do |s|
+      servers.each do |s|
-          sr = {}
+        sr = {}
-          t1 = Time.now
+        t1 = Time.now
-          out << "\n=== Create server ===\n"
+        out << "\n=== Create server ===\n"
+        out.flush
+        if provider.create_server(s, out)
+          out << "\n=== Create server - OK ===\n"
           out.flush
-          if provider.create_server(s, out)
+          t2 = Time.now
-            out << "\n=== Create server - OK ===\n"
+          sr[:id] = s.id
+          sr[:create] = {:status => true}
+          sr[:create][:time] = time_diff_s(t1, t2)
+          s.chef_node_name = provider.create_default_chef_node_name(s)
+          logger.info "Server with parameters: #{s.to_hash.inspect} is running"
+          key = mongo.key(s.key)
+          out << "\n=== Bootstrap ===\n"
+          out.flush
+          r = bootstrap(s, out, key.path, logger)
+          t1 = Time.now
+          sr[:chef_node_name] = s.chef_node_name
+          if r == 0
+            out << "\n=== Bootstrap - OK ===\n"
             out.flush
-            t2 = Time.now
+            sr[:bootstrap] = {:status => true}
-            sr[:id] = s.id
+            sr[:bootstrap][:time] = time_diff_s(t2, t1)
-            sr[:create] = {:status => true}
+            logger.info "Server with id '#{s.id}' is bootstraped"
-            sr[:create][:time] = time_diff_s(t1, t2)
+            out << "\n=== Check server ===\n"
-            s.chef_node_name = provider.create_default_chef_node_name(s)
-            logger.info "Server with parameters: #{s.to_hash.inspect} is running"
-            key = mongo.key(s.key)
-            out << "\n=== Bootstrap ===\n"
             out.flush
-            r = bootstrap(s, out, key.path, logger)
+            if check_server(s)
-            t1 = Time.now
+              mongo.server_insert s
-            sr[:chef_node_name] = s.chef_node_name
+              out << "\n=== OK, server has been inserted ===\n"
-            if r == 0
-              out << "\n=== Bootstrap - OK ===\n"
               out.flush
-              sr[:bootstrap] = {:status => true}
-              sr[:bootstrap][:time] = time_diff_s(t2, t1)
-              logger.info "Server with id '#{s.id}' is bootstraped"
-              out << "\n=== Check server ===\n"
-              out.flush
-              if check_server(s)
-                mongo.server_insert s
-                out << "\n=== OK, server has been inserted ===\n"
-                out.flush
-              end
-            else
-              status = 2
-              out << "\n=== Bootstrap - FAIL ===\n"
-              out.flush
-              sr[:bootstrap] = {:status => false}
-              sr[:bootstrap][:return_code] = r
             end
-
-            t1 = Time.now
-            out << "\n=== Delete server ===\n"
-            out.flush
-            r = delete_from_chef_server(s.chef_node_name)
-            begin
-              r[:server] = provider.delete_server s
-              out << "\n=== Delete server - OK ===\n"
-              out.flush
-            rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error
-              status = 3
-              out << "\n=== Delete server - FAIL ===\n"
-              out.flush
-              r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
-              logger.warn r[:server]
-            end
-            mongo.server_delete s.id
-            t2 = Time.now
-            sr[:delete] = {:status => true}
-            sr[:delete][:time] = time_diff_s(t1, t2)
           else
-            status = 1
+            status = 2
-            out << "\n=== Create server - FAIL ===\n"
+            out << "\n=== Bootstrap - FAIL ===\n"
             out.flush
-            sr[:create] = {:status => false}
+            sr[:bootstrap] = {:status => false}
+            sr[:bootstrap][:return_code] = r
           end
-          result[:servers].push sr
-        end
-        out << "\n\n#{result.to_json}"
 
-        set_status jid, (status == 0 ? "completed" : "failed")
+          t1 = Time.now
-      rescue Exception => e
+          out << "\n=== Delete server ===\n"
-        out << "\n"
+          out.flush
-        out << e.message
+          r = delete_from_chef_server(s.chef_node_name)
-        out << "\n"
+          begin
-        out << e.backtrace.join("\n")
+            r[:server] = provider.delete_server s
-        set_status jid, "failed"
+            out << "\n=== Delete server - OK ===\n"
+            out.flush
+          rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error
+            status = 3
+            out << "\n=== Delete server - FAIL ===\n"
+            out.flush
+            r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
+            logger.warn r[:server]
+          end
+          mongo.server_delete s.id
+          t2 = Time.now
+          sr[:delete] = {:status => true}
+          sr[:delete][:time] = time_diff_s(t1, t2)
+        else
+          status = 1
+          out << "\n=== Create server - FAIL ===\n"
+          out.flush
+          sr[:create] = {:status => false}
+        end
+        result[:servers].push sr
       end
+      out << "\n\n#{result.to_json}"
+      status
     end
   end
 end

devops-service/workers/worker.rb

@@ -4,11 +4,21 @@ $LOAD_PATH.push root unless $LOAD_PATH.include? root
 require "sidekiq"
 require "sidekiq/api"
 
+require "fileutils"
+
 require "db/mongo/mongo_connector"
+require "providers/provider_factory"
 
 class Worker
   include Sidekiq::Worker
 
+  module STATUS
+    INIT = "init"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+  end
+
   def convert_config conf
     config = {}
     conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}
@@ -26,4 +36,41 @@ class Worker
     Sidekiq.redis {|con| con.hset "devops", id, status}
   end
 
+  def call conf, e_provider, dir
+    FileUtils.mkdir_p(dir) unless File.exists?(dir)
+    set_status jid, "init"
+    config = convert_config(conf)
+    file = File.join(dir, jid)
+    error = nil
+    mongo = nil
+    provider = nil
+    begin
+      mongo = mongo_connector(config)
+      unless e_provider.nil?
+        ::Provider::ProviderFactory.init(config)
+        provider = ::Provider::ProviderFactory.get(e_provider)
+      end
+    rescue Exception => e
+      error = e
+    end
+    File.open(file, "w") do |out|
+      begin
+        set_status jid, STATUS::RUNNING
+        raise error unless error.nil?
+        status = yield(mongo, provider, out, file)
+        status = (status == 0 ? STATUS::COMPLETED : STATUS::FAILED)
+        set_status jid, status
+        mongo.set_report_status(jid, status)
+        status
+      rescue Exception => e
+        out << "\n"
+        out << e.message
+        out << "\n"
+        out << e.backtrace.join("\n")
+        set_status jid, STATUS::FAILED
+        mongo.set_report_status(jid, STATUS::FAILED) unless mongo.nil?
+      end
+    end
+  end
+
 end