server and deploy commands migrated to server_executor

devops-service/app/api2/handlers/project.rb

@@ -1,6 +1,4 @@
-require "commands/deploy"
 require "commands/status"
-require "commands/server"
 require "db/mongo/models/project"
 require "workers/project_test_worker"
 require "app/api2/parsers/project"
@@ -17,9 +15,7 @@ module Devops
 
         include Devops::API2_0::Helpers
 
-        extend DeployCommands
         extend StatusCommands
-        extend ServerCommands
 
         def project_types
           Devops::TypesFactory.types_names
@@ -132,18 +128,15 @@ module Devops
           out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n")
           status = []
           servers.each do |s|
-            begin
+            project = begin
-              Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
+              Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
             rescue InvalidPrivileges, RecordNotFound  => e
               out << e.message + "\n"
               status.push 2
               next
             end
-            unless keys.key? s.key
+            deploy_info = project.deploy_info(s.deploy_env, nil)
-              k = Devops::Db.connector.key s.key
+            status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info))
-              keys[s.key] = k.path
-            end
-            status.push(deploy_server(out, s, keys[s.key]))
           end
           status
         end
@@ -155,12 +148,11 @@ module Devops
           uri = URI.parse(request.url)
           servers(id, deploy_env, servers).each do |s|
             project = begin
-              Devops::Db.connector.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
+              Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
             rescue InvalidPrivileges, RecordNotFound  => e
               next
             end
             jid = DeployWorker.perform_async(dir, s.to_hash, [], DevopsConfig.config)
-            #DevopsLogger.logger.info "Job '#{jid}' has been started"
             uri.path =  "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid
             files.push uri.to_s
           end

devops-service/app/api2/handlers/server.rb

@@ -1,8 +1,6 @@
-require 'rufus-scheduler'
 require "uri"
 
 require "commands/status"
-#require "commands/server"
 require "commands/bootstrap_templates"
 
 require "lib/executors/server_executor"
@@ -23,10 +21,9 @@ module Devops
         set_parser Devops::API2_0::Parser::ServerParser
 
         extend StatusCommands
-#        extend ServerCommands
         extend BootstrapTemplatesCommands
 
-        scheduler = Rufus::Scheduler.new
+        #scheduler = Rufus::Scheduler.new
 
         def servers
           fields, reserved = parser.servers
@@ -286,6 +283,7 @@ module Devops
 
       end
 
+=begin
       class ExpireHandler
         include ServerCommands
 
@@ -303,6 +301,7 @@ module Devops
           end
         end
       end
+=end
 
     end
   end

devops-service/commands/deploy.rb

@@ -1,81 +0,0 @@
-require "commands/knife_commands"
-require "commands/ssh"
-
-module DeployCommands
-
-  def deploy_server_proc
-    lambda do |out, s, mongo, tags, deploy_info|
-      old_tags_str = nil
-      new_tags_str = nil
-      unless tags.empty?
-        old_tags_str = KnifeCommands.tags_list(s.chef_node_name).join(" ")
-        out << "Server tags: #{old_tags_str}\n"
-        KnifeCommands.tags_delete(s.chef_node_name, old_tags_str)
-
-        new_tags_str = tags.join(" ")
-        out << "Server new tags: #{new_tags_str}\n"
-        cmd = KnifeCommands.tags_create(s.chef_node_name, new_tags_str)
-        unless cmd[1]
-          m = "Error: Cannot add tags '#{new_tags_str}' to server '#{s.chef_node_name}'"
-          DevopsLogger.logger.error(m)
-          out << m + "\n"
-          return 3
-        end
-        DevopsLogger.logger.info("Set tags for '#{s.chef_node_name}': #{new_tags_str}")
-      end
-
-      k = mongo.key s.key
-      r = deploy_server out, s, k.path, deploy_info
-
-      unless tags.empty?
-        out << "Restore tags\n"
-        cmd = KnifeCommands.tags_delete(s.chef_node_name, new_tags_str)
-        DevopsLogger.logger.info("Deleted tags for #{s.chef_node_name}: #{new_tags_str}")
-        cmd = KnifeCommands.tags_create(s.chef_node_name, old_tags_str)
-        DevopsLogger.logger.info("Set tags for #{s.chef_node_name}: #{old_tags_str}")
-      end
-      return r
-    end
-  end
-
-  def deploy_server out, server, cert_path, deploy_info
-    out << "Before deploy hooks...\n"
-    res = server.run_hook(:before_deploy, out, deploy_info)
-    out << "Done\n"
-    out << "\nRun chef-client on '#{server.chef_node_name}'\n"
-    cmd = "chef-client --no-color"
-    if deploy_info["use_json_file"]
-      deploy_info.delete["use_json_file"]
-      out << "Build information:\n"
-      json = JSON.pretty_generate(deploy_info)
-      out << json
-      out << "\n"
-      file = "#{server.project}_#{server.deploy_env}_#{Time.new.to_i}"
-      dir = DevopsConfig.config[:project_info_dir]
-      File.open(File.join(dir, file), "w") do |f|
-        f.write json
-      end
-      out.flush if out.respond_to?(:flush)
-      cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
-    end
-    ip = if server.public_ip.nil?
-      server.private_ip
-    else
-      out << "Public IP detected\n"
-      server.public_ip
-    end
-    out.flush if out.respond_to?(:flush)
-    lline = KnifeCommands.ssh_stream(out, cmd, ip, server.remote_user, cert_path)
-    r = /Chef\sClient\sfinished/i
-    if lline[r].nil?
-      1
-    else
-      out << "After deploy hooks...\n"
-      res = server.run_hook(:after_deploy, out, deploy_info)
-      out << "Done\n"
-      0
-    end
-
-  end
-
-end

devops-service/commands/server.rb

@@ -1,52 +0,0 @@
-require 'set'
-require "commands/knife_commands"
-require "commands/deploy"
-require "exceptions/record_not_found"
-
-module ServerCommands
-
-  include DeployCommands
-
-  def create_server project, env, params, user, out
-    provider = ::Provider::ProviderFactory.get(env.provider)
-    mongo = ::Devops::Db.connector
-    begin
-      out << "Create server...\n"
-      out.flush if out.respond_to?(:flush)
-
-      s = Devops::Model::Server.new
-      s.provider = provider.name
-      s.project = project.id
-      s.deploy_env = env.identifier
-      s.run_list = params["run_list"] || []
-      s.chef_node_name = params["name"]
-      s.key = params["key"] || provider.ssh_key
-
-      i = mongo.image env.image
-      s.remote_user = i.remote_user
-      s.created_by = user
-
-      return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out)
-      out.flush if out.respond_to?(:flush)
-      DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
-      unless params["without_bootstrap"]
-        s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
-        key = mongo.key(s.key)
-        s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
-        options = {
-          bootstrap_template: i.bootstrap_template,
-          cert_path: key.path
-        }
-        return two_phase_bootstrap(s, options, out)
-      else
-        return 0
-      end
-    rescue => e
-      DevopsLogger.logger.error e.message
-      DevopsLogger.logger.warn roll_back(s, provider)
-      mongo.server_delete s.id
-      return 5
-    end
-  end
-
-end

devops-service/commands/stack.rb

@@ -1,8 +1,6 @@
-require "commands/server"
 require 'commands/commands_storage'
 
 module StackCommands
-  include ServerCommands
   extend self
 
   def sync_bootstrap_proc
@@ -29,12 +27,14 @@ module StackCommands
     end
   end
 
+=begin
   def bootstrap_stack_servers_proc(out, stack, mongo, provider, logger)
     mongo.stack_servers(stack.id).each do |server|
       key = mongo.key(server.key)
       two_phase_bootstrap(server, out, provider, mongo, key.path, logger)
     end
   end
+=end
 
   CommandsStorage.add_job_lambda(
     sync_bootstrap: sync_bootstrap_proc

devops-service/lib/executors/server_executor.rb

@@ -39,6 +39,10 @@ module Devops
         @deploy_env = e
       end
 
+      def server
+        @server
+      end
+
       def create_server options
         @server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider})
         provider = @server.provider_instance
@@ -57,11 +61,14 @@ module Devops
           res = {}
           @out << "\nBefore create hooks...\n"
           res[:before] = self.run_hook :before_create
+          @out << "Done\n"
+
           return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
           mongo.server_insert @server
+
           @out << "\nAfter create hooks...\n"
           res[:after] = self.run_hook :after_create
-
+          @out << "Done\n"
           @out.flush
           DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
           unless options["without_bootstrap"]
@@ -80,7 +87,6 @@ module Devops
           mongo.server_delete @server.id
           return 5
         end
-
       end
 
       def bootstrap options

devops-service/lib/expire_handler.rb

@@ -1,8 +1,6 @@
-require "commands/server"
 require 'rufus-scheduler'
 
 class ExpireHandler
-  include ServerCommands
 
   @@scheduler = Rufus::Scheduler.start_new
 

devops-service/workers/bootstrap_worker.rb

@@ -3,13 +3,12 @@
 
 require File.join(File.dirname(__FILE__), "worker")
 
+require "lib/executors/server_executor"
 require "providers/provider_factory"
-require "commands/server"
 require "db/mongo/models/server"
 require "db/mongo/models/report"
 
 class BootstrapWorker < Worker
-  include ServerCommands
 
   def perform(dir, e_provider, server, bootstrap_template, owner, conf)
     call(conf, e_provider, dir) do |provider, out, file|
@@ -26,10 +25,10 @@ class BootstrapWorker < Worker
       }
       mongo.save_report(Devops::Model::Report.new(o))
 
-      key = mongo.key(s.key)
+      options = {
-#      out << "\nBootstrap with run list: #{s.options[:run_list].inspect}"
+        :bootstrap_template => bootstrap_template
-      status = two_phase_bootstrap s, provider.run_list, bootstrap_template, key.path, out
+      }
-#      status = bootstrap(s, out, key.path)
+      status = Devops::Executor::ServerExecutor.new(s, out).two_phase_bootstrap(options)
       mongo.set_report_server_data(jid, s.chef_node_name, s.public_ip || s.private_ip)
       status
     end

devops-service/workers/project_test_worker.rb

@@ -1,7 +1,6 @@
 require File.join(File.dirname(__FILE__), "worker")
 
 require "providers/provider_factory"
-require "commands/server"
 require "db/mongo/models/server"
 require "json"
 require "fileutils"
@@ -9,16 +8,15 @@ require "commands/status"
 require "db/mongo/models/report"
 
 class ProjectTestWorker < Worker
-  include ServerCommands
   include StatusCommands
 
   def perform(dir, params, conf)
     call(conf, nil, dir) do |provider, out, file|
-      DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{params["user"]})"
+      user = params["user"]
+      DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{user})"
       mongo = Devops::Db.connector
       project = mongo.project(params["project"])
       env = project.deploy_env(params["env"])
-      user = params["user"]
       o = {
         "file" => file,
         "_id" => jid,
@@ -29,80 +27,81 @@ class ProjectTestWorker < Worker
       }
       mongo.save_report(Report.new(o))
 
-      provider = ::Provider::ProviderFactory.get(env.provider)
+      executor = Devops::Executor::ServerExecutor.new(nil, out)
-      servers = extract_servers(provider, project, env, {}, user)
+      executor.project = project
+      executor.deploy_env = env
+      params["created_by"] = user
+
+
       result = {:servers => []}
       project.deploy_envs = [ env ]
       result[:project] = project.to_hash
       status = 0
-      servers.each do |s|
+      sr = {}
-        sr = {}
+      t1 = Time.now
-        t1 = Time.now
+      out << "\n=== Create server ===\n"
-        out << "\n=== Create server ===\n"
+      out.flush
+      if executor.create_server({"without_bootstrap" => true})
+        out << "\n=== Create server - OK ===\n"
         out.flush
-        if provider.create_server(s, out)
+        t2 = Time.now
-          out << "\n=== Create server - OK ===\n"
+        s = executor.server
+        sr[:id] = s.id
+        sr[:create] = {:status => true}
+        sr[:create][:time] = time_diff_s(t1, t2)
+        DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
+        out << "\n=== Bootstrap ===\n"
+        out.flush
+        r = executor.bootstrap({})
+        t1 = Time.now
+        sr[:chef_node_name] = s.chef_node_name
+        if r == 0
+          out << "\n=== Bootstrap - OK ===\n"
           out.flush
-          t2 = Time.now
+          sr[:bootstrap] = {:status => true}
-          sr[:id] = s.id
+          sr[:bootstrap][:time] = time_diff_s(t2, t1)
-          sr[:create] = {:status => true}
+          DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
-          sr[:create][:time] = time_diff_s(t1, t2)
+          out << "\n=== Check server ===\n"
-          s.chef_node_name = provider.create_default_chef_node_name(s)
-          logger.info "Server with parameters: #{s.to_hash.inspect} is running"
-          key = mongo.key(s.key)
-          out << "\n=== Bootstrap ===\n"
           out.flush
-          r = bootstrap(s, out, key.path)
+          if executor.check_server
-          t1 = Time.now
+#            mongo.server_insert s
-          sr[:chef_node_name] = s.chef_node_name
+            out << "\n=== OK, server has been inserted ===\n"
-          if r == 0
-            out << "\n=== Bootstrap - OK ===\n"
             out.flush
-            sr[:bootstrap] = {:status => true}
-            sr[:bootstrap][:time] = time_diff_s(t2, t1)
-            logger.info "Server with id '#{s.id}' is bootstraped"
-            out << "\n=== Check server ===\n"
-            out.flush
-            if check_server(s)
-              mongo.server_insert s
-              out << "\n=== OK, server has been inserted ===\n"
-              out.flush
-            end
-          else
-            status = 2
-            out << "\n=== Bootstrap - FAIL ===\n"
-            out.flush
-            sr[:bootstrap] = {:status => false}
-            sr[:bootstrap][:return_code] = r
           end
-
-          t1 = Time.now
-          out << "\n=== Delete server ===\n"
-          out.flush
-          r = delete_from_chef_server(s.chef_node_name)
-          begin
-            r[:server] = provider.delete_server s
-            out << "\n=== Delete server - OK ===\n"
-            out.flush
-          rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error
-            status = 3
-            out << "\n=== Delete server - FAIL ===\n"
-            out.flush
-            r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
-            logger.warn r[:server]
-          end
-          mongo.server_delete s.id
-          t2 = Time.now
-          sr[:delete] = {:status => true}
-          sr[:delete][:time] = time_diff_s(t1, t2)
         else
-          status = 1
+          status = 2
-          out << "\n=== Create server - FAIL ===\n"
+          out << "\n=== Bootstrap - FAIL ===\n"
           out.flush
-          sr[:create] = {:status => false}
+          sr[:bootstrap] = {:status => false}
+          sr[:bootstrap][:return_code] = r
         end
-        result[:servers].push sr
+
+        t1 = Time.now
+        out << "\n=== Delete server ===\n"
+        out.flush
+        r = executor.delete_from_chef_server
+        begin
+          r[:server] = s.provider_instance.delete_server s
+          out << "\n=== Delete server - OK ===\n"
+          out.flush
+        rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error
+          status = 3
+          out << "\n=== Delete server - FAIL ===\n"
+          out.flush
+          r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
+          DevopsLogger.logger.warn r[:server]
+        end
+        mongo.server_delete s.id
+        t2 = Time.now
+        sr[:delete] = {:status => true}
+        sr[:delete][:time] = time_diff_s(t1, t2)
+      else
+        status = 1
+        out << "\n=== Create server - FAIL ===\n"
+        out.flush
+        sr[:create] = {:status => false}
       end
+      result[:servers].push sr
       out << "\n\n#{result.to_json}"
       status
     end