server create refactored

devops-service/app/api2/handlers/server.rb

@@ -54,8 +54,8 @@ module Devops
 
         def create_server_stream out, body
           status = []
-          prepare_create_server(body).each do |s|
+          prepare_create_server do |project, env, user, body|
-            res = create_server_proc.call(out, s, provider)
+            res = create_server(project, env, body, user, out)
             status.push res
           end
           status
@@ -63,19 +63,16 @@ module Devops
 
         def create_server
           dir = DevopsConfig[:report_dir_v2]
-          files = []
           uri = URI.parse(parser.request.url)
-          prepare_create_server.each do |s|
+          file = prepare_create_server do |project, env, user, body|
-            h = s.to_hash
+            jid = CreateServerWorker.perform_async(dir, body, user, DevopsConfig.config)
-            h["options"] = s.options
-            jid = CreateServerWorker.perform_async(dir, s.provider, h, parser.current_user, DevopsConfig.config)
             Worker.set_status jid, Worker::STATUS::IN_QUEUE
-            #logger.info "Job '#{jid}' has been started"
+            DevopsLogger.logger.info "Job '#{jid}' has been started"
             uri.path =  "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid
-            files.push uri.to_s
+            uri.to_s
           end
           sleep 1
-          files
+          [file]
         end
 
         def prepare_create_server
@@ -96,7 +93,9 @@ module Devops
             halt_response("Invalid security groups '#{buf.join("', '")}' for provider '#{provider.name}'") if buf.empty?
           end
 
-          Server.extract_servers(provider, p, env, body, user)
+          yield p, env, user, body
+
+          #Server.extract_servers(provider, p, env, body, user)
         end
 
         def pause_server node_name

devops-service/commands/server.rb

@@ -1,3 +1,4 @@
+require 'set'
 require "commands/knife_commands"
 require "commands/deploy"
 require "exceptions/record_not_found"
@@ -28,80 +29,51 @@ module ServerCommands
       end
     end
   end
-=begin
 
-  def create_server_proc
+  def create_server project, env, params, user, out
-    lambda do |out, s, provider|
+    provider = ::Provider::ProviderFactory.get(env.provider)
-      mongo = ::Devops::Db.connector
+    mongo = ::Devops::Db.connector
-      begin
+    begin
-        out << "Create server...\n"
+      out << "Create server...\n"
-        out.flush if out.respond_to?(:flush)
+      out.flush if out.respond_to?(:flush)
-        unless provider.create_server(s, out)
+
-          return 3
+      s = Devops::Model::Server.new
-        end
+      s.provider = provider.name
-        s.create
+      s.project = project.id
-        out.flush if out.respond_to?(:flush)
+      s.deploy_env = env.identifier
-        DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
+      s.run_list = params["run_list"] || []
+      s.chef_node_name = params["name"]
+      s.key = params["key"] || provider.ssh_key
+
+      i = mongo.image env.image
+      puts "Image: #{i.inspect}"
+      s.remote_user = i.remote_user
+      s.created_by = user
+
+      return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out)
+      out.flush if out.respond_to?(:flush)
+      DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
+      unless params["without_bootstrap"]
+        s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
         key = mongo.key(s.key)
         s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
-        out << "\n\nBootstrap..."
+        return two_phase_bootstrap(s, provider.run_list, i.bootstrap_template, key.path, out)
-        out.flush if out.respond_to?(:flush)
+      else
-        run_list = s.options[:run_list]
+        return 0
-        s.options[:run_list] = provider.run_list
-        out << "\nBootstrap with provider run list: #{s.options[:run_list].inspect}"
-        status = bootstrap(s, out, key.path)
-        out.flush if out.respond_to?(:flush)
-        if status == 0
-          mongo.server_set_chef_node_name s
-          DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
-          if check_server(s)
-            out << "Server #{s.chef_node_name} is created"
-          else
-            out << roll_back(s, provider)
-            mongo.server_delete s.id
-            return 5
-          end
-          out << "\n"
-          out.flush if out.respond_to?(:flush)
-
-          out << "\nAdd project run list: #{run_list.inspect}"
-          s.options[:run_list] += run_list
-          KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list])
-          status = deploy_server(out, s, key.path)
-          if status != 0
-            msg = "Failed on chef-client with project run list, server with id '#{s.id}'"
-            DevopsLogger.logger.error msg
-            out << "\n" + msg + "\n"
-            mongo.server_delete s.id
-          end
-          return status
-        else
-          msg = "Failed while bootstraping server with id '#{s.id}'"
-          DevopsLogger.logger.error msg
-          out << "\n" + msg + "\n"
-          out << roll_back(s, provider)
-          mongo.server_delete s.id
-          status
-        end
-      rescue IOError => e
-        DevopsLogger.logger.error e.message
-        DevopsLogger.logger.warn roll_back(s, provider)
-        mongo.server_delete s.id
-        return 5
       end
+    rescue => e
+      DevopsLogger.logger.error e.message
+      DevopsLogger.logger.warn roll_back(s, provider)
+      mongo.server_delete s.id
+      return 5
     end
   end
-=end
 
-  def two_phase_bootstrap s, out, provider, cert_path
+  def two_phase_bootstrap s, provider_run_list, bootstrap_template, cert_path, out
     mongo = ::Devops::Db.connector
     out << "\n\nBootstrap...\n"
     out.flush if out.respond_to?(:flush)
-    run_list = s.options[:run_list]
+    status = bootstrap(s, out, bootstrap_template, provider_run_list, cert_path)
-    s.options[:run_list] = provider.run_list
-    out << "Bootstrap with provider run list: #{s.options[:run_list].inspect}\n"
-    s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
-    status = bootstrap(s, out, cert_path)
     out.flush if out.respond_to?(:flush)
     if status == 0
       DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
@@ -115,9 +87,10 @@ module ServerCommands
       out << "\n"
       out.flush if out.respond_to?(:flush)
 
-      out << "\nAdd project run list: #{run_list.inspect}"
+      run_list = s.run_list + provider_run_list
-      s.options[:run_list] += run_list
+      out << "\nRun list: #{run_list.inspect}"
-      KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list])
+#      s.options[:run_list] += run_list
+      KnifeCommands.set_run_list(s.chef_node_name, run_list)
       status = deploy_server(out, s, cert_path)
       if status != 0
         msg = "Failed on chef-client with project run list, server with id '#{s.id}'"
@@ -172,12 +145,14 @@ module ServerCommands
       i = mongo.image env.image
       flavor = flavors.detect {|f| f["id"] == env.flavor}
       raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil?
+      rl = Set.new
+      rl.merge(project.run_list).merge(env.run_list)
       o = {
         :image => i,
         :name => params["name"],
         :flavor => flavor["id"],
         :groups => params["groups"] || env.groups,
-        :run_list => env.run_list,
+        :run_list => rl,
         :subnets => env.subnets,
         :key => params["key"]
       }
@@ -191,15 +166,16 @@ module ServerCommands
       s.provider = provider.name
       s.project = project_name
       s.deploy_env = env_name
+      s.run_list = params["run_list"] || []
       s.remote_user = image.remote_user
-      s.chef_node_name = info[:name]# || provider.create_default_chef_node_name(s)
+      s.chef_node_name = info[:name]
       s.key = info[:key] || provider.ssh_key
       s.options = {
         :image => image.id,
         :flavor => info[:flavor],
         :name => info[:name],
         :groups => info[:groups],
-        :run_list => info[:run_list],
+        :run_list => info[:run_list].merge(s.run_list),
         :bootstrap_template => image.bootstrap_template,
         :subnets => info[:subnets]
       }
@@ -220,7 +196,7 @@ module ServerCommands
     KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
   end
 
-  def bootstrap s, out, cert_path
+  def bootstrap s, out, bootstrap_template, run_list, cert_path
     out << "Before bootstrap hooks...\n"
     res = s.run_hook(:before_bootstrap, out)
     out << "Done\n"
@@ -228,6 +204,7 @@ module ServerCommands
       out << "Error: Private IP is null"
       return false
     end
+    out << "\nBootstrap with run list: #{run_list.inspect}\n"
     ja = {
       :provider => s.provider,
       :devops_host => `hostname`.strip
@@ -235,12 +212,12 @@ module ServerCommands
     bootstrap_options = [
       "-x #{s.remote_user}",
       "-i #{cert_path}",
-      "--json-attributes '#{ja.to_json}'"
+      "--json-attributes '#{ja.to_json}'",
+      "-N #{s.chef_node_name}"
     ]
     bootstrap_options.push "--sudo" unless s.remote_user == "root"
-    bootstrap_options.push "-N #{s.chef_node_name}" if s.chef_node_name
+    bootstrap_options.push "-d #{bootstrap_template}" if bootstrap_template
-    bootstrap_options.push "-d #{s.options[:bootstrap_template]}" if s.options[:bootstrap_template]
+    bootstrap_options.push "-r #{run_list.join(",")}" unless run_list.empty?
-    bootstrap_options.push "-r #{s.options[:run_list].join(",")}" unless s.options[:run_list].empty?
     ip = s.private_ip
     unless s.public_ip.nil? || s.public_ip.strip.empty?
       ip = s.public_ip

devops-service/db/mongo/connectors/report.rb

@@ -11,7 +11,11 @@ module Connectors
 
     def save_report r
       r.created_at = Time.new
-      collection.insert(r.to_mongo_hash)
+      collection.insert(r.to_mongo_hash) unless report_exists?(r.id)
+    end
+
+    def report_exists? id
+      !collection.find({"_id" => id}, fields: ["_id" => true]).to_a.empty?
     end
 
     def reports options={}

devops-service/db/mongo/models/model_with_provider.rb

@@ -0,0 +1,15 @@
+require "providers/provider_factory"
+
+module Devops
+  module Model
+    module ModelWithProvider
+
+      attr_accessor :provider
+
+      def provider_instance
+        @provider_instance ||= Provider::ProviderFactory.get(self.provider)
+      end
+    end
+  end
+end
+

devops-service/db/mongo/models/project.rb

@@ -41,8 +41,8 @@ module Devops
         self.archived = p["archived"] || false
         self.run_list = p["run_list"] || []
 
-        handler = Devops::TypesFactory.type self.type
+        @handler = Devops::TypesFactory.type self.type
-        handler.prepare(self) unless handler.nil?
+        @handler.prepare(self)
 
         env_class = ( self.multi? ? DeployEnvMulti : DeployEnvFactory )
         unless p["deploy_envs"].nil?
@@ -171,6 +171,10 @@ module Devops
         res
       end
 
+      def extract_servers env, params
+        @handler.extract_servers(self, env, params)
+      end
+
       def self.build_from_bson p
         p["name"] = p["_id"]
         Project.new p

devops-service/db/mongo/models/server.rb

@@ -24,7 +24,7 @@ module Devops
       define_hook :after_bootstrap
 
       attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list
-      attr_accessor :options, :key
+      attr_accessor :key
 
       types :id => {:type => String, :empty => false},
             :provider => {:type => String, :empty => false},
@@ -63,9 +63,10 @@ module Devops
         self.run_list = s["run_list"] || []
       end
 
-      def create
+      def create provider, image, flavor, subnets, groups, out
         res = {}
         res[:before] = self.run_hook :before_create
+        return false unless provider.create_server(self, image, flavor, subnets, groups, out)
         Devops::Db.connector.server_insert self
         res[:after] = self.run_hook :after_create
         res

devops-service/lib/project/type/project_type.rb

@@ -17,5 +17,55 @@ module Devops
 
     end
 
+    def extract_servers project, env, params
+=begin
+      mongo = ::Devops::Db.connector
+      env_name = env.identifier
+      project_name = project.id
+      servers_info = []
+      i = mongo.image env.image
+      rl = Set.new
+      rl.merge(project.run_list).merge(env.run_list)
+      o = {
+        :image => i,
+        :name => ,
+        :flavor => flavor["id"],
+        :groups => params["groups"] || env.groups,
+        :run_list => rl,
+        :subnets => env.subnets,
+        :key =>
+      }
+      servers_info.push(o)
+
+      servers = []
+      servers_info.each do |info|
+        servers.push s
+      end
+      return servers
+
+        image = info[:image]
+=end
+        s = Devops::Model::Server.new
+        s.provider = provider.name
+        s.project = project_name
+        s.deploy_env = env_name
+        s.run_list = params["run_list"] || []
+        s.remote_user = image.remote_user
+        s.chef_node_name = params["name"]
+        s.key = params["key"] || provider.ssh_key
+=begin
+        s.options = {
+          :image => image.id,
+          :flavor => info[:flavor],
+          :name => info[:name],
+          :groups => info[:groups],
+          :run_list => info[:run_list].merge(s.run_list),
+          :bootstrap_template => image.bootstrap_template,
+          :subnets => info[:subnets]
+        }
+=end
+        #s.created_by = user
+      [s]
+    end
   end
 end

devops-service/providers/ec2.rb

@@ -91,28 +91,28 @@ module Provider
       convert_server list[0]["instancesSet"][0]
     end
 
-    def create_server s, out
+    def create_server s, image, flavor, subnets, groups, out
       out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
       options = {
-        "InstanceType" =>  s.options[:flavor],
+        "InstanceType" => flavor,
-        "Placement.AvailabilityZone" => s.options[:availability_zone],
+#        "Placement.AvailabilityZone" => s.options[:availability_zone],
         "KeyName" => self.ssh_key
       }
       vpcId = nil
-      unless s.options[:subnets].empty?
+      unless subnets.empty?
-        options["SubnetId"] = s.options[:subnets][0]
+        options["SubnetId"] = subnets[0]
         vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
         if vpcId.nil?
           out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
           return false
         end
       end
-      options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
+      options["SecurityGroupId"] = extract_group_ids(groups, vpcId).join(",")
 
       aws_server = nil
       compute = self.compute
       begin
-        aws_server = compute.run_instances(s.options[:image], 1, 1, options)
+        aws_server = compute.run_instances(image, 1, 1, options)
       rescue Excon::Errors::Unauthorized => ue
         #root = XML::Parser.string(ue.response.body).parse.root
         #msg = root.children.find { |node| node.name == "Message" }
@@ -130,8 +130,6 @@ module Provider
       instance = abody["instancesSet"][0]
       s.id = instance["instanceId"]
 
-      out << "\nInstance Name: #{s.chef_node_name}"
-      out << "\nInstance ID: #{s.id}\n"
       out << "\nWaiting for server..."
 
       details, state = nil, instance["instanceState"]["name"]

devops-service/providers/openstack.rb

@@ -91,34 +91,34 @@ module Provider
       end
     end
 
-    def create_server s, out
+    def create_server s, image, flavor, subnets, groups, out
       out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
       if s.chef_node_name.nil?
         out << "Generate new instance name: "
         out << s.chef_node_name = create_default_chef_node_name(s)
         out << "\n"
       end
-      networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
+      networks = self.networks.select{|n| subnets.include?(n["name"])}
-      buf = s.options[:subnets] - networks.map{|n| n["name"]}
+      buf = subnets - networks.map{|n| n["name"]}
       unless buf.empty?
         out << "No networks with names '#{buf.join("', '")}' found"
         return false
       end
-      s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
+      flavor = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == flavor}["id"]
-      out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
+      out << "Creating server with name '#{s.chef_node_name}', image '#{image}', flavor '#{flavor}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
       compute = self.compute
       begin
-        o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
+        o_server = compute.create_server(s.chef_node_name, image, flavor,
                            "nics" => networks.map{|n| {"net_id" => n["id"]}},
-                           "security_groups" => s.options[:groups],
+                           "security_groups" => groups,
                            "key_name" => s.key)
       rescue Excon::Errors::BadRequest => e
         response = ::Chef::JSONCompat.from_json(e.response.body)
         if response['badRequest']['code'] == 400
           if response['badRequest']['message'] =~ /Invalid flavorRef/
-            out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
+            out << "\nERROR: Bad request (400): Invalid flavor id specified: #{flavor}"
           elsif response['badRequest']['message'] =~ /Invalid imageRef/
-            out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
+            out << "\nERROR: Bad request (400): Invalid image specified: #{image}"
           else
             out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
           end
@@ -141,8 +141,6 @@ module Provider
       sbody = o_server.body
       s.id = sbody["server"]["id"]
 
-      out << "\nInstance Name: #{s.chef_node_name}"
-      out << "\nInstance ID: #{s.id}\n"
       out << "\nWaiting for server..."
 
       details, status = nil, nil

devops-service/providers/static.rb

@@ -47,7 +47,7 @@ module Provider
       "static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
     end
 
-    def create_server s, out
+    def create_server s, image, flavor, subnets, groups, out
       out << "Unsupported operation: ca not create server for provider 'static'"
       false
     end

devops-service/workers/create_server_worker.rb

@@ -11,23 +11,25 @@ require "db/mongo/models/report"
 class CreateServerWorker < Worker
   include ServerCommands
 
-  def perform(dir, e_provider, server, owner, conf)
+  def perform(dir, params, owner, conf)
     logger.info "Create server"
-    call(conf, e_provider, dir) do |provider, out, file|
+    call(conf, nil, dir) do |provider, out, file|
       mongo = Devops::Db.connector
-      s = Devops::Model::Server.new(server)
+      project = mongo.project(params["project"])
-      s.options = convert_config(server["options"])
+      env = project.deploy_env(params["deploy_env"])
+#      s = Devops::Model::Server.new(server)
+#      s.options = convert_config(server["options"])
       o = {
         "file" => file,
         "_id" => jid,
         "created_by" => owner,
-        "project" => s.project,
+        "project" => project.id,
-        "deploy_env" => s.deploy_env,
+        "deploy_env" => env.identifier,
         "type" => Devops::Model::Report::SERVER_TYPE
       }
       mongo.save_report(Devops::Model::Report.new(o))
 
-      status = create_server_proc.call(out, s, provider)
+      status = create_server(project, env, params, owner, out)
       status
     end
   end

devops-service/workers/worker.rb

@@ -51,8 +51,8 @@ class Worker
     provider = nil
     begin
       Devops::Db.init
+      ::Provider::ProviderFactory.init(DevopsConfig.config)
       unless e_provider.nil?
-        ::Provider::ProviderFactory.init(DevopsConfig.config)
         provider = ::Provider::ProviderFactory.get(e_provider)
       end
     rescue Exception => e