From 5f9a22498f51bb7e417cd98d1c4e58ceec187c78 Mon Sep 17 00:00:00 2001 From: amartynov Date: Wed, 5 Aug 2015 14:05:14 +0300 Subject: [PATCH] server create refactored --- devops-service/app/api2/handlers/server.rb | 21 ++- devops-service/commands/server.rb | 127 +++++++----------- devops-service/db/mongo/connectors/report.rb | 6 +- .../db/mongo/models/model_with_provider.rb | 15 +++ devops-service/db/mongo/models/project.rb | 8 +- devops-service/db/mongo/models/server.rb | 5 +- .../lib/project/type/project_type.rb | 50 +++++++ devops-service/providers/ec2.rb | 16 +-- devops-service/providers/openstack.rb | 20 ++- devops-service/providers/static.rb | 2 +- .../workers/create_server_worker.rb | 16 ++- devops-service/workers/worker.rb | 2 +- 12 files changed, 168 insertions(+), 120 deletions(-) create mode 100644 devops-service/db/mongo/models/model_with_provider.rb diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb index cb2acdf..fb50f57 100644 --- a/devops-service/app/api2/handlers/server.rb +++ b/devops-service/app/api2/handlers/server.rb @@ -54,8 +54,8 @@ module Devops def create_server_stream out, body status = [] - prepare_create_server(body).each do |s| - res = create_server_proc.call(out, s, provider) + prepare_create_server do |project, env, user, body| + res = create_server(project, env, body, user, out) status.push res end status @@ -63,19 +63,16 @@ module Devops def create_server dir = DevopsConfig[:report_dir_v2] - files = [] uri = URI.parse(parser.request.url) - prepare_create_server.each do |s| - h = s.to_hash - h["options"] = s.options - jid = CreateServerWorker.perform_async(dir, s.provider, h, parser.current_user, DevopsConfig.config) + file = prepare_create_server do |project, env, user, body| + jid = CreateServerWorker.perform_async(dir, body, user, DevopsConfig.config) Worker.set_status jid, Worker::STATUS::IN_QUEUE - #logger.info "Job '#{jid}' has been started" + DevopsLogger.logger.info "Job '#{jid}' has been started" uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid - files.push uri.to_s + uri.to_s end sleep 1 - files + [file] end def prepare_create_server @@ -96,7 +93,9 @@ module Devops halt_response("Invalid security groups '#{buf.join("', '")}' for provider '#{provider.name}'") if buf.empty? end - Server.extract_servers(provider, p, env, body, user) + yield p, env, user, body + + #Server.extract_servers(provider, p, env, body, user) end def pause_server node_name diff --git a/devops-service/commands/server.rb b/devops-service/commands/server.rb index d8cfeda..311aec9 100644 --- a/devops-service/commands/server.rb +++ b/devops-service/commands/server.rb @@ -1,3 +1,4 @@ +require 'set' require "commands/knife_commands" require "commands/deploy" require "exceptions/record_not_found" @@ -28,80 +29,51 @@ module ServerCommands end end end -=begin - def create_server_proc - lambda do |out, s, provider| - mongo = ::Devops::Db.connector - begin - out << "Create server...\n" - out.flush if out.respond_to?(:flush) - unless provider.create_server(s, out) - return 3 - end - s.create - out.flush if out.respond_to?(:flush) - DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running" + def create_server project, env, params, user, out + provider = ::Provider::ProviderFactory.get(env.provider) + mongo = ::Devops::Db.connector + begin + out << "Create server...\n" + out.flush if out.respond_to?(:flush) + + s = Devops::Model::Server.new + s.provider = provider.name + s.project = project.id + s.deploy_env = env.identifier + s.run_list = params["run_list"] || [] + s.chef_node_name = params["name"] + s.key = params["key"] || provider.ssh_key + + i = mongo.image env.image + puts "Image: #{i.inspect}" + s.remote_user = i.remote_user + s.created_by = user + + return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out) + out.flush if out.respond_to?(:flush) + DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running" + unless params["without_bootstrap"] + s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list) key = mongo.key(s.key) s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil? - out << "\n\nBootstrap..." - out.flush if out.respond_to?(:flush) - run_list = s.options[:run_list] - s.options[:run_list] = provider.run_list - out << "\nBootstrap with provider run list: #{s.options[:run_list].inspect}" - status = bootstrap(s, out, key.path) - out.flush if out.respond_to?(:flush) - if status == 0 - mongo.server_set_chef_node_name s - DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped" - if check_server(s) - out << "Server #{s.chef_node_name} is created" - else - out << roll_back(s, provider) - mongo.server_delete s.id - return 5 - end - out << "\n" - out.flush if out.respond_to?(:flush) - - out << "\nAdd project run list: #{run_list.inspect}" - s.options[:run_list] += run_list - KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list]) - status = deploy_server(out, s, key.path) - if status != 0 - msg = "Failed on chef-client with project run list, server with id '#{s.id}'" - DevopsLogger.logger.error msg - out << "\n" + msg + "\n" - mongo.server_delete s.id - end - return status - else - msg = "Failed while bootstraping server with id '#{s.id}'" - DevopsLogger.logger.error msg - out << "\n" + msg + "\n" - out << roll_back(s, provider) - mongo.server_delete s.id - status - end - rescue IOError => e - DevopsLogger.logger.error e.message - DevopsLogger.logger.warn roll_back(s, provider) - mongo.server_delete s.id - return 5 + return two_phase_bootstrap(s, provider.run_list, i.bootstrap_template, key.path, out) + else + return 0 end + rescue => e + DevopsLogger.logger.error e.message + DevopsLogger.logger.warn roll_back(s, provider) + mongo.server_delete s.id + return 5 end end -=end - def two_phase_bootstrap s, out, provider, cert_path + def two_phase_bootstrap s, provider_run_list, bootstrap_template, cert_path, out mongo = ::Devops::Db.connector out << "\n\nBootstrap...\n" out.flush if out.respond_to?(:flush) - run_list = s.options[:run_list] - s.options[:run_list] = provider.run_list - out << "Bootstrap with provider run list: #{s.options[:run_list].inspect}\n" - s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil? - status = bootstrap(s, out, cert_path) + status = bootstrap(s, out, bootstrap_template, provider_run_list, cert_path) out.flush if out.respond_to?(:flush) if status == 0 DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped" @@ -115,9 +87,10 @@ module ServerCommands out << "\n" out.flush if out.respond_to?(:flush) - out << "\nAdd project run list: #{run_list.inspect}" - s.options[:run_list] += run_list - KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list]) + run_list = s.run_list + provider_run_list + out << "\nRun list: #{run_list.inspect}" +# s.options[:run_list] += run_list + KnifeCommands.set_run_list(s.chef_node_name, run_list) status = deploy_server(out, s, cert_path) if status != 0 msg = "Failed on chef-client with project run list, server with id '#{s.id}'" @@ -172,12 +145,14 @@ module ServerCommands i = mongo.image env.image flavor = flavors.detect {|f| f["id"] == env.flavor} raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil? + rl = Set.new + rl.merge(project.run_list).merge(env.run_list) o = { :image => i, :name => params["name"], :flavor => flavor["id"], :groups => params["groups"] || env.groups, - :run_list => env.run_list, + :run_list => rl, :subnets => env.subnets, :key => params["key"] } @@ -191,15 +166,16 @@ module ServerCommands s.provider = provider.name s.project = project_name s.deploy_env = env_name + s.run_list = params["run_list"] || [] s.remote_user = image.remote_user - s.chef_node_name = info[:name]# || provider.create_default_chef_node_name(s) + s.chef_node_name = info[:name] s.key = info[:key] || provider.ssh_key s.options = { :image => image.id, :flavor => info[:flavor], :name => info[:name], :groups => info[:groups], - :run_list => info[:run_list], + :run_list => info[:run_list].merge(s.run_list), :bootstrap_template => image.bootstrap_template, :subnets => info[:subnets] } @@ -220,7 +196,7 @@ module ServerCommands KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name) end - def bootstrap s, out, cert_path + def bootstrap s, out, bootstrap_template, run_list, cert_path out << "Before bootstrap hooks...\n" res = s.run_hook(:before_bootstrap, out) out << "Done\n" @@ -228,6 +204,7 @@ module ServerCommands out << "Error: Private IP is null" return false end + out << "\nBootstrap with run list: #{run_list.inspect}\n" ja = { :provider => s.provider, :devops_host => `hostname`.strip @@ -235,12 +212,12 @@ module ServerCommands bootstrap_options = [ "-x #{s.remote_user}", "-i #{cert_path}", - "--json-attributes '#{ja.to_json}'" + "--json-attributes '#{ja.to_json}'", + "-N #{s.chef_node_name}" ] bootstrap_options.push "--sudo" unless s.remote_user == "root" - bootstrap_options.push "-N #{s.chef_node_name}" if s.chef_node_name - bootstrap_options.push "-d #{s.options[:bootstrap_template]}" if s.options[:bootstrap_template] - bootstrap_options.push "-r #{s.options[:run_list].join(",")}" unless s.options[:run_list].empty? + bootstrap_options.push "-d #{bootstrap_template}" if bootstrap_template + bootstrap_options.push "-r #{run_list.join(",")}" unless run_list.empty? ip = s.private_ip unless s.public_ip.nil? || s.public_ip.strip.empty? ip = s.public_ip diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb index b5ea994..6e43683 100644 --- a/devops-service/db/mongo/connectors/report.rb +++ b/devops-service/db/mongo/connectors/report.rb @@ -11,7 +11,11 @@ module Connectors def save_report r r.created_at = Time.new - collection.insert(r.to_mongo_hash) + collection.insert(r.to_mongo_hash) unless report_exists?(r.id) + end + + def report_exists? id + !collection.find({"_id" => id}, fields: ["_id" => true]).to_a.empty? end def reports options={} diff --git a/devops-service/db/mongo/models/model_with_provider.rb b/devops-service/db/mongo/models/model_with_provider.rb new file mode 100644 index 0000000..41fe401 --- /dev/null +++ b/devops-service/db/mongo/models/model_with_provider.rb @@ -0,0 +1,15 @@ +require "providers/provider_factory" + +module Devops + module Model + module ModelWithProvider + + attr_accessor :provider + + def provider_instance + @provider_instance ||= Provider::ProviderFactory.get(self.provider) + end + end + end +end + diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb index e791f34..a5fb3f0 100644 --- a/devops-service/db/mongo/models/project.rb +++ b/devops-service/db/mongo/models/project.rb @@ -41,8 +41,8 @@ module Devops self.archived = p["archived"] || false self.run_list = p["run_list"] || [] - handler = Devops::TypesFactory.type self.type - handler.prepare(self) unless handler.nil? + @handler = Devops::TypesFactory.type self.type + @handler.prepare(self) env_class = ( self.multi? ? DeployEnvMulti : DeployEnvFactory ) unless p["deploy_envs"].nil? @@ -171,6 +171,10 @@ module Devops res end + def extract_servers env, params + @handler.extract_servers(self, env, params) + end + def self.build_from_bson p p["name"] = p["_id"] Project.new p diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb index 8bcd484..9055fdd 100644 --- a/devops-service/db/mongo/models/server.rb +++ b/devops-service/db/mongo/models/server.rb @@ -24,7 +24,7 @@ module Devops define_hook :after_bootstrap attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list - attr_accessor :options, :key + attr_accessor :key types :id => {:type => String, :empty => false}, :provider => {:type => String, :empty => false}, @@ -63,9 +63,10 @@ module Devops self.run_list = s["run_list"] || [] end - def create + def create provider, image, flavor, subnets, groups, out res = {} res[:before] = self.run_hook :before_create + return false unless provider.create_server(self, image, flavor, subnets, groups, out) Devops::Db.connector.server_insert self res[:after] = self.run_hook :after_create res diff --git a/devops-service/lib/project/type/project_type.rb b/devops-service/lib/project/type/project_type.rb index af51ced..ffab1a7 100644 --- a/devops-service/lib/project/type/project_type.rb +++ b/devops-service/lib/project/type/project_type.rb @@ -17,5 +17,55 @@ module Devops end + def extract_servers project, env, params +=begin + mongo = ::Devops::Db.connector + env_name = env.identifier + project_name = project.id + servers_info = [] + i = mongo.image env.image + rl = Set.new + rl.merge(project.run_list).merge(env.run_list) + o = { + :image => i, + :name => , + :flavor => flavor["id"], + :groups => params["groups"] || env.groups, + :run_list => rl, + :subnets => env.subnets, + :key => + } + servers_info.push(o) + + servers = [] + servers_info.each do |info| + servers.push s + end + return servers + + image = info[:image] +=end + s = Devops::Model::Server.new + s.provider = provider.name + s.project = project_name + s.deploy_env = env_name + s.run_list = params["run_list"] || [] + s.remote_user = image.remote_user + s.chef_node_name = params["name"] + s.key = params["key"] || provider.ssh_key +=begin + s.options = { + :image => image.id, + :flavor => info[:flavor], + :name => info[:name], + :groups => info[:groups], + :run_list => info[:run_list].merge(s.run_list), + :bootstrap_template => image.bootstrap_template, + :subnets => info[:subnets] + } +=end + #s.created_by = user + [s] + end end end diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 29311d5..3c9455f 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -91,28 +91,28 @@ module Provider convert_server list[0]["instancesSet"][0] end - def create_server s, out + def create_server s, image, flavor, subnets, groups, out out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" options = { - "InstanceType" => s.options[:flavor], - "Placement.AvailabilityZone" => s.options[:availability_zone], + "InstanceType" => flavor, +# "Placement.AvailabilityZone" => s.options[:availability_zone], "KeyName" => self.ssh_key } vpcId = nil - unless s.options[:subnets].empty? - options["SubnetId"] = s.options[:subnets][0] + unless subnets.empty? + options["SubnetId"] = subnets[0] vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"] if vpcId.nil? out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n" return false end end - options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",") + options["SecurityGroupId"] = extract_group_ids(groups, vpcId).join(",") aws_server = nil compute = self.compute begin - aws_server = compute.run_instances(s.options[:image], 1, 1, options) + aws_server = compute.run_instances(image, 1, 1, options) rescue Excon::Errors::Unauthorized => ue #root = XML::Parser.string(ue.response.body).parse.root #msg = root.children.find { |node| node.name == "Message" } @@ -130,8 +130,6 @@ module Provider instance = abody["instancesSet"][0] s.id = instance["instanceId"] - out << "\nInstance Name: #{s.chef_node_name}" - out << "\nInstance ID: #{s.id}\n" out << "\nWaiting for server..." details, state = nil, instance["instanceState"]["name"] diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb index d51fffa..c8844cc 100644 --- a/devops-service/providers/openstack.rb +++ b/devops-service/providers/openstack.rb @@ -91,34 +91,34 @@ module Provider end end - def create_server s, out + def create_server s, image, flavor, subnets, groups, out out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" if s.chef_node_name.nil? out << "Generate new instance name: " out << s.chef_node_name = create_default_chef_node_name(s) out << "\n" end - networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])} - buf = s.options[:subnets] - networks.map{|n| n["name"]} + networks = self.networks.select{|n| subnets.include?(n["name"])} + buf = subnets - networks.map{|n| n["name"]} unless buf.empty? out << "No networks with names '#{buf.join("', '")}' found" return false end - s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"] - out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n" + flavor = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == flavor}["id"] + out << "Creating server with name '#{s.chef_node_name}', image '#{image}', flavor '#{flavor}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n" compute = self.compute begin - o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor], + o_server = compute.create_server(s.chef_node_name, image, flavor, "nics" => networks.map{|n| {"net_id" => n["id"]}}, - "security_groups" => s.options[:groups], + "security_groups" => groups, "key_name" => s.key) rescue Excon::Errors::BadRequest => e response = ::Chef::JSONCompat.from_json(e.response.body) if response['badRequest']['code'] == 400 if response['badRequest']['message'] =~ /Invalid flavorRef/ - out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}" + out << "\nERROR: Bad request (400): Invalid flavor id specified: #{flavor}" elsif response['badRequest']['message'] =~ /Invalid imageRef/ - out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}" + out << "\nERROR: Bad request (400): Invalid image specified: #{image}" else out << "\nERROR: Bad request (400): #{response['badRequest']['message']}" end @@ -141,8 +141,6 @@ module Provider sbody = o_server.body s.id = sbody["server"]["id"] - out << "\nInstance Name: #{s.chef_node_name}" - out << "\nInstance ID: #{s.id}\n" out << "\nWaiting for server..." details, status = nil, nil diff --git a/devops-service/providers/static.rb b/devops-service/providers/static.rb index 31bd748..9de6570 100644 --- a/devops-service/providers/static.rb +++ b/devops-service/providers/static.rb @@ -47,7 +47,7 @@ module Provider "static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" end - def create_server s, out + def create_server s, image, flavor, subnets, groups, out out << "Unsupported operation: ca not create server for provider 'static'" false end diff --git a/devops-service/workers/create_server_worker.rb b/devops-service/workers/create_server_worker.rb index e9892e2..93a56d8 100644 --- a/devops-service/workers/create_server_worker.rb +++ b/devops-service/workers/create_server_worker.rb @@ -11,23 +11,25 @@ require "db/mongo/models/report" class CreateServerWorker < Worker include ServerCommands - def perform(dir, e_provider, server, owner, conf) + def perform(dir, params, owner, conf) logger.info "Create server" - call(conf, e_provider, dir) do |provider, out, file| + call(conf, nil, dir) do |provider, out, file| mongo = Devops::Db.connector - s = Devops::Model::Server.new(server) - s.options = convert_config(server["options"]) + project = mongo.project(params["project"]) + env = project.deploy_env(params["deploy_env"]) +# s = Devops::Model::Server.new(server) +# s.options = convert_config(server["options"]) o = { "file" => file, "_id" => jid, "created_by" => owner, - "project" => s.project, - "deploy_env" => s.deploy_env, + "project" => project.id, + "deploy_env" => env.identifier, "type" => Devops::Model::Report::SERVER_TYPE } mongo.save_report(Devops::Model::Report.new(o)) - status = create_server_proc.call(out, s, provider) + status = create_server(project, env, params, owner, out) status end end diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb index db21c27..251c41a 100644 --- a/devops-service/workers/worker.rb +++ b/devops-service/workers/worker.rb @@ -51,8 +51,8 @@ class Worker provider = nil begin Devops::Db.init + ::Provider::ProviderFactory.init(DevopsConfig.config) unless e_provider.nil? - ::Provider::ProviderFactory.init(DevopsConfig.config) provider = ::Provider::ProviderFactory.get(e_provider) end rescue Exception => e