tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

server create refactored

Browse Source
This commit is contained in:
amartynov 2015-08-05 14:05:14 +03:00
parent f9db126e0c
commit 5f9a22498f
12 changed files with 168 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
devops-service/app/api2/handlers/server.rb
View File

@ -54,8 +54,8 @@ module Devops
def create_server_stream out, body
status = []
prepare_create_server(body).each do |s|
res = create_server_proc.call(out, s, provider)
prepare_create_server do |project, env, user, body|
res = create_server(project, env, body, user, out)
status.push res
end
status
@ -63,19 +63,16 @@ module Devops
def create_server
dir = DevopsConfig[:report_dir_v2]
files = []
uri = URI.parse(parser.request.url)
prepare_create_server.each do |s|
h = s.to_hash
h["options"] = s.options
jid = CreateServerWorker.perform_async(dir, s.provider, h, parser.current_user, DevopsConfig.config)
file = prepare_create_server do |project, env, user, body|
jid = CreateServerWorker.perform_async(dir, body, user, DevopsConfig.config)
Worker.set_status jid, Worker::STATUS::IN_QUEUE
#logger.info "Job '#{jid}' has been started"
DevopsLogger.logger.info "Job '#{jid}' has been started"
uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
uri.to_s
end
sleep 1
files
[file]
end
def prepare_create_server
@ -96,7 +93,9 @@ module Devops
halt_response("Invalid security groups '#{buf.join("', '")}' for provider '#{provider.name}'") if buf.empty?
end
Server.extract_servers(provider, p, env, body, user)
yield p, env, user, body
#Server.extract_servers(provider, p, env, body, user)
end
def pause_server node_name

127
devops-service/commands/server.rb
View File

@ -1,3 +1,4 @@
require 'set'
require "commands/knife_commands"
require "commands/deploy"
require "exceptions/record_not_found"
@ -28,80 +29,51 @@ module ServerCommands
end
end
end
=begin
def create_server_proc
lambda do |out, s, provider|
mongo = ::Devops::Db.connector
begin
out << "Create server...\n"
out.flush if out.respond_to?(:flush)
unless provider.create_server(s, out)
return 3
end
s.create
out.flush if out.respond_to?(:flush)
DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
def create_server project, env, params, user, out
provider = ::Provider::ProviderFactory.get(env.provider)
mongo = ::Devops::Db.connector
begin
out << "Create server...\n"
out.flush if out.respond_to?(:flush)
s = Devops::Model::Server.new
s.provider = provider.name
s.project = project.id
s.deploy_env = env.identifier
s.run_list = params["run_list"] || []
s.chef_node_name = params["name"]
s.key = params["key"] || provider.ssh_key
i = mongo.image env.image
puts "Image: #{i.inspect}"
s.remote_user = i.remote_user
s.created_by = user
return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out)
out.flush if out.respond_to?(:flush)
DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
unless params["without_bootstrap"]
s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
key = mongo.key(s.key)
s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
out << "\n\nBootstrap..."
out.flush if out.respond_to?(:flush)
run_list = s.options[:run_list]
s.options[:run_list] = provider.run_list
out << "\nBootstrap with provider run list: #{s.options[:run_list].inspect}"
status = bootstrap(s, out, key.path)
out.flush if out.respond_to?(:flush)
if status == 0
mongo.server_set_chef_node_name s
DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
if check_server(s)
out << "Server #{s.chef_node_name} is created"
else
out << roll_back(s, provider)
mongo.server_delete s.id
return 5
end
out << "\n"
out.flush if out.respond_to?(:flush)
out << "\nAdd project run list: #{run_list.inspect}"
s.options[:run_list] += run_list
KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list])
status = deploy_server(out, s, key.path)
if status != 0
msg = "Failed on chef-client with project run list, server with id '#{s.id}'"
DevopsLogger.logger.error msg
out << "\n" + msg + "\n"
mongo.server_delete s.id
end
return status
else
msg = "Failed while bootstraping server with id '#{s.id}'"
DevopsLogger.logger.error msg
out << "\n" + msg + "\n"
out << roll_back(s, provider)
mongo.server_delete s.id
status
end
rescue IOError => e
DevopsLogger.logger.error e.message
DevopsLogger.logger.warn roll_back(s, provider)
mongo.server_delete s.id
return 5
return two_phase_bootstrap(s, provider.run_list, i.bootstrap_template, key.path, out)
else
return 0
end
rescue => e
DevopsLogger.logger.error e.message
DevopsLogger.logger.warn roll_back(s, provider)
mongo.server_delete s.id
return 5
end
end
=end
def two_phase_bootstrap s, out, provider, cert_path
def two_phase_bootstrap s, provider_run_list, bootstrap_template, cert_path, out
mongo = ::Devops::Db.connector
out << "\n\nBootstrap...\n"
out.flush if out.respond_to?(:flush)
run_list = s.options[:run_list]
s.options[:run_list] = provider.run_list
out << "Bootstrap with provider run list: #{s.options[:run_list].inspect}\n"
s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
status = bootstrap(s, out, cert_path)
status = bootstrap(s, out, bootstrap_template, provider_run_list, cert_path)
out.flush if out.respond_to?(:flush)
if status == 0
DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
@ -115,9 +87,10 @@ module ServerCommands
out << "\n"
out.flush if out.respond_to?(:flush)
out << "\nAdd project run list: #{run_list.inspect}"
s.options[:run_list] += run_list
KnifeCommands.set_run_list(s.chef_node_name, s.options[:run_list])
run_list = s.run_list + provider_run_list
out << "\nRun list: #{run_list.inspect}"
# s.options[:run_list] += run_list
KnifeCommands.set_run_list(s.chef_node_name, run_list)
status = deploy_server(out, s, cert_path)
if status != 0
msg = "Failed on chef-client with project run list, server with id '#{s.id}'"
@ -172,12 +145,14 @@ module ServerCommands
i = mongo.image env.image
flavor = flavors.detect {|f| f["id"] == env.flavor}
raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil?
rl = Set.new
rl.merge(project.run_list).merge(env.run_list)
o = {
:image => i,
:name => params["name"],
:flavor => flavor["id"],
:groups => params["groups"] || env.groups,
:run_list => env.run_list,
:run_list => rl,
:subnets => env.subnets,
:key => params["key"]
}
@ -191,15 +166,16 @@ module ServerCommands
s.provider = provider.name
s.project = project_name
s.deploy_env = env_name
s.run_list = params["run_list"] || []
s.remote_user = image.remote_user
s.chef_node_name = info[:name]# || provider.create_default_chef_node_name(s)
s.chef_node_name = info[:name]
s.key = info[:key] || provider.ssh_key
s.options = {
:image => image.id,
:flavor => info[:flavor],
:name => info[:name],
:groups => info[:groups],
:run_list => info[:run_list],
:run_list => info[:run_list].merge(s.run_list),
:bootstrap_template => image.bootstrap_template,
:subnets => info[:subnets]
}
@ -220,7 +196,7 @@ module ServerCommands
KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
end
def bootstrap s, out, cert_path
def bootstrap s, out, bootstrap_template, run_list, cert_path
out << "Before bootstrap hooks...\n"
res = s.run_hook(:before_bootstrap, out)
out << "Done\n"
@ -228,6 +204,7 @@ module ServerCommands
out << "Error: Private IP is null"
return false
end
out << "\nBootstrap with run list: #{run_list.inspect}\n"
ja = {
:provider => s.provider,
:devops_host => `hostname`.strip
@ -235,12 +212,12 @@ module ServerCommands
bootstrap_options = [
"-x #{s.remote_user}",
"-i #{cert_path}",
"--json-attributes '#{ja.to_json}'"
"--json-attributes '#{ja.to_json}'",
"-N #{s.chef_node_name}"
]
bootstrap_options.push "--sudo" unless s.remote_user == "root"
bootstrap_options.push "-N #{s.chef_node_name}" if s.chef_node_name
bootstrap_options.push "-d #{s.options[:bootstrap_template]}" if s.options[:bootstrap_template]
bootstrap_options.push "-r #{s.options[:run_list].join(",")}" unless s.options[:run_list].empty?
bootstrap_options.push "-d #{bootstrap_template}" if bootstrap_template
bootstrap_options.push "-r #{run_list.join(",")}" unless run_list.empty?
ip = s.private_ip
unless s.public_ip.nil? || s.public_ip.strip.empty?
ip = s.public_ip

6
devops-service/db/mongo/connectors/report.rb
View File

@ -11,7 +11,11 @@ module Connectors
def save_report r
r.created_at = Time.new
collection.insert(r.to_mongo_hash)
collection.insert(r.to_mongo_hash) unless report_exists?(r.id)
end
def report_exists? id
!collection.find({"_id" => id}, fields: ["_id" => true]).to_a.empty?
end
def reports options={}

15
devops-service/db/mongo/models/model_with_provider.rb Normal file
View File

@ -0,0 +1,15 @@
require "providers/provider_factory"
module Devops
module Model
module ModelWithProvider
attr_accessor :provider
def provider_instance
@provider_instance ||= Provider::ProviderFactory.get(self.provider)
end
end
end
end

8
devops-service/db/mongo/models/project.rb
View File

@ -41,8 +41,8 @@ module Devops
self.archived = p["archived"] || false
self.run_list = p["run_list"] || []
handler = Devops::TypesFactory.type self.type
handler.prepare(self) unless handler.nil?
@handler = Devops::TypesFactory.type self.type
@handler.prepare(self)
env_class = ( self.multi? ? DeployEnvMulti : DeployEnvFactory )
unless p["deploy_envs"].nil?
@ -171,6 +171,10 @@ module Devops
res
end
def extract_servers env, params
@handler.extract_servers(self, env, params)
end
def self.build_from_bson p
p["name"] = p["_id"]
Project.new p

5
devops-service/db/mongo/models/server.rb
View File

@ -24,7 +24,7 @@ module Devops
define_hook :after_bootstrap
attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list
attr_accessor :options, :key
attr_accessor :key
types :id => {:type => String, :empty => false},
:provider => {:type => String, :empty => false},
@ -63,9 +63,10 @@ module Devops
self.run_list = s["run_list"] || []
end
def create
def create provider, image, flavor, subnets, groups, out
res = {}
res[:before] = self.run_hook :before_create
return false unless provider.create_server(self, image, flavor, subnets, groups, out)
Devops::Db.connector.server_insert self
res[:after] = self.run_hook :after_create
res

50
devops-service/lib/project/type/project_type.rb
View File

@ -17,5 +17,55 @@ module Devops
end
def extract_servers project, env, params
=begin
mongo = ::Devops::Db.connector
env_name = env.identifier
project_name = project.id
servers_info = []
i = mongo.image env.image
rl = Set.new
rl.merge(project.run_list).merge(env.run_list)
o = {
:image => i,
:name => ,
:flavor => flavor["id"],
:groups => params["groups"] || env.groups,
:run_list => rl,
:subnets => env.subnets,
:key =>
}
servers_info.push(o)
servers = []
servers_info.each do |info|
servers.push s
end
return servers
image = info[:image]
=end
s = Devops::Model::Server.new
s.provider = provider.name
s.project = project_name
s.deploy_env = env_name
s.run_list = params["run_list"] || []
s.remote_user = image.remote_user
s.chef_node_name = params["name"]
s.key = params["key"] || provider.ssh_key
=begin
s.options = {
:image => image.id,
:flavor => info[:flavor],
:name => info[:name],
:groups => info[:groups],
:run_list => info[:run_list].merge(s.run_list),
:bootstrap_template => image.bootstrap_template,
:subnets => info[:subnets]
}
=end
#s.created_by = user
[s]
end
end
end

16
devops-service/providers/ec2.rb
View File

@ -91,28 +91,28 @@ module Provider
convert_server list[0]["instancesSet"][0]
end
def create_server s, out
def create_server s, image, flavor, subnets, groups, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
options = {
"InstanceType" => s.options[:flavor],
"Placement.AvailabilityZone" => s.options[:availability_zone],
"InstanceType" => flavor,
# "Placement.AvailabilityZone" => s.options[:availability_zone],
"KeyName" => self.ssh_key
}
vpcId = nil
unless s.options[:subnets].empty?
options["SubnetId"] = s.options[:subnets][0]
unless subnets.empty?
options["SubnetId"] = subnets[0]
vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
if vpcId.nil?
out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
return false
end
end
options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
options["SecurityGroupId"] = extract_group_ids(groups, vpcId).join(",")
aws_server = nil
compute = self.compute
begin
aws_server = compute.run_instances(s.options[:image], 1, 1, options)
aws_server = compute.run_instances(image, 1, 1, options)
rescue Excon::Errors::Unauthorized => ue
#root = XML::Parser.string(ue.response.body).parse.root
#msg = root.children.find { |node| node.name == "Message" }
@ -130,8 +130,6 @@ module Provider
instance = abody["instancesSet"][0]
s.id = instance["instanceId"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, state = nil, instance["instanceState"]["name"]

20
devops-service/providers/openstack.rb
View File

@ -91,34 +91,34 @@ module Provider
end
end
def create_server s, out
def create_server s, image, flavor, subnets, groups, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
if s.chef_node_name.nil?
out << "Generate new instance name: "
out << s.chef_node_name = create_default_chef_node_name(s)
out << "\n"
end
networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
buf = s.options[:subnets] - networks.map{|n| n["name"]}
networks = self.networks.select{|n| subnets.include?(n["name"])}
buf = subnets - networks.map{|n| n["name"]}
unless buf.empty?
out << "No networks with names '#{buf.join("', '")}' found"
return false
end
s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
flavor = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == flavor}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{image}', flavor '#{flavor}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
compute = self.compute
begin
o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
o_server = compute.create_server(s.chef_node_name, image, flavor,
"nics" => networks.map{|n| {"net_id" => n["id"]}},
"security_groups" => s.options[:groups],
"security_groups" => groups,
"key_name" => s.key)
rescue Excon::Errors::BadRequest => e
response = ::Chef::JSONCompat.from_json(e.response.body)
if response['badRequest']['code'] == 400
if response['badRequest']['message'] =~ /Invalid flavorRef/
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{flavor}"
elsif response['badRequest']['message'] =~ /Invalid imageRef/
out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
out << "\nERROR: Bad request (400): Invalid image specified: #{image}"
else
out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
end
@ -141,8 +141,6 @@ module Provider
sbody = o_server.body
s.id = sbody["server"]["id"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, status = nil, nil

2
devops-service/providers/static.rb
View File

@ -47,7 +47,7 @@ module Provider
"static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def create_server s, out
def create_server s, image, flavor, subnets, groups, out
out << "Unsupported operation: ca not create server for provider 'static'"
false
end

16
devops-service/workers/create_server_worker.rb
View File

@ -11,23 +11,25 @@ require "db/mongo/models/report"
class CreateServerWorker < Worker
include ServerCommands
def perform(dir, e_provider, server, owner, conf)
def perform(dir, params, owner, conf)
logger.info "Create server"
call(conf, e_provider, dir) do |provider, out, file|
call(conf, nil, dir) do |provider, out, file|
mongo = Devops::Db.connector
s = Devops::Model::Server.new(server)
s.options = convert_config(server["options"])
project = mongo.project(params["project"])
env = project.deploy_env(params["deploy_env"])
# s = Devops::Model::Server.new(server)
# s.options = convert_config(server["options"])
o = {
"file" => file,
"_id" => jid,
"created_by" => owner,
"project" => s.project,
"deploy_env" => s.deploy_env,
"project" => project.id,
"deploy_env" => env.identifier,
"type" => Devops::Model::Report::SERVER_TYPE
}
mongo.save_report(Devops::Model::Report.new(o))
status = create_server_proc.call(out, s, provider)
status = create_server(project, env, params, owner, out)
status
end
end

2
devops-service/workers/worker.rb
View File

@ -51,8 +51,8 @@ class Worker
provider = nil
begin
Devops::Db.init
::Provider::ProviderFactory.init(DevopsConfig.config)
unless e_provider.nil?
::Provider::ProviderFactory.init(DevopsConfig.config)
provider = ::Provider::ProviderFactory.get(e_provider)
end
rescue Exception => e