tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

server and deploy commands migrated to server_executor

Browse Source
This commit is contained in:
amartynov 2015-08-12 16:39:41 +03:00
parent 0f83e3d9e6
commit 5b1b7da21b
9 changed files with 87 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
devops-service/app/api2/handlers/project.rb
View File

@ -1,6 +1,4 @@
require "commands/deploy"
require "commands/status"
require "commands/server"
require "db/mongo/models/project"
require "workers/project_test_worker"
require "app/api2/parsers/project"
@ -17,9 +15,7 @@ module Devops
include Devops::API2_0::Helpers
extend DeployCommands
extend StatusCommands
extend ServerCommands
def project_types
Devops::TypesFactory.types_names
@ -132,18 +128,15 @@ module Devops
out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n")
status = []
servers.each do |s|
begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
project = begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
rescue InvalidPrivileges, RecordNotFound => e
out << e.message + "\n"
status.push 2
next
end
unless keys.key? s.key
k = Devops::Db.connector.key s.key
keys[s.key] = k.path
end
status.push(deploy_server(out, s, keys[s.key]))
deploy_info = project.deploy_info(s.deploy_env, nil)
status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info))
end
status
end
@ -155,12 +148,11 @@ module Devops
uri = URI.parse(request.url)
servers(id, deploy_env, servers).each do |s|
project = begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
rescue InvalidPrivileges, RecordNotFound => e
next
end
jid = DeployWorker.perform_async(dir, s.to_hash, [], DevopsConfig.config)
#DevopsLogger.logger.info "Job '#{jid}' has been started"
uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
end

7
devops-service/app/api2/handlers/server.rb
View File

@ -1,8 +1,6 @@
require 'rufus-scheduler'
require "uri"
require "commands/status"
#require "commands/server"
require "commands/bootstrap_templates"
require "lib/executors/server_executor"
@ -23,10 +21,9 @@ module Devops
set_parser Devops::API2_0::Parser::ServerParser
extend StatusCommands
# extend ServerCommands
extend BootstrapTemplatesCommands
scheduler = Rufus::Scheduler.new
#scheduler = Rufus::Scheduler.new
def servers
fields, reserved = parser.servers
@ -286,6 +283,7 @@ module Devops
end
=begin
class ExpireHandler
include ServerCommands
@ -303,6 +301,7 @@ module Devops
end
end
end
=end
end
end

81
devops-service/commands/deploy.rb
View File

@ -1,81 +0,0 @@
require "commands/knife_commands"
require "commands/ssh"
module DeployCommands
def deploy_server_proc
lambda do |out, s, mongo, tags, deploy_info|
old_tags_str = nil
new_tags_str = nil
unless tags.empty?
old_tags_str = KnifeCommands.tags_list(s.chef_node_name).join(" ")
out << "Server tags: #{old_tags_str}\n"
KnifeCommands.tags_delete(s.chef_node_name, old_tags_str)
new_tags_str = tags.join(" ")
out << "Server new tags: #{new_tags_str}\n"
cmd = KnifeCommands.tags_create(s.chef_node_name, new_tags_str)
unless cmd[1]
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{s.chef_node_name}'"
DevopsLogger.logger.error(m)
out << m + "\n"
return 3
end
DevopsLogger.logger.info("Set tags for '#{s.chef_node_name}': #{new_tags_str}")
end
k = mongo.key s.key
r = deploy_server out, s, k.path, deploy_info
unless tags.empty?
out << "Restore tags\n"
cmd = KnifeCommands.tags_delete(s.chef_node_name, new_tags_str)
DevopsLogger.logger.info("Deleted tags for #{s.chef_node_name}: #{new_tags_str}")
cmd = KnifeCommands.tags_create(s.chef_node_name, old_tags_str)
DevopsLogger.logger.info("Set tags for #{s.chef_node_name}: #{old_tags_str}")
end
return r
end
end
def deploy_server out, server, cert_path, deploy_info
out << "Before deploy hooks...\n"
res = server.run_hook(:before_deploy, out, deploy_info)
out << "Done\n"
out << "\nRun chef-client on '#{server.chef_node_name}'\n"
cmd = "chef-client --no-color"
if deploy_info["use_json_file"]
deploy_info.delete["use_json_file"]
out << "Build information:\n"
json = JSON.pretty_generate(deploy_info)
out << json
out << "\n"
file = "#{server.project}_#{server.deploy_env}_#{Time.new.to_i}"
dir = DevopsConfig.config[:project_info_dir]
File.open(File.join(dir, file), "w") do |f|
f.write json
end
out.flush if out.respond_to?(:flush)
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
end
ip = if server.public_ip.nil?
server.private_ip
else
out << "Public IP detected\n"
server.public_ip
end
out.flush if out.respond_to?(:flush)
lline = KnifeCommands.ssh_stream(out, cmd, ip, server.remote_user, cert_path)
r = /Chef\sClient\sfinished/i
if lline[r].nil?
1
else
out << "After deploy hooks...\n"
res = server.run_hook(:after_deploy, out, deploy_info)
out << "Done\n"
0
end
end
end

52
devops-service/commands/server.rb
View File

@ -1,52 +0,0 @@
require 'set'
require "commands/knife_commands"
require "commands/deploy"
require "exceptions/record_not_found"
module ServerCommands
include DeployCommands
def create_server project, env, params, user, out
provider = ::Provider::ProviderFactory.get(env.provider)
mongo = ::Devops::Db.connector
begin
out << "Create server...\n"
out.flush if out.respond_to?(:flush)
s = Devops::Model::Server.new
s.provider = provider.name
s.project = project.id
s.deploy_env = env.identifier
s.run_list = params["run_list"] || []
s.chef_node_name = params["name"]
s.key = params["key"] || provider.ssh_key
i = mongo.image env.image
s.remote_user = i.remote_user
s.created_by = user
return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out)
out.flush if out.respond_to?(:flush)
DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
unless params["without_bootstrap"]
s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
key = mongo.key(s.key)
s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
options = {
bootstrap_template: i.bootstrap_template,
cert_path: key.path
}
return two_phase_bootstrap(s, options, out)
else
return 0
end
rescue => e
DevopsLogger.logger.error e.message
DevopsLogger.logger.warn roll_back(s, provider)
mongo.server_delete s.id
return 5
end
end
end

4
devops-service/commands/stack.rb
View File

@ -1,8 +1,6 @@
require "commands/server"
require 'commands/commands_storage'
module StackCommands
include ServerCommands
extend self
def sync_bootstrap_proc
@ -29,12 +27,14 @@ module StackCommands
end
end
=begin
def bootstrap_stack_servers_proc(out, stack, mongo, provider, logger)
mongo.stack_servers(stack.id).each do |server|
key = mongo.key(server.key)
two_phase_bootstrap(server, out, provider, mongo, key.path, logger)
end
end
=end
CommandsStorage.add_job_lambda(
sync_bootstrap: sync_bootstrap_proc

10
devops-service/lib/executors/server_executor.rb
View File

@ -39,6 +39,10 @@ module Devops
@deploy_env = e
end
def server
@server
end
def create_server options
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider})
provider = @server.provider_instance
@ -57,11 +61,14 @@ module Devops
res = {}
@out << "\nBefore create hooks...\n"
res[:before] = self.run_hook :before_create
@out << "Done\n"
return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
mongo.server_insert @server
@out << "\nAfter create hooks...\n"
res[:after] = self.run_hook :after_create
@out << "Done\n"
@out.flush
DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
unless options["without_bootstrap"]
@ -80,7 +87,6 @@ module Devops
mongo.server_delete @server.id
return 5
end
end
def bootstrap options

2
devops-service/lib/expire_handler.rb
View File

@ -1,8 +1,6 @@
require "commands/server"
require 'rufus-scheduler'
class ExpireHandler
include ServerCommands
@@scheduler = Rufus::Scheduler.start_new

11
devops-service/workers/bootstrap_worker.rb
View File

@ -3,13 +3,12 @@
require File.join(File.dirname(__FILE__), "worker")
require "lib/executors/server_executor"
require "providers/provider_factory"
require "commands/server"
require "db/mongo/models/server"
require "db/mongo/models/report"
class BootstrapWorker < Worker
include ServerCommands
def perform(dir, e_provider, server, bootstrap_template, owner, conf)
call(conf, e_provider, dir) do |provider, out, file|
@ -26,10 +25,10 @@ class BootstrapWorker < Worker
}
mongo.save_report(Devops::Model::Report.new(o))
key = mongo.key(s.key)
# out << "\nBootstrap with run list: #{s.options[:run_list].inspect}"
status = two_phase_bootstrap s, provider.run_list, bootstrap_template, key.path, out
# status = bootstrap(s, out, key.path)
options = {
:bootstrap_template => bootstrap_template
}
status = Devops::Executor::ServerExecutor.new(s, out).two_phase_bootstrap(options)
mongo.set_report_server_data(jid, s.chef_node_name, s.public_ip || s.private_ip)
status
end

37
devops-service/workers/project_test_worker.rb
View File

@ -1,7 +1,6 @@
require File.join(File.dirname(__FILE__), "worker")
require "providers/provider_factory"
require "commands/server"
require "db/mongo/models/server"
require "json"
require "fileutils"
@ -9,16 +8,15 @@ require "commands/status"
require "db/mongo/models/report"
class ProjectTestWorker < Worker
include ServerCommands
include StatusCommands
def perform(dir, params, conf)
call(conf, nil, dir) do |provider, out, file|
DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{params["user"]})"
user = params["user"]
DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{user})"
mongo = Devops::Db.connector
project = mongo.project(params["project"])
env = project.deploy_env(params["env"])
user = params["user"]
o = {
"file" => file,
"_id" => jid,
@ -29,30 +27,32 @@ class ProjectTestWorker < Worker
}
mongo.save_report(Report.new(o))
provider = ::Provider::ProviderFactory.get(env.provider)
servers = extract_servers(provider, project, env, {}, user)
executor = Devops::Executor::ServerExecutor.new(nil, out)
executor.project = project
executor.deploy_env = env
params["created_by"] = user
result = {:servers => []}
project.deploy_envs = [ env ]
result[:project] = project.to_hash
status = 0
servers.each do |s|
sr = {}
t1 = Time.now
out << "\n=== Create server ===\n"
out.flush
if provider.create_server(s, out)
if executor.create_server({"without_bootstrap" => true})
out << "\n=== Create server - OK ===\n"
out.flush
t2 = Time.now
s = executor.server
sr[:id] = s.id
sr[:create] = {:status => true}
sr[:create][:time] = time_diff_s(t1, t2)
s.chef_node_name = provider.create_default_chef_node_name(s)
logger.info "Server with parameters: #{s.to_hash.inspect} is running"
key = mongo.key(s.key)
DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
out << "\n=== Bootstrap ===\n"
out.flush
r = bootstrap(s, out, key.path)
r = executor.bootstrap({})
t1 = Time.now
sr[:chef_node_name] = s.chef_node_name
if r == 0
@ -60,11 +60,11 @@ class ProjectTestWorker < Worker
out.flush
sr[:bootstrap] = {:status => true}
sr[:bootstrap][:time] = time_diff_s(t2, t1)
logger.info "Server with id '#{s.id}' is bootstraped"
DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
out << "\n=== Check server ===\n"
out.flush
if check_server(s)
mongo.server_insert s
if executor.check_server
# mongo.server_insert s
out << "\n=== OK, server has been inserted ===\n"
out.flush
end
@ -79,9 +79,9 @@ class ProjectTestWorker < Worker
t1 = Time.now
out << "\n=== Delete server ===\n"
out.flush
r = delete_from_chef_server(s.chef_node_name)
r = executor.delete_from_chef_server
begin
r[:server] = provider.delete_server s
r[:server] = s.provider_instance.delete_server s
out << "\n=== Delete server - OK ===\n"
out.flush
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error
@ -89,7 +89,7 @@ class ProjectTestWorker < Worker
out << "\n=== Delete server - FAIL ===\n"
out.flush
r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
logger.warn r[:server]
DevopsLogger.logger.warn r[:server]
end
mongo.server_delete s.id
t2 = Time.now
@ -102,7 +102,6 @@ class ProjectTestWorker < Worker
sr[:create] = {:status => false}
end
result[:servers].push sr
end
out << "\n\n#{result.to_json}"
status
end