tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

#856: subreports for stack reports

Browse Source
This commit is contained in:
Anton Martynov 2015-11-03 12:05:07 +03:00
parent 4800efc987
commit 2f2f446def
14 changed files with 97 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
devops-service/app/api2/handlers/deploy.rb
View File

@ -19,10 +19,8 @@ module Devops
names = body["names"]
tags = body["tags"] || []
run_list = body["run_list"]
dir = DevopsConfig.config[:report_dir_v2]
files = []
jid = nil
uri = URI.parse(parser.request.url)
owner = parser.current_user
@deploy_info_buf = {}
servers(names).each do |s|
@ -36,7 +34,7 @@ module Devops
deploy_info = create_deploy_info(s, project, body["build_number"])
deploy_info["run_list"] = run_list if run_list
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -63,7 +61,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push(uri)
files.push(jid)
end
files
end

8
devops-service/app/api2/handlers/project.rb
View File

@ -251,13 +251,13 @@ module Devops
deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil)
end
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: parser.current_user,
tags: [],
deploy_info: deploy_info
)
files.push uri
files.push jid
end
files
end
@ -288,14 +288,14 @@ module Devops
raise InvalidRecord.new(msg)
end
uri = Worker.start_async(ProjectTestWorker, @request,
jid = Worker.start_async(ProjectTestWorker,
project: project.id,
deploy_env: env.identifier,
user: @request.env['REMOTE_USER']
)
sleep 1
return [uri]
return [jid]
end
end

12
devops-service/app/api2/handlers/server.rb
View File

@ -71,11 +71,11 @@ module Devops
user = parser.current_user
check_if_server_attrs_are_valid(body, user)
uri = Worker.start_async(CreateServerWorker, @request,
jid = Worker.start_async(CreateServerWorker,
server_attrs: body,
owner: user
)
[uri]
[jid]
end
def pause_server node_name
@ -153,19 +153,15 @@ module Devops
def bootstrap_server
server, rl, bootstrap_template = prepare_bootstrap_server
dir = DevopsConfig[:report_dir_v2]
files = []
uri = URI.parse(@request.url)
uri = Worker.start_async(BootstrapWorker, @request,
provider_name: server.provider,
jid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: bootstrap_template,
owner: parser.current_user
)
sleep 1
[uri]
[jid]
end
def prepare_bootstrap_server

12
devops-service/app/api2/handlers/stack.rb
View File

@ -28,13 +28,10 @@ module Devops
object["provider"] = env.provider
object["provider_account"] = env.provider_account
# TODO: without provider_name
uri = Worker.start_async(StackBootstrapWorker, @request,
provider_name: env.provider,
jid = Worker.start_async(StackBootstrapWorker,
stack_attributes: object
)
puts "Syncing report is located here: #{uri}"
[uri]
[jid]
end
def stack id
@ -102,7 +99,6 @@ module Devops
def deploy id
stack = self.stack(id)
owner = parser.current_user
status = []
project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner)
deploy_env_model = project.deploy_env(stack.deploy_env)
body = parser.deploy
@ -114,7 +110,7 @@ module Devops
servers.each do |s|
begin
deploy_info = project.deploy_info(deploy_env_model, nil)
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -140,7 +136,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push uri
files.push jid
end
files
end

14
devops-service/db/mongo/connectors/report.rb
View File

@ -51,8 +51,18 @@ module Connectors
collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)}
end
def set_report_status(jid, status)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}})
def add_report_subreports(jid, subreports)
collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}})
end
def set_report_status(jid, status, job_result_code)
set = {"status" => status, "updated_at" => Time.new}
set["job_result_code"] = job_result_code unless job_result_code.nil?
collection.update({"_id" => jid}, {"$set" => set})
end
def set_report_status(jid, status, job_result_code)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}})
end
def set_report_server_data id, chef_node_name, host

10
devops-service/db/mongo/models/report.rb
View File

@ -12,7 +12,7 @@ module Devops
DEPLOY_STACK_TYPE = 6
DELETE_SERVER_TYPE = 7
attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack
attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
def initialize r
self.id = r["_id"]
@ -21,13 +21,15 @@ module Devops
self.project = r["project"]
self.deploy_env = r["deploy_env"]
self.type = r["type"]
self.created_at = r["created_at"]
# self.created_at = r["created_at"]
self.chef_node_name = r["chef_node_name"]
self.host = r["host"]
self.stack = r["stack"]
self.created_at = r["created_at"].localtime unless r["created_at"].nil?
self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil?
self.status = r["status"]
self.subreports = r["subreports"]
self.job_result_code = r["job_result_code"]
end
def to_hash_without_id
@ -42,7 +44,9 @@ module Devops
"chef_node_name" => self.chef_node_name,
"host" => self.host,
"status" => self.status,
"stack" => self.stack
"stack" => self.stack,
"subreports" => self.subreports,
"job_result_code" => self.job_result_code
}
end

2
devops-service/db/mongo/mongo_connector.rb
View File

@ -31,7 +31,7 @@ class MongoConnector
[:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
:user_update, :create_root_user, :check_user_privileges] => :users_connector,
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector,
[:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector
)

4
devops-service/workers/bootstrap_worker.rb
View File

@ -1,19 +1,17 @@
require File.join(File.dirname(__FILE__), "worker")
require "lib/executors/server_executor"
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
class BootstrapWorker < Worker
def perform(options)
provider_name = options.fetch('provider_name')
server_attrs = options.fetch('server_attrs')
bootstrap_template = options.fetch('bootstrap_template')
owner = options.fetch('owner')
call(provider_name) do |provider, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)

3
devops-service/workers/create_server_worker.rb
View File

@ -1,6 +1,5 @@
require File.join(File.dirname(__FILE__), "worker")
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
require "lib/executors/server_executor"
@ -11,7 +10,7 @@ class CreateServerWorker < Worker
server_attrs = options.fetch('server_attrs')
owner = options.fetch('owner')
call(nil) do |not_used, out, file|
call() do |out, file|
project = mongo.project(server_attrs["project"])
env = project.deploy_env(server_attrs["deploy_env"])
report = save_report(file, project, env, owner)

2
devops-service/workers/delete_server_worker.rb
View File

@ -8,7 +8,7 @@ class DeleteServerWorker < Worker
chef_node_name = options.fetch('server_chef_node_name')
puts "Expire server '#{chef_node_name}'."
call(nil) do |not_used, out, file|
call() do |out, file|
server = mongo.server_by_chef_node_name(chef_node_name)
report = save_report(file, server)

2
devops-service/workers/deploy_worker.rb
View File

@ -12,7 +12,7 @@ class DeployWorker < Worker
tags = options.fetch('tags')
deploy_info = options.fetch('deploy_info')
call(nil) do |not_used, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)

2
devops-service/workers/project_test_worker.rb
View File

@ -15,7 +15,7 @@ class ProjectTestWorker < Worker
project_name = params.fetch('project')
deploy_env_name = params.fetch('deploy_env')
call(nil) do |not_used, out, file|
call() do |out, file|
DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})"
project = mongo.project(project_name)
env = project.deploy_env(deploy_env_name)

54
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -1,4 +1,3 @@
require "providers/provider_factory"
require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker
include StackCommands
def perform(options)
provider_name = options.fetch('provider_name')
stack_attrs = options.fetch('stack_attributes')
call(provider_name) do |provider, out, file|
call() do |out, file|
@out = out
without_bootstrap = stack_attrs.delete('without_bootstrap')
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
@ -23,13 +21,13 @@ class StackBootstrapWorker < Worker
report = save_report(file, stack_attrs)
begin
stack = create_stack(provider_name, stack_attrs)
stack = create_stack(stack_attrs)
rescue StackCreatingError
return 1
end
begin
servers = persist_stack_servers!(stack, provider)
servers = persist_stack_servers!(stack)
bootstrap_servers!(servers, report) unless without_bootstrap
rescue BootstrapingStackServerError
@out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
@ -57,8 +55,8 @@ class StackBootstrapWorker < Worker
@out.puts "Rollback has been completed"
end
def create_stack(provider_name, stack_attrs)
stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out)
def create_stack(stack_attrs)
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
mongo.stack_insert(stack)
operation_result = sync_stack_proc.call(@out, stack, mongo)
@ -76,15 +74,40 @@ class StackBootstrapWorker < Worker
def bootstrap_servers!(servers, report)
@out << "\nStart bootstraping stack servers\n"
bootstraping_results = {}
report.subreports = subreports = []
data = {}
servers.each do |server|
executor = Devops::Executor::ServerExecutor.new(server, @out)
executor.report = report
#TODO: can stack choose bootstrap template?
bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'})
@out.flush
sjid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
subreports << sjid
@out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
data[server.id] = sjid
end
check_bootstrap_results!(bootstraping_results)
@out.puts
@out.flush
mongo.add_report_subreports(jid, subreports)
results = []
data.each do |server_id, subreport_id|
begin
sleep(5)
subreport = mongo.report(subreport_id)
status = subreport.status
if status == Worker::STATUS::COMPLETED
@out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
break
elsif status == Worker::STATUS::FAILED
results << subreport.job_result_code
@out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
break
end
end while(true)
end
@out.flush
results.empty? ? 0 : -5
end
def check_bootstrap_results!(results)
@ -130,11 +153,12 @@ class StackBootstrapWorker < Worker
report
end
def persist_stack_servers!(stack, provider)
def persist_stack_servers!(stack)
@out.puts "Start syncing stack servers with CID"
@out.flush
project = mongo.project(stack.project)
deploy_env = project.deploy_env(stack.deploy_env)
provider = stack.provider_instance
servers = provider.stack_servers(stack).map do |extended_info|
server_attrs = {

42
devops-service/workers/worker.rb
View File

@ -26,14 +26,16 @@ class Worker
IN_QUEUE = "queued"
end
def self.start_async(worker_class, request, job_options)
def self.start_async(worker_class, job_options)
jid = worker_class.perform_async(job_options.dup)
Worker.set_status jid, Worker::STATUS::IN_QUEUE
DevopsLogger.logger.info "Job '#{jid}' has been queued"
jid
=begin
uri = URI.parse(request.url)
uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}"
uri.to_s
=end
end
def self.start_sync(worker_class, request, job_options, out)
@ -50,14 +52,13 @@ class Worker
Sidekiq.redis {|con| con.hset "devops", id, status}
end
def call provider_name, &block
def call &block
begin
initialize_devops(provider_name)
provider = ::Provider::ProviderFactory.get(provider_name) if provider_name
initialize_devops()
if jid
call_async(provider, &block)
call_async(&block)
else
call_sync(provider, &block)
call_sync(&block)
end
rescue StandardError => e
puts e.message
@ -68,7 +69,7 @@ class Worker
private
def initialize_devops(provider_name)
def initialize_devops()
DevopsLogger.logger = logger
DevopsConfig.read
DevopsService.init
@ -76,33 +77,33 @@ class Worker
end
# outputs to file
def call_async(provider)
def call_async()
dir = DevopsConfig[:report_dir_v2]
# directory is created on server start in config.ru
file = File.join(dir, jid)
update_job_status(STATUS::INIT)
update_job_status(STATUS::INIT, nil)
File.open(file, "w") do |out|
begin
update_job_status(STATUS::RUNNING)
update_job_status(STATUS::RUNNING, nil)
self.out = out
job_result = yield(provider, out, file)
job_result = yield(out, file)
canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED)
update_job_status(canonical_status)
update_job_status(canonical_status, job_result)
rescue StandardError, RecordNotFound => e
out << "\n #{e.class}\n #{e.message}\n"
out << e.backtrace.join("\n")
update_job_status(STATUS::FAILED)
update_job_status(STATUS::FAILED, -100)
end
end
end
# outputs to STDOUT
def call_sync(provider)
def call_sync()
out = STDOUT
begin
yield(provider, out, '')
yield(out, '')
rescue StandardError, RecordNotFound => e
out << "\n"
out << e.message
@ -115,17 +116,12 @@ class Worker
::Devops::Db.connector
end
def update_job_status(status)
def update_job_status(status, job_result_code)
set_status(jid, status)
mongo.set_report_status(jid, status)
mongo.set_report_status(jid, status, job_result_code)
status
end
def init_provider(provider_name)
::Provider::ProviderFactory.init(DevopsConfig.config)
::Provider::ProviderFactory.get(provider_name) if provider_name
end
def convert_config conf
config = {}
conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}