diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index e61536d..579240f 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -1,6 +1,4 @@ -require "commands/deploy" require "commands/status" -require "commands/server" require "db/mongo/models/project" require "workers/project_test_worker" require "app/api2/parsers/project" @@ -17,9 +15,7 @@ module Devops include Devops::API2_0::Helpers - extend DeployCommands extend StatusCommands - extend ServerCommands def project_types Devops::TypesFactory.types_names @@ -132,18 +128,15 @@ module Devops out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") status = [] servers.each do |s| - begin - Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER'] + project = begin + Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e out << e.message + "\n" status.push 2 next end - unless keys.key? s.key - k = Devops::Db.connector.key s.key - keys[s.key] = k.path - end - status.push(deploy_server(out, s, keys[s.key])) + deploy_info = project.deploy_info(s.deploy_env, nil) + status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info)) end status end @@ -155,12 +148,11 @@ module Devops uri = URI.parse(request.url) servers(id, deploy_env, servers).each do |s| project = begin - Devops::Db.connector.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e next end jid = DeployWorker.perform_async(dir, s.to_hash, [], DevopsConfig.config) - #DevopsLogger.logger.info "Job '#{jid}' has been started" uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/" + jid files.push uri.to_s end diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb index dfe58ad..83fd486 100644 --- a/devops-service/app/api2/handlers/server.rb +++ b/devops-service/app/api2/handlers/server.rb @@ -1,8 +1,6 @@ -require 'rufus-scheduler' require "uri" require "commands/status" -#require "commands/server" require "commands/bootstrap_templates" require "lib/executors/server_executor" @@ -23,10 +21,9 @@ module Devops set_parser Devops::API2_0::Parser::ServerParser extend StatusCommands -# extend ServerCommands extend BootstrapTemplatesCommands - scheduler = Rufus::Scheduler.new + #scheduler = Rufus::Scheduler.new def servers fields, reserved = parser.servers @@ -286,6 +283,7 @@ module Devops end +=begin class ExpireHandler include ServerCommands @@ -303,6 +301,7 @@ module Devops end end end +=end end end diff --git a/devops-service/commands/deploy.rb b/devops-service/commands/deploy.rb deleted file mode 100644 index 8b725a2..0000000 --- a/devops-service/commands/deploy.rb +++ /dev/null @@ -1,81 +0,0 @@ -require "commands/knife_commands" -require "commands/ssh" - -module DeployCommands - - def deploy_server_proc - lambda do |out, s, mongo, tags, deploy_info| - old_tags_str = nil - new_tags_str = nil - unless tags.empty? - old_tags_str = KnifeCommands.tags_list(s.chef_node_name).join(" ") - out << "Server tags: #{old_tags_str}\n" - KnifeCommands.tags_delete(s.chef_node_name, old_tags_str) - - new_tags_str = tags.join(" ") - out << "Server new tags: #{new_tags_str}\n" - cmd = KnifeCommands.tags_create(s.chef_node_name, new_tags_str) - unless cmd[1] - m = "Error: Cannot add tags '#{new_tags_str}' to server '#{s.chef_node_name}'" - DevopsLogger.logger.error(m) - out << m + "\n" - return 3 - end - DevopsLogger.logger.info("Set tags for '#{s.chef_node_name}': #{new_tags_str}") - end - - k = mongo.key s.key - r = deploy_server out, s, k.path, deploy_info - - unless tags.empty? - out << "Restore tags\n" - cmd = KnifeCommands.tags_delete(s.chef_node_name, new_tags_str) - DevopsLogger.logger.info("Deleted tags for #{s.chef_node_name}: #{new_tags_str}") - cmd = KnifeCommands.tags_create(s.chef_node_name, old_tags_str) - DevopsLogger.logger.info("Set tags for #{s.chef_node_name}: #{old_tags_str}") - end - return r - end - end - - def deploy_server out, server, cert_path, deploy_info - out << "Before deploy hooks...\n" - res = server.run_hook(:before_deploy, out, deploy_info) - out << "Done\n" - out << "\nRun chef-client on '#{server.chef_node_name}'\n" - cmd = "chef-client --no-color" - if deploy_info["use_json_file"] - deploy_info.delete["use_json_file"] - out << "Build information:\n" - json = JSON.pretty_generate(deploy_info) - out << json - out << "\n" - file = "#{server.project}_#{server.deploy_env}_#{Time.new.to_i}" - dir = DevopsConfig.config[:project_info_dir] - File.open(File.join(dir, file), "w") do |f| - f.write json - end - out.flush if out.respond_to?(:flush) - cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}" - end - ip = if server.public_ip.nil? - server.private_ip - else - out << "Public IP detected\n" - server.public_ip - end - out.flush if out.respond_to?(:flush) - lline = KnifeCommands.ssh_stream(out, cmd, ip, server.remote_user, cert_path) - r = /Chef\sClient\sfinished/i - if lline[r].nil? - 1 - else - out << "After deploy hooks...\n" - res = server.run_hook(:after_deploy, out, deploy_info) - out << "Done\n" - 0 - end - - end - -end diff --git a/devops-service/commands/server.rb b/devops-service/commands/server.rb deleted file mode 100644 index be31a95..0000000 --- a/devops-service/commands/server.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'set' -require "commands/knife_commands" -require "commands/deploy" -require "exceptions/record_not_found" - -module ServerCommands - - include DeployCommands - - def create_server project, env, params, user, out - provider = ::Provider::ProviderFactory.get(env.provider) - mongo = ::Devops::Db.connector - begin - out << "Create server...\n" - out.flush if out.respond_to?(:flush) - - s = Devops::Model::Server.new - s.provider = provider.name - s.project = project.id - s.deploy_env = env.identifier - s.run_list = params["run_list"] || [] - s.chef_node_name = params["name"] - s.key = params["key"] || provider.ssh_key - - i = mongo.image env.image - s.remote_user = i.remote_user - s.created_by = user - - return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out) - out.flush if out.respond_to?(:flush) - DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running" - unless params["without_bootstrap"] - s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list) - key = mongo.key(s.key) - s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil? - options = { - bootstrap_template: i.bootstrap_template, - cert_path: key.path - } - return two_phase_bootstrap(s, options, out) - else - return 0 - end - rescue => e - DevopsLogger.logger.error e.message - DevopsLogger.logger.warn roll_back(s, provider) - mongo.server_delete s.id - return 5 - end - end - -end diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb index 38cbd2d..86168bc 100644 --- a/devops-service/commands/stack.rb +++ b/devops-service/commands/stack.rb @@ -1,8 +1,6 @@ -require "commands/server" require 'commands/commands_storage' module StackCommands - include ServerCommands extend self def sync_bootstrap_proc @@ -29,12 +27,14 @@ module StackCommands end end +=begin def bootstrap_stack_servers_proc(out, stack, mongo, provider, logger) mongo.stack_servers(stack.id).each do |server| key = mongo.key(server.key) two_phase_bootstrap(server, out, provider, mongo, key.path, logger) end end +=end CommandsStorage.add_job_lambda( sync_bootstrap: sync_bootstrap_proc diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 854a193..6c70d47 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -39,6 +39,10 @@ module Devops @deploy_env = e end + def server + @server + end + def create_server options @server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider}) provider = @server.provider_instance @@ -57,11 +61,14 @@ module Devops res = {} @out << "\nBefore create hooks...\n" res[:before] = self.run_hook :before_create + @out << "Done\n" + return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out) mongo.server_insert @server + @out << "\nAfter create hooks...\n" res[:after] = self.run_hook :after_create - + @out << "Done\n" @out.flush DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running" unless options["without_bootstrap"] @@ -80,7 +87,6 @@ module Devops mongo.server_delete @server.id return 5 end - end def bootstrap options diff --git a/devops-service/lib/expire_handler.rb b/devops-service/lib/expire_handler.rb index 3a00ea0..1eba9df 100644 --- a/devops-service/lib/expire_handler.rb +++ b/devops-service/lib/expire_handler.rb @@ -1,8 +1,6 @@ -require "commands/server" require 'rufus-scheduler' class ExpireHandler - include ServerCommands @@scheduler = Rufus::Scheduler.start_new diff --git a/devops-service/workers/bootstrap_worker.rb b/devops-service/workers/bootstrap_worker.rb index 9f25d1c..1fe177e 100644 --- a/devops-service/workers/bootstrap_worker.rb +++ b/devops-service/workers/bootstrap_worker.rb @@ -3,13 +3,12 @@ require File.join(File.dirname(__FILE__), "worker") +require "lib/executors/server_executor" require "providers/provider_factory" -require "commands/server" require "db/mongo/models/server" require "db/mongo/models/report" class BootstrapWorker < Worker - include ServerCommands def perform(dir, e_provider, server, bootstrap_template, owner, conf) call(conf, e_provider, dir) do |provider, out, file| @@ -26,10 +25,10 @@ class BootstrapWorker < Worker } mongo.save_report(Devops::Model::Report.new(o)) - key = mongo.key(s.key) -# out << "\nBootstrap with run list: #{s.options[:run_list].inspect}" - status = two_phase_bootstrap s, provider.run_list, bootstrap_template, key.path, out -# status = bootstrap(s, out, key.path) + options = { + :bootstrap_template => bootstrap_template + } + status = Devops::Executor::ServerExecutor.new(s, out).two_phase_bootstrap(options) mongo.set_report_server_data(jid, s.chef_node_name, s.public_ip || s.private_ip) status end diff --git a/devops-service/workers/project_test_worker.rb b/devops-service/workers/project_test_worker.rb index bca1cb0..cf9c6db 100644 --- a/devops-service/workers/project_test_worker.rb +++ b/devops-service/workers/project_test_worker.rb @@ -1,7 +1,6 @@ require File.join(File.dirname(__FILE__), "worker") require "providers/provider_factory" -require "commands/server" require "db/mongo/models/server" require "json" require "fileutils" @@ -9,16 +8,15 @@ require "commands/status" require "db/mongo/models/report" class ProjectTestWorker < Worker - include ServerCommands include StatusCommands def perform(dir, params, conf) call(conf, nil, dir) do |provider, out, file| - DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{params["user"]})" + user = params["user"] + DevopsLogger.logger.info "Test project '#{params["project"]}' and env '#{params["env"]}' (user - #{user})" mongo = Devops::Db.connector project = mongo.project(params["project"]) env = project.deploy_env(params["env"]) - user = params["user"] o = { "file" => file, "_id" => jid, @@ -29,80 +27,81 @@ class ProjectTestWorker < Worker } mongo.save_report(Report.new(o)) - provider = ::Provider::ProviderFactory.get(env.provider) - servers = extract_servers(provider, project, env, {}, user) + executor = Devops::Executor::ServerExecutor.new(nil, out) + executor.project = project + executor.deploy_env = env + params["created_by"] = user + + result = {:servers => []} project.deploy_envs = [ env ] result[:project] = project.to_hash status = 0 - servers.each do |s| - sr = {} - t1 = Time.now - out << "\n=== Create server ===\n" + sr = {} + t1 = Time.now + out << "\n=== Create server ===\n" + out.flush + if executor.create_server({"without_bootstrap" => true}) + out << "\n=== Create server - OK ===\n" out.flush - if provider.create_server(s, out) - out << "\n=== Create server - OK ===\n" + t2 = Time.now + s = executor.server + sr[:id] = s.id + sr[:create] = {:status => true} + sr[:create][:time] = time_diff_s(t1, t2) + DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running" + out << "\n=== Bootstrap ===\n" + out.flush + r = executor.bootstrap({}) + t1 = Time.now + sr[:chef_node_name] = s.chef_node_name + if r == 0 + out << "\n=== Bootstrap - OK ===\n" out.flush - t2 = Time.now - sr[:id] = s.id - sr[:create] = {:status => true} - sr[:create][:time] = time_diff_s(t1, t2) - s.chef_node_name = provider.create_default_chef_node_name(s) - logger.info "Server with parameters: #{s.to_hash.inspect} is running" - key = mongo.key(s.key) - out << "\n=== Bootstrap ===\n" + sr[:bootstrap] = {:status => true} + sr[:bootstrap][:time] = time_diff_s(t2, t1) + DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped" + out << "\n=== Check server ===\n" out.flush - r = bootstrap(s, out, key.path) - t1 = Time.now - sr[:chef_node_name] = s.chef_node_name - if r == 0 - out << "\n=== Bootstrap - OK ===\n" + if executor.check_server +# mongo.server_insert s + out << "\n=== OK, server has been inserted ===\n" out.flush - sr[:bootstrap] = {:status => true} - sr[:bootstrap][:time] = time_diff_s(t2, t1) - logger.info "Server with id '#{s.id}' is bootstraped" - out << "\n=== Check server ===\n" - out.flush - if check_server(s) - mongo.server_insert s - out << "\n=== OK, server has been inserted ===\n" - out.flush - end - else - status = 2 - out << "\n=== Bootstrap - FAIL ===\n" - out.flush - sr[:bootstrap] = {:status => false} - sr[:bootstrap][:return_code] = r end - - t1 = Time.now - out << "\n=== Delete server ===\n" - out.flush - r = delete_from_chef_server(s.chef_node_name) - begin - r[:server] = provider.delete_server s - out << "\n=== Delete server - OK ===\n" - out.flush - rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error - status = 3 - out << "\n=== Delete server - FAIL ===\n" - out.flush - r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" - logger.warn r[:server] - end - mongo.server_delete s.id - t2 = Time.now - sr[:delete] = {:status => true} - sr[:delete][:time] = time_diff_s(t1, t2) else - status = 1 - out << "\n=== Create server - FAIL ===\n" + status = 2 + out << "\n=== Bootstrap - FAIL ===\n" out.flush - sr[:create] = {:status => false} + sr[:bootstrap] = {:status => false} + sr[:bootstrap][:return_code] = r end - result[:servers].push sr + + t1 = Time.now + out << "\n=== Delete server ===\n" + out.flush + r = executor.delete_from_chef_server + begin + r[:server] = s.provider_instance.delete_server s + out << "\n=== Delete server - OK ===\n" + out.flush + rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error + status = 3 + out << "\n=== Delete server - FAIL ===\n" + out.flush + r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" + DevopsLogger.logger.warn r[:server] + end + mongo.server_delete s.id + t2 = Time.now + sr[:delete] = {:status => true} + sr[:delete][:time] = time_diff_s(t1, t2) + else + status = 1 + out << "\n=== Create server - FAIL ===\n" + out.flush + sr[:create] = {:status => false} end + result[:servers].push sr out << "\n\n#{result.to_json}" status end