diff --git a/devops-service/app/api2/handlers/deploy.rb b/devops-service/app/api2/handlers/deploy.rb index 2e42e2f..acaf5b1 100644 --- a/devops-service/app/api2/handlers/deploy.rb +++ b/devops-service/app/api2/handlers/deploy.rb @@ -19,10 +19,8 @@ module Devops names = body["names"] tags = body["tags"] || [] run_list = body["run_list"] - dir = DevopsConfig.config[:report_dir_v2] files = [] jid = nil - uri = URI.parse(parser.request.url) owner = parser.current_user @deploy_info_buf = {} servers(names).each do |s| @@ -36,7 +34,7 @@ module Devops deploy_info = create_deploy_info(s, project, body["build_number"]) deploy_info["run_list"] = run_list if run_list - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: owner, tags: tags, @@ -63,7 +61,7 @@ module Devops } Devops::Db.connector.save_report(Report.new(o)) end - files.push(uri) + files.push(jid) end files end diff --git a/devops-service/app/api2/handlers/flavor.rb b/devops-service/app/api2/handlers/flavor.rb index 421bb1a..34ddb12 100644 --- a/devops-service/app/api2/handlers/flavor.rb +++ b/devops-service/app/api2/handlers/flavor.rb @@ -7,9 +7,12 @@ module Devops class Flavor < RequestHandler def flavors provider - ::Provider::ProviderFactory.get(provider).flavors + flavors_with_account(provider, nil) end + def flavors_with_account provider, account + ::Provider::ProviderFactory.get(provider, account).flavors + end end end end diff --git a/devops-service/app/api2/handlers/group.rb b/devops-service/app/api2/handlers/group.rb index 26f2e36..82ed2d7 100644 --- a/devops-service/app/api2/handlers/group.rb +++ b/devops-service/app/api2/handlers/group.rb @@ -8,7 +8,11 @@ module Devops # TODO: vpc support for ec2 def groups provider - ::Provider::ProviderFactory.get(provider).groups()#params + groups_with_account(provider, nil) + end + + def groups_with_account provider, account + ::Provider::ProviderFactory.get(provider, account).groups()#params end end end diff --git a/devops-service/app/api2/handlers/network.rb b/devops-service/app/api2/handlers/network.rb index 81964c3..25c89ec 100644 --- a/devops-service/app/api2/handlers/network.rb +++ b/devops-service/app/api2/handlers/network.rb @@ -7,7 +7,11 @@ module Devops class Network < RequestHandler def networks provider - p = ::Provider::ProviderFactory.get provider + networks_with_account provider, nil + end + + def networks_with_account provider, account + p = ::Provider::ProviderFactory.get(provider, account) p.networks_detail end end diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index e82616c..68d0b39 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -251,13 +251,13 @@ module Devops deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil) end - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: parser.current_user, tags: [], deploy_info: deploy_info ) - files.push uri + files.push jid end files end @@ -288,14 +288,14 @@ module Devops raise InvalidRecord.new(msg) end - uri = Worker.start_async(ProjectTestWorker, @request, + jid = Worker.start_async(ProjectTestWorker, project: project.id, deploy_env: env.identifier, user: @request.env['REMOTE_USER'] ) sleep 1 - return [uri] + return [jid] end end diff --git a/devops-service/app/api2/handlers/provider.rb b/devops-service/app/api2/handlers/provider.rb index 51f1167..a054f2c 100644 --- a/devops-service/app/api2/handlers/provider.rb +++ b/devops-service/app/api2/handlers/provider.rb @@ -1,15 +1,44 @@ +require "app/api2/parsers/provider" require "providers/provider_factory" require_relative "request_handler" +require "db/mongo/models/provider_accounts/static_provider_account" +require "db/mongo/models/provider_accounts/openstack_provider_account" +require "db/mongo/models/provider_accounts/ec2_provider_account" module Devops module API2_0 module Handler class Provider < RequestHandler + set_parser Devops::API2_0::Parser::ProviderParser + def providers ::Provider::ProviderFactory.providers end + def accounts provider + ::Provider::ProviderFactory.get_accounts_factory(provider).accounts + end + + def account_fields provider + ::Provider::ProviderFactory.get_account_class(provider).account_fields + end + + def add_account provider + account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account) + account.validate_fields! + Devops::Db.connector.provider_accounts_insert(account) + ::Provider::ProviderFactory.add_account(provider, account) + account.to_hash + end + + def delete_account name, provider + account = Devops::Db.connector.provider_accounts_show(name) + Devops::Db.connector.provider_accounts_delete(name) + ::Provider::ProviderFactory.delete_account(provider, account) + account.to_hash + end + end end end diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb index a43ec4b..c7bcedc 100644 --- a/devops-service/app/api2/handlers/server.rb +++ b/devops-service/app/api2/handlers/server.rb @@ -35,7 +35,11 @@ module Devops end def provider_servers provider - ::Provider::ProviderFactory.get(provider).servers + provider_servers_with_account provider, nil + end + + def provider_servers_with_account provider, account + ::Provider::ProviderFactory.get(provider, account).servers end def server id @@ -67,20 +71,57 @@ module Devops user = parser.current_user check_if_server_attrs_are_valid(body, user) - uri = Worker.start_async(CreateServerWorker, @request, + jid = Worker.start_async(CreateServerWorker, server_attrs: body, owner: user ) - [uri] + [jid] + end + + def deploy node_name + call_deploy(node_name) do |options| + [ Worker.start_async(DeployWorker, @request, options) ] + end + end + + def deploy_stream node_name, out + call_deploy(node_name) do |options| + Worker.start_sync(DeployWorker, @request, options, out) + end + rescue RecordNotFound => e + out << e.message + -10 + end + + def call_deploy node_name + body = parser.deploy + names = body["names"] + tags = body["tags"] || [] + run_list = body["run_list"] + dir = DevopsConfig.config[:report_dir_v2] + owner = parser.current_user + s = Server.get_server_by_key(node_name, parser.instance_key) + project = Devops::Db.connector.check_project_auth s.project, s.deploy_env, owner + deploy_info = create_deploy_info(s, project, body["build_number"]) + deploy_info["run_list"] = run_list if run_list + + yield({ + server_attrs: s.to_hash, + owner: owner, + tags: tags, + deploy_info: deploy_info + }) end def pause_server node_name - s = Server.get_server_by_key(node_name, parser.instance_key) + s = get_server_by_key(node_name, parser.instance_key) ## Authorization Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user provider = s.provider_instance r = provider.pause_server s if r.nil? + set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::PAUSE) + "Server with instance ID '#{s.id}' and node name '#{node_name}' is paused" else raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be paused, It in state '#{r}'", 409) @@ -88,12 +129,14 @@ module Devops end def unpause_server node_name - s = Server.get_server_by_key(node_name, parser.instance_key) + s = get_server_by_key(node_name, parser.instance_key) ## Authorization Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user provider = s.provider_instance r = provider.unpause_server s if r.nil? + set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNPAUSE) + "Server with instance ID '#{s.id}' and node name '#{node_name}' is unpaused" else raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be unpaused, It in state '#{r}'") @@ -106,7 +149,7 @@ module Devops Devops::Db.connector.check_project_auth s.project, s.deploy_env, user raise ConflictException.new("Server '#{node_name}' already reserved") unless s.reserved_by.nil? s.reserved_by = user - Devops::Db.connector.server_update(s) + set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::RESERVE) end def unreserve_server node_name @@ -114,22 +157,20 @@ module Devops Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user raise ConflictException.new("Server '#{node_name}' is not reserved") if s.reserved_by.nil? s.reserved_by = nil - Devops::Db.connector.server_update(s) + set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNRESERVE) end # TODO: check bootstrap template name def bootstrap_server_stream out - s, rl, bt = prepare_bootstrap_server + options = prepare_bootstrap_server + s = options.delete(:server) status = [] cert = Devops::Db.connector.key s.key DevopsLogger.logger.debug "Bootstrap certificate path: #{cert.path}" #bootstrap s, out, cert.path, logger - options = { - :bootstrap_template => bt, - :cert_path => cert.path, - :run_list => rl - } - r = two_phase_bootstrap s, options, out + options[:cert_path] = cert.path + executor = Devops::Executor::ServerExecutor.new(s, out) + r = executor.two_phase_bootstrap(options) str = nil r = if check_server(s) Devops::Db.connector.server_set_chef_node_name s @@ -149,19 +190,15 @@ module Devops def bootstrap_server server, rl, bootstrap_template = prepare_bootstrap_server - dir = DevopsConfig[:report_dir_v2] - files = [] - uri = URI.parse(@request.url) - uri = Worker.start_async(BootstrapWorker, @request, - provider_name: server.provider, + jid = Worker.start_async(BootstrapWorker, server_attrs: server.to_mongo_hash, bootstrap_template: bootstrap_template, owner: parser.current_user ) sleep 1 - [uri] + [jid] end def prepare_bootstrap_server @@ -171,6 +208,7 @@ module Devops rl = body["run_list"] t = body["bootstrap_template"] s = Devops::Db.connector.server_by_instance_id(id) + res = {server: s} p = Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user d = p.deploy_env(s.deploy_env) @@ -181,9 +219,18 @@ module Devops unless t.nil? templates = get_templates halt_response("Invalid bootstrap template '#{t}', available values: #{templates.join(", ")}", 400) unless templates.include?(t) + res[:bootstrap_template] = t end s.chef_node_name = name || provider.create_default_chef_node_name(s) - return s, rl || d.run_list, t + res[:run_list] = rl || d.run_list + return res + end + + def unbootstrap_server id + s = get_server_by_key(id, parser.instance_key) + ### Authorization + Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user + Devops::Executor::ServerExecutor.new(s, "").unbootstrap end def add_server @@ -273,6 +320,11 @@ module Devops end end + def set_last_operation_type_and_save(server, operation_type) + server.set_last_operation(operation_type) + Devops::Db.connector.server_update(server) + end + end end end diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 315e8a6..ffc1fe0 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -25,13 +25,13 @@ module Devops raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil? object["stack_template"] = env.stack_template object["owner"] = parser.current_user + object["provider"] = env.provider + object["provider_account"] = env.provider_account - uri = Worker.start_async(StackBootstrapWorker, @request, - provider_name: env.provider, + jid = Worker.start_async(StackBootstrapWorker, stack_attributes: object ) - puts "Syncing report is located here: #{uri}" - [uri] + [jid] end def stack id @@ -99,7 +99,6 @@ module Devops def deploy id stack = self.stack(id) owner = parser.current_user - status = [] project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner) deploy_env_model = project.deploy_env(stack.deploy_env) body = parser.deploy @@ -111,7 +110,7 @@ module Devops servers.each do |s| begin deploy_info = project.deploy_info(deploy_env_model, nil) - uri = Worker.start_async(DeployWorker, @request, + jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, owner: owner, tags: tags, @@ -137,7 +136,7 @@ module Devops } Devops::Db.connector.save_report(Report.new(o)) end - files.push uri + files.push jid end files end diff --git a/devops-service/app/api2/helpers/version_2.rb b/devops-service/app/api2/helpers/version_2.rb index 08a538b..2e4ce94 100644 --- a/devops-service/app/api2/helpers/version_2.rb +++ b/devops-service/app/api2/helpers/version_2.rb @@ -30,7 +30,7 @@ module Devops def check_provider provider list = ::Provider::ProviderFactory.providers - halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider) + halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 400) unless list.include?(provider) end # Save information about requests with methods POST, PUT, DELETE diff --git a/devops-service/app/api2/parsers/project.rb b/devops-service/app/api2/parsers/project.rb index dc2a8e9..53b3cec 100644 --- a/devops-service/app/api2/parsers/project.rb +++ b/devops-service/app/api2/parsers/project.rb @@ -23,11 +23,11 @@ module Devops end def project_servers - @params[:deploy_env] + @params["deploy_env"] end def project_stacks - @params[:deploy_env] + @params["deploy_env"] end def create_project diff --git a/devops-service/app/api2/parsers/provider.rb b/devops-service/app/api2/parsers/provider.rb new file mode 100644 index 0000000..38d3670 --- /dev/null +++ b/devops-service/app/api2/parsers/provider.rb @@ -0,0 +1,16 @@ +require_relative "request_parser" + +module Devops + module API2_0 + module Parser + class ProviderParser < RequestParser + + def account + create_object_from_json_body + end + + end + end + end +end + diff --git a/devops-service/app/api2/routes/flavor.rb b/devops-service/app/api2/routes/flavor.rb index d0e5ea4..67db888 100644 --- a/devops-service/app/api2/routes/flavor.rb +++ b/devops-service/app/api2/routes/flavor.rb @@ -37,6 +37,13 @@ module Devops json Devops::API2_0::Handler::Flavor.new(request).flavors(provider) end + # TODO: check account + app.get_with_headers "/flavors/:provider/:account", :headers => [:accept] do |provider, account| + check_privileges("flavor", "r") + check_provider(provider) + json Devops::API2_0::Handler::Flavor.new(request).flavors_with_account(provider, account) + end + puts "Flavor routes initialized" end diff --git a/devops-service/app/api2/routes/group.rb b/devops-service/app/api2/routes/group.rb index 6ba87a7..989c4fb 100644 --- a/devops-service/app/api2/routes/group.rb +++ b/devops-service/app/api2/routes/group.rb @@ -50,6 +50,12 @@ module Devops json Devops::API2_0::Handler::Group.new(request).groups(provider) end + app.get_with_headers "/groups/:provider/:account", :headers => [:accept] do |provider, account| + check_privileges("group", "r") + check_provider(provider) + json Devops::API2_0::Handler::Group.new(request).groups_with_account(provider, account) + end + puts "Group routes initialized" end diff --git a/devops-service/app/api2/routes/network.rb b/devops-service/app/api2/routes/network.rb index 3d31d64..5da6724 100644 --- a/devops-service/app/api2/routes/network.rb +++ b/devops-service/app/api2/routes/network.rb @@ -38,6 +38,12 @@ module Devops json Devops::API2_0::Handler::Network.new(request).networks(provider) end + app.get_with_headers "/networks/:provider/:account", :headers => [:accept] do |provider, account| + check_privileges("network", "r") + check_provider(provider) + json Devops::API2_0::Handler::Network.new(request).networks_with_account(provider, account) + end + puts "Network routes initialized" end diff --git a/devops-service/app/api2/routes/provider.rb b/devops-service/app/api2/routes/provider.rb index 0c1f627..49a797e 100644 --- a/devops-service/app/api2/routes/provider.rb +++ b/devops-service/app/api2/routes/provider.rb @@ -26,6 +26,98 @@ module Devops json Devops::API2_0::Handler::Provider.new(request).providers end + # Get list of provider account fields + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : hash + # key - field + # value - description + app.get_with_headers "/provider/:provider/account/fields", :headers => [:accept] do |provider| + check_privileges("provider", "r") + check_provider(provider) + json Devops::API2_0::Handler::Provider.new(request).account_fields(provider) + end + + # Get list of provider accounts + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of strings + # - ec2: + # { + # "account_name": "", + # "description": "", + # "access_key_id" : "", + # "ssh_key": "", + # "certificate" : "path to file", + # "availability_zone": "" + # + # } + # - openstack: + # { + # "account_name": "", + # "description": "", + # "username": "", + # "auth_url": "", + # "tenant": "", + # "ssh_key": "", + # "certificate" : "path to file" + # } + # - static: + # { + # "account_name": "", + # "description": "", + # "ssh_key": "", + # "certificate" : "path to file" + # } + app.get_with_headers "/provider/:provider/accounts", :headers => [:accept] do |provider| + check_privileges("provider", "r") + check_provider(provider) + json Devops::API2_0::Handler::Provider.new(request).accounts(provider) + end + + # Create provider account for :provider + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # + # + # } + # + # * *Returns* : 201 + app.post_with_headers "/provider/:provider/account", :headers => [:accept, :content_type] do |provider| + check_privileges("provider", "w") + check_provider(provider) + create_response("Created", {:account => Devops::API2_0::Handler::Provider.new(request).add_account(provider)}, 201) + end + + # Delete account with name :account_name for :provider + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # + # * *Returns* : 200 + app.delete_with_headers "/provider/:provider/account/:account_name", :headers => [:accept, :content_type] do |provider, account_name| + check_privileges("provider", "w") + check_provider(provider) + create_response("Deleted", {:account => Devops::API2_0::Handler::Provider.new(request).delete_account(provider)}) + end + puts "Provider routes initialized" end end diff --git a/devops-service/app/api2/routes/server.rb b/devops-service/app/api2/routes/server.rb index d50ba0e..b00d983 100644 --- a/devops-service/app/api2/routes/server.rb +++ b/devops-service/app/api2/routes/server.rb @@ -85,6 +85,11 @@ module Devops json Devops::API2_0::Handler::Server.new(request).provider_servers(provider) end + app.get_with_headers "/servers/provider/:provider/:account", :headers => [:accept] do |provider, account| + check_privileges("server", "r") + json Devops::API2_0::Handler::Server.new(request).provider_servers_with_account(provider, account) + end + # Get server info by :name # # * *Request* @@ -127,6 +132,53 @@ module Devops } app.multi_routes "/server/:id", {:headers => [:accept, :content_type]}, hash + # Run deploy command on reserved server + # + # * *Request* + # - method : POST + # - headers : + # - Content-Type: application/json + # - body : + # { + # "names": [], -> array of servers names to run chef-client + # "tags": [], -> array of tags to apply on each server before running chef-client + # "build_number": "", -> string, build number to deploy + # "run_list": [], -> array of strings to set run_list for chef-client + # } + # + # * *Returns* : text stream + app.post_with_headers "/server/:node_name/deploy", :headers => [:content_type, :accept] do |node_name| + check_privileges("server", "x") + + if request["HTTP_X_STREAM"] + stream() do |out| + status = [] + begin + status = Devops::API2_0::Handler::Server.new(request).deploy_stream(node_name, out) + out << create_status(status) + rescue DeployInfoError => e + msg = "Can not get deploy info: " + e.message + DevopsLogger.logger.error "msg:\n#{e.backtrace.join('\n')}" + out.puts msg + rescue IOError => e + logger.error e.message + break + end + end # stream + else + files = begin + Devops::API2_0::Handler::Server.new(request).deploy(node_name) + rescue DeployInfoError => e + msg = "Can not get deploy info: " + e.message + DevopsLogger.logger.error "#{msg}:\n#{e.backtrace.join("\n")}" + out << "\nError - " + out.puts msg + end + sleep 1 + json files + end + end + # Create devops server # # * *Request* @@ -279,6 +331,25 @@ module Devops end end + # Unbootstrap devops server + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "key": "instance", -> search server by instance_id rather then chef_node_name + # } + # + # * *Returns* : 200 + app.post_with_headers "/server/:id/unbootstrap", :headers => [:accept, :content_type] do |id| + check_privileges("server", "w") + info = Devops::API2_0::Handler::Server.new(request).unbootstrap_server(id) + create_response("Unbootstrap", info) + end + # Add external server to devops # # * *Request* diff --git a/devops-service/app/api2/routes/v2.0.rb b/devops-service/app/api2/routes/v2.0.rb index fcf126a..3645afc 100644 --- a/devops-service/app/api2/routes/v2.0.rb +++ b/devops-service/app/api2/routes/v2.0.rb @@ -61,13 +61,23 @@ module Devops DevopsLogger.logger = @@logger begin res = super(env) - rescue DevopsError => e + rescue ::Devops::Exception::DevopsError => e + return [e.code, {}, e.message] + rescue InvalidRecord => e return [e.code, {}, e.message] end - @@access_logger.info(env["REQUEST_METHOD"] + " " + env["REQUEST_URI"] + " - from #{env["HTTP_USER_AGENT"]} (#{env["REMOTE_USER"]}) / #{res.inspect}") + @@access_logger.info(env["REQUEST_METHOD"] + " " + env["REQUEST_URI"] + " - from #{env["HTTP_USER_AGENT"]} (#{env["REMOTE_USER"]}) / #{res[0]}") res end + def handle_exception!(boom) + if boom.is_a?(::Devops::Exception::DevopsError) + boom.http_response + else + super(boom) + end + end + error Devops::ValidationError do e = env["sinatra.error"] #logger.warn e.message @@ -92,12 +102,14 @@ module Devops halt_response(e.message, 404) end +=begin error InvalidRecord do e = env["sinatra.error"] logger.warn e.message logger.warn "Request body: #{request.body.read}" halt_response(e.message, 400) end +=end error InvalidCommand do e = env["sinatra.error"] diff --git a/devops-service/app/devops-api2.rb b/devops-service/app/devops-api2.rb index 4eb8832..5e650de 100644 --- a/devops-service/app/devops-api2.rb +++ b/devops-service/app/devops-api2.rb @@ -78,7 +78,7 @@ module Devops Devops::Api2.register r end Routes.route "/v2.0", Devops::Api2 - Devops::Api2.routes_list + #Devops::Api2.routes_list end private diff --git a/devops-service/app/version/version.rb b/devops-service/app/version/version.rb index b32fcd3..5e423f6 100644 --- a/devops-service/app/version/version.rb +++ b/devops-service/app/version/version.rb @@ -2,7 +2,7 @@ require 'sinatra/base' class DevopsVersion < Sinatra::Base - VERSION = "2.3.0" + VERSION = "2.3.2" get "/" do VERSION diff --git a/devops-service/commands/knife_commands.rb b/devops-service/commands/knife_commands.rb index ccaa811..45c4835 100644 --- a/devops-service/commands/knife_commands.rb +++ b/devops-service/commands/knife_commands.rb @@ -73,7 +73,9 @@ EOH end def knife cmd - o = `bundle exec knife #{cmd} -c #{self.config} 2>&1` + cmd = "bundle exec knife #{cmd} -c #{self.config} 2>&1" + DevopsLogger.logger.info("Going to invoke command: #{cmd}") + o = `#{cmd}` return o, $?.success? end diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb index c00cc48..be380a9 100644 --- a/devops-service/db/mongo/connectors/helpers/insert_command.rb +++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb @@ -17,10 +17,13 @@ module Connectors def insert(record) begin record.validate! - collection.insert(record.to_mongo_hash) + hash = record.to_mongo_hash + hash["created_at"] = Time.now.to_i + collection.insert(hash) record rescue Mongo::OperationFailure => e - if e.message =~ /^11000/ + # exception's message doesn't always start from error code + if e.message =~ /11000/ resource_name = StringHelper.underscore_class(record.class) raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") end diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index e6c0996..4f10fe2 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -94,7 +94,7 @@ module Connectors def check_project_auth(project_id, env, user_id) project = show(project_id) - raise InvalidPrivileges.new("User '#{user_id}' unauthorized to work with project '#{project_id}'") unless project.check_authorization(user_id, env) + raise InvalidPrivileges.new("User '#{user_id}' is unauthorized to work with project '#{project_id}' and environment '#{env}'") unless project.check_authorization(user_id, env) project end diff --git a/devops-service/db/mongo/connectors/provider_accounts.rb b/devops-service/db/mongo/connectors/provider_accounts.rb new file mode 100644 index 0000000..3627562 --- /dev/null +++ b/devops-service/db/mongo/connectors/provider_accounts.rb @@ -0,0 +1,30 @@ +module Connectors + class ProviderAccounts < Base + + include Helpers::InsertCommand, + Helpers::DeleteCommand + + def initialize(db) + super(db) + end + + def provider_accounts provider + c = Provider::ProviderFactory.get_account_class(provider) + collection.find({provider: provider}).to_a.map{|bson| c.build_from_bson(bson)} + end + + def collection_name + 'provider_accounts' + end + + private + + def model_from_bson(bson) + c = Provider::ProviderFactory.get_account_class(bson["provider"]) + raise InvalidRecord.new "Invalid provider '#{bson["provider"]}' for account '#{bson["_id"]}'" if c.nil? + c.build_from_bson(bson) + end + end + +end + diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb index 8c5b199..fcc363f 100644 --- a/devops-service/db/mongo/connectors/report.rb +++ b/devops-service/db/mongo/connectors/report.rb @@ -51,8 +51,18 @@ module Connectors collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)} end - def set_report_status(jid, status) - collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}}) + def add_report_subreports(jid, subreports) + collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}}) + end + + def set_report_status(jid, status, job_result_code) + set = {"status" => status, "updated_at" => Time.new} + set["job_result_code"] = job_result_code unless job_result_code.nil? + collection.update({"_id" => jid}, {"$set" => set}) + end + + def set_report_status(jid, status, job_result_code) + collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}}) end def set_report_server_data id, chef_node_name, host diff --git a/devops-service/db/mongo/connectors/server.rb b/devops-service/db/mongo/connectors/server.rb index 4ab4fea..12f9e2a 100644 --- a/devops-service/db/mongo/connectors/server.rb +++ b/devops-service/db/mongo/connectors/server.rb @@ -63,17 +63,14 @@ module Connectors end def server_insert(server) - #server.validate! server.created_at = Time.now collection.insert(server.to_mongo_hash) end - # somewhy servers are not validated in previous version of code. I leave this until I know, why. def server_update(server) collection.update({'_id' => server.id}, server.to_hash_without_id) end - # somewhy servers are not validated in previous version of code. I leave this until I know, why. def server_set_chef_node_name(server) collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}}) end diff --git a/devops-service/db/mongo/models/deploy_env/cloud_deploy_env.rb b/devops-service/db/mongo/models/deploy_env/cloud_deploy_env.rb index c187b7e..30adbf2 100644 --- a/devops-service/db/mongo/models/deploy_env/cloud_deploy_env.rb +++ b/devops-service/db/mongo/models/deploy_env/cloud_deploy_env.rb @@ -26,6 +26,7 @@ module Devops :stack_template => {:type => String, :empty => false, :nil => true} =end + #TODO: account validator set_validators ::Validators::DeployEnv::Flavor, ::Validators::DeployEnv::Image, ::Validators::DeployEnv::Groups, diff --git a/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb b/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb index d69caff..b5aec77 100644 --- a/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb +++ b/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb @@ -32,10 +32,10 @@ module Devops def initialize d={} self.identifier = d["identifier"] + set_provider(d) b = d["run_list"] || [] self.run_list = b.uniq self.expires = d["expires"] - self.provider = d["provider"] b = d["users"] || [] self.users = b.uniq end @@ -45,9 +45,8 @@ module Devops "identifier" => self.identifier, "run_list" => self.run_list, "expires" => self.expires, - "provider" => self.provider, "users" => self.users - } + }.merge(provider_hash) end def add_users users diff --git a/devops-service/db/mongo/models/model_with_provider.rb b/devops-service/db/mongo/models/model_with_provider.rb index 5314bae..850b7ee 100644 --- a/devops-service/db/mongo/models/model_with_provider.rb +++ b/devops-service/db/mongo/models/model_with_provider.rb @@ -4,7 +4,7 @@ module Devops module Model module ModelWithProvider - attr_accessor :provider + attr_accessor :provider, :provider_account def ModelWithProvider.included(mod) @@ -15,8 +15,21 @@ module Devops end def provider_instance - @provider_instance ||= Provider::ProviderFactory.get(self.provider) + @provider_instance ||= Provider::ProviderFactory.get(self.provider, self.provider_account) end + + def set_provider hash + self.provider = hash["provider"] + self.provider_account = hash["provider_account"] + end + + def provider_hash + { + "provider" => self.provider, + "provider_account" => self.provider_account + } + end + end end end diff --git a/devops-service/db/mongo/models/mongo_model.rb b/devops-service/db/mongo/models/mongo_model.rb index 7be13e7..3721a5c 100644 --- a/devops-service/db/mongo/models/mongo_model.rb +++ b/devops-service/db/mongo/models/mongo_model.rb @@ -1,4 +1,4 @@ -require "providers/provider_factory" +#require "providers/provider_factory" require "exceptions/invalid_record" require "json" require 'db/validators/all' @@ -7,6 +7,8 @@ module Devops module Model class MongoModel + attr_accessor :created_at + # multi_json sends argument to 'to_json' method def to_json arg=nil JSON.pretty_generate self.to_hash @@ -64,6 +66,21 @@ module Devops end end + def validate_fields! + result = [] + self.class.field_validators.each do |field, validation_method| + begin + self.send(validation_method) + rescue InvalidRecord => e + result << {key: field, message: e.message} + end + end + unless result.empty? + raise InvalidRecord.new(error_data: result) + end + true + end + def build_error_message(message) # overrided in descendants message @@ -76,6 +93,7 @@ module Devops # :empty - can param be empty? (false) # :nil - can param be nil? (false) # :value_type - type of array element (String) + # TODO: @deprecated def self.types types define_method :validate_fields_types do t = types.keys @@ -120,17 +138,16 @@ module Devops end @validators = [] -# @field_validators = [] + @field_validators = {} class << self attr_accessor :validators -# attr_accessor :field_validators + attr_accessor :field_validators def inherited(subclass) subclass.validators = [] subclass.validators += self.validators -# subclass.field_validators = [] -# subclass.field_validators += self.field_validators + subclass.field_validators = self.field_validators.clone end # all exceptions are handled in @validate! method @@ -143,11 +160,13 @@ module Devops # validate field value # if method validate! returns false, then stop validation without error def set_field_validators field, *validators - define_method("validate_" + field.to_s + "!") do + method_name = "validate_" + field.to_s + "!" + define_method(method_name) do validators.each do |validator| break unless validator.new(self, send(field)).validate! end end + self.field_validators[field] = method_name end # private class methods diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb index 6e9e674..d6cc327 100644 --- a/devops-service/db/mongo/models/project.rb +++ b/devops-service/db/mongo/models/project.rb @@ -148,7 +148,7 @@ module Devops } end - def deploy_info deploy_env, build_number + def deploy_info deploy_env, build_number=nil { "use_json_file" => true, # "run_list" => Set.new.merge(self.run_list).merge(deploy_env.run_list).to_a, diff --git a/devops-service/db/mongo/models/provider_accounts/ec2_provider_account.rb b/devops-service/db/mongo/models/provider_accounts/ec2_provider_account.rb new file mode 100644 index 0000000..3f2fb41 --- /dev/null +++ b/devops-service/db/mongo/models/provider_accounts/ec2_provider_account.rb @@ -0,0 +1,59 @@ +require "db/mongo/models/provider_accounts/provider_account" + +module Devops + module Model + class Ec2ProviderAccount < ProviderAccount + + attr_accessor :access_key_id, :availability_zone, :secret_access_key + + set_field_validators :access_key_id, ::Validators::FieldValidator::NotNil, + ::Validators::FieldValidator::FieldType::String, + ::Validators::FieldValidator::NotEmpty + + set_field_validators :secret_access_key, ::Validators::FieldValidator::NotNil, + ::Validators::FieldValidator::FieldType::String, + ::Validators::FieldValidator::NotEmpty + def initialize a={} + super(a) + self.provider = Provider::Ec2::PROVIDER + self.availability_zone = a["availability_zone"] + self.access_key_id = a["access_key_id"] + self.secret_access_key = a["secret_access_key"] + end + + def to_list_hash + s = super + s["availability_zone"] = self.availability_zone + s["access_key_id"] = self.access_key_id + s + end + + def to_mongo_hash + s = super + s["availability_zone"] = self.availability_zone + s["access_key_id"] = self.access_key_id + s["secret_access_key"] = self.secret_access_key + s + end + + def self.build_from_bson a + a["account_name"] = a["_id"] + Ec2ProviderAccount.new a + end + + def self.account_fields + { + access_key_id: "AWS account access key", + secret_access_key: "AWS account secret key", + availability_zone: "Availability zone, todo: remove field?" + }.merge(ProviderAccount::ACCOUNT_FIELDS) + end + + # TODO: remove + def validate_fields_types + + end + end + end +end + diff --git a/devops-service/db/mongo/models/provider_accounts/openstack_provider_account.rb b/devops-service/db/mongo/models/provider_accounts/openstack_provider_account.rb new file mode 100644 index 0000000..9aa6ca2 --- /dev/null +++ b/devops-service/db/mongo/models/provider_accounts/openstack_provider_account.rb @@ -0,0 +1,49 @@ +require "db/mongo/models/provider_accounts/provider_account" + +module Devops + module Model + class OpenstackProviderAccount < ProviderAccount + + attr_accessor :username, :auth_url, :tenant, :api_key + + def self.account_fields + { + username: "Openstack user name", + auth_url: "Identity API endpoint", + tenant: "Tenant to access", + api_key: "Openstack user password" + }.merge(ProviderAccount::ACCOUNT_FIELDS) + end + + def initialize a={} + super(a) + self.username = a["username"] + self.auth_url = a["auth_url"] + self.tenant = a["tenant"] + self.api_key = a["api_key"] + end + + def to_list_hash + s = super + s["username"] = self.username + s["auth_url"] = self.auth_url + s["tenant"] = self.tenant + end + + def to_mongo_hash + s = super + s["username"] = self.username + s["auth_url"] = self.auth_url + s["tenant"] = self.tenant + s["api_key"] = self.api_key + end + + def self.build_from_bson a + a["account_name"] = a["_id"] + OpenstackProviderAccount.new a + end + + end + end +end + diff --git a/devops-service/db/mongo/models/provider_accounts/provider_account.rb b/devops-service/db/mongo/models/provider_accounts/provider_account.rb new file mode 100644 index 0000000..3a6fb59 --- /dev/null +++ b/devops-service/db/mongo/models/provider_accounts/provider_account.rb @@ -0,0 +1,66 @@ +require "db/mongo/models/mongo_model" +require "db/mongo/models/model_with_provider" + +module Devops + module Model + class ProviderAccount < MongoModel + + include ModelWithProvider + + attr_accessor :account_name, :description, :ssh_key + + set_field_validators :account_name, ::Validators::FieldValidator::NotNil, + ::Validators::FieldValidator::FieldType::String, + ::Validators::FieldValidator::NotEmpty, + ::Validators::FieldValidator::Name + + set_field_validators :description, ::Validators::FieldValidator::Nil, + ::Validators::FieldValidator::FieldType::String, + ::Validators::FieldValidator::NotEmpty, + ::Validators::FieldValidator::Description + + set_field_validators :ssh_key, ::Validators::FieldValidator::NotNil, + ::Validators::FieldValidator::FieldType::String, + ::Validators::FieldValidator::NotEmpty, + ::Validators::FieldValidator::SshKey + + ACCOUNT_FIELDS = { + account_name: "Account name (id)", + description: "Account description", + ssh_key: "Ssh key id" + } + + def initialize a={} + self.account_name = a["account_name"] + self.description = a["description"] + self.ssh_key = a["ssh_key"] + self.provider = a["provider"] + self.created_at = a["created_at"] + end + + def to_list_hash + to_hash + end + + def to_hash + { + "account_name" => self.account_name, + "description" => self.description, + "ssh_key" => self.ssh_key, + "provider" => self.provider, + "created_at" => self.created_at + } + end + + def to_mongo_hash + { + "_id" => self.account_name, + "description" => self.description, + "ssh_key" => self.ssh_key, + "provider" => self.provider + } + end + + end + end +end diff --git a/devops-service/db/mongo/models/provider_accounts/static_provider_account.rb b/devops-service/db/mongo/models/provider_accounts/static_provider_account.rb new file mode 100644 index 0000000..1359165 --- /dev/null +++ b/devops-service/db/mongo/models/provider_accounts/static_provider_account.rb @@ -0,0 +1,18 @@ +require "db/mongo/models/provider_accounts/provider_account" + +module Devops + module Model + class StaticProviderAccount < ProviderAccount + + def self.build_from_bson a + a["account_name"] = a["_id"] + StaticProviderAccount.new a + end + + def self.account_fields + ProviderAccount::ACCOUNT_FIELDS + end + end + end +end + diff --git a/devops-service/db/mongo/models/report.rb b/devops-service/db/mongo/models/report.rb index c7ca2c4..c557b22 100644 --- a/devops-service/db/mongo/models/report.rb +++ b/devops-service/db/mongo/models/report.rb @@ -12,7 +12,7 @@ module Devops DEPLOY_STACK_TYPE = 6 DELETE_SERVER_TYPE = 7 - attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack + attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code def initialize r self.id = r["_id"] @@ -21,13 +21,15 @@ module Devops self.project = r["project"] self.deploy_env = r["deploy_env"] self.type = r["type"] - self.created_at = r["created_at"] +# self.created_at = r["created_at"] self.chef_node_name = r["chef_node_name"] self.host = r["host"] self.stack = r["stack"] self.created_at = r["created_at"].localtime unless r["created_at"].nil? self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil? self.status = r["status"] + self.subreports = r["subreports"] + self.job_result_code = r["job_result_code"] end def to_hash_without_id @@ -42,7 +44,9 @@ module Devops "chef_node_name" => self.chef_node_name, "host" => self.host, "status" => self.status, - "stack" => self.stack + "stack" => self.stack, + "subreports" => self.subreports, + "job_result_code" => self.job_result_code } end diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb index f6296fe..91ec775 100644 --- a/devops-service/db/mongo/models/server.rb +++ b/devops-service/db/mongo/models/server.rb @@ -9,8 +9,17 @@ module Devops include ModelWithProvider + module OperationType + CREATION = :creation + DEPLOY = :deploy + RESERVE = :reserve + UNRESERVE = :unreserve + PAUSE = :pause + UNPAUSE = :unpause + end + attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list - attr_accessor :key + attr_accessor :key, :last_operation_at, :last_operation_type types :id => {:type => String, :empty => false}, :provider => {:type => String, :empty => false}, @@ -29,11 +38,12 @@ module Devops set_validators ::Validators::DeployEnv::RunList def self.fields - ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack"] + ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack", + "last_operation_type", "last_operation_at"] end def initialize s={} - self.provider = s["provider"] + self.set_provider(s) self.chef_node_name = s["chef_node_name"] self.id = s["_id"] self.remote_user = s["remote_user"] @@ -47,6 +57,8 @@ module Devops self.reserved_by = s["reserved_by"] self.stack = s["stack"] self.run_list = s["run_list"] || [] + self.last_operation_at = s["last_operation_at"] + self.last_operation_type = s["last_operation_type"] self end @@ -66,7 +78,6 @@ module Devops def to_hash_without_id { - "provider" => self.provider, "chef_node_name" => self.chef_node_name, "remote_user" => self.remote_user, "project" => self.project, @@ -78,8 +89,10 @@ module Devops "key" => self.key, "reserved_by" => self.reserved_by, "stack" => stack, - "run_list" => self.run_list - }.delete_if { |k,v| v.nil? } + "run_list" => self.run_list, + "last_operation_at" => self.last_operation_at, + "last_operation_type" => self.last_operation_type + }.merge(provider_hash).delete_if { |k,v| v.nil? } end def self.build_from_bson s @@ -98,7 +111,12 @@ module Devops end def static? - (self.provider == Provider::Static::PROVIDER) || false + self.provider == Provider::Static::PROVIDER + end + + def set_last_operation(operation_type) + self.last_operation_type = operation_type + self.last_operation_at = Time.now end end diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index 0d71b8b..b8cb4c6 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -21,7 +21,7 @@ module Devops def initialize attrs={} # self.provider = self.class.provider - + self.set_provider(attrs) self.id = attrs['id'] self.project = attrs['project'] self.deploy_env = attrs['deploy_env'] @@ -36,7 +36,6 @@ module Devops def to_hash_without_id { - provider: provider, project: project, deploy_env: deploy_env, stack_template: stack_template, @@ -46,7 +45,7 @@ module Devops details: bson_safe_details, stack_status: stack_status, owner: owner - } + }.merge(provider_hash) end # overrided in ec2 @@ -106,6 +105,8 @@ module Devops # - id (String) # - deploy_env (String) # - stack_template (String) + # - provider (String) + # - provider_account (String) def create(attrs, out) model = new(attrs) model.create_stack_in_cloud!(out) diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index e188e66..bda584e 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -31,8 +31,9 @@ class MongoConnector [:user_auth, :user, :users, :users_names, :user_insert, :user_delete, :user_update, :create_root_user, :check_user_privileges] => :users_connector, [:keys, :key, :key_insert, :key_delete] => :keys_connector, - [:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector, - [:insert_statistic, :search_statistic] => :statistics_connector + [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector, + [:insert_statistic, :search_statistic] => :statistics_connector, + [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector ) def initialize(db, host, port=27017, user=nil, password=nil) @@ -46,6 +47,10 @@ class MongoConnector private + def provider_accounts_connector + @provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db) + end + def images_connector @image_connector ||= Connectors::Image.new(@db) end diff --git a/devops-service/db/validators/all.rb b/devops-service/db/validators/all.rb index 03257dd..e64b6d0 100644 --- a/devops-service/db/validators/all.rb +++ b/devops-service/db/validators/all.rb @@ -15,5 +15,5 @@ require "db/validators/base" 'db/validators/image/*.rb', 'db/validators/field_validators/*.rb' ].each do |files_regexp| - Dir[files_regexp].each {|file| require file } + Dir[File.join(Devops::Application.root, files_regexp)].each {|file| require file } end diff --git a/devops-service/db/validators/field_validators/description.rb b/devops-service/db/validators/field_validators/description.rb new file mode 100644 index 0000000..c414e3a --- /dev/null +++ b/devops-service/db/validators/field_validators/description.rb @@ -0,0 +1,17 @@ +require_relative "base" +module Validators + module FieldValidator + class Description < Base + + MAX_LEN = 500 + + def valid? + @value.size <= 500 + end + + def message + "Invalid value '#{@value}': it should be less or equals then #{MAX_LEN} symbols" + end + end + end +end diff --git a/devops-service/db/validators/field_validators/name.rb b/devops-service/db/validators/field_validators/name.rb index 1319d85..4603934 100644 --- a/devops-service/db/validators/field_validators/name.rb +++ b/devops-service/db/validators/field_validators/name.rb @@ -11,7 +11,7 @@ module Validators end def message - "Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN}" + "Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN} symbols" end end end diff --git a/devops-service/db/validators/field_validators/ssh_key.rb b/devops-service/db/validators/field_validators/ssh_key.rb new file mode 100644 index 0000000..fce76e3 --- /dev/null +++ b/devops-service/db/validators/field_validators/ssh_key.rb @@ -0,0 +1,20 @@ +require_relative "base" +module Validators + module FieldValidator + class SshKey < Base + + MAX_LEN = 500 + + def valid? + Devops::Db.connector.key @value + true + rescue RecordNotFound + false + end + + def message + "Invalid value '#{@value}': ssh key '#{@value}' not found" + end + end + end +end diff --git a/devops-service/exceptions/devops_error.rb b/devops-service/exceptions/devops_error.rb index 996ec04..fdcb6a2 100644 --- a/devops-service/exceptions/devops_error.rb +++ b/devops-service/exceptions/devops_error.rb @@ -1,12 +1,25 @@ +require "json" module Devops module Exception class DevopsError < StandardError - def code + def http_status 500 end + def http_response + [self.http_status, self.http_headers, self.http_body] + end + + def http_body + JSON.pretty_generate(message: self.message) + end + + def http_headers + {"Content-Type" => "application/json"} + end + end end diff --git a/devops-service/exceptions/invalid_record.rb b/devops-service/exceptions/invalid_record.rb index 5ca7cb1..48d3750 100644 --- a/devops-service/exceptions/invalid_record.rb +++ b/devops-service/exceptions/invalid_record.rb @@ -1,3 +1,23 @@ -class InvalidRecord < StandardError +require_relative "devops_error" +class InvalidRecord < ::Devops::Exception::DevopsError + def initialize msg + if msg.is_a?(String) + super(msg) + else + @object = msg + end + end + + def http_status + 400 + end + + def http_body + if @object.nil? + super + else + JSON.pretty_generate(@object) + end + end end diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 7c502c9..1d57df3 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -16,7 +16,8 @@ module Devops deploy_failed: 8 } - MAX_SSH_RETRIES_AMOUNT = 20 + # waiting for 5*60 seconds (5 min) + MAX_SSH_RETRIES_AMOUNT = 60 #params: # out - container for output data @@ -74,11 +75,13 @@ module Devops end def create_server options - @server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider}) - provider = @server.provider_instance + @server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider, "provider_account" => @deploy_env.provider_account}) + @server.set_last_operation(Devops::Model::Server::OperationType::CREATION) + provider = @deploy_env.provider_instance mongo = ::Devops::Db.connector begin - @out << "Create server...\n" + @out.puts "Using '#{@deploy_env.provider}' account '#{@deploy_env.provider_account}'\n" if @deploy_env.provider_account + @out.puts "Create server..." @out.flush @server.run_list = options["run_list"] || [] @@ -137,6 +140,7 @@ module Devops end ja = { :provider => @server.provider, + :provider_account => @server.provider_account, :devops_host => `hostname`.strip } ip = @server.private_ip @@ -190,8 +194,10 @@ module Devops @out << "After bootstrap hooks...\n" res = self.run_hook(:after_bootstrap, @out) @out << "Done\n" + msg = "Server with id '#{@server.id}' is bootstraped" + @out.puts msg @out.flush - DevopsLogger.logger.info "Server with id '#{@server.id}' is bootstraped" + DevopsLogger.logger.info msg r else @out << "Can not bootstrap node '#{@server.id}', error code: #{r}" @@ -210,12 +216,18 @@ module Devops bootstrap_options.push "--sudo" unless @server.remote_user == "root" bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template] rl = options[:run_list] - bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil?# rl.empty? + bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil? or rl.empty? bootstrap_options.push "-c #{options[:config]}" if options[:config] bootstrap_options end + def prepare_two_phase_bootstrap options + @out << "Prepare bootstrap...\n" + @out << "Done\n" + end + def two_phase_bootstrap options + prepare_two_phase_bootstrap(options) # bootstrap phase begin provider = @server.provider_instance @@ -227,7 +239,8 @@ module Devops if check_server @out << "Server #{@server.chef_node_name} is created" else - @out << roll_back + @out.puts "Can not find client or node on chef-server" + roll_back @out.flush mongo.server_delete @server.id return result_code(:server_not_in_chef_nodes) @@ -256,7 +269,7 @@ module Devops @out << "\nComputed run list: #{run_list.join(", ")}" @out.flush @knife_instance.set_run_list(@server.chef_node_name, run_list) - deploy_info = @project.deploy_info(@deploy_env) + deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env) deploy_status = deploy_server(deploy_info) if deploy_status == 0 0 @@ -269,6 +282,7 @@ module Devops end rescue => e @out << "\nError: #{e.message}\n" + DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n")) result_code(:deploy_unknown_error) end end @@ -281,17 +295,27 @@ module Devops k = Devops::Db.connector.key(@server.key) cert_path = k.path i = 0 + res = delete_from_chef_server(@server.chef_node_name) begin - r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef` - raise(r) unless $?.success? + new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}" +# r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef` + cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} \"/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name}; else echo not found; fi'\"" + DevopsLogger.logger.info("Trying to run command '#{cmd}'") + r = `#{cmd}`.strip + if r == 'not found' + res[:server] = "Directory '/etc/chef' does not exists" + else + raise(r) unless $?.success? + res[:server] = "'/etc/chef' renamed to '#{new_name}'" + end rescue => e DevopsLogger.logger.error "Unbootstrap error: " + e.message i += 1 sleep(1) retry unless i == 5 - return e.message + return {error: e.message} end - nil + res end def deploy_server_with_tags tags, deploy_info @@ -334,15 +358,20 @@ module Devops cmd = "chef-client --no-color" if deploy_info["use_json_file"] deploy_info.delete("use_json_file") - @out << "Deploy Input Parameters:\n" - json = JSON.pretty_generate(deploy_info) - @out << json - @out << "\n" - file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}" + json = nil dir = DevopsConfig.config[:project_info_dir] - File.open(File.join(dir, file), "w") do |f| - f.write json + file = deploy_info["json_file"] || "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}" + path = File.join(dir, file) + if File.exists?(path) + json = File.read(path) + else + json = JSON.pretty_generate(deploy_info) + File.open(File.join(dir, file), "w") do |f| + f.write json + end end + @out << "Deploy Input Parameters:\n" + @out.puts json @out.flush cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}" else @@ -364,6 +393,10 @@ module Devops @out << "\nAfter deploy hooks...\n" res = self.run_hook(:after_deploy, @out, deploy_info) @out << "Done\n" + + @server.set_last_operation(Devops::Model::Server::OperationType::DEPLOY) + Devops::Db.connector.server_update(@server) + 0 end end @@ -412,13 +445,14 @@ module Devops end def roll_back + @out.puts "Trying to roll back..." unless @server.id.nil? - @out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n" - @out << delete_from_chef_server(@server.chef_node_name).values.join("\n") + @out.puts "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created" + @out.puts delete_from_chef_server(@server.chef_node_name).values.join("\n") begin - @out << @server.provider_instance.delete_server(@server) + @out.puts @server.provider_instance.delete_server(@server) rescue => e - @out << e.message + @out.puts e.message end @out << "\nRolled back\n" end diff --git a/devops-service/providers/accounts_factory.rb b/devops-service/providers/accounts_factory.rb new file mode 100644 index 0000000..7f212e9 --- /dev/null +++ b/devops-service/providers/accounts_factory.rb @@ -0,0 +1,33 @@ +module Provider + class AccountsFactory + + def init config + + end + + # providers instances + def connection account_name + @connections[account_name] + end + + def add_connection name, conn + @connections[name] = conn + end + + def delete_connection name + @connections.delete(name) + end + + def create_connection_from_account config, account + end + + def accounts + Devops::Db.connector.provider_accounts(provider_name) + end + + def create_account hash + raise "override me" + end + end +end + diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index ed3f07d..52644f8 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -1,5 +1,7 @@ require "exceptions/conflict_exception" require "providers/base_provider" +require "db/mongo/models/provider_accounts/ec2_provider_account" +require_relative "ec2_accounts_factory" module Provider # Provider for Amazon EC2 @@ -88,7 +90,7 @@ module Provider def servers list = self.compute.describe_instances.body["reservationSet"] - list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server| + list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s == "running"}.map do |server| convert_server server["instancesSet"][0] end end @@ -317,6 +319,7 @@ module Provider 'key_name' => instance["keyName"], 'private_ip' => instance["privateIpAddress"], 'public_ip' => instance["ipAddress"], + 'tags' => instance["tagSet"] } end end diff --git a/devops-service/providers/ec2_accounts_factory.rb b/devops-service/providers/ec2_accounts_factory.rb new file mode 100644 index 0000000..58eb3a8 --- /dev/null +++ b/devops-service/providers/ec2_accounts_factory.rb @@ -0,0 +1,37 @@ +require_relative "accounts_factory" +module Provider + class Ec2AccountsFactory < AccountsFactory + + def init config + @connections = {} + accounts.each do |account| + create_connection_from_account(config, account) + puts "\tFound ec2 account '#{account.account_name}'" + end + ProviderFactory.add_provider Ec2::PROVIDER unless @connections.empty? + end + + def provider_name + Ec2::PROVIDER + end + + def create_connection_from_account config, account + options = { + aws_ssh_key: account.ssh_key, + aws_access_key_id: account.access_key_id, + aws_secret_access_key: account.secret_access_key, + aws_availability_zone: account.availability_zone, + + aws_proxy: config[:aws_proxy], + aws_no_proxy: config[:aws_no_proxy], + aws_integration_run_list: config[:aws_integration_run_list] + } + add_connection(account.account_name, Ec2.new(options)) + end + + def create_account hash + Devops::Model::Ec2ProviderAccount.new(hash) + end + + end +end diff --git a/devops-service/providers/openstack_accounts_factory.rb b/devops-service/providers/openstack_accounts_factory.rb new file mode 100644 index 0000000..72bfa0f --- /dev/null +++ b/devops-service/providers/openstack_accounts_factory.rb @@ -0,0 +1,6 @@ +require_relative "accounts_factory" +module Provider + class OpenstackAccountsFactory < AccountsFactory + + end +end diff --git a/devops-service/providers/provider_factory.rb b/devops-service/providers/provider_factory.rb index ff89d75..921cceb 100644 --- a/devops-service/providers/provider_factory.rb +++ b/devops-service/providers/provider_factory.rb @@ -3,14 +3,21 @@ require "sinatra" module Provider module ProviderFactory + @@available_providers = [] @@providers = {} + @@providers_with_accounts_factories = {} def self.providers - @@providers.keys + @@available_providers end - def self.get provider - p = @@providers[provider] + def self.get provider, account=nil + p = if account.nil? + @@providers[provider] + else + @@providers_with_accounts_factories[provider].connection(account) + end + # TODO: new exception raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil? p end @@ -19,6 +26,10 @@ module Provider @@providers.values end + def self.add_provider provider + @@available_providers << provider unless @@available_providers.include?(provider) + end + def self.init conf # require providers here to get access to debug properties require_all @@ -28,24 +39,59 @@ module Provider o = Provider.const_get(p.capitalize).new(conf) if o.configured? @@providers[p] = o + @@available_providers << p puts "Provider '#{p}' has been loaded" end + factory = Provider.const_get(p.capitalize + "AccountsFactory").new + factory.init(conf) + @@providers_with_accounts_factories[p] = factory rescue => e puts "Error while loading provider '#{p}': " + e.message next end end + puts "Available providers: #{@@available_providers}" + end + + def self.add_account provider, account + factory = @@providers_with_accounts_factories[provider] + factory.create_connection_from_account(DevopsConfig.config, account) + DevopsLogger.logger.info("Added #{provider} account '#{account.account_name}'") + end + + def self.delete_account provider, account + factory = @@providers_with_accounts_factories[provider] + factory.delete_connection(account.account_name) + DevopsLogger.logger.info("Removed #{provider} account '#{account.account_name}'") end def self.require_all ["ec2", "openstack", "static"].each do |provider| begin require_relative provider + require_relative provider + "_accounts_factory" rescue LoadError => e puts "Can not load provider '#{provider}': " + e.message end end end + def self.get_accounts_factory provider + @@providers_with_accounts_factories[provider] + end + + def self.get_account_class provider + case(provider) + when ::Provider::Static::PROVIDER + ::Devops::Model::StaticProviderAccount + when ::Provider::Ec2::PROVIDER + ::Devops::Model::Ec2ProviderAccount + when ::Provider::Openstack::PROVIDER + ::Devops::Model::OpenstackProviderAccount + else + nil + end + end + end end diff --git a/devops-service/providers/static_accounts_factory.rb b/devops-service/providers/static_accounts_factory.rb new file mode 100644 index 0000000..630921b --- /dev/null +++ b/devops-service/providers/static_accounts_factory.rb @@ -0,0 +1,6 @@ +require_relative "accounts_factory" +module Provider + class StaticAccountsFactory < AccountsFactory + + end +end diff --git a/devops-service/workers/bootstrap_worker.rb b/devops-service/workers/bootstrap_worker.rb index 5fe0856..a261900 100644 --- a/devops-service/workers/bootstrap_worker.rb +++ b/devops-service/workers/bootstrap_worker.rb @@ -1,25 +1,26 @@ require File.join(File.dirname(__FILE__), "worker") require "lib/executors/server_executor" -require "providers/provider_factory" require "db/mongo/models/server" require "db/mongo/models/report" class BootstrapWorker < Worker def perform(options) - provider_name = options.fetch('provider_name') server_attrs = options.fetch('server_attrs') - bootstrap_template = options.fetch('bootstrap_template') +# bootstrap_template = options.fetch('bootstrap_template') owner = options.fetch('owner') + options = convert_config(options) - call(provider_name) do |provider, out, file| + call() do |out, file| server = Devops::Model::Server.new(server_attrs) report = save_report(file, owner, server) +=begin options = { bootstrap_template: bootstrap_template } +=end executor = Devops::Executor::ServerExecutor.new(server, out) executor.report = report status = executor.two_phase_bootstrap(options) diff --git a/devops-service/workers/create_server_worker.rb b/devops-service/workers/create_server_worker.rb index 192de26..bc2655a 100644 --- a/devops-service/workers/create_server_worker.rb +++ b/devops-service/workers/create_server_worker.rb @@ -1,6 +1,5 @@ require File.join(File.dirname(__FILE__), "worker") -require "providers/provider_factory" require "db/mongo/models/server" require "db/mongo/models/report" require "lib/executors/server_executor" @@ -11,7 +10,7 @@ class CreateServerWorker < Worker server_attrs = options.fetch('server_attrs') owner = options.fetch('owner') - call(nil) do |not_used, out, file| + call() do |out, file| project = mongo.project(server_attrs["project"]) env = project.deploy_env(server_attrs["deploy_env"]) report = save_report(file, project, env, owner) diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb index 66c8f48..c8fe476 100644 --- a/devops-service/workers/delete_server_worker.rb +++ b/devops-service/workers/delete_server_worker.rb @@ -8,7 +8,7 @@ class DeleteServerWorker < Worker chef_node_name = options.fetch('server_chef_node_name') puts "Expire server '#{chef_node_name}'." - call(nil) do |not_used, out, file| + call() do |out, file| server = mongo.server_by_chef_node_name(chef_node_name) report = save_report(file, server) diff --git a/devops-service/workers/deploy_worker.rb b/devops-service/workers/deploy_worker.rb index e47d384..9916ee2 100644 --- a/devops-service/workers/deploy_worker.rb +++ b/devops-service/workers/deploy_worker.rb @@ -12,7 +12,7 @@ class DeployWorker < Worker tags = options.fetch('tags') deploy_info = options.fetch('deploy_info') - call(nil) do |not_used, out, file| + call() do |out, file| server = Devops::Model::Server.new(server_attrs) report = save_report(file, owner, server) diff --git a/devops-service/workers/project_test_worker.rb b/devops-service/workers/project_test_worker.rb index 5f70be4..760b06c 100644 --- a/devops-service/workers/project_test_worker.rb +++ b/devops-service/workers/project_test_worker.rb @@ -15,7 +15,7 @@ class ProjectTestWorker < Worker project_name = params.fetch('project') deploy_env_name = params.fetch('deploy_env') - call(nil) do |not_used, out, file| + call() do |out, file| DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})" project = mongo.project(project_name) env = project.deploy_env(deploy_env_name) diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 780606a..442e793 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -1,4 +1,3 @@ -require "providers/provider_factory" require "commands/stack" require "db/mongo/models/stack/stack_factory" require "db/mongo/models/project" @@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker include StackCommands def perform(options) - provider_name = options.fetch('provider_name') stack_attrs = options.fetch('stack_attributes') - call(provider_name) do |provider, out, file| + call() do |out, file| @out = out without_bootstrap = stack_attrs.delete('without_bootstrap') @out.puts "Received 'without_bootstrap' option" if without_bootstrap @@ -23,14 +21,23 @@ class StackBootstrapWorker < Worker report = save_report(file, stack_attrs) begin - stack = create_stack(provider_name, stack_attrs) + stack = create_stack(stack_attrs) rescue StackCreatingError return 1 end + #TODO: errors begin - servers = persist_stack_servers!(stack, provider) - bootstrap_servers!(servers, report) unless without_bootstrap + servers_with_priority = persist_stack_servers!(stack) + unless without_bootstrap + sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x} + sorted_keys.each do |key| + @out.puts "Servers with priority '#{key}':" + bootstrap_servers!(servers_with_priority[key], report) + end + end + @out.puts "Done." + 0 rescue BootstrapingStackServerError @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback." rollback_stack!(stack) @@ -57,8 +64,8 @@ class StackBootstrapWorker < Worker @out.puts "Rollback has been completed" end - def create_stack(provider_name, stack_attrs) - stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out) + def create_stack(stack_attrs) + stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) mongo.stack_insert(stack) operation_result = sync_stack_proc.call(@out, stack, mongo) @@ -76,15 +83,40 @@ class StackBootstrapWorker < Worker def bootstrap_servers!(servers, report) @out << "\nStart bootstraping stack servers\n" - bootstraping_results = {} + + subreports = [] + data = {} servers.each do |server| - executor = Devops::Executor::ServerExecutor.new(server, @out) - executor.report = report - #TODO: can stack choose bootstrap template? - bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'}) - @out.flush + sjid = Worker.start_async(BootstrapWorker, + server_attrs: server.to_mongo_hash, + bootstrap_template: 'omnibus', + owner: server.created_by + ) + subreports << sjid + @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}" + data[server.id] = sjid end - check_bootstrap_results!(bootstraping_results) + @out.puts + @out.flush + mongo.add_report_subreports(jid, subreports) + results = [] + data.each do |server_id, subreport_id| + begin + sleep(5) + subreport = mongo.report(subreport_id) + status = subreport.status + if status == Worker::STATUS::COMPLETED + @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}" + break + elsif status == Worker::STATUS::FAILED + results << subreport.job_result_code + @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'" + break + end + end while(true) + end + @out.flush + results.empty? ? 0 : -5 end def check_bootstrap_results!(results) @@ -124,42 +156,60 @@ class StackBootstrapWorker < Worker "created_by" => stack_attrs['owner'], "project" => stack_attrs["project"], "deploy_env" => stack_attrs["deploy_env"], - "type" => ::Devops::Model::Report::STACK_TYPE + "type" => ::Devops::Model::Report::STACK_TYPE, + "subreports" => [], + "stack" => stack_attrs['name'] ) mongo.save_report(report) report end - def persist_stack_servers!(stack, provider) + # returns + # { + # "priority" => [Servers] + # } + def persist_stack_servers!(stack) @out.puts "Start syncing stack servers with CID" @out.flush project = mongo.project(stack.project) deploy_env = project.deploy_env(stack.deploy_env) + provider = stack.provider_instance - servers = provider.stack_servers(stack).map do |extended_info| - server_attrs = { - 'provider' => provider.name, - 'project' => project.id, - 'deploy_env' => deploy_env.identifier, - 'remote_user' => mongo.image(deploy_env.image).remote_user, - 'key' => extended_info["key_name"] || provider.ssh_key, - '_id' => extended_info["id"], - 'chef_node_name' => extended_info["name"], - 'private_ip' => extended_info["private_ip"], - 'public_ip' => extended_info["public_ip"], - 'created_by' => stack.owner, - 'run_list' => stack.run_list || [], - 'stack' => stack.name - } + stack_servers = provider.stack_servers(stack) + stack_servers.each do |info| + info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i + end + stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]} + stack_servers_with_priority = {} + stack_servers_info.each do |priority, info_array| + stack_servers_with_priority[priority] = info_array.map do |extended_info| + server_attrs = { + 'provider' => provider.name, + 'project' => project.id, + 'deploy_env' => deploy_env.identifier, + 'remote_user' => mongo.image(deploy_env.image).remote_user, + 'key' => extended_info["key_name"] || provider.ssh_key, + '_id' => extended_info["id"], + 'chef_node_name' => extended_info["name"], + 'private_ip' => extended_info["private_ip"], + 'public_ip' => extended_info["public_ip"], + 'created_by' => stack.owner, + 'run_list' => stack.run_list || [], + 'stack' => stack.name + } - server = ::Devops::Model::Server.new(server_attrs) - mongo.server_insert(server) - # server.chef_node_name = provider.create_default_chef_node_name(server) - server + server = ::Devops::Model::Server.new(server_attrs) + mongo.server_insert(server) + # server.chef_node_name = provider.create_default_chef_node_name(server) + server + end end @out.puts "Stack servers have been synced with CID" + stack_servers_with_priority.each do |priority, servers| + @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" + end @out.flush - servers + stack_servers_with_priority end end diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb index ca91940..3c80a63 100644 --- a/devops-service/workers/worker.rb +++ b/devops-service/workers/worker.rb @@ -13,6 +13,8 @@ require "core/devops-db" require "providers/provider_factory" require "lib/knife/knife_factory" + +# All options keys MUST be a symbol!!! class Worker include Sidekiq::Worker @@ -26,21 +28,25 @@ class Worker IN_QUEUE = "queued" end - def self.start_async(worker_class, request, job_options) + def self.start_async(worker_class, job_options) jid = worker_class.perform_async(job_options.dup) Worker.set_status jid, Worker::STATUS::IN_QUEUE DevopsLogger.logger.info "Job '#{jid}' has been queued" - + jid +=begin uri = URI.parse(request.url) uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}" uri.to_s +=end end def self.start_sync(worker_class, request, job_options, out) - stringified_options = {} + stringified_options = job_options +=begin job_options.each do |key, value| stringified_options[key.to_s] = value end +=end w = worker_class.new w.out = out w.perform(stringified_options) @@ -50,25 +56,22 @@ class Worker Sidekiq.redis {|con| con.hset "devops", id, status} end - def call provider_name, &block + def call &block begin - initialize_devops(provider_name) - provider = ::Provider::ProviderFactory.get(provider_name) if provider_name + initialize_devops() if jid - call_async(provider, &block) + call_async(&block) else - call_sync(provider, &block) + call_sync(&block) end rescue StandardError => e - puts e.message - puts e.backtrace.join("\n") - DevopsLogger.logger.error e.message + DevopsLogger.logger.error "#{e.message}:\n#{e.backtrace.join("\n")}" end end private - def initialize_devops(provider_name) + def initialize_devops() DevopsLogger.logger = logger DevopsConfig.read DevopsService.init @@ -76,33 +79,33 @@ class Worker end # outputs to file - def call_async(provider) + def call_async() dir = DevopsConfig[:report_dir_v2] # directory is created on server start in config.ru file = File.join(dir, jid) - update_job_status(STATUS::INIT) + update_job_status(STATUS::INIT, nil) File.open(file, "w") do |out| begin - update_job_status(STATUS::RUNNING) + update_job_status(STATUS::RUNNING, nil) self.out = out - job_result = yield(provider, out, file) + job_result = yield(out, file) canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED) - update_job_status(canonical_status) + update_job_status(canonical_status, job_result) rescue StandardError, RecordNotFound => e out << "\n #{e.class}\n #{e.message}\n" out << e.backtrace.join("\n") - update_job_status(STATUS::FAILED) + update_job_status(STATUS::FAILED, -100) end end end # outputs to STDOUT - def call_sync(provider) + def call_sync() out = STDOUT begin - yield(provider, out, '') + yield(out, '') rescue StandardError, RecordNotFound => e out << "\n" out << e.message @@ -115,17 +118,12 @@ class Worker ::Devops::Db.connector end - def update_job_status(status) + def update_job_status(status, job_result_code) set_status(jid, status) - mongo.set_report_status(jid, status) + mongo.set_report_status(jid, status, job_result_code) status end - def init_provider(provider_name) - ::Provider::ProviderFactory.init(DevopsConfig.config) - ::Provider::ProviderFactory.get(provider_name) if provider_name - end - def convert_config conf config = {} conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}