diff --git a/.gitignore b/.gitignore index 49cc974..c742b09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ devops-service/tests/features/support/config.yml .devops_files/ devops-service/plugins -devops-service/spec/examples.txt \ No newline at end of file +devops-service/spec/examples.txt +devops-service/coverage +devops-service/tmp \ No newline at end of file diff --git a/devops-client/lib/devops-client.rb b/devops-client/lib/devops-client.rb index fd6c6e7..9c581eb 100644 --- a/devops-client/lib/devops-client.rb +++ b/devops-client/lib/devops-client.rb @@ -49,7 +49,7 @@ module DevopsClient if result.is_a?(Hash) puts result["message"] else - puts result + puts result if result end rescue OptionParser::InvalidOption => e puts e.message diff --git a/devops-client/lib/devops-client/handler/deploy.rb b/devops-client/lib/devops-client/handler/deploy.rb index f5c06ed..7ab707c 100644 --- a/devops-client/lib/devops-client/handler/deploy.rb +++ b/devops-client/lib/devops-client/handler/deploy.rb @@ -26,7 +26,8 @@ class Deploy < Handler @options_parser.invalid_deploy_command abort() end - post_chunk("/deploy", :names => names, :tags => tags) + job_ids = post("/deploy", :names => names, :tags => tags) + reports_urls(job_ids) end end diff --git a/devops-client/lib/devops-client/handler/deploy_envs/deploy_env.rb b/devops-client/lib/devops-client/handler/deploy_envs/deploy_env.rb index e12c6cf..7994799 100644 --- a/devops-client/lib/devops-client/handler/deploy_envs/deploy_env.rb +++ b/devops-client/lib/devops-client/handler/deploy_envs/deploy_env.rb @@ -87,7 +87,7 @@ class DeployEnv end def fetcher - @fetcher ||= Helpers::ResourcesFetcher.new(host: @host, handler_object_options: @options, auth: @auth) + @fetcher ||= Helpers::ResourcesFetcher.new(host: @host, handler_options: @options, auth: @auth) end diff --git a/devops-client/lib/devops-client/handler/handler.rb b/devops-client/lib/devops-client/handler/handler.rb index edc2b69..cfd75d3 100644 --- a/devops-client/lib/devops-client/handler/handler.rb +++ b/devops-client/lib/devops-client/handler/handler.rb @@ -26,7 +26,11 @@ class Handler attr_accessor :auth def host - "http://#{@host}" + if @host.start_with?('http') + @host + else + "http://#{@host}" + end end #TODO: only basic auth now @@ -50,30 +54,9 @@ protected end def fetcher - @fetcher ||= Helpers::ResourcesFetcher.new(host: @host, handler_object_options: @options, auth: @auth) + @fetcher ||= Helpers::ResourcesFetcher.new(host: @host, handler_options: @options, auth: @auth) end - def params_filter params - r = [] - return params if params.kind_of?(String) - params.each do |k,v| - key = k.to_s - if v.kind_of?(Array) - v.each do |val| - r.push "#{key}[]=#{val}" - end - elsif v.kind_of?(Hash) - buf = {} - v.each do |k1,v1| - buf["#{key}[#{k1}]"] = v1 - end - r = r + params_filter(buf) - else - r.push "#{key}=#{v}" - end - end - r - end def inspect_parameters names, *args names.each_with_index do |name, i| diff --git a/devops-client/lib/devops-client/handler/helpers/http_utils.rb b/devops-client/lib/devops-client/handler/helpers/http_utils.rb index a5119e5..6fcaebc 100644 --- a/devops-client/lib/devops-client/handler/helpers/http_utils.rb +++ b/devops-client/lib/devops-client/handler/helpers/http_utils.rb @@ -143,4 +143,26 @@ module HttpUtils params_filter(params.select{|k,v| k != :cmd and !v.nil?}).join("&") end + def params_filter params + r = [] + return params if params.kind_of?(String) + params.each do |k,v| + key = k.to_s + if v.kind_of?(Array) + v.each do |val| + r.push "#{key}[]=#{val}" + end + elsif v.kind_of?(Hash) + buf = {} + v.each do |k1,v1| + buf["#{key}[#{k1}]"] = v1 + end + r = r + params_filter(buf) + else + r.push "#{key}=#{v}" + end + end + r + end + end \ No newline at end of file diff --git a/devops-client/lib/devops-client/handler/helpers/outputtable.rb b/devops-client/lib/devops-client/handler/helpers/outputtable.rb index 1f78db3..aed1478 100644 --- a/devops-client/lib/devops-client/handler/helpers/outputtable.rb +++ b/devops-client/lib/devops-client/handler/helpers/outputtable.rb @@ -13,6 +13,17 @@ module Outputtable outputter.output(options) end + def report_url(job_id) + create_url "report/#{job_id}" + end + + def reports_urls(job_ids) + raise "Parameter should be an array of strings" unless job_ids.is_a?(Array) + job_ids.map do |job_id| + report_url(job_id) + end.join("\n") + end + def self.included(base) base.extend(ClassMethods) diff --git a/devops-client/lib/devops-client/handler/helpers/resources_fetcher.rb b/devops-client/lib/devops-client/handler/helpers/resources_fetcher.rb index 5e50ade..4514741 100644 --- a/devops-client/lib/devops-client/handler/helpers/resources_fetcher.rb +++ b/devops-client/lib/devops-client/handler/helpers/resources_fetcher.rb @@ -1,13 +1,22 @@ +require "devops-client/handler/helpers/http_utils" require 'devops-client/helpers/string_helper' # fetches resources list along with table +# Rewrite this to avoid dependency on handlers module Helpers class ResourcesFetcher + include HttpUtils + + # have the same meaning as in handlers + attr_reader :username, :password, :options, :host def initialize(options) - @host = options.fetch(:host) - @handler_object_options = options.fetch(:handler_object_options) + @host = "http://#{options.fetch(:host)}" + @options = options.fetch(:handler_options) @auth = options.fetch(:auth) + + # username, password and options are used to perform http queries with module HttpUtils + @username, @password = @auth[:username], @auth[:password] end def fetch(collection_name, *args) @@ -19,12 +28,21 @@ module Helpers [handler.list_handler(*args), handler.outputter.table] end + def fetch_project(project_id) + @fetched_projects = {} + if cached = @fetched_projects[project_id] + cached + else + @fetched_projects[project_id] = get("/project/#{project_id}") + end + end + private def build_handler(collection_name) require_handler_file(collection_name) - handler = resource_handler_klass(collection_name).new(@host, @handler_object_options) + handler = resource_handler_klass(collection_name).new(host, options) handler.auth = @auth handler end diff --git a/devops-client/lib/devops-client/handler/helpers/resources_selector.rb b/devops-client/lib/devops-client/handler/helpers/resources_selector.rb index 1b2ed8e..33d76c6 100644 --- a/devops-client/lib/devops-client/handler/helpers/resources_selector.rb +++ b/devops-client/lib/devops-client/handler/helpers/resources_selector.rb @@ -1,38 +1,36 @@ require 'devops-client/handler/helpers/resources_fetcher' require 'devops-client/handler/helpers/input_utils' +require 'devops-client/output/project' module Helpers class ResourcesSelector include InputUtils - # fetcher_instance_or_attrs should be: - # instance of ResourcesFetcher - # OR - # hash with these keys: - # :host - # :handler_object_options - # :auth - def initialize(fetcher_instance_or_attrs) - if fetcher_instance_or_attrs.is_a?(ResourcesFetcher) - @fetcher = fetcher_instance_or_attrs - else - @fetcher = ResourcesFetcher.new(fetcher_instance_or_attrs) - end + def initialize(fetcher) + raise "fetcher should be instance of ResourcesFetcher" unless fetcher.is_a?(ResourcesFetcher) + @fetcher = fetcher end - def select_available_provider(options={}) + def select_available_provider providers, table = @fetcher.fetch_with_table('provider') # somewhy returns provider name as String. select_item_from_table(I18n.t("headers.provider"), providers, table) end - def select_available_project(options={}) + def select_available_project projects, table = @fetcher.fetch_with_table('project') project = select_item_from_table(I18n.t("headers.project"), projects, table) project['name'] end + def select_available_env(project_id) + project = @fetcher.fetch_project(project_id) + outputter = Output::Project.new(project, {output_type: :show, with_num: true}) + env = select_item_from_table("Select deploy env", project['deploy_envs'], outputter.table) + env['identifier'] + end + def select_available_stack_template(options={}) stack_templates, table = @fetcher.fetch_with_table('stack_template', options[:provider]) stack_template = select_item_from_table(I18n.t("headers.stack_template"), stack_templates, table) diff --git a/devops-client/lib/devops-client/handler/project.rb b/devops-client/lib/devops-client/handler/project.rb index dd261d3..f1657a4 100644 --- a/devops-client/lib/devops-client/handler/project.rb +++ b/devops-client/lib/devops-client/handler/project.rb @@ -87,7 +87,6 @@ class Project < Handler when "delete_servers" self.options = @options_parser.delete_servers_options delete_servers_handler @options_parser.args - output(output_type: :delete_servers) else @options_parser.invalid_command end @@ -279,7 +278,8 @@ class Project < Handler q = {} q[:servers] = options[:servers] unless options[:servers].nil? q[:deploy_env] = args[3] unless args[3].nil? - post_chunk "/project/#{args[2]}/deploy", q + job_ids = post "/project/#{args[2]}/deploy", q + reports_urls(job_ids) end def test_handler args @@ -288,8 +288,8 @@ class Project < Handler @options_parser.invalid_test_command abort(r) end - response = post "/project/test/#{args[2]}/#{args[3]}" - puts response.inspect + job_ids = post "/project/test/#{args[2]}/#{args[3]}" + reports_urls(job_ids) end protected @@ -456,7 +456,8 @@ protected deploy_env: env, dry_run: false } - @list = delete("/project/#{project}/servers", body) + response = delete("/project/#{project}/servers", body) + reports_urls(response['reports']) end private diff --git a/devops-client/lib/devops-client/handler/server.rb b/devops-client/lib/devops-client/handler/server.rb index d22f528..790597a 100644 --- a/devops-client/lib/devops-client/handler/server.rb +++ b/devops-client/lib/devops-client/handler/server.rb @@ -25,6 +25,8 @@ class Server < Handler create_handler when :delete delete_handler + when :delete_list + delete_list_handler when :bootstrap bootstrap_handler when :sync @@ -77,7 +79,8 @@ class Server < Handler q[k] = self.options[k] unless self.options[k].nil? end - post_chunk "/server", q + job_ids = post "/server", q + reports_urls(job_ids) end def delete_handler @@ -88,11 +91,24 @@ class Server < Handler abort(r) end if question(I18n.t("handler.server.question.delete", :name => name)) - puts "Server '#{name}', deleting..." - o = delete("/server/#{name}", options) - ["server", "chef_node", "chef_client", "message"].each do |k| - puts o[k] unless o[k].nil? - end + jobs_ids = delete("/server/#{name}", options) # returns array with one job id, actually + puts reports_urls(jobs_ids) + end + end + nil + end + + # this method differs from #delete_handler in this: + # it deletes multiple servers at once and takes servers ids instead of node names. + # Timur said we planned to transfer all server requests to using ids by default, that's why + # later we could get rid of #delete_list method. + def delete_list_handler + abort "Please specify at least one server id" if @args.length < 3 + server_ids = @args[2..-1] + if question(I18n.t("handler.server.question.delete_list", ids: server_ids.join(', '))) + servers_jobs = post("/server/delete_list", {servers_ids: server_ids}) + servers_jobs.each do |server_id, job_id| + puts "Report for deleting #{server_id}: #{report_url(job_id)}" end end nil @@ -122,7 +138,8 @@ class Server < Handler if q.has_key?(:run_list) abort unless Project.validate_run_list(q[:run_list]) end - post_chunk "/server/bootstrap", q + job_ids = post "/server/bootstrap", q + reports_urls(job_ids) end def add_static_handler # add --public-ip diff --git a/devops-client/lib/devops-client/handler/stack.rb b/devops-client/lib/devops-client/handler/stack.rb index 7385024..d294c14 100644 --- a/devops-client/lib/devops-client/handler/stack.rb +++ b/devops-client/lib/devops-client/handler/stack.rb @@ -51,19 +51,19 @@ class Stack < Handler q = {} q[:without_bootstrap] = options[:without_bootstrap] - q[:provider] = options[:provider] || resources_selector.select_available_provider - # q[:id] = options[:id] || enter_parameter(I18n.t('handler.stack.create.id')) + # q[:provider] = options[:provider] || resources_selector.select_available_provider q[:project] = options[:project] || resources_selector.select_available_project - q[:deploy_env] = options[:deploy_env] || enter_parameter(I18n.t('handler.stack.create.deploy_env')) - # q[:run_list] = options[:run_list] || enter_parameter_or_empty(I18n.t('handler.stack.create.run_list')) - # q[:run_list] = q[:run_list].split(',') + q[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(q[:project]) + env = fetcher.fetch_project(q[:project])['deploy_envs'].detect {|env| env['identifier'] == q[:deploy_env]} + q[:provider] = env['provider'] filepath = options[:parameters_file] || enter_parameter(I18n.t('handler.stack.create.parameters_file')) q[:parameters] = JSON.parse(File.read(filepath)) json = JSON.pretty_generate(q) if question(I18n.t("handler.stack.question.create")) {puts json} - post_body "/stack", json + job_ids = post_body "/stack", json + reports_urls(job_ids) end end @@ -123,8 +123,8 @@ class Stack < Handler def deploy_handler stack_id = @args[2] - response = post "/stack/#{stack_id}/deploy" - puts response.inspect + job_ids = post "/stack/#{stack_id}/deploy" + reports_urls(job_ids) end def reserve_handler diff --git a/devops-client/lib/devops-client/options/server_options.rb b/devops-client/lib/devops-client/options/server_options.rb index 879bf91..6b844f6 100644 --- a/devops-client/lib/devops-client/options/server_options.rb +++ b/devops-client/lib/devops-client/options/server_options.rb @@ -2,7 +2,7 @@ require "devops-client/options/common_options" class ServerOptions < CommonOptions - commands :add, :bootstrap, :create, :delete, :list, :pause, :reserve, :show, :unpause, :unreserve, :add_and_bootstrap_list + commands :add, :bootstrap, :create, :delete, :list, :pause, :reserve, :show, :unpause, :unreserve, :delete_list, :add_and_bootstrap_list def initialize args, def_options super(args, def_options) @@ -127,7 +127,7 @@ class ServerOptions < CommonOptions options[:groups] = groups.split(",") end - parser.recognize_option_value(:private_ip, 'server', short: '-N', i18n_scope: 'create') + parser.recognize_option_value(:private_ip, short: '-N', i18n_scope: 'create') # it was disabled somewhy # parser.on('--public-ip', "Associate public IP with server") do @@ -172,4 +172,8 @@ class ServerOptions < CommonOptions self.banner_header + " delete NODE_NAME [NODE_NAME ...]\n" end + def delete_list_banner + self.banner_header + " delete_list INSTANCE_ID [INSTANCE_ID ...]\n" + end + end diff --git a/devops-client/lib/devops-client/output/base.rb b/devops-client/lib/devops-client/output/base.rb index d5c8e71..82f8650 100644 --- a/devops-client/lib/devops-client/output/base.rb +++ b/devops-client/lib/devops-client/output/base.rb @@ -38,7 +38,7 @@ module Output end def with_num? - outputting_list? + @options[:with_num] || outputting_list? end def create_table headers, rows, title=nil, with_num=true, separator=false diff --git a/devops-client/lib/devops-client/output/project.rb b/devops-client/lib/devops-client/output/project.rb index b137aad..1123c51 100644 --- a/devops-client/lib/devops-client/output/project.rb +++ b/devops-client/lib/devops-client/output/project.rb @@ -27,8 +27,6 @@ module Output when :test title = I18n.t("output.title.project.test", :project => ARGV[2], :env => ARGV[3]) create_test(@data) - when :delete_servers - return delete_servers_output else title = I18n.t("output.title.project.list") create_list(@data) @@ -62,7 +60,7 @@ module Output def create_show show rows = [] - headers = if show["type"] == "multi" + if show["type"] == "multi" show["deploy_envs"].each do |de| subprojects = [] nodes = [] @@ -74,14 +72,14 @@ module Output end rows.push [ de["identifier"], subprojects.join("\n"), nodes.join("\n"), de["users"].join("\n") ] end - [ + headers = [ I18n.t("output.table_header.deploy_env"), I18n.t("output.table_header.subproject") + " - " + I18n.t("output.table_header.deploy_env"), I18n.t("output.table_header.node_number"), I18n.t("output.table_header.users") ] else - show["deploy_envs"].each do |de| + show["deploy_envs"].each_with_index do |de, i| rows.push [ show["name"], de["identifier"], @@ -93,7 +91,7 @@ module Output (de["users"] || []).join("\n") ] end - [ + headers = [ I18n.t("output.table_header.id"), I18n.t("output.table_header.deploy_env"), I18n.t("output.table_header.image_id"), @@ -151,22 +149,5 @@ module Output headers_and_rows(stacks, fields_to_output) end - def delete_servers_output - output = '' - - if @data['deleted'].empty? - output << 'There are no deleted servers.' - else - output << "Deleted servers:\n----\n" - output << @data['deleted'].join("\n") - end - - if !@data['failed'].empty? - output << "\nThere were errors during deleting these servers:\n----\n" - output << @data['failed'].join("\n") - end - output - end - end end diff --git a/devops-client/locales/en.yml b/devops-client/locales/en.yml index 119e419..64f97e2 100644 --- a/devops-client/locales/en.yml +++ b/devops-client/locales/en.yml @@ -43,6 +43,7 @@ en: user: "User" stack: "Stack" stack_template: "Stack template" + env: "Deploy environment" handler: flavor: list: @@ -102,6 +103,7 @@ en: server: question: delete: "Are you sure to delete server '%{name}'?" + delete_list: "Are you sure to delete these servers: %{ids}?" stack_template: create: id: "Id: " @@ -214,6 +216,7 @@ en: show: "Project '%{name}' information" servers: "Project '%{title}' servers" test: "Project test: %{project} - %{env}" + envs: "Project '%{name}' deploy envs" provider: list: "Providers" script: diff --git a/devops-service/Gemfile b/devops-service/Gemfile index 13b88c3..9a51237 100644 --- a/devops-service/Gemfile +++ b/devops-service/Gemfile @@ -36,4 +36,6 @@ end group :devepoment do gem 'byebug' gem 'guard-rspec', require: false + gem 'simplecov', require: false + gem 'simplecov-rcov', require: false end diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock index 31da4a0..7ee4373 100644 --- a/devops-service/Gemfile.lock +++ b/devops-service/Gemfile.lock @@ -60,6 +60,7 @@ GEM gherkin (~> 2.12.0) daemons (1.2.3) diff-lcs (1.2.5) + docile (1.1.5) em-websocket (0.3.8) addressable (>= 2.1.1) eventmachine (>= 0.12.9) @@ -291,6 +292,13 @@ GEM json redis (>= 3.0.6) redis-namespace (>= 1.3.1) + simplecov (0.11.1) + docile (~> 1.1.0) + json (~> 1.8) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + simplecov-rcov (0.2.3) + simplecov (>= 0.4.1) sinatra (1.4.5) rack (~> 1.4) rack-protection (~> 1.4) @@ -352,9 +360,14 @@ DEPENDENCIES rspec (~> 3.3) rspec_junit_formatter sidekiq (= 3.2.6) + simplecov + simplecov-rcov sinatra (= 1.4.5) sinatra-contrib sinatra-websocket test-unit thin (~> 1.5.1) wisper + +BUNDLED WITH + 1.11.2 diff --git a/devops-service/Guardfile b/devops-service/Guardfile index 13d5834..a66e388 100644 --- a/devops-service/Guardfile +++ b/devops-service/Guardfile @@ -42,4 +42,5 @@ guard :rspec, cmd: "rspec" do # Devops files watch(%r{db/.+\.rb}) { rspec.spec_dir } + watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" } end diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index 3e36bb9..5f8f918 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -4,6 +4,7 @@ require "workers/project_test_worker" require "app/api2/parsers/project" require "lib/project/type/types_factory" require "lib/executors/server_executor" +require "workers/delete_server_worker" require_relative "../helpers/version_2.rb" require_relative "request_handler" @@ -314,16 +315,11 @@ module Devops private def delete_chosen_servers!(servers) - deleted, failed = [], [] - servers.each do |server| - begin - Devops::Executor::ServerExecutor.new(server, '').delete_server - deleted << server.id - rescue - failed << server.id - end + current_user = parser.current_user + reports = servers.map do |server| + Worker.start_async(DeleteServerWorker, 'server_id' => server.id, 'current_user' => current_user) end - {deleted: deleted, failed: failed} + {reports: reports} end end diff --git a/devops-service/app/api2/handlers/provider.rb b/devops-service/app/api2/handlers/provider.rb index 8e5445d..7e74f27 100644 --- a/devops-service/app/api2/handlers/provider.rb +++ b/devops-service/app/api2/handlers/provider.rb @@ -27,14 +27,14 @@ module Devops def add_account provider account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account) account.validate_fields! - Devops::Db.connector.provider_accounts_insert(account) + Devops::Db.connector.provider_account_insert(account) ::Provider::ProviderFactory.add_account(provider, account) account.to_hash end def delete_account name, provider account = Devops::Db.connector.provider_account(provider, name) - Devops::Db.connector.provider_accounts_delete(name) + Devops::Db.connector.provider_account_delete(name) ::Provider::ProviderFactory.delete_account(provider, account) account.to_hash end diff --git a/devops-service/app/api2/handlers/server.rb b/devops-service/app/api2/handlers/server.rb index 06dfcdd..5b01192 100644 --- a/devops-service/app/api2/handlers/server.rb +++ b/devops-service/app/api2/handlers/server.rb @@ -10,6 +10,7 @@ require "db/mongo/models/server" require "workers/create_server_worker" require "workers/bootstrap_worker" +require "workers/delete_server_worker" require "app/api2/parsers/server" require_relative "request_handler" @@ -47,10 +48,22 @@ module Devops end def delete id - s = get_server_by_key(id, parser.instance_key) - ### Authorization - Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user - Devops::Executor::ServerExecutor.new(s, "").delete_server + server = get_server_by_key(id, parser.instance_key) + current_user = parser.current_user + Devops::Db.connector.check_project_auth server.project, server.deploy_env, current_user + jid = Worker.start_async(DeleteServerWorker, 'server_id' => server.id, 'current_user' => current_user) + [jid] + end + + def delete_list + server_ids = parser.delete_list.uniq + servers = server_ids.map { |id| Devops::Db.connector.server_by_instance_id(id) } + current_user = parser.current_user + check_servers_list_auth(servers, current_user) + server_ids.inject({}) do |hash, server_id| + hash[server_id] = Worker.start_async(DeleteServerWorker, 'server_id' => server_id, 'current_user' => current_user) + hash + end end def create_server_stream out @@ -238,8 +251,8 @@ module Devops project, deploy_env, server_attrs = parser.add_server Devops::Db.connector.check_project_auth project, deploy_env, parser.current_user - add_static_server(server_attrs) - "Server '#{s.id}' has been added" + server = add_static_server(server_attrs) + "Server '#{server.id}' has been added" end # returns jobs ids @@ -352,6 +365,16 @@ module Devops Devops::Db.connector.server_update(server) end + def check_servers_list_auth(servers, current_user) + project_with_env_pairs = servers.map do |server| + [server.project, server.deploy_env] + end + project_with_env_pairs.uniq.each do |pair| + project, env = *pair + Devops::Db.connector.check_project_auth project, env, current_user + end + end + end end end diff --git a/devops-service/app/api2/parsers/server.rb b/devops-service/app/api2/parsers/server.rb index b3afbb3..c6e9c1a 100644 --- a/devops-service/app/api2/parsers/server.rb +++ b/devops-service/app/api2/parsers/server.rb @@ -109,6 +109,11 @@ module Devops rl end + def delete_list + @body ||= create_object_from_json_body + check_array(@body["servers_ids"], "Parameter 'servers_ids' should be a not empty array of string", String, false) + end + private def parse_list_of_ips_with_names(text) diff --git a/devops-service/app/api2/routes/deploy.rb b/devops-service/app/api2/routes/deploy.rb index 8302e35..7a2a208 100644 --- a/devops-service/app/api2/routes/deploy.rb +++ b/devops-service/app/api2/routes/deploy.rb @@ -14,7 +14,7 @@ module Devops # - Content-Type: application/json # - body : # { - # "names": [], -> array of servers names to run chef-client + # "names": [], -> array of servers chef node names to run chef-client # "tags": [], -> array of tags to apply on each server before running chef-client # "build_number": "", -> string, build number to deploy # "run_list": [], -> array of strings to set run_list for chef-client diff --git a/devops-service/app/api2/routes/server.rb b/devops-service/app/api2/routes/server.rb index b0f89bb..b2d5c17 100644 --- a/devops-service/app/api2/routes/server.rb +++ b/devops-service/app/api2/routes/server.rb @@ -127,8 +127,7 @@ module Devops # 200 - Deleted hash["DELETE"] = lambda {|id| check_privileges("server", "w") - info, r = Devops::API2_0::Handler::Server.new(request).delete(id) - create_response(info, r) + json Devops::API2_0::Handler::Server.new(request).delete(id) } app.multi_routes "/server/:id", {:headers => [:accept, :content_type]}, hash @@ -470,6 +469,29 @@ module Devops create_response("Run list has been changed") end + + # Delete list of servers + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "servers_ids": [ "server1", "server2"] + # } + # + # * *Returns* : + # { + # "server1": "report_1", + # "server2": "report_2" + # } + app.post_with_headers "/server/delete_list", :headers => [:accept, :content_type] do + check_privileges("server", "w") + json Devops::API2_0::Handler::Server.new(request).delete_list + end + puts "Server routes initialized" end diff --git a/devops-service/commands/knife_commands.rb b/devops-service/commands/knife_commands.rb index 45c4835..73c41bf 100644 --- a/devops-service/commands/knife_commands.rb +++ b/devops-service/commands/knife_commands.rb @@ -38,6 +38,13 @@ class KnifeCommands knife("tag delete #{name} #{tagsStr}") end + # extracted from server_executor.rb + def swap_tags(name, from, to) + tags_delete(name, from) + result = tags_create(name, to) + !!result[1] + end + def create_role role_name, project, env file = "/tmp/new_role.json" File.open(file, "w") do |f| diff --git a/devops-service/core/devops-service.rb b/devops-service/core/devops-service.rb index cd33364..7bf32f1 100644 --- a/devops-service/core/devops-service.rb +++ b/devops-service/core/devops-service.rb @@ -1,7 +1,7 @@ require "wisper" -require "lib/hash_ext" -require "lib/nil_class_ext" -require "lib/string_ext" +require "lib/core_ext/hash" +require "lib/core_ext/nil_class" +require "lib/core_ext/string" require_relative "devops-loader" require_relative "devops-application" diff --git a/devops-service/db/mongo/connectors/helpers/delete_command.rb b/devops-service/db/mongo/connectors/helpers/delete_command.rb index b66be9c..8841aaa 100644 --- a/devops-service/db/mongo/connectors/helpers/delete_command.rb +++ b/devops-service/db/mongo/connectors/helpers/delete_command.rb @@ -1,5 +1,3 @@ -require 'lib/string_helper' - module Connectors module Helpers module DeleteCommand @@ -9,7 +7,7 @@ module Connectors # We need this alias to forward methods from MongoConnector to resources connectors. def self.included(base) - resource_name = StringHelper.underscore_class(base) + resource_name = base.to_s.underscore_class method_name = "#{resource_name}_delete".to_sym alias_method method_name, :delete end diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb index be380a9..446ca5f 100644 --- a/devops-service/db/mongo/connectors/helpers/insert_command.rb +++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb @@ -1,5 +1,3 @@ -require 'lib/string_helper' - module Connectors module Helpers module InsertCommand @@ -9,7 +7,7 @@ module Connectors # We need this alias to forward methods from MongoConnector to resources connectors. def self.included(base) - resource_name = StringHelper.underscore_class(base) + resource_name = base.to_s.underscore_class method_name = "#{resource_name}_insert".to_sym alias_method method_name, :insert end @@ -24,7 +22,7 @@ module Connectors rescue Mongo::OperationFailure => e # exception's message doesn't always start from error code if e.message =~ /11000/ - resource_name = StringHelper.underscore_class(record.class) + resource_name = record.class.to_s.underscore_class raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") end end diff --git a/devops-service/db/mongo/connectors/helpers/list_command.rb b/devops-service/db/mongo/connectors/helpers/list_command.rb index aaab3b8..a53dcb0 100644 --- a/devops-service/db/mongo/connectors/helpers/list_command.rb +++ b/devops-service/db/mongo/connectors/helpers/list_command.rb @@ -1,5 +1,3 @@ -require 'lib/string_helper' - module Connectors module Helpers module ListCommand @@ -9,9 +7,7 @@ module Connectors # We need this alias to forward methods from MongoConnector to resources connectors. def self.included(base) - resource_name = StringHelper.underscore_class(base).to_sym - method_name = StringHelper.pluralize(resource_name) - alias_method method_name, :list + alias_method base.to_s.underscore_class.pluralize, :list end # query options is needed, for example, for fields limiting diff --git a/devops-service/db/mongo/connectors/helpers/show_command.rb b/devops-service/db/mongo/connectors/helpers/show_command.rb index cb06d74..2889d72 100644 --- a/devops-service/db/mongo/connectors/helpers/show_command.rb +++ b/devops-service/db/mongo/connectors/helpers/show_command.rb @@ -1,5 +1,3 @@ -require 'lib/string_helper' - module Connectors module Helpers module ShowCommand @@ -9,7 +7,7 @@ module Connectors # We need this alias to forward methods from MongoConnector to resources connectors. def self.included(base) - method_name = StringHelper.underscore_class(base).to_sym + method_name = base.to_s.underscore_class.to_sym alias_method method_name, :show end diff --git a/devops-service/db/mongo/connectors/helpers/update_command.rb b/devops-service/db/mongo/connectors/helpers/update_command.rb index 674eb60..a8f88b6 100644 --- a/devops-service/db/mongo/connectors/helpers/update_command.rb +++ b/devops-service/db/mongo/connectors/helpers/update_command.rb @@ -1,5 +1,3 @@ -require 'lib/string_helper' - module Connectors module Helpers module UpdateCommand @@ -8,8 +6,7 @@ module Connectors # We need second method name to forward methods from MongoConnector to resources connectors. def self.included(base) - resource_name = StringHelper.underscore_class(base) - method_name = "#{resource_name}_update".to_sym + method_name = "#{base.to_s.underscore_class}_update".to_sym alias_method method_name, :update end diff --git a/devops-service/db/mongo/connectors/provider_accounts.rb b/devops-service/db/mongo/connectors/provider_account.rb similarity index 90% rename from devops-service/db/mongo/connectors/provider_accounts.rb rename to devops-service/db/mongo/connectors/provider_account.rb index 4bd235a..931f99f 100644 --- a/devops-service/db/mongo/connectors/provider_accounts.rb +++ b/devops-service/db/mongo/connectors/provider_account.rb @@ -1,5 +1,5 @@ module Connectors - class ProviderAccounts < Base + class ProviderAccount < Base include Helpers::InsertCommand, Helpers::DeleteCommand @@ -16,7 +16,7 @@ module Connectors def provider_account provider, account c = Provider::ProviderFactory.get_account_class(provider) bson = collection.find({provider: provider, _id: account}).to_a.first - raise RecordNotFound.new("'Account #{account}' for provider '#{provider}' not found") unless bson + raise RecordNotFound.new("Account '#{account}' for provider '#{provider}' not found") unless bson c.build_from_bson(bson) end diff --git a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb index 49cdfad..f4259cf 100644 --- a/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb +++ b/devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb @@ -53,7 +53,7 @@ module Devops def subnets_filter networks = provider_instance.networks - unless self.subnets.empty? + if subnets && !subnets.empty? network = networks.detect {|n| n["name"] == self.subnets[0]} if network {"vpc-id" => network["vpcId"] } diff --git a/devops-service/db/mongo/models/image.rb b/devops-service/db/mongo/models/image.rb index 0263562..8ccfc59 100644 --- a/devops-service/db/mongo/models/image.rb +++ b/devops-service/db/mongo/models/image.rb @@ -73,10 +73,6 @@ module Devops } end - def self.create_from_json! json - Image.new( JSON.parse(json) ) - end - end end end diff --git a/devops-service/db/mongo/models/key.rb b/devops-service/db/mongo/models/key.rb index 32332ff..edaac58 100644 --- a/devops-service/db/mongo/models/key.rb +++ b/devops-service/db/mongo/models/key.rb @@ -30,10 +30,6 @@ module Devops key end - def self.create_from_json json - Key.new( JSON.parse(json) ) - end - def filename File.basename(self.path) end diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb index c4edc04..0bb61ac 100644 --- a/devops-service/db/mongo/models/project.rb +++ b/devops-service/db/mongo/models/project.rb @@ -22,6 +22,7 @@ module Devops #define_hook :after_add_deploy_env attr_accessor :id, :deploy_envs, :type, :archived, :description, :run_list + attr_accessor :components MULTI_TYPE = "multi" diff --git a/devops-service/db/mongo/models/provider_accounts/provider_account.rb b/devops-service/db/mongo/models/provider_accounts/provider_account.rb index d3c9c42..efc6dd1 100644 --- a/devops-service/db/mongo/models/provider_accounts/provider_account.rb +++ b/devops-service/db/mongo/models/provider_accounts/provider_account.rb @@ -61,6 +61,12 @@ module Devops } end + # absent of "id" attribute can cause some inconviniences. + # for example, we have "record.id" call in InsertCommand + def id + account_name + end + end end end diff --git a/devops-service/db/mongo/models/report.rb b/devops-service/db/mongo/models/report.rb index c557b22..e0d03db 100644 --- a/devops-service/db/mongo/models/report.rb +++ b/devops-service/db/mongo/models/report.rb @@ -11,6 +11,7 @@ module Devops STACK_TYPE = 5 DEPLOY_STACK_TYPE = 6 DELETE_SERVER_TYPE = 7 + EXPIRE_SERVER_TYPE = 8 attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb index 20b3d03..4cd52f3 100644 --- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb +++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb @@ -16,10 +16,6 @@ module Devops super.merge(template_url: template_url) end - def delete_template_file_from_storage - raise 'Implement me' - end - def update_template_url self.template_url = generate_template_file_and_upload_to_storage(id, template_body) end diff --git a/devops-service/db/mongo/models/user.rb b/devops-service/db/mongo/models/user.rb index c24247e..c96b3dd 100644 --- a/devops-service/db/mongo/models/user.rb +++ b/devops-service/db/mongo/models/user.rb @@ -75,17 +75,12 @@ module Devops user end - def self.create_from_json json - User.new( JSON.parse(json) ) - end - def to_hash_without_id - o = { + { "email" => self.email, "password" => self.password, "privileges" => self.privileges } - o end def check_privileges cmd, required_privelege diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index baa47ff..60deaa7 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -33,7 +33,7 @@ class MongoConnector [:keys, :key, :key_insert, :key_delete] => :keys_connector, [:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector, [:insert_statistic, :search_statistic] => :statistics_connector, - [:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_account] => :provider_accounts_connector + [:provider_accounts, :provider_account_insert, :provider_account_delete, :provider_account] => :provider_accounts_connector ) def initialize(db, host, port=27017, user=nil, password=nil) @@ -48,7 +48,7 @@ class MongoConnector private def provider_accounts_connector - @provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db) + @provider_accounts_connector ||= Connectors::ProviderAccount.new(@db) end def images_connector diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb index 677e9f2..bec8722 100644 --- a/devops-service/db/validators/base.rb +++ b/devops-service/db/validators/base.rb @@ -13,6 +13,7 @@ module Validators raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}") end + # :nocov: def valid? raise 'override me' end @@ -20,6 +21,7 @@ module Validators def message raise 'override me' end + # :nocov: class << self private diff --git a/devops-service/db/validators/deploy_env/flavor.rb b/devops-service/db/validators/deploy_env/flavor.rb index e0ff359..b763af5 100644 --- a/devops-service/db/validators/deploy_env/flavor.rb +++ b/devops-service/db/validators/deploy_env/flavor.rb @@ -3,7 +3,7 @@ module Validators def valid? return true unless @model.flavor - available_flavors.detect do |flavor| + @model.provider_instance.flavors.detect do |flavor| flavor['id'] == @model.flavor end end @@ -11,11 +11,5 @@ module Validators def message "Invalid flavor '#{@model.flavor}'." end - - private - - def available_flavors - @model.provider_instance.flavors - end end end \ No newline at end of file diff --git a/devops-service/db/validators/deploy_env/groups.rb b/devops-service/db/validators/deploy_env/groups.rb index 43f7027..8e3db24 100644 --- a/devops-service/db/validators/deploy_env/groups.rb +++ b/devops-service/db/validators/deploy_env/groups.rb @@ -3,6 +3,9 @@ module Validators def valid? return true if @model.groups.nil? + subnets_filter = @model.subnets_filter + available_groups = @model.provider_instance.groups(subnets_filter).keys + @invalid_groups = @model.groups - available_groups @invalid_groups.empty? end @@ -10,12 +13,5 @@ module Validators def message "Invalid groups '#{@invalid_groups.join("', '")}'." end - - private - - def available_groups - subnets_filter = @model.subnets_filter - @model.provider_instance.groups(subnets_filter).keys - end end end \ No newline at end of file diff --git a/devops-service/db/validators/deploy_env/image.rb b/devops-service/db/validators/deploy_env/image.rb index 6056d88..ee69561 100644 --- a/devops-service/db/validators/deploy_env/image.rb +++ b/devops-service/db/validators/deploy_env/image.rb @@ -6,7 +6,7 @@ module Validators def valid? return true unless @model.image - available_images.detect do |image| + get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image| image["id"] == @model.image end end @@ -14,11 +14,5 @@ module Validators def message "Invalid image '#{@model.image}'." end - - private - - def available_images - get_available_provider_images(::Devops::Db.connector, @model.provider) - end end end diff --git a/devops-service/db/validators/deploy_env/stack_template.rb b/devops-service/db/validators/deploy_env/stack_template.rb index 9b20d6d..eee09de 100644 --- a/devops-service/db/validators/deploy_env/stack_template.rb +++ b/devops-service/db/validators/deploy_env/stack_template.rb @@ -4,21 +4,13 @@ module Validators def valid? return true unless @model.stack_template - available_stack_templates.detect do |template| - template['id'] == @model.stack_template + Devops::Db.connector.stack_templates.detect do |template| + template.id == @model.stack_template end end def message "Invalid stack template '#{@model.stack_template}'." end - - private - - def available_stack_templates - # map to hash to simplify mocks. Later replace this method with something more suitable - Devops::Db.connector.stack_templates.map(&:to_hash) - end - end end diff --git a/devops-service/db/validators/field_validators/flavor.rb b/devops-service/db/validators/field_validators/flavor.rb index 4f86df4..a06d104 100644 --- a/devops-service/db/validators/field_validators/flavor.rb +++ b/devops-service/db/validators/field_validators/flavor.rb @@ -4,7 +4,7 @@ module Validators class Flavor < Base def valid? - available_flavors.detect do |flavor| + @model.provider_instance.flavors.detect do |flavor| flavor['id'] == @value end end @@ -12,12 +12,6 @@ module Validators def message "Invalid flavor '#{@value}'." end - - private - - def available_flavors - @model.provider_instance.flavors - end end end end diff --git a/devops-service/db/validators/field_validators/image.rb b/devops-service/db/validators/field_validators/image.rb index 3dd727b..d8df841 100644 --- a/devops-service/db/validators/field_validators/image.rb +++ b/devops-service/db/validators/field_validators/image.rb @@ -7,7 +7,7 @@ module Validators include ::ImageCommands def valid? - available_images.detect do |image| + get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image| image["id"] == @value end end @@ -15,12 +15,6 @@ module Validators def message "Invalid image '#{@value}'." end - - private - - def available_images - get_available_provider_images(::Devops::Db.connector, @model.provider) - end end end end diff --git a/devops-service/db/validators/helpers/users.rb b/devops-service/db/validators/helpers/users.rb index 7b534bf..430a7bc 100644 --- a/devops-service/db/validators/helpers/users.rb +++ b/devops-service/db/validators/helpers/users.rb @@ -2,6 +2,7 @@ module Validators class Helpers::Users < Base def valid? + available_users = ::Devops::Db.connector.users_names(@model) @nonexistent_users = (@model || []) - available_users @nonexistent_users.empty? end @@ -9,11 +10,5 @@ module Validators def message Devops::Messages.t("project.deploy_env.validation.users.not_exist", users: @nonexistent_users.join("', '")) end - - private - - def available_users - ::Devops::Db.connector.users_names(@model) - end end end diff --git a/devops-service/db/validators/image/bootstrap_template.rb b/devops-service/db/validators/image/bootstrap_template.rb index 2ced224..67bc070 100644 --- a/devops-service/db/validators/image/bootstrap_template.rb +++ b/devops-service/db/validators/image/bootstrap_template.rb @@ -8,22 +8,12 @@ module Validators include BootstrapTemplatesCommands def valid? - if @model.bootstrap_template - available_templates.include?(@model.bootstrap_template) - else - true - end + get_templates.include?(@model.bootstrap_template) end def message "Invalid bootstrap template '#{@model.bootstrap_template}' for image '#{@model.id}'" end - - private - - def available_templates - get_templates - end end end end diff --git a/devops-service/lib/hash_ext.rb b/devops-service/lib/core_ext/hash.rb similarity index 100% rename from devops-service/lib/hash_ext.rb rename to devops-service/lib/core_ext/hash.rb diff --git a/devops-service/lib/nil_class_ext.rb b/devops-service/lib/core_ext/nil_class.rb similarity index 100% rename from devops-service/lib/nil_class_ext.rb rename to devops-service/lib/core_ext/nil_class.rb diff --git a/devops-service/lib/core_ext/string.rb b/devops-service/lib/core_ext/string.rb new file mode 100644 index 0000000..54475e3 --- /dev/null +++ b/devops-service/lib/core_ext/string.rb @@ -0,0 +1,27 @@ +class String + def present? + !empty? + end + + def blank? + empty? + end + + # from ActiveSupport + def underscore + gsub(/::/, '/'). + gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). + gsub(/([a-z\d])([A-Z])/,'\1_\2'). + tr("-", "_"). + downcase + end + + def underscore_class + split('::').last.underscore + end + + # rough simplification + def pluralize + "#{self}s" + end +end \ No newline at end of file diff --git a/devops-service/lib/executors/expiration_scheduler.rb b/devops-service/lib/executors/expiration_scheduler.rb new file mode 100644 index 0000000..d84702d --- /dev/null +++ b/devops-service/lib/executors/expiration_scheduler.rb @@ -0,0 +1,35 @@ +require "workers/delete_expired_server_worker" + +module Devops + module Executor + class ExpirationScheduler + def initialize(expires, server) + @expires, @server = expires, server + end + + def schedule_expiration! + return unless @expires + DeleteExpiredServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name) + end + + def interval_in_seconds + interval = @expires.to_i + measure_unit = @expires.chars.last + case measure_unit + when 's' + interval + when 'm' + interval * 60 + when 'h' + interval * 60 * 60 + when 'd' + interval * 60 * 60 * 24 + when 'w' + interval * 60 * 60 * 24 * 7 + else + raise 'Wrong interval format' + end + end + end + end +end \ No newline at end of file diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 88ac412..e861220 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -1,6 +1,5 @@ require "lib/knife/knife_factory" -require "workers/worker" -require "workers/delete_server_worker" +require "lib/executors/expiration_scheduler" require "hooks" require 'net/ssh' @@ -9,12 +8,16 @@ module Devops class ServerExecutor include Hooks - RESULT_CODES = { + ERROR_CODES = { server_bootstrap_fail: 2, + server_cannot_update_tags: 3, + server_bootstrap_private_ip_unset: 4, server_not_in_chef_nodes: 5, server_bootstrap_unknown_error: 7, deploy_unknown_error: 6, - deploy_failed: 8 + deploy_failed: 8, + creating_server_unknown_error: 9, + creating_server_in_cloud_failed: 10 } # waiting for 5*60 seconds (5 min) @@ -34,46 +37,32 @@ module Devops define_hook :before_bootstrap define_hook :after_bootstrap - before_deploy :create_run_list + before_deploy :add_run_list_to_deploy_info + + + attr_accessor :server, :deploy_env, :report, :project def initialize server, out, options={} if server @project = Devops::Db.connector.project(server.project) @deploy_env = @project.deploy_env(server.deploy_env) end - @knife_instance = KnifeFactory.instance @server = server @out = out @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush) @current_user = options[:current_user] end - def self.result_code(symbolic_code) - RESULT_CODES.fetch(symbolic_code) + def self.error_code(symbolic_code) + ERROR_CODES.fetch(symbolic_code) end - def self.symbolic_result_code(integer_code) - RESULT_CODES.key(integer_code) || :unknown_error + def self.symbolic_error_code(integer_code) + ERROR_CODES.key(integer_code) || :unknown_error end - def result_code(symbolic_code) - self.class.result_code(symbolic_code) - end - - def report= r - @report = r - end - - def project= p - @project = p - end - - def deploy_env= e - @deploy_env = e - end - - def server - @server + def error_code(symbolic_code) + self.class.error_code(symbolic_code) end def create_server_object options @@ -109,7 +98,9 @@ module Devops res[:before] = self.run_hook :before_create @out << "Done\n" - return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out) + unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out) + return error_code(:creating_server_in_cloud_failed) + end mongo.server_insert @server @out << "\nAfter create hooks...\n" @@ -118,7 +109,7 @@ module Devops @out.flush DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running" - schedule_expiration(@server) + schedule_expiration() unless options["without_bootstrap"] bootstrap_options = { @@ -135,11 +126,15 @@ module Devops DevopsLogger.logger.error e.message roll_back mongo.server_delete @server.id - # return 5 - return result_code(:server_not_in_chef_nodes) + error_code(:creating_server_unknown_error) end end + # options: + # :run_list (optional) + # :bootstrap_template (optional) + # :chef_environment (optional) + # :config (optional) def bootstrap options @out << "\n\nBootstrap...\n" @out.flush @@ -151,7 +146,7 @@ module Devops @out << "Done\n" if @server.private_ip.nil? @out << "Error: Private IP is null" - return false + return error_code(:server_bootstrap_private_ip_unset) end ja = { :provider => @server.provider, @@ -166,13 +161,7 @@ module Devops address = "#{@server.remote_user}@#{ip}" - cmd = 'ssh ' - cmd << "-i #{cert_path} " - cmd << '-q ' - cmd << '-o StrictHostKeyChecking=no ' - cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 ' - cmd << "#{address} 'exit'" - cmd << " 2>&1" + cmd = check_ssh_command(cert_path, address) @out << "\nWaiting for SSH..." @out << "\nTest command: '#{cmd}'\n" @@ -181,16 +170,16 @@ module Devops retries_amount = 0 begin sleep(5) - res = `#{cmd}` + res = execute_system_command(cmd) retries_amount += 1 - if retries_amount > MAX_SSH_RETRIES_AMOUNT + if retries_amount >= MAX_SSH_RETRIES_AMOUNT @out.puts "Can not connect to #{address}" @out.puts res @out.flush DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}" - return result_code(:server_bootstrap_fail) + return error_code(:server_bootstrap_fail) end - raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success? + raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless last_command_successful? rescue ArgumentError => e @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})" @out.flush @@ -200,7 +189,7 @@ module Devops provider = @server.provider_instance @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil? - r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options)) + r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options)) if r == 0 @out << "Chef node name: #{@server.chef_node_name}\n" @@ -217,10 +206,16 @@ module Devops else @out << "Can not bootstrap node '#{@server.id}', error code: #{r}" @out.flush - result_code(:server_bootstrap_fail) + error_code(:server_bootstrap_fail) end end + # options: + # :cert_path (required) + # :run_list (optional) + # :bootstrap_template (optional) + # :chef_environment (optional) + # :config (optional) def bootstrap_options attributes, options bootstrap_options = [ "-x #{@server.remote_user}", @@ -242,6 +237,7 @@ module Devops @out << "Done\n" end + # essentially, it just bootstrap and then deploy def two_phase_bootstrap options prepare_two_phase_bootstrap(options) # bootstrap phase @@ -252,14 +248,14 @@ module Devops bootstrap_status = bootstrap(options) if bootstrap_status == 0 - if check_server + if check_server_on_chef_server @out << "Server #{@server.chef_node_name} is created" else @out.puts "Can not find client or node on chef-server" roll_back @out.flush mongo.server_delete @server.id - return result_code(:server_not_in_chef_nodes) + return error_code(:server_not_in_chef_nodes) end else # @out << roll_back @@ -269,22 +265,20 @@ module Devops DevopsLogger.logger.error msg @out.puts msg @out.flush - return result_code(:server_bootstrap_fail) + return error_code(:server_bootstrap_fail) end rescue => e @out << "\nError: #{e.message}\n" @out.flush - return result_code(:server_bootstrap_unknown_error) + return error_code(:server_bootstrap_unknown_error) end # deploy phase. Assume that all servers are bootstraped successfully here. begin - #raise "hello" - @out << "\n" run_list = compute_run_list - @out << "\nComputed run list: #{run_list.join(", ")}" + @out << "\n\nComputed run list: #{run_list.join(", ")}" @out.flush - @knife_instance.set_run_list(@server.chef_node_name, run_list) + knife_instance.set_run_list(@server.chef_node_name, run_list) deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env) deploy_status = deploy_server(deploy_info) if deploy_status == 0 @@ -294,19 +288,20 @@ module Devops msg << "\nDeploing server operation status was #{deploy_status}" DevopsLogger.logger.error msg @out << "\n" + msg + "\n" - result_code(:deploy_failed) + error_code(:deploy_failed) end rescue => e @out << "\nError: #{e.message}\n" DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n")) - result_code(:deploy_unknown_error) + error_code(:deploy_unknown_error) end end - def check_server - @knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name) + def check_server_on_chef_server + knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name) end + # returns a hash with :chef_node, :chef_client and :server keys def unbootstrap k = Devops::Db.connector.key(@server.key) cert_path = k.path @@ -346,35 +341,28 @@ module Devops end def deploy_server_with_tags tags, deploy_info - old_tags_str = nil - new_tags_str = nil - unless tags.empty? - old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ") - @out << "Server tags: #{old_tags_str}\n" - @knife_instance.tags_delete(@server.chef_node_name, old_tags_str) + return deploy_server(deploy_info) if tags.empty? - new_tags_str = tags.join(" ") - @out << "Server new tags: #{new_tags_str}\n" - cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str) - unless cmd[1] - m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'" - DevopsLogger.logger.error(m) - @out << m + "\n" - return 3 - end - DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}") + old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ") + new_tags_str = tags.join(" ") + + @out.puts "Temporarily changing tags (#{old_tags_str}) to (#{new_tags_str})" + unless knife_instance.swap_tags(@server.chef_node_name, old_tags_str, new_tags_str) + m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'" + DevopsLogger.logger.error(m) + @out.puts m + return error_code(:server_cannot_update_tags) end + DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}") - r = deploy_server deploy_info - - unless tags.empty? - @out << "Restore tags\n" - cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str) - DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}") - cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str) - DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}") + begin + deploy_result = deploy_server deploy_info + ensure + @out.puts "Restoring tags" + knife_instance.swap_tags(@server.chef_node_name, new_tags_str, old_tags_str) + DevopsLogger.logger.info("Restoring tags for #{@server.chef_node_name}: from #{new_tags_str} back to (#{old_tags_str})") end - return r + deploy_result end def deploy_server deploy_info @@ -397,7 +385,7 @@ module Devops f.write json end end - @out << "Deploy Input Parameters:\n" + @out.puts "Deploy Input Parameters:" @out.puts json @out.flush cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}" @@ -412,7 +400,7 @@ module Devops end @out.flush k = Devops::Db.connector.key(@server.key) - lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path) + lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path) r = /Chef\sClient\sfinished/i if lline && lline[r] @@ -432,21 +420,11 @@ module Devops def delete_from_chef_server node_name { - :chef_node => @knife_instance.chef_node_delete(node_name), - :chef_client => @knife_instance.chef_client_delete(node_name) + :chef_node => knife_instance.chef_node_delete(node_name), + :chef_client => knife_instance.chef_client_delete(node_name) } end -=begin - def delete_etc_chef s, cert_path - cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}" - cmd += " sudo " unless s.remote_user == "root" - cmd += "rm -Rf /etc/chef" - r = `#{cmd}` - raise(r) unless $?.success? - end -=end - def delete_server mongo = ::Devops::Db.connector if @server.static? @@ -454,23 +432,25 @@ module Devops unbootstrap end mongo.server_delete @server.id - msg = "Static server '#{@server.id}' is removed" - DevopsLogger.logger.info msg - return msg, nil + puts_and_flush "Static server '#{@server.id}' is removed" + return 0 end - r = delete_from_chef_server(@server.chef_node_name) + + puts_and_flush "Deleting from chef server:" + delete_from_chef_server(@server.chef_node_name).each do |key, result| + @out.puts "#{key} - #{result}" + end + + puts_and_flush "Deleting from cloud:" provider = @server.provider_instance begin - r[:server] = provider.delete_server @server + puts_and_flush provider.delete_server @server rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound - r[:server] = "Server with id '#{@server.id}' not found in '#{provider.name}' servers" - DevopsLogger.logger.warn r[:server] + puts_and_flush "Server with id '#{@server.id}' not found among '#{provider.name}' servers" end mongo.server_delete @server.id - info = "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed" - DevopsLogger.logger.info info - r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")} - return info, r + puts_and_flush "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed." + 0 end def roll_back @@ -487,7 +467,7 @@ module Devops end end - def create_run_list out, deploy_info + def add_run_list_to_deploy_info out, deploy_info out << "\nGenerate run list hook...\n" if deploy_info["run_list"] out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n" @@ -496,14 +476,6 @@ module Devops out << "Project run list: #{@project.run_list.join(", ")}\n" out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n" out << "Server run list: #{@server.run_list.join(", ")}\n" -=begin - rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list) - if @server.stack - stack = Devops::Db.connector.stack(@server.stack) - out << "Stack run list: #{stack.run_list.join(", ")}\n" - rlist.merge(stack.run_list) - end -=end deploy_info["run_list"] = compute_run_list out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n" end @@ -511,47 +483,56 @@ module Devops def compute_run_list rlist = [] [@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list| - rlist += sub_run_list if sub_run_list.is_a?(Array) + rlist += sub_run_list if sub_run_list end - rlist = Set.new(rlist) if @server.stack stack = Devops::Db.connector.stack(@server.stack) -# out << "Stack run list: #{stack.run_list.join(", ")}\n" srl = stack.run_list - rlist.merge(srl) if srl.is_a?(Array) + rlist += srl if srl end - rlist.to_a + rlist.uniq end private - def schedule_expiration(server) - expires = @deploy_env.expires - return unless expires - interval = interval_in_seconds(expires) - @out << "Planning expiration in #{expires}" - DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name) + def puts_and_flush(message) + @out.puts message + @out.flush end - def interval_in_seconds(interval_as_string) - interval = interval_as_string.to_i - measure_unit = interval_as_string.chars.last - case measure_unit - when 's' - interval - when 'm' - interval * 60 - when 'h' - interval * 60 * 60 - when 'd' - interval * 60 * 60 * 24 - when 'w' - interval * 60 * 60 * 24 * 7 - else - raise 'Wrong interval format' + def schedule_expiration + if @deploy_env.expires + @out << "Planning expiration in #{@deploy_env.expires}" + ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration! end end + def check_ssh_command(cert_path, address) + cmd = 'ssh ' + cmd << "-i #{cert_path} " + cmd << '-q ' + cmd << '-o StrictHostKeyChecking=no ' + cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 ' + cmd << "#{address} 'exit'" + cmd << " 2>&1" + cmd + end + + # to simplify testing + # :nocov: + def execute_system_command(cmd) + `#{cmd}` + end + + def last_command_successful? + $?.success? + end + + def knife_instance + @knife_instance ||= KnifeFactory.instance + end + # :nocov: + end end end diff --git a/devops-service/lib/string_ext.rb b/devops-service/lib/string_ext.rb deleted file mode 100644 index ce2e9ca..0000000 --- a/devops-service/lib/string_ext.rb +++ /dev/null @@ -1,9 +0,0 @@ -class String - def present? - !empty? - end - - def blank? - empty? - end -end \ No newline at end of file diff --git a/devops-service/lib/string_helper.rb b/devops-service/lib/string_helper.rb deleted file mode 100644 index 74a6727..0000000 --- a/devops-service/lib/string_helper.rb +++ /dev/null @@ -1,35 +0,0 @@ -module StringHelper - extend self - - # from Rails' ActiveSupport - def underscore(string) - string.gsub(/::/, '/'). - gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). - gsub(/([a-z\d])([A-Z])/,'\1_\2'). - tr("-", "_"). - downcase - end - - def underscore_class(klass, without_ancestors=true) - class_name = if without_ancestors - klass.to_s.split('::').last - else - klass.to_s - end - StringHelper.underscore(class_name) - end - - # from Rails' ActiveSupport - def camelize(term) - string = term.to_s - string = string.sub(/^[a-z\d]*/) { $&.capitalize } - string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" } - string.gsub!(/\//, '::') - string - end - - # rough simplification - def pluralize(string) - "#{string}s" - end -end diff --git a/devops-service/lib/users_permissions_updater.rb b/devops-service/migrations/users_permissions_updater.rb similarity index 100% rename from devops-service/lib/users_permissions_updater.rb rename to devops-service/migrations/users_permissions_updater.rb diff --git a/devops-service/spec/connectors/provider_accounts_connector_spec.rb b/devops-service/spec/connectors/provider_accounts_connector_spec.rb new file mode 100644 index 0000000..2500179 --- /dev/null +++ b/devops-service/spec/connectors/provider_accounts_connector_spec.rb @@ -0,0 +1,58 @@ +require 'db/mongo/models/provider_accounts/ec2_provider_account' +require 'db/mongo/models/provider_accounts/openstack_provider_account' +require 'db/mongo/connectors/provider_account' +require 'spec/connectors/tester_connector/provider_account' + +RSpec.describe Connectors::ProviderAccount, type: :connector do + set_tester_connector TesterConnector::ProviderAccount + + include_examples 'mongo connector', { + model_name: :provider_account, + factory_name: :ec2_provider_account, + only: [:insert, :delete], + field_to_update: :description + } + + describe '#provider_accounts', cleanup_after: :all do + before(:all) do + @tester_connector.create(id: 'foo', provider: 'ec2') + @tester_connector.create(id: 'bar', provider: 'openstack') + end + + it 'returns array of Ec2ProviderAccount if @provider is ec2' do + expect( + @connector.provider_accounts('ec2') + ).to be_an_array_of(Devops::Model::Ec2ProviderAccount).and have_size(1) + end + + it 'returns array of Ec2ProviderAccount if @provider is openstack' do + expect( + @connector.provider_accounts('openstack') + ).to be_an_array_of(Devops::Model::OpenstackProviderAccount).and have_size(1) + end + end + + describe '#provider_account', cleanup_after: :all do + before(:all) do + @tester_connector.create(id: 'foo', provider: 'ec2') + @tester_connector.create(id: 'bar', provider: 'openstack') + end + + it 'returns ec2 provider account' do + acc = @connector.provider_account('ec2', 'foo') + expect(acc).to be_a(Devops::Model::Ec2ProviderAccount) + expect(acc.account_name).to eq 'foo' + end + + it 'returns openstack provider account' do + acc = @connector.provider_account('openstack', 'bar') + expect(acc).to be_a(Devops::Model::OpenstackProviderAccount) + expect(acc.account_name).to eq 'bar' + end + + it 'raises error if account is missing' do + expect{@connector.provider_account('ec2', 'missing')}.to raise_error(RecordNotFound) + end + end + +end diff --git a/devops-service/spec/connectors/tester_connector/provider_account.rb b/devops-service/spec/connectors/tester_connector/provider_account.rb new file mode 100644 index 0000000..38a8feb --- /dev/null +++ b/devops-service/spec/connectors/tester_connector/provider_account.rb @@ -0,0 +1,6 @@ +require_relative 'base' + +module TesterConnector + class ProviderAccount < Base + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/expiration_scheduler_spec.rb new file mode 100644 index 0000000..f8009f8 --- /dev/null +++ b/devops-service/spec/executors/expiration_scheduler_spec.rb @@ -0,0 +1,47 @@ +require 'lib/executors/expiration_scheduler' + +RSpec.describe Devops::Executor::ExpirationScheduler do + let(:server) { build(:server) } + + describe '#schedule_expiration!' do + it 'schedules server deleting at given time' do + expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name') + described_class.new('2m', server).schedule_expiration! + end + + it "doesn't schedule job if expires is nil" do + expect(DeleteExpiredServerWorker).not_to receive(:perform_in) + described_class.new(nil, server).schedule_expiration! + end + end + + describe '#interval_in_seconds' do + def interval_in_seconds(expires) + described_class.new(expires, server).interval_in_seconds + end + + it 'recognizes seconds' do + expect(interval_in_seconds('2s')).to eq 2 + end + + it 'recognizes minutes' do + expect(interval_in_seconds('3m')).to eq 180 + end + + it 'recognizes hours' do + expect(interval_in_seconds('1h')).to eq 3600 + end + + it 'recognizes days' do + expect(interval_in_seconds('1d')).to eq 86400 + end + + it 'recognizes weeks' do + expect(interval_in_seconds('1w')).to eq 604800 + end + + it 'raises on wrong format' do + expect { interval_in_seconds('wrong') }.to raise_error(StandardError) + end + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb new file mode 100644 index 0000000..bc19b58 --- /dev/null +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -0,0 +1,790 @@ +require 'lib/executors/server_executor' + +RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do + let(:project) { build(:project) } + let(:deploy_env) { project.deploy_env('foo') } + let(:server) { build(:server, project: project.id, deploy_env: 'foo') } + let(:output) { File.open(File::NULL, "w") } + let(:provider) { double('Provider instance') } + let(:executor) { described_class.new(server, output) } + + + before do + allow(stubbed_connector).to receive(:project) { project } + allow(executor.deploy_env).to receive(:provider_instance) { provider } + allow(server).to receive(:provider_instance) { provider } + end + + describe '#initialize' do + it 'sets server, project, deploy_env, out instance variables' do + expect(executor.server).to eq server + expect(executor.deploy_env).to eq deploy_env + expect(executor).to have_instance_variable_value(:project, project) + expect(executor).to have_instance_variable_value(:out, output) + end + + it 'defines :flush method on @out if it is absent' do + out = Class.new.new + expect(out).not_to respond_to(:flush) + described_class.new(server, out) + expect(out).to respond_to(:flush) + end + + it 'sets current_user from options' do + user = double + executor = described_class.new(server, '', {current_user: user}) + expect(executor).to have_instance_variable_value(:current_user, user) + end + end + + + describe '#report=' do + it 'sets report instance variable' do + executor.report= 'foo' + expect(executor).to have_instance_variable_value(:report, 'foo') + end + end + + describe '#project=' do + it 'sets project instance variable' do + executor.project= 'foo' + expect(executor).to have_instance_variable_value(:project, 'foo') + end + end + + + describe '.symbolic_error_code' do + it 'returns symbol given an integer' do + expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail + end + + it "returns :unknown_error if can't recognize error code" do + expect(described_class.symbolic_error_code(123)).to eq :unknown_error + end + end + + + + describe '#create_server_object' do + it 'builds Server object' do + server = executor.create_server_object('created_by' => 'me') + expect(server).to be_a(Devops::Model::Server) + expect(server.project).to eq 'my_project' + expect(server.deploy_env).to eq 'foo' + expect(server.created_by).to eq 'me' + end + end + + + + describe '#create_server' do + let(:image) { double('Image instance', remote_user: 'remote_user') } + let(:create_server) { + executor.create_server( + 'created_by' => 'user', + 'run_list' => @run_list, + 'name' => 'node_name', + 'key' => @key, + 'without_bootstrap' => @without_bootstrap + ) + } + + before do + allow(provider).to receive(:create_server) { true } + allow(stubbed_connector).to receive(:image) { image } + allow(stubbed_connector).to receive(:server_insert) + @without_bootstrap = true + @run_list = %w(role[asd]) + @key = 'key' + end + + it 'builds server model from given options' do + create_server + expect(executor.server.created_by).to eq 'user' + expect(executor.server.chef_node_name).to eq 'node_name' + expect(executor.server.key).to eq @key + expect(executor.server.run_list).to eq @run_list + end + + it 'sets run list to an empty array by default' do + @run_list = nil + create_server + expect(executor.server.run_list).to eq [] + end + + it 'sets key to default provider ssh key by default' do + @key = nil + allow(provider).to receive(:ssh_key) { 'default_key' } + create_server + expect(executor.server.key).to eq 'default_key' + end + + it 'runs hooks' do + expect(executor).to receive(:run_hook).with(:before_create).ordered + expect(executor).to receive(:run_hook).with(:after_create).ordered + create_server + end + + it 'creates server in cloud' do + expect(provider).to receive(:create_server).with( + an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output + ) + create_server + end + + it 'inserts built server into mongo' do + expect(stubbed_connector).to receive(:server_insert) + create_server + end + + it 'schedules expiration for server' do + deploy_env.expires = '2m' + allow(DeleteExpiredServerWorker).to receive(:perform_in) + expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'}) + create_server + end + + it "doesn't schedule expiration if deploy_env.expires is nil" do + deploy_env.expires = nil + expect(DeleteExpiredServerWorker).not_to receive(:perform_in) + create_server + end + + context 'without_bootstrap option is false' do + it 'launches bootstrap' do + @without_bootstrap = false + allow(image).to receive(:bootstrap_template) { 'template' } + allow(executor).to receive(:two_phase_bootstrap) + expect(executor).to receive(:two_phase_bootstrap) + create_server + end + end + + context 'without_bootstrap option is nil' do + it 'launches bootstrap' do + @without_bootstrap = nil + allow(image).to receive(:bootstrap_template) { 'template' } + allow(executor).to receive(:two_phase_bootstrap) + expect(executor).to receive(:two_phase_bootstrap) + create_server + end + end + + context 'without_bootstrap option is true' do + it "doesn't launch bootstrap" do + @without_bootstrap = true + expect(executor).not_to receive(:two_phase_bootstrap) + create_server + end + end + + context 'if error has been raised during execution' do + before do + allow(stubbed_connector).to receive(:server_delete) + allow(provider).to receive(:create_server) { raise } + end + + it 'rollbacks server creating' do + expect(executor).to receive(:roll_back) + create_server + end + + it 'deletes server from mongo' do + expect(stubbed_connector).to receive(:server_delete) + create_server + end + end + + context "if creating server in cloud wasn't successful" do + it 'returns creating_server_in_cloud_failed error code' do + allow(provider).to receive(:create_server) { false } + expect(create_server).to eq 10 + end + end + end + + + + describe '#bootstrap', stubbed_knife: true do + let(:image) { double('Key instance', path: 'path') } + let(:bootstrap) { executor.bootstrap({}) } + + before do + allow(executor).to receive(:sleep) + allow(executor).to receive(:last_command_successful?).and_return(true) + allow(executor).to receive(:execute_system_command) + allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node') + allow(stubbed_connector).to receive(:key).and_return(image) + allow(stubbed_connector).to receive(:server_set_chef_node_name) + allow(stubbed_knife).to receive(:knife_bootstrap).and_return(0) + end + + it 'run before hook' do + expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered + expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered + bootstrap + end + + context "when server's private ip is unset" do + it 'returns server_bootstrap_private_ip_unset error code' do + server.private_ip = nil + expect(bootstrap).to eq 4 + end + end + + it 'tries to ssh to server' do + expect(executor).to receive(:execute_system_command).with(/ssh/) + bootstrap + end + + context "couldn't ssh to server" do + before { allow(executor).to receive(:last_command_successful?) { false } } + + it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do + max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT + expect(executor).to receive(:execute_system_command).exactly(max_retries).times + bootstrap + end + + it 'returns server_bootstrap_fail error code' do + expect(bootstrap).to eq 2 + end + end + + + context 'after successful ssh check' do + before { allow(executor).to receive(:last_command_successful?).and_return(false, true) } + + it "sets default chef node name if it's nil" do + executor.server.chef_node_name = nil + expect {bootstrap}.to change {executor.server.chef_node_name}.to 'chef_node' + end + + it 'executes knife bootstrap' do + expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array)) + bootstrap + end + + it "bootstraps to public ip if it's set" do + server.public_ip = '8.8.8.8' + expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array)) + bootstrap + end + + context 'after successful bootstrap' do + it "updates server's chef node name in db" do + expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server)) + bootstrap + end + end + + context "if bootstraping wasn't successful" do + before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) } + + it 'returns :server_bootstrap_fail code' do + expect(bootstrap).to eq 2 + end + + it "doesn't run after hook" do + expect(executor).to receive(:run_hook).with(:before_bootstrap, output) + bootstrap + end + end + end + end + + + + describe '#two_phase_bootstrap', stubbed_knife: true do + let(:two_phase_bootstrap) { executor.two_phase_bootstrap({}) } + + before do + allow(provider).to receive(:run_list) {[]} + allow(stubbed_connector).to receive(:server_delete) + end + + context 'when bootstrap was successful' do + before do + allow(executor).to receive(:bootstrap) { 0 } + allow(executor).to receive(:check_server_on_chef_server) { false } + end + + context 'if node presents on chef server' do + before do + allow(executor).to receive(:check_server_on_chef_server) { true } + allow(executor).to receive(:deploy_server) + allow(stubbed_knife).to receive(:set_run_list) + end + + it 'builds run list' do + expect(executor).to receive(:compute_run_list) + two_phase_bootstrap + end + + it 'sets run list to chef node' do + expect(stubbed_knife).to receive(:set_run_list) + two_phase_bootstrap + end + + it 'deploys server' do + expect(executor).to receive(:deploy_server) + two_phase_bootstrap + end + + context 'if deploy was successful' do + it 'returns 0' do + allow(executor).to receive(:deploy_server) { 0 } + expect(two_phase_bootstrap).to eq 0 + end + end + + context "if deploy wasn't successful" do + it 'returns :deploy_failed code' do + allow(executor).to receive(:deploy_server) { 1 } + expect(two_phase_bootstrap).to eq 8 + end + end + + context 'when an error occured during deploy' do + it 'returns :deploy_unknown_error code' do + allow(executor).to receive(:deploy_server) { raise } + expect(two_phase_bootstrap).to eq 6 + end + end + end + + context "if node doesn't present on chef server" do + it 'roll backs and then deletes server from mongo' do + allow(executor).to receive(:check_server_on_chef_server) { false } + allow(executor).to receive(:roll_back) + allow(stubbed_connector).to receive(:server_delete) + expect(executor).to receive(:roll_back).ordered + expect(stubbed_connector).to receive(:server_delete).ordered + two_phase_bootstrap + end + end + end + + context "when bootstrap wasn't successful" do + it 'returns :server_bootstrap_fail error code' do + allow(executor).to receive(:bootstrap) { 1 } + expect(two_phase_bootstrap).to eq 2 + end + end + + context 'when an error occured during bootstrap' do + it 'returns :server_bootstrap_unknown_error error code' do + allow(executor).to receive(:bootstrap) { raise } + expect(two_phase_bootstrap).to eq 7 + end + end + end + + describe '#check_server_on_chef_server', stubbed_knife: true do + before do + server.chef_node_name = 'a' + allow(stubbed_knife).to receive(:chef_node_list) { @node_list } + allow(stubbed_knife).to receive(:chef_client_list) { @client_list } + end + + it 'returns true when node_name in node list and in client list' do + @node_list = %w(a); @client_list = %w(a) + expect(executor.check_server_on_chef_server).to be true + end + + it "returns false if node name isn't in node list" do + @node_list = []; @client_list = %w(a) + expect(executor.check_server_on_chef_server).to be false + end + + it "returns false if node name isn't in node list" do + @node_list = %w(a); @client_list = [] + expect(executor.check_server_on_chef_server).to be false + end + end + + + describe '#unbootstrap', stubbed_knife: true do + before do + allow(stubbed_connector).to receive_message_chain('key.path') { 'path_to_key' } + allow(stubbed_knife).to receive(:chef_node_delete) + allow(stubbed_knife).to receive(:chef_client_delete) + allow(executor).to receive(:execute_system_command) { '' } + allow(executor).to receive(:last_command_successful?) { true } + allow(executor).to receive(:sleep) + allow(Net::SSH).to receive(:start) + end + + it 'connects by ssh' do + expect(Net::SSH).to receive(:start) + executor.unbootstrap + end + + it 'returns hash with error after 5 unsuccessful retries' do + allow(Net::SSH).to receive(:start) { raise } + expect(Net::SSH).to receive(:start).exactly(5).times + expect(executor.unbootstrap).to be_a(Hash).and include(:error) + end + end + + + describe '#deploy_server_with_tags', stubbed_knife: true do + let(:current_tags) { @current_tags } + let(:initial_tags) { %w(a b) } + let(:joined_initial_tags) { initial_tags.join(' ') } + let(:given_tags) { %w(c d) } + let(:joined_given_tags) { given_tags.join(' ') } + let(:deploy_server_with_tags) { executor.deploy_server_with_tags(given_tags, {}) } + + before do + @current_tags = initial_tags.dup + allow(stubbed_knife).to receive(:tags_list) { @current_tags } + allow(stubbed_knife).to receive(:swap_tags) do |_, tags_to_delete, tags_to_add| + @current_tags -= tags_to_delete.split + @current_tags += tags_to_add.split + end + allow(executor).to receive(:deploy_server) + end + + context 'when tags are empty' do + it 'just deploys server' do + expect(executor).to receive(:deploy_server) + expect(stubbed_knife).not_to receive(:swap_tags) + executor.deploy_server_with_tags([], {}) + end + end + + context 'when tags are not empty' do + it 'temporarily swaps current_tags with given ones, deploys server and then restores tags' do + expect(stubbed_knife).to receive(:tags_list).ordered + expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_initial_tags, joined_given_tags).ordered + expect(executor).to receive(:deploy_server).ordered + expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_given_tags, joined_initial_tags).ordered + deploy_server_with_tags + end + end + + context 'if error occures during deploy' do + it 'restores tags anyway' do + allow(executor).to receive(:deploy_server) { raise } + expect { + deploy_server_with_tags + }.to raise_error StandardError + expect(current_tags).to eq initial_tags + end + end + + context 'if cannot add tags to server' do + it 'returns :server_cannot_update_tags code' do + allow(stubbed_knife).to receive(:swap_tags) { false } + expect(deploy_server_with_tags).to eq 3 + end + end + end + + + + describe '#deploy_server', stubbed_knife: true do + let(:deploy_info) { @deploy_info } + let(:json_file_name) { 'json.json' } + let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) } + + let(:deploy_server) { executor.deploy_server(deploy_info) } + + before do + allow(executor).to receive(:run_hook).with(:before_deploy, any_args) + allow(executor).to receive(:run_hook).with(:after_deploy, any_args) + allow(stubbed_knife).to receive(:ssh_stream) { 'Chef Client finished'} + allow(stubbed_connector).to receive(:key) { double('Key', path: 'path_to_key') } + allow(stubbed_connector).to receive(:server_update) + @deploy_info = {} + end + + + it 'runs before_deploy and after_deploy hooks' do + expect(executor).to receive(:run_hook).with(:before_deploy, any_args).ordered + expect(executor).to receive(:run_hook).with(:after_deploy, any_args).ordered + deploy_server + end + + context 'when uses json file' do + before(:all) do + @tmp_files_at_start = Dir.entries(SpecSupport.tmp_dir) + end + before do + allow(DevopsConfig).to receive(:config).and_return({ + project_info_dir: SpecSupport.tmp_dir, + address: 'host.com', + port: '8080', + url_prefix: 'api' + }) + deploy_info['use_json_file'] = true + end + + after(:all) do + diff = Dir.entries(SpecSupport.tmp_dir) - @tmp_files_at_start + diff.each do |file| + FileUtils.rm(File.join(SpecSupport.tmp_dir, file)) + end + end + + + it 'writes deploy_info to json file if it not exists' do + expect { deploy_server }.to change { Dir.entries(SpecSupport.tmp_dir)} + end + + it "writes deploy_info to given json file name if it doesn't exist" do + FileUtils.rm(json_file_path) if File.exists?(json_file_path) + deploy_info['json_file'] = json_file_name + expect { deploy_server }.to change { + Dir.entries(SpecSupport.tmp_dir) + } + FileUtils.rm(json_file_path) + end + + it 'reads json from file if it exists' do + deploy_info['json_file'] = json_file_name + File.open(json_file_path, 'w') { |file| file.puts '{"foo": "bar"'} + expect { deploy_server }.not_to change { + Dir.entries(SpecSupport.tmp_dir) + } + FileUtils.rm(json_file_path) + end + + it 'adds link to json to deploy command' do + deploy_info['json_file'] = json_file_name + regexp = %r(-j http://host.com:8080/api/v2.0/deploy/data/#{json_file_name}) + expect(stubbed_knife).to receive(:ssh_stream).with(anything, regexp, any_args) + deploy_server + end + end + + context "doesn't use json file" do + before do + deploy_info['use_json_file'] = false + deploy_info['run_list'] = %w(foo bar) + end + + it "adds run list to command if server's stack is set" do + server.stack = 'stack' + expect(stubbed_knife).to receive(:ssh_stream).with(anything, %r(-r foo,bar), any_args) + deploy_server + end + + it "doesn't add run list to command if server's stack is unset" do + expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args) + deploy_server + end + end + + it "uses server's key" do + expect(stubbed_connector).to receive(:key).with('key_id') + expect(stubbed_knife).to receive(:ssh_stream).with(any_args, 'path_to_key') + deploy_server + end + + it "uses public ip if it's set" do + server.public_ip = '127.0.0.1' + expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, '127.0.0.1', any_args) + deploy_server + end + + it "uses private_ip if public_ip isn't set" do + expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, server.private_ip, any_args) + deploy_server + end + + context 'if deploy was successful' do + it "updates server's last operation" do + expect(server).to receive(:set_last_operation).with('deploy', anything) + expect(stubbed_connector).to receive(:server_update).with(server) + deploy_server + end + + it 'returns 0' do + expect(deploy_server).to eq 0 + end + end + + context "when deploy wasn't successful" do + before { allow(stubbed_knife).to receive(:ssh_stream) { 'fail'} } + it "doesn't run after_deploy hook" do + expect(executor).to receive(:run_hook).with(:before_deploy, any_args) + expect(executor).not_to receive(:run_hook).with(:after_deploy, any_args) + deploy_server + end + + it 'returns 1' do + expect(deploy_server).to eq 1 + end + end + end + + + describe '#delete_from_chef_server', stubbed_knife: true do + let(:delete_from_chef_server) { executor.delete_from_chef_server('foo') } + before do + allow(stubbed_knife).to receive(:chef_client_delete) + allow(stubbed_knife).to receive(:chef_node_delete) + delete_from_chef_server + end + + it 'returns hash with :chef_node and :chef_client keys' do + expect(delete_from_chef_server).to be_a(Hash).and include(:chef_node, :chef_client) + end + + it 'calls to :chef_node_delete and :chef_client_delete' do + expect(stubbed_knife).to have_received(:chef_client_delete) + expect(stubbed_knife).to have_received(:chef_node_delete) + end + end + + + describe '#delete_server' do + let(:delete_server) { executor.delete_server } + + context 'when server is static' do + before do + server.provider = 'static' + allow(stubbed_connector).to receive(:server_delete).with(server.id) + allow(executor).to receive(:unbootstrap) + end + + it 'performs unbootstrap' do + expect(executor).to receive(:unbootstrap) + delete_server + end + + it 'deletes server from mongo' do + expect(stubbed_connector).to receive(:server_delete).with(server.id) + delete_server + end + + it 'returns 0' do + expect(delete_server).to eq 0 + end + + it "doesn't try to remove it from cloud" do + expect{delete_server}.not_to raise_error + end + end + + context "when server isn't static", stubbed_knife: true do + before do + allow(server).to receive_message_chain('provider_instance.delete_server') + allow(stubbed_connector).to receive(:server_delete).with(server.id) + allow(stubbed_knife).to receive(:chef_node_delete) + allow(stubbed_knife).to receive(:chef_client_delete) + end + + it 'deletes from info about note chef server' do + allow(executor).to receive(:delete_from_chef_server).and_call_original + expect(executor).to receive(:delete_from_chef_server) + delete_server + end + + it "doesn't unbootstrap server" do + expect(executor).not_to receive(:unbootstrap) + delete_server + end + + it 'deletes server from cloud' do + expect(server).to receive_message_chain('provider_instance.delete_server').with(server) + delete_server + end + + it "doesn't raise error if server wasn't found in cloud" do + allow(server).to receive_message_chain('provider_instance.name') + allow(server).to receive_message_chain('provider_instance.delete_server') { + raise Fog::Compute::OpenStack::NotFound + } + expect { delete_server }.not_to raise_error + end + + it 'deletes server from mongo' do + expect(stubbed_connector).to receive(:server_delete).with(server.id) + delete_server + end + + it 'returns 0' do + expect(delete_server).to eq 0 + end + end + end + + + describe '#rollback' do + before do + allow(executor).to receive(:delete_from_chef_server) { {} } + allow(server).to receive_message_chain('provider_instance.delete_server') + end + + it "does nothing if server.id is nil" do + server.id = nil + expect(executor).not_to receive(:delete_from_chef_server) + expect(server).not_to receive(:provider_instance) + executor.roll_back + end + + it 'deletes node from chef server and instance from cloud' do + expect(executor).to receive(:delete_from_chef_server) + expect(server).to receive_message_chain('provider_instance.delete_server') + executor.roll_back + end + + it "doesn't raise if deleting server in cloud raises an error" do + allow(server).to receive_message_chain('provider_instance.delete_server') { raise } + expect { executor.roll_back }.not_to raise_error + end + end + + + describe '#add_run_list_to_deploy_info' do + it "doesn't change deploy info if it already includes run list" do + deploy_info = {'run_list' => %w(foo)} + expect { + executor.add_run_list_to_deploy_info(output, deploy_info) + }.not_to change { deploy_info } + end + + it 'computes and adds run_list to deploy_info' do + deploy_info = {} + allow(executor).to receive(:compute_run_list) { %w(foo) } + expect(executor).to receive(:compute_run_list) + executor.add_run_list_to_deploy_info(output, deploy_info) + expect(deploy_info['run_list']).to eq %w(foo) + end + end + + describe '#compute_run_list' do + before do + allow(deploy_env).to receive_message_chain('provider_instance.run_list') { %w(a) } + project.run_list = %w(b) + deploy_env.run_list = %w(c) + server.run_list = %w(d) + end + + it "returns array with run list merged from provider's, project's, env's and server's run lists" do + expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d)) + end + + it "includes stack's run list if stack is set", stubbed_connector: true do + server.stack = 'stack' + allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: %w(e)) } + expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d e)) + end + + it "doesn't contain nils" do + server.run_list = nil + server.stack = 'stack' + allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: nil) } + expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c)) + end + + it 'returns uniq elements' do + project.run_list = %w(a) + deploy_env.run_list = %w(a) + expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a d)) + end + end +end \ No newline at end of file diff --git a/devops-service/spec/factories/key.rb b/devops-service/spec/factories/key.rb index 06fb96e..ccc0085 100644 --- a/devops-service/spec/factories/key.rb +++ b/devops-service/spec/factories/key.rb @@ -3,7 +3,7 @@ require 'db/mongo/models/key' FactoryGirl.define do factory :key, class: Devops::Model::Key do id 'user_key' - path SpecSupport::BLANK_FILE + path SpecSupport.blank_file scope 'user' end end \ No newline at end of file diff --git a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb index 1886a1c..dd3c181 100644 --- a/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb +++ b/devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb @@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs' RSpec.describe Devops::Model::DeployEnvEc2, type: :model do let(:env) { build(:deploy_env_ec2) } - describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do + describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do + before do + provider_double = instance_double('Provider::Ec2', + flavors: [{'id' => 'flavor'}], + networks: [{'default' => {'vpcId' => 'foo'}}], + groups: {'default' => nil}, + images: [{'id' => 'image'}] + ) + allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) } + allow(Provider::ProviderFactory).to receive(:get) { provider_double } + allow(stubbed_connector).to receive(:users_names) { %w(root) } + allow(stubbed_connector).to receive(:available_images) { %w(image) } + allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] } + end + it_behaves_like 'deploy env' it_behaves_like 'cloud deploy env' end diff --git a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb index 96fdfd7..a7f180b 100644 --- a/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb +++ b/devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb @@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs' RSpec.describe Devops::Model::DeployEnvOpenstack, type: :model do let(:env) { build(:deploy_env_openstack) } - describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do + describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do + before do + provider_double = instance_double('Provider::Openstack', + flavors: [{'id' => 'flavor'}], + networks: [{'default' => {'vpcId' => 'foo'}}], + groups: {'default' => nil}, + images: [{'id' => 'image'}] + ) + allow(Provider::ProviderFactory).to receive(:providers) { %w(openstack) } + allow(Provider::ProviderFactory).to receive(:get) { provider_double } + allow(stubbed_connector).to receive(:users_names) { %w(root) } + allow(stubbed_connector).to receive(:available_images) { %w(image) } + allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_openstack, id: 'template')] } + end + it_behaves_like 'deploy env' it_behaves_like 'cloud deploy env' end diff --git a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb index 45ad446..58f170f 100644 --- a/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb +++ b/devops-service/spec/models/deploy_env/deploy_env_static_spec.rb @@ -5,10 +5,10 @@ RSpec.describe Devops::Model::DeployEnvStatic, type: :model do let(:env) { build(:deploy_env_static) } - describe 'it inherits from deploy env', stubbed_logger: true do + describe 'it inherits from deploy env', stubbed_logger: true, stubbed_connector: true do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(static)) - allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root']) + allow(stubbed_connector).to receive(:users_names) { %w(root) } end it_behaves_like 'deploy env' diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb index 6457ebb..5fc0521 100644 --- a/devops-service/spec/models/project_spec.rb +++ b/devops-service/spec/models/project_spec.rb @@ -3,7 +3,21 @@ require 'db/mongo/models/project' RSpec.describe Devops::Model::Project, type: :model do let(:project) { build(:project) } - describe 'validation rules:', stubbed_env_validators: true, stubbed_logger: true do + describe 'validation rules:', stubbed_connector: true, stubbed_logger: true do + before do + provider_double = instance_double('Provider::Ec2', + flavors: [{'id' => 'flavor'}], + networks: [{'default' => {'vpcId' => 'foo'}}], + groups: {'default' => nil}, + images: [{'id' => 'image'}] + ) + allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) } + allow(Provider::ProviderFactory).to receive(:get) { provider_double } + allow(stubbed_connector).to receive(:users_names) { %w(root) } + allow(stubbed_connector).to receive(:available_images) { %w(image) } + allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] } + end + include_examples 'field type validation', :id, :not_nil, :non_empty_string include_examples 'field type validation', :deploy_envs, :not_nil, :non_empty_array include_examples 'field type validation', :description, :maybe_nil, :maybe_empty_string @@ -23,6 +37,23 @@ RSpec.describe Devops::Model::Project, type: :model do project = build(:project, with_deploy_env_identifiers: ['foo', nil]) expect(project).not_to be_valid end + + describe 'components validation' do + it 'is valid with components with filenames' do + project.components = {'foo' => {'filename' => 'bar'}} + expect{project.validate_components}.not_to raise_error + end + + it "isn't valid if components isn't a hash" do + project.components = [] + expect{project.validate_components}.to raise_error InvalidRecord + end + + it "raises InvalidRecord if one of componentsц hasn't filename" do + project.components = {'foo' => {}} + expect{project.validate_components}.to raise_error InvalidRecord + end + end end describe '.fields' do diff --git a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb index 4e31025..0c17dce 100644 --- a/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb +++ b/devops-service/spec/models/provider_account/openstack_provider_account_spec.rb @@ -1,6 +1,5 @@ require 'spec_helper' -# не пытайся выделить в shared_specs, фигня выйдет RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do let(:provider_account) { build(:openstack_provider_account) } diff --git a/devops-service/spec/models/server_spec.rb b/devops-service/spec/models/server_spec.rb index c744618..0beb489 100644 --- a/devops-service/spec/models/server_spec.rb +++ b/devops-service/spec/models/server_spec.rb @@ -44,6 +44,14 @@ RSpec.describe Devops::Model::Server, type: :model do end end + describe '.build_from_bson' do + it 'takes a hash and returns instance of Server model' do + model = described_class.build_from_bson('id' => 'foo') + expect(model).to be_an_instance_of(described_class) + expect(model.id).to eq 'foo' + end + end + it '#to_hash_without_id returns not nil fields' do server = described_class.new('run_list' => [], 'project' => 'asd') expect(server.to_hash_without_id.keys).to match_array(%w(run_list project)) diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb index 3386445..b94506f 100644 --- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb +++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb @@ -6,21 +6,23 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) - allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true } - allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} } + provider_double = instance_double('Provider::Ec2', + validate_stack_template: true, + store_stack_template: {'url' => 'template_url'} + ) + allow(Provider::ProviderFactory).to receive(:get) { provider_double } end it_behaves_like 'stack template' it 'uploads file to S3' do - expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') - params = { - 'id' => 'foo', + result = described_class.create('id' => 'foo', 'template_body' => '{}', 'owner' => 'root', 'provider' => 'ec2' - } - expect(described_class.create(params)).to be_an_instance_of(described_class) + ) + expect(result).to be_an_instance_of(described_class) + expect(result.template_url).to eq 'template_url' end end \ No newline at end of file diff --git a/devops-service/spec/models/user_spec.rb b/devops-service/spec/models/user_spec.rb index c060ff4..ad9bc69 100644 --- a/devops-service/spec/models/user_spec.rb +++ b/devops-service/spec/models/user_spec.rb @@ -79,13 +79,25 @@ RSpec.describe Devops::Model::User, type: :model do end end + describe '.build_from_bson' do + it 'builds User model from given hash and assigns id' do + model = described_class.build_from_bson('_id' => 'foo', 'username' => 'not shown', 'email' => 'baz') + expect(model.id).to eq 'foo' + expect(model.email).to eq 'baz' + end + end + describe '#check_privileges' do it "raises InvalidPrivileges if user hasn't specified privilege" do expect { user.check_privileges('key', 'w') }.to raise_error(InvalidPrivileges) end it 'does nothing is user has specified privilege' do - user.check_privileges('key', 'r') + expect{user.check_privileges('key', 'r')}.not_to raise_error + end + + it 'raises InvalidPrivileges if given privelege is wrong' do + expect{user.check_privileges('key', 't')}.to raise_error InvalidPrivileges end end diff --git a/devops-service/spec/shared_contexts/stubbed_connector.rb b/devops-service/spec/shared_contexts/stubbed_connector.rb index 7d6dabb..49b202a 100644 --- a/devops-service/spec/shared_contexts/stubbed_connector.rb +++ b/devops-service/spec/shared_contexts/stubbed_connector.rb @@ -1,5 +1,5 @@ RSpec.shared_context 'stubbed calls to connector', stubbed_connector: true do - let(:stubbed_connector) { double() } + let(:stubbed_connector) { instance_double(MongoConnector) } before do allow(Devops::Db).to receive(:connector) { stubbed_connector } end diff --git a/devops-service/spec/shared_contexts/stubbed_env_validators.rb b/devops-service/spec/shared_contexts/stubbed_env_validators.rb index 69f081c..998f27d 100644 --- a/devops-service/spec/shared_contexts/stubbed_env_validators.rb +++ b/devops-service/spec/shared_contexts/stubbed_env_validators.rb @@ -1,9 +1,10 @@ RSpec.shared_context 'stubbed calls to connector in env validators', stubbed_env_validators: true do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2 openstack)) + allow_any_instance_of(env_class).to receive_message_chain('provider_instance.flavors').and_return [{'id' => 'flavor'}] allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root']) - allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}]) - allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}]) + # allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}]) + # allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}]) allow_any_instance_of(Validators::DeployEnv::Groups).to receive(:available_groups).and_return(['default']) allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}]) allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}]) diff --git a/devops-service/spec/shared_contexts/stubbed_knife.rb b/devops-service/spec/shared_contexts/stubbed_knife.rb new file mode 100644 index 0000000..042bcc7 --- /dev/null +++ b/devops-service/spec/shared_contexts/stubbed_knife.rb @@ -0,0 +1,6 @@ +RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do + let(:stubbed_knife) { instance_double(KnifeCommands) } + before do + allow(KnifeFactory).to receive(:instance) { stubbed_knife } + end +end \ No newline at end of file diff --git a/devops-service/spec/shared_contexts/stubbed_logger.rb b/devops-service/spec/shared_contexts/stubbed_logger.rb index f2d79e9..486caa1 100644 --- a/devops-service/spec/shared_contexts/stubbed_logger.rb +++ b/devops-service/spec/shared_contexts/stubbed_logger.rb @@ -3,5 +3,6 @@ RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do allow(DevopsLogger).to receive_message_chain('logger.debug') allow(DevopsLogger).to receive_message_chain('logger.info') allow(DevopsLogger).to receive_message_chain('logger.error') + allow(DevopsLogger).to receive_message_chain('logger.warn') end end \ No newline at end of file diff --git a/devops-service/spec/spec_helper.rb b/devops-service/spec/spec_helper.rb index 9f2500f..10f8ebf 100644 --- a/devops-service/spec/spec_helper.rb +++ b/devops-service/spec/spec_helper.rb @@ -4,32 +4,53 @@ require 'factory_girl' require 'active_support/core_ext/hash/indifferent_access' require 'active_support/inflector' -# setup load_path and require support files -root = File.join(File.dirname(__FILE__), "..") -$LOAD_PATH.push root unless $LOAD_PATH.include? root - -# suppress output -original_stdout = $stdout -$stdout = File.open(File::NULL, "w") - -Dir[("./spec/support/**/*.rb")].each { |f| require f } -Dir[("./spec/shared_contexts/**/*.rb")].each { |f| require f } - -# Factory girl configuration -FactoryGirl.define do - # do not try to persist, but raise validation errors - to_create { |model| model.validate! } +def suppress_output! + original_stdout = $stdout + $stdout = File.open(File::NULL, "w") + RSpec.configure do |config| + config.after(:all) do + $stdout = original_stdout + end + end end -FactoryGirl.find_definitions + +def check_coverage + require 'simplecov' + if ENV['JENKINS'] + require 'simplecov-rcov' + SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter + end + SimpleCov.start do + add_filter { |src| src.filename =~ /spec\// } + end +end + +def require_support_files + root = File.join(File.dirname(__FILE__), "..") + $LOAD_PATH.push root unless $LOAD_PATH.include?(root) + Dir[("#{root}/spec/support/**/*.rb")].each { |f| require f } + Dir[("#{root}/spec/shared_contexts/**/*.rb")].each { |f| require f } +end + +def setup_factory_girl + FactoryGirl.define do + # do not persist, but raise validation errors + to_create { |model| model.validate! } + end + FactoryGirl.find_definitions + RSpec.configure { |config| config.include FactoryGirl::Syntax::Methods } +end + + +# extra configuration +suppress_output! +check_coverage if ENV['COVERAGE'] +require_support_files +setup_factory_girl + # RSpec configuration RSpec.configure do |config| - config.include FactoryGirl::Syntax::Methods - - config.after(:all) do - $stdout = original_stdout - end - # rspec-expectations config goes here. You can use an alternate # assertion/expectation library such as wrong or the stdlib/minitest # assertions if you prefer. diff --git a/devops-service/spec/support/instance_variable_matcher.rb b/devops-service/spec/support/instance_variable_matcher.rb new file mode 100644 index 0000000..51f5097 --- /dev/null +++ b/devops-service/spec/support/instance_variable_matcher.rb @@ -0,0 +1,6 @@ +RSpec::Matchers.define :have_instance_variable_value do |name, value| + match do |actual| + actual.instance_variable_defined?("@#{name}") && + actual.instance_variable_get("@#{name}") == value + end +end \ No newline at end of file diff --git a/devops-service/spec/support/spec_support.rb b/devops-service/spec/support/spec_support.rb index ab47bc6..69bd0e4 100644 --- a/devops-service/spec/support/spec_support.rb +++ b/devops-service/spec/support/spec_support.rb @@ -2,7 +2,15 @@ require 'yaml' module SpecSupport ROOT = File.join(__dir__, '../../') - BLANK_FILE = File.join(ROOT, 'spec/support/blank_file') + + def self.blank_file + File.join(ROOT, 'spec/support/templates/blank_file') + end + + # for specs which write files + def self.tmp_dir + File.join(ROOT, 'spec/support/tmp/') + end def self.db_params @db_params ||= begin @@ -32,4 +40,8 @@ module SpecSupport end end end + + def self.root + File.join(__dir__, '../../') + end end \ No newline at end of file diff --git a/devops-service/spec/support/blank_file b/devops-service/spec/support/templates/blank_file similarity index 100% rename from devops-service/spec/support/blank_file rename to devops-service/spec/support/templates/blank_file diff --git a/devops-service/spec/support/tmp/.gitkeep b/devops-service/spec/support/tmp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/devops-service/workers/delete_expired_server_worker.rb b/devops-service/workers/delete_expired_server_worker.rb new file mode 100644 index 0000000..a749e73 --- /dev/null +++ b/devops-service/workers/delete_expired_server_worker.rb @@ -0,0 +1,37 @@ +require "db/mongo/models/server" +require "db/mongo/models/report" +require "lib/executors/server_executor" +require "workers/worker" + +class DeleteExpiredServerWorker < Worker + + def perform(options) + chef_node_name = options.fetch('server_chef_node_name') + + call() do |out, file| + out.puts "Expire server '#{chef_node_name}'." + server = mongo.server_by_chef_node_name(chef_node_name) + report = save_report(file, server) + + e = Devops::Executor::ServerExecutor.new(server, out) + e.report = report + e.delete_server + end + end + + private + + def save_report(file, server) + report = Devops::Model::Report.new( + "file" => file, + "_id" => jid, + "created_by" => 'SYSTEM', + "project" => server.project, + "deploy_env" => server.deploy_env, + "type" => Devops::Model::Report::EXPIRE_SERVER_TYPE + ) + mongo.save_report(report) + report + end + +end diff --git a/devops-service/workers/delete_server_worker.rb b/devops-service/workers/delete_server_worker.rb index c8fe476..77f3e5c 100644 --- a/devops-service/workers/delete_server_worker.rb +++ b/devops-service/workers/delete_server_worker.rb @@ -1,18 +1,21 @@ require "db/mongo/models/server" require "db/mongo/models/report" require "lib/executors/server_executor" +require "workers/worker" class DeleteServerWorker < Worker + # options should contain 'server_id' def perform(options) - chef_node_name = options.fetch('server_chef_node_name') - puts "Expire server '#{chef_node_name}'." + server_id = options.fetch('server_id') + current_user = options.fetch('current_user') call() do |out, file| - server = mongo.server_by_chef_node_name(chef_node_name) - report = save_report(file, server) + out.puts "Deleting server with id #{server_id}" and out.flush + @server = mongo.server_by_instance_id(server_id) + report = save_report(file, current_user) - e = Devops::Executor::ServerExecutor.new(server, out) + e = Devops::Executor::ServerExecutor.new(@server, out) e.report = report e.delete_server end @@ -20,13 +23,13 @@ class DeleteServerWorker < Worker private - def save_report(file, server) + def save_report(file, current_user) report = Devops::Model::Report.new( "file" => file, "_id" => jid, - "created_by" => 'SYSTEM', - "project" => server.project, - "deploy_env" => server.deploy_env, + "created_by" => current_user, + "project" => @server.project, + "deploy_env" => @server.deploy_env, "type" => Devops::Model::Report::DELETE_SERVER_TYPE ) mongo.save_report(report) diff --git a/devops-service/workers/run_workers.rb b/devops-service/workers/run_workers.rb index 7dac55e..6e25e39 100644 --- a/devops-service/workers/run_workers.rb +++ b/devops-service/workers/run_workers.rb @@ -5,6 +5,8 @@ require File.join(root, "deploy_worker") require File.join(root, "bootstrap_worker") require File.join(root, "project_test_worker") require File.join(root, "stack_bootstrap_worker") +require File.join(root, "delete_server_worker") +require File.join(root, "delete_expired_server_worker") config = {} #require File.join(root, "../proxy") diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 6ad25d7..da7f712 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -130,7 +130,7 @@ class StackBootstrapWorker < Worker @out.puts results.each do |chef_node_name, code| - human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code) + human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code) @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}" end @@ -144,7 +144,7 @@ class StackBootstrapWorker < Worker def errors_in_bootstrapping_present?(result_codes) bootstrap_error_codes = [] [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code| - bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code) + bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code) end (bootstrap_error_codes & result_codes).size > 0