diff --git a/devops-client/lib/devops-client/handler/deploy.rb b/devops-client/lib/devops-client/handler/deploy.rb index 7ab707c..d8860ae 100644 --- a/devops-client/lib/devops-client/handler/deploy.rb +++ b/devops-client/lib/devops-client/handler/deploy.rb @@ -20,13 +20,17 @@ class Deploy < Handler end def deploy_handler args - tags = options[:tags] names = args[1..-1] if names.empty? @options_parser.invalid_deploy_command abort() end - job_ids = post("/deploy", :names => names, :tags => tags) + job_ids = post("/deploy", + names: names, + tags: options[:tags], + chef_client_options: options[:chef_client_options], + named_task: options[:named_task] + ) reports_urls(job_ids) end diff --git a/devops-client/lib/devops-client/options/deploy_options.rb b/devops-client/lib/devops-client/options/deploy_options.rb index abb657d..aee6340 100644 --- a/devops-client/lib/devops-client/options/deploy_options.rb +++ b/devops-client/lib/devops-client/options/deploy_options.rb @@ -13,11 +13,13 @@ class DeployOptions < CommonOptions def deploy_options options do |parser, options| parser.banner << self.banner + parser.resource_name = :deploy - parser.recognize_option_value(:tag, resource_name: :deploy, variable_name: 'TAG1,TAG2...') do |tags| + parser.recognize_option_value(:tag, variable: 'TAG1,TAG2...') do |tags| options[:tags] = tags.split(",") end - + parser.recognize_option_value(:chef_client_options) + parser.recognize_option_value(:named_task) end end diff --git a/devops-client/locales/en.yml b/devops-client/locales/en.yml index a4a968b..20f390b 100644 --- a/devops-client/locales/en.yml +++ b/devops-client/locales/en.yml @@ -314,6 +314,8 @@ en: descriptions: deploy: tag: 'Tag names, comma separated list' + chef_client_options: 'String like "-o role[foo]"' + named_task: Name of task to run image: provider: Image provider image_id: Image identifier diff --git a/devops-service/app/api2/handlers/deploy.rb b/devops-service/app/api2/handlers/deploy.rb index b0e3045..71b6059 100644 --- a/devops-service/app/api2/handlers/deploy.rb +++ b/devops-service/app/api2/handlers/deploy.rb @@ -33,6 +33,8 @@ module Devops begin deploy_info = create_deploy_info(s, project, body["build_number"]) deploy_info["run_list"] = run_list if run_list + set_chef_client_options(deploy_info, s, project, body['chef_client_options']) + deploy_info["named_task"] = body["named_task"] jid = Worker.start_async(DeployWorker, server_attrs: s.to_hash, @@ -117,6 +119,17 @@ module Devops @deploy_info_buf[buf_key] = project.deploy_info(deploy_env_model, build_number) end end + + private + + # env's chef client options may be nil or empty string; it's OK. + def set_chef_client_options(deploy_info, server, project, single_run_options) + if single_run_options + deploy_info['chef_client_options'] = single_run_options + else + deploy_info['chef_client_options'] = project.deploy_env(server.deploy_env).chef_client_options + end + end end end end diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index 5f8f918..0ccd928 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -5,7 +5,6 @@ require "app/api2/parsers/project" require "lib/project/type/types_factory" require "lib/executors/server_executor" require "workers/delete_server_worker" - require_relative "../helpers/version_2.rb" require_relative "request_handler" diff --git a/devops-service/app/api2/parsers/deploy.rb b/devops-service/app/api2/parsers/deploy.rb index 74a92db..084070a 100644 --- a/devops-service/app/api2/parsers/deploy.rb +++ b/devops-service/app/api2/parsers/deploy.rb @@ -12,6 +12,7 @@ module Devops build_number = check_string(r["build_number"], "Parameter 'build_number' should be a not empty string", true) rl = check_array(r["run_list"], "Parameter 'run_list' should be an array of string", String, true) Validators::Helpers::RunList.new(rl).validate! unless rl.nil? + check_string(r["named_task"], "Parameter 'named_task' should be a not empty string", true) r end diff --git a/devops-service/app/api2/routes/deploy.rb b/devops-service/app/api2/routes/deploy.rb index 7a2a208..c42a69b 100644 --- a/devops-service/app/api2/routes/deploy.rb +++ b/devops-service/app/api2/routes/deploy.rb @@ -18,6 +18,7 @@ module Devops # "tags": [], -> array of tags to apply on each server before running chef-client # "build_number": "", -> string, build number to deploy # "run_list": [], -> array of strings to set run_list for chef-client + # "chef_client_options": "", String, optional. May be used to redefine run_list # } # # * *Returns* : text stream diff --git a/devops-service/app/api2/routes/project.rb b/devops-service/app/api2/routes/project.rb index 5d16ca0..ee1aae8 100644 --- a/devops-service/app/api2/routes/project.rb +++ b/devops-service/app/api2/routes/project.rb @@ -87,28 +87,12 @@ module Devops # - Content-Type: application/json # - body : # { - # "deploy_envs": [ - # { - # "identifier": "dev", - # "provider": "openstack", - # "flavor": "m1.small", - # "image": "image id", - # "subnets": [ - # "private" - # ], - # "groups": [ - # "default" - # ], - # "users": [ - # "user" - # ], - # "run_list": [ - # - # ], - # "expires": null - # } - # ], - # "name": "project_1" + # "run_list": [], + # "description": "Description", + # "named_tasks": [{ + # "name": "restart", + # "run_list": ["role[restart_service]"] + # }] # } # # * *Returns* : diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index abe74f9..d42792f 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -135,13 +135,16 @@ module Connectors end def project_update id, params - keys = %w(run_list description) - params.delete_if{|k,v| !keys.include?(k)} + params.keep_if {|k,v| permitted_project_fields.include?(k)} @collection.update({"_id" => id}, {'$set' => params }) end private + def permitted_project_fields + %w(run_list description named_tasks) + end + def list(query={}, query_options={}) @collection.find(query, query_options).to_a.map {|bson| model_from_bson(bson)} end diff --git a/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb b/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb index 3c3bbf1..f2885da 100644 --- a/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb +++ b/devops-service/db/mongo/models/deploy_env/deploy_env_base.rb @@ -31,8 +31,7 @@ module Devops ::Validators::FieldValidator::Expires] set_field_validators :chef_client_options, [::Validators::FieldValidator::Nil, - ::Validators::FieldValidator::FieldType::String, - ::Validators::FieldValidator::NotEmpty] + ::Validators::FieldValidator::FieldType::String] def initialize d={} self.identifier = d["identifier"] diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb index 0bb61ac..32b71c1 100644 --- a/devops-service/db/mongo/models/project.rb +++ b/devops-service/db/mongo/models/project.rb @@ -41,12 +41,23 @@ module Devops ::Validators::FieldValidator::FieldType::Array, ::Validators::FieldValidator::RunList] + # should be an array of hashes like: + # { + # 'name' => 'restart', + # 'run_list' => [ + # 'role[restart_service]' + # ] + # } + set_field_validators :named_tasks, [::Validators::FieldValidator::Nil, + ::Validators::FieldValidator::FieldType::Array, + ::Validators::FieldValidator::NamedTasks] + set_validators ::Validators::DeployEnv::RunList, ::Validators::DeployEnv::DeployEnvs def self.fields - ["deploy_envs", "type", "description"] + ["deploy_envs", "type", "description", "named_tasks"] end def initialize p={} @@ -56,6 +67,7 @@ module Devops self.description = p["description"] self.archived = p["archived"] || false self.run_list = p["run_list"] || [] + self.named_tasks = p["named_tasks"] || [] @handler = Devops::TypesFactory.type self.type @handler.prepare(self) @@ -174,6 +186,7 @@ module Devops h["archived"] = self.archived if self.archived h["description"] = self.description h["run_list"] = self.run_list + h["named_tasks"] = self.named_tasks if self.multi? h["type"] = MULTI_TYPE end diff --git a/devops-service/db/validators/field_validators/named_tasks.rb b/devops-service/db/validators/field_validators/named_tasks.rb new file mode 100644 index 0000000..d7934bf --- /dev/null +++ b/devops-service/db/validators/field_validators/named_tasks.rb @@ -0,0 +1,65 @@ +require_relative "base" + +module Validators + module FieldValidator + class NamedTasks < Base + + def valid? + @value.each do |named_task| + break unless check_name!(named_task) + break unless check_run_list!(named_task) + break unless check_additional_keys!(named_task) + end + @message.nil? + end + + def message + @message + end + + private + + def check_name!(task) + if task.key?('name') + true + else + @message = "One of tasks doesn't have a name" + false + end + end + + def check_run_list!(task) + if !task.key?('run_list') + @message = "Task #{task['name']} doesn't have run_list" + return false + end + + if !task['run_list'].is_a?(Array) + @message = "Run list of #{task['name']} isn't an array" + return false + end + + wrong_elements = task['run_list'].select do |element| + Validators::Helpers::RunList::RUN_LIST_REGEX.match(element).nil? + end + + if wrong_elements.empty? + true + else + @message = "Invalid run list elements: '#{wrong_elements.join(', ')}' for task #{task['name']}." + false + end + end + + def check_additional_keys!(task) + if task.keys.length > 2 + @message = "Task hash should contain only name and run_list" + false + else + true + end + end + + end + end +end diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 7cf84e8..297e0d4 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -1,5 +1,6 @@ require "lib/knife/knife_factory" require "lib/executors/expiration_scheduler" +require "lib/puts_and_flush" require "hooks" require 'net/ssh' @@ -7,6 +8,7 @@ module Devops module Executor class ServerExecutor include Hooks + include PutsAndFlush ERROR_CODES = { server_bootstrap_fail: 2, @@ -41,6 +43,7 @@ module Devops attr_accessor :server, :deploy_env, :report, :project + attr_reader :out def initialize server, out, options={} if server @@ -394,8 +397,13 @@ module Devops @out.flush cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}" else - if @deploy_env.chef_client_options - cmd << " #{@deploy_env.chef_client_options}" + if deploy_info['chef_client_options'].present? + cmd << " #{deploy_info['chef_client_options']}" + elsif deploy_info['named_task'].present? + named_task = @project.named_tasks.detect {|task| task['name'] == deploy_info['named_task']} + raise "Named task #{deploy_info['named_task']} doesn't exist." unless named_task + puts_and_flush "Using named task #{deploy_info['named_task']}." + cmd << " -o #{named_task['run_list'].join(',')}" else cmd << " -r #{deploy_info["run_list"].join(",")}" unless @server.stack.nil? end @@ -503,11 +511,6 @@ module Devops private - def puts_and_flush(message) - @out.puts message - @out.flush - end - def schedule_expiration if @deploy_env.expires @out << "Planning expiration in #{@deploy_env.expires}" diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb index db48152..1372bd7 100644 --- a/devops-service/spec/executors/server_executor_spec.rb +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -576,11 +576,24 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec deploy_server end - it "uses deploy_env's chef_client_options if they are set" do - deploy_env.chef_client_options = '-r role' + it "uses chef_client_options from deploy_info if it is set" do + deploy_info['chef_client_options'] = '-r role' expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color -r role', any_args) deploy_server end + + it "doesn't use chef_client_options from deploy_info if it's blank string" do + deploy_info['chef_client_options'] = '' + expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args) + deploy_server + end + + it "uses run list from named_task if it's set" do + project.named_tasks = [{'name' => 'foo', 'run_list' => ['role[backend]', 'role[frontend]']}] + deploy_info['named_task'] = 'foo' + expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color -o role[backend],role[frontend]', any_args) + deploy_server + end end it "uses server's key" do diff --git a/devops-service/spec/models/deploy_env/shared_deploy_env_specs.rb b/devops-service/spec/models/deploy_env/shared_deploy_env_specs.rb index e8e8b42..ffdd708 100644 --- a/devops-service/spec/models/deploy_env/shared_deploy_env_specs.rb +++ b/devops-service/spec/models/deploy_env/shared_deploy_env_specs.rb @@ -10,7 +10,7 @@ RSpec.shared_examples 'deploy env' do include_examples 'field type validation', :run_list, :not_nil, :maybe_empty_array, :run_list, :field_validator include_examples 'field type validation', :users, :not_nil, :maybe_empty_array, :field_validator include_examples 'field type validation', :expires, :maybe_nil, :non_empty_string, :field_validator - include_examples 'field type validation', :chef_client_options, :maybe_nil, :non_empty_string, :field_validator + include_examples 'field type validation', :chef_client_options, :maybe_nil, :maybe_empty_string, :field_validator it 'should be valid only with all users available' do expect(build(validated_model_name, users: ['root'])).to be_valid diff --git a/devops-service/spec/models/project_spec.rb b/devops-service/spec/models/project_spec.rb index 5fc0521..4d8a2fa 100644 --- a/devops-service/spec/models/project_spec.rb +++ b/devops-service/spec/models/project_spec.rb @@ -22,20 +22,33 @@ RSpec.describe Devops::Model::Project, type: :model do include_examples 'field type validation', :deploy_envs, :not_nil, :non_empty_array include_examples 'field type validation', :description, :maybe_nil, :maybe_empty_string include_examples 'field type validation', :run_list, :not_nil, :maybe_empty_array, :run_list + include_examples 'field type validation', :named_tasks, :maybe_nil, :maybe_empty_array it "isn't valid when has envs with same identifier" do - project = build(:project, with_deploy_env_identifiers: %w(foo foo)) - expect(project).not_to be_valid + expect(build(:project, with_deploy_env_identifiers: %w(foo foo))).not_to be_valid end it "is valid when all envs have uniq identifiers" do - project = build(:project, with_deploy_env_identifiers: %w(foo bar)) - expect(project).to be_valid + expect(build(:project, with_deploy_env_identifiers: %w(foo bar))).to be_valid end it "isn't valid when at least one of envs isn't valid" do - project = build(:project, with_deploy_env_identifiers: ['foo', nil]) - expect(project).not_to be_valid + expect(build(:project, with_deploy_env_identifiers: ['foo', nil])).not_to be_valid + end + + describe 'named_tasks validation' do + it 'is valid with array of hashes like {"name" => "foo", "run_list" => ["role[bar]"]}' do + expect(build(:project, named_tasks: [{"name" => "foo", "run_list" => ["role[bar]"]}] )).to be_valid + end + + it "isn't valid with array of hashes with invalid run_list elements" do + expect(build(:project, named_tasks: [{"name" => "foo", "run_list" => ["bar"]}] )).not_to be_valid + end + + it "isn't valid with array of hashes without name or run_list" do + expect(build(:project, named_tasks: [{run_list: ["role[bar]"]}] )).not_to be_valid + expect(build(:project, named_tasks: [{"name" => "foo"}] )).not_to be_valid + end end describe 'components validation' do @@ -58,7 +71,7 @@ RSpec.describe Devops::Model::Project, type: :model do describe '.fields' do subject { described_class.fields } - it { should eq %w(deploy_envs type description) } + it { should eq %w(deploy_envs type description named_tasks) } end describe '#initialize' do @@ -286,8 +299,8 @@ RSpec.describe Devops::Model::Project, type: :model do expect(subject['deploy_envs']).to be_an_array_of(Hash) end - it 'also contains descriptions and run_list' do - expect(subject).to include('description', 'run_list') + it 'also contains descriptions, run_list and named_tasks' do + expect(subject).to include('description', 'run_list', 'named_tasks') end it 'contains archived key if project is archived' do