diff --git a/devops-service/Guardfile b/devops-service/Guardfile index a66e388..82943a3 100644 --- a/devops-service/Guardfile +++ b/devops-service/Guardfile @@ -43,4 +43,5 @@ guard :rspec, cmd: "rspec" do # Devops files watch(%r{db/.+\.rb}) { rspec.spec_dir } watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" } + watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" } end diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb deleted file mode 100644 index ba98ded..0000000 --- a/devops-service/commands/stack.rb +++ /dev/null @@ -1,62 +0,0 @@ -module StackCommands - extend self - - RESULT_CODES = { - stack_rolled_back: 1, - unkown_status: 2, - timeout: 3, - error: 5 - } - - def self.result_codes - RESULT_CODES - end - - def self.result_code(code) - result_codes.fetch(code) - end - - def sync_stack_proc - lambda do |out, stack, mongo| - # 5 tries each 5 seconds, then 200 tries each 10 seconds - sleep_times = [5]*5 + [10]*200 - - begin - out << "Syncing stack '#{stack.id}'...\n" - events_keys = [] - sleep_times.each do |sleep_time| - sleep sleep_time - stack.sync_details! - stack.events.each do |event| - unless events_keys.include?(event["event_id"]) - events_keys << event["event_id"] - out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}" - end - end - case stack.stack_status - when 'CREATE_IN_PROGRESS' - out.flush - when 'CREATE_COMPLETE' - mongo.stack_update(stack) - out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n" - out.flush - return 0 - when 'ROLLBACK_COMPLETE' - out << "\nStack '#{stack.id}' status is rolled back\n" - return StackCommands.result_code(:stack_rolled_back) - else - out.puts "\nUnknown stack status: '#{stack.stack_status}'" - return StackCommands.result_code(:unkown_status) - end - end - out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds." - return StackCommands.result_code(:timeout) - rescue StandardError => e - logger.error e.message - out << "Error: #{e.message}\n" - return StackCommands.result_code(:error) - end - end - end - -end diff --git a/devops-service/db/validators/base.rb b/devops-service/db/validators/base.rb index bec8722..677e9f2 100644 --- a/devops-service/db/validators/base.rb +++ b/devops-service/db/validators/base.rb @@ -13,7 +13,6 @@ module Validators raise InvalidRecord.new("An error raised during validation with #{self.class}: #{e.class}: #{e.message}") end - # :nocov: def valid? raise 'override me' end @@ -21,7 +20,6 @@ module Validators def message raise 'override me' end - # :nocov: class << self private diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index a8aebc6..ff4c9f6 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -53,16 +53,20 @@ module Devops @current_user = options[:current_user] end - def self.error_code(symbolic_code) - ERROR_CODES.fetch(symbolic_code) + def self.error_code(reason) + ERROR_CODES.fetch(reason) end - def self.symbolic_error_code(integer_code) + def self.reason_from_error_code(integer_code) ERROR_CODES.key(integer_code) || :unknown_error end - def error_code(symbolic_code) - self.class.error_code(symbolic_code) + def self.bootstrap_errors_reasons + [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error] + end + + def error_code(reason) + self.class.error_code(reason) end def create_server_object options @@ -511,8 +515,6 @@ module Devops cmd end - # to simplify testing - # :nocov: def execute_system_command(cmd) `#{cmd}` end @@ -524,7 +526,6 @@ module Devops def knife_instance @knife_instance ||= KnifeFactory.instance end - # :nocov: end end diff --git a/devops-service/lib/puts_and_flush.rb b/devops-service/lib/puts_and_flush.rb new file mode 100644 index 0000000..a0af609 --- /dev/null +++ b/devops-service/lib/puts_and_flush.rb @@ -0,0 +1,9 @@ +module PutsAndFlush + private + + # out stream should be defined + def puts_and_flush(message) + out.puts message + out.flush + end +end \ No newline at end of file diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb index de70722..3ba6eb9 100644 --- a/devops-service/spec/executors/server_executor_spec.rb +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -53,13 +53,13 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec end - describe '.symbolic_error_code' do + describe '.reason_from_error_code' do it 'returns symbol given an integer' do - expect(described_class.symbolic_error_code(2)).to eq :server_bootstrap_fail + expect(described_class.reason_from_error_code(2)).to eq :server_bootstrap_fail end it "returns :unknown_error if can't recognize error code" do - expect(described_class.symbolic_error_code(123)).to eq :unknown_error + expect(described_class.reason_from_error_code(123)).to eq :unknown_error end end diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb index 3386445..b94506f 100644 --- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb +++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb @@ -6,21 +6,23 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) - allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true } - allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} } + provider_double = instance_double('Provider::Ec2', + validate_stack_template: true, + store_stack_template: {'url' => 'template_url'} + ) + allow(Provider::ProviderFactory).to receive(:get) { provider_double } end it_behaves_like 'stack template' it 'uploads file to S3' do - expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') - params = { - 'id' => 'foo', + result = described_class.create('id' => 'foo', 'template_body' => '{}', 'owner' => 'root', 'provider' => 'ec2' - } - expect(described_class.create(params)).to be_an_instance_of(described_class) + ) + expect(result).to be_an_instance_of(described_class) + expect(result.template_url).to eq 'template_url' end end \ No newline at end of file diff --git a/devops-service/spec/shared_contexts/stubbed_logger.rb b/devops-service/spec/shared_contexts/stubbed_logger.rb index 486caa1..6105101 100644 --- a/devops-service/spec/shared_contexts/stubbed_logger.rb +++ b/devops-service/spec/shared_contexts/stubbed_logger.rb @@ -1,8 +1,6 @@ RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do before do - allow(DevopsLogger).to receive_message_chain('logger.debug') - allow(DevopsLogger).to receive_message_chain('logger.info') - allow(DevopsLogger).to receive_message_chain('logger.error') - allow(DevopsLogger).to receive_message_chain('logger.warn') + logger = double('logger', debug: nil, info: nil, error: nil, warn: nil) + allow(DevopsLogger).to receive(:logger) { logger } end end \ No newline at end of file diff --git a/devops-service/spec/workers/chef_node_name_builder_spec.rb b/devops-service/spec/workers/chef_node_name_builder_spec.rb new file mode 100644 index 0000000..246dcab --- /dev/null +++ b/devops-service/spec/workers/chef_node_name_builder_spec.rb @@ -0,0 +1,45 @@ +require 'workers/stack_bootstrap/chef_node_name_builder' +RSpec.describe ChefNodeNameBuilder do + let(:server_info) do + { + 'id' => 'server1', + 'name' => 'server_name', + 'key_name' => 'key', + 'private_ip' => '127.0.0.1', + 'public_ip' => '127.0.0.2', + 'tags' => { + 'cid:priority' => '3' + } + } + end + let(:project) { build(:project, id: 'proj', with_deploy_env_identifiers: %w(dev)) } + let(:env) { project.deploy_env('dev') } + let(:name_builder) { described_class.new(server_info, project, env) } + let(:build_name) { name_builder.build_node_name } + + def set_mask(mask) + server_info['tags']['cid:node-name-mask'] = mask + end + + describe '#build_node_name' do + it 'uses default mask ("$project-$nodename-$env")' do + expect(build_name).to eq 'proj-server1-dev' + end + + it 'substitutes project, env and nodename' do + set_mask('$project/$env/$nodename') + expect(build_name).to eq 'proj/dev/server1' + end + + it 'substitutes $time' do + set_mask('$nodename-$time') + expect(build_name).to match /server1-\d+/ + end + + it 'substitutes underscores to dashes' do + server_info['id'] = 'server_1' + expect(build_name).to match 'proj-server-1-dev' + end + end + +end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb new file mode 100644 index 0000000..b2ede82 --- /dev/null +++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb @@ -0,0 +1,128 @@ +require 'workers/stack_bootstrap_worker' + +RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:file) { 'temp.txt' } + + let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys } + let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } + let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) } + let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) } + let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) } + let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) } + let(:worker) { described_class.new } + + before do + allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) + allow(stubbed_connector).to receive(:save_report) + allow(stubbed_connector).to receive(:stack_insert) + + allow(worker).to receive(:stack_synchronizer) { stack_synchronizer } + allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper } + allow(worker).to receive(:stack_servers_persister) { stack_servers_persister } + allow(worker).to receive(:call).and_yield(out, file) + allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) } + end + + + it 'requires "stack_attributes" in options' do + expect{ + worker.perform({}) + }.to raise_error KeyError + end + + it 'saves report about operation' do + expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)) + perform_without_bootstrap + end + + it 'saves report about operation, creates stack and persists stack servers' do + allow(worker).to receive(:create_stack).and_call_original + expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)).ordered + expect(worker).to receive(:create_stack).ordered + expect(stack_servers_persister).to receive(:persist).ordered + perform_without_bootstrap + end + + context 'if without_bootstrap is true' do + it "doesn't bootstrap servers" do + expect(stack_servers_bootstrapper).not_to receive(:bootstrap) + perform_without_bootstrap + end + + it 'returns 0' do + expect(perform_without_bootstrap).to eq 0 + end + end + + context 'if without_bootstrap is false or not set' do + it 'bootstraps servers in order by priorities, separately' do + first_servers = build_list(:server, 2) + last_servers = build_list(:server, 3) + allow(stack_servers_persister).to receive(:persist) { + {3 => first_servers, 1 => last_servers} + } + expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered + expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered + perform_with_bootstrap + end + + context 'when bootstraping servers was successful' do + it 'returns 0' do + expect(perform_with_bootstrap).to eq 0 + end + end + + context 'when a known error occured during servers bootstrap' do + before do + allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError } + end + + it 'rollbacks stack and returns 2' do + expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!) + expect(stubbed_connector).to receive(:stack_servers_delete) + expect(stubbed_connector).to receive(:stack_delete) + perform_with_bootstrap + end + + it 'returns 2' do + allow(worker).to receive(:rollback_stack!) + expect(perform_with_bootstrap).to eq 2 + end + end + + context 'when a known error occured during servers deploy' do + it "doesn't rollback stack and returns 3" do + allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError } + expect(worker).not_to receive(:rollback_stack!) + expect(perform_with_bootstrap).to eq 3 + end + end + + context "when a servers bootstrap & deploy haven't been finished due to timeout" do + it "doesn't rollback stack and returns 3" do + allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout } + expect(worker).not_to receive(:rollback_stack!) + expect(perform_with_bootstrap).to eq 4 + end + end + + context 'when an unknown error occured during servers bootsrap and deploy' do + it 'rollbacks stack and reraises that error' do + error = StandardError.new + allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error } + allow(worker).to receive(:rollback_stack!) + expect(worker).to receive(:rollback_stack!) + expect{perform_with_bootstrap}.to raise_error(error) + end + end + end + + context "when stack creation wasn't successful" do + it 'returns 1' do + allow(stack_synchronizer).to receive(:sync) { 5 } + allow(stack_synchronizer).to receive(:reason_from_error_code) { :error } + expect(perform_without_bootstrap).to eq 1 + end + end +end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb b/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb new file mode 100644 index 0000000..374f57c --- /dev/null +++ b/devops-service/spec/workers/stack_servers_bootstrapper_spec.rb @@ -0,0 +1,59 @@ +require 'workers/stack_bootstrap_worker' + +RSpec.describe StackServersBootstrapper, stubbed_connector: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:jid) { 1000 } + let(:bootstrapper) { described_class.new(out, jid) } + let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] } + let(:bootstrap_job_ids) { %w(100 200) } + let(:subreport1) { build(:report, id: bootstrap_job_ids.first) } + let(:subreport2) { build(:report, id: bootstrap_job_ids.last) } + + describe '#bootstrap' do + let(:bootstrap!) { bootstrapper.bootstrap(servers) } + + before do + allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids) + allow(stubbed_connector).to receive(:add_report_subreports) + allow(stubbed_connector).to receive(:report) do |subreport_id| + subreport_id == '100' ? subreport1 : subreport2 + end + allow(bootstrapper).to receive(:sleep) + end + + it 'start bootstrap workers' do + expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner)) + bootstrap! + end + + it 'add subreports' do + expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids) + bootstrap! + end + + it 'waits for job to end' do + allow(subreport1).to receive(:status).and_return('running', 'running', 'running', 'completed') + allow(subreport2).to receive(:status).and_return('running', 'running', 'running', 'completed') + expect(bootstrapper).to receive(:sleep).exactly(2*4).times + bootstrap! + end + + it 'raises StackServerBootstrapError if an error occured during bootstrap' do + allow(subreport1).to receive(:status) {'failed'} + allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:server_bootstrap_fail) } + expect { bootstrap! }.to raise_error StackServerBootstrapError + end + + it 'raises StackServerDeployError if an error occured during deploy' do + allow(subreport1).to receive(:status) {'failed'} + allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:deploy_failed) } + expect { bootstrap! }.to raise_error StackServerDeployError + end + + it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do + allow(subreport1).to receive(:status) {'running'} + expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout + end + end + +end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_servers_persister_spec.rb b/devops-service/spec/workers/stack_servers_persister_spec.rb new file mode 100644 index 0000000..c24ba8b --- /dev/null +++ b/devops-service/spec/workers/stack_servers_persister_spec.rb @@ -0,0 +1,116 @@ +require 'workers/stack_bootstrap/stack_servers_persister' + +RSpec.describe StackServersPersister, stubbed_connector: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:run_list) { ['role[asd]'] } + let(:stack) { build(:stack, deploy_env: 'foo', run_list: run_list) } + let(:project) { build(:project, id: 'name') } + let(:persister) { described_class.new(stack, out) } + let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } + let(:server_info_hash) do + { + 'id' => 'server1', + 'name' => 'server_name', + 'key_name' => 'key', + 'private_ip' => '127.0.0.1', + 'public_ip' => '127.0.0.2', + 'tags' => { + 'cid:priority' => '3' + } + } + end + + before do + allow(stubbed_connector).to receive(:project) { project } + allow(stubbed_connector).to receive(:image) { + instance_double(Devops::Model::Image, remote_user: 'user') + } + allow(stubbed_connector).to receive(:server_insert) + allow(stack).to receive(:provider_instance) { provider } + allow(provider).to receive(:stack_servers) {[server_info_hash]} + end + + describe '#persist' do + it 'fetches stack servers info' do + expect(provider).to receive(:stack_servers).with(stack) + persister.persist + end + + it "doesn't raise error if cid:priority tag is absent" do + server_info_hash['tags'].delete('cid:priority') + expect {persister.persist}.not_to raise_error + end + + it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do + result = persister.persist + expect(result).to be_a(Hash) + expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1) + end + + it 'takes id, key_name, private_ip and public_ip attrs from info hash' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.id).to eq 'server1' + expect(server.key).to eq 'key' + expect(server.private_ip).to eq '127.0.0.1' + expect(server.public_ip).to eq '127.0.0.2' + end + persister.persist + end + + it 'takes created_by, run_list and stack attrs from stack' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.created_by).to eq 'root' + expect(server.run_list).to eq run_list + expect(server.stack).to eq 'iamstack' + end + persister.persist + end + + it 'takes remote_user from image user' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.remote_user).to eq 'user' + end + persister.persist + end + + it "takes deploy_env from project's deploy_env identifier" do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.deploy_env).to eq 'foo' + end + persister.persist + end + + it "takes default provider's ssh key if info doesn't contain it" do + allow(provider).to receive(:ssh_key) { 'default_key' } + server_info_hash.delete('key_name') + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.key).to eq 'default_key' + end + persister.persist + end + + it "sets server's run list to empty array if stack's run_list is nil" do + stack.run_list = nil + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.run_list).to eq [] + end + persister.persist + end + + it 'build chef_node_name with default mask "$project-$nodename-$env"' do + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'name-server1-foo' + end + persister.persist + end + + it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do + server_info_hash['tags']['cid:node-name-mask'] = '$project-$nodename-123' + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.chef_node_name).to eq 'name-server1-123' + end + persister.persist + end + end + +end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_synchronizer_spec.rb b/devops-service/spec/workers/stack_synchronizer_spec.rb new file mode 100644 index 0000000..83a43cf --- /dev/null +++ b/devops-service/spec/workers/stack_synchronizer_spec.rb @@ -0,0 +1,82 @@ +require 'workers/stack_bootstrap/stack_synchronizer' +RSpec.describe StackSynchronizer, stubbed_connector: true do + let(:out) { double(:out, puts: nil, flush: nil) } + let(:stack) { build(:stack) } + let(:syncer) { described_class.new(stack, out) } + + before do + allow(stack).to receive(:sync_details!) + allow(stack).to receive(:events).and_return( [{'event_id' => 1}] ) + allow(syncer).to receive(:sleep) + allow(stubbed_connector).to receive(:stack_update) + + lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'] + allow(stack).to receive(:stack_status).and_return(*lots_of_statuses) + end + + describe '#sync' do + it 'waits for stack creating to be finished' do + expect(syncer).to receive(:sleep).at_least(10).times + expect(stack).to receive(:sync_details!).at_least(10).times + syncer.sync + end + + it 'prints each message only once' do + event1 = {'event_id' => 1, 'timestamp' => 't1'} + event2 = {'event_id' => 2, 'timestamp' => 't2'} + event3 = {'event_id' => 3, 'timestamp' => 't3'} + + allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3]) + syncer.sync + expect(out).to have_received(:puts).with(/t1/).once.ordered + expect(out).to have_received(:puts).with(/t2/).once.ordered + expect(out).to have_received(:puts).with(/t3/).once.ordered + end + + context 'when stack creating was successful' do + it 'updates stack in DB when stack creating is finished and returns 0' do + expect(stubbed_connector).to receive(:stack_update).with(stack) + expect(syncer.sync).to eq 0 + end + end + + context 'when stack was rollbacked' do + it 'returns 1 (:stack_rolled_back)' do + allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE') + expect(syncer.sync).to eq 1 + end + end + + context 'when unkown stack status was found' do + it 'returns 2 (:unkown_status)' do + allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown') + expect(syncer.sync).to eq 2 + end + end + + context "when stack hasn't been synced in an hour" do + it 'returns 3 (:timeout)' do + allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} + expect(syncer.sync).to eq 3 + end + end + + context 'when an error occured during syncing', stubbed_logger: true do + it 'returns 5 (:error)' do + allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE') + allow(stubbed_connector).to receive(:stack_update) { raise } + expect(syncer.sync).to eq 5 + end + end + end + + describe '#reason_from_error_code' do + it 'returns reason as symbol for integer error_code' do + expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back + expect(syncer.reason_from_error_code(2)).to eq :unkown_status + expect(syncer.reason_from_error_code(3)).to eq :timeout + expect(syncer.reason_from_error_code(5)).to eq :error + end + end + +end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb b/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb new file mode 100644 index 0000000..660b23a --- /dev/null +++ b/devops-service/workers/stack_bootstrap/chef_node_name_builder.rb @@ -0,0 +1,15 @@ +class ChefNodeNameBuilder + def initialize(server_info, project, env) + @server_info, @project, @env = server_info, project, env + @mask = server_info['tags']['cid:node-name-mask'] || '$project-$nodename-$env' + end + + def build_node_name + @mask.gsub!('$project', @project.id) + @mask.gsub!('$env', @env.identifier) + @mask.gsub!('$nodename', @server_info['id']) + @mask.gsub!('$time', Time.now.to_i.to_s) + @mask.gsub!('_', '-') + @mask + end +end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/errors.rb b/devops-service/workers/stack_bootstrap/errors.rb new file mode 100644 index 0000000..e9d6035 --- /dev/null +++ b/devops-service/workers/stack_bootstrap/errors.rb @@ -0,0 +1,4 @@ +class StackCreatingError < StandardError; end +class StackServerBootstrapError < StandardError; end +class StackServerDeployError < StandardError; end +class StackServerBootstrapDeployTimeout < StandardError; end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb new file mode 100644 index 0000000..77baab6 --- /dev/null +++ b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb @@ -0,0 +1,78 @@ +require 'workers/bootstrap_worker' +require "workers/stack_bootstrap/errors" + +class StackServersBootstrapper + include PutsAndFlush + attr_reader :out + + def initialize(out, jid) + @out, @jid = out, jid + end + + def bootstrap(servers) + @servers = servers + puts_and_flush "\nStart bootstraping stack servers" + + servers_jobs_ids = start_workers + ::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values) + + out.puts + servers_jobs_ids.each do |server_id, subreport_id| + job_result_code = wait_for_job(server_id, subreport_id) + check_job_result!(server_id, job_result_code) + end + puts_and_flush "Stack servers have been bootstraped" + end + + private + + def check_job_result!(server_id, job_result_code) + return if job_result_code == 0 + + reason = Devops::Executor::ServerExecutor.reason_from_error_code(job_result_code) + puts_and_flush "Operation result for #{server_id}: #{reason}" + + if error_occured_during_bootstrap?(reason) + raise StackServerBootstrapError # will cause rollback of a stack + else + raise StackServerDeployError #will not cause rollback of a stack + end + end + + def error_occured_during_bootstrap?(reason) + Devops::Executor::ServerExecutor.bootstrap_errors_reasons.include?(reason) + end + + def wait_for_job(server_id, subreport_id) + 1000.times do + sleep(5) + subreport = ::Devops::Db.connector.report(subreport_id) + case subreport.status + when Worker::STATUS::COMPLETED + puts_and_flush "Server '#{server_id}' has been bootstraped with job #{subreport_id}" + return 0 + when Worker::STATUS::FAILED + puts_and_flush "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'" + return subreport.job_result_code + end + end + puts_and_flush "Waiting for job #{subreport_id} halted: timeout reached." + raise StackServerBootstrapDeployTimeout + end + + # returns hash: {server_id => worker_job_id} + def start_workers + servers_jobs_ids = {} + @servers.each do |server| + job_id = Worker.start_async(::BootstrapWorker, + server_attrs: server.to_mongo_hash, + bootstrap_template: 'omnibus', + owner: server.created_by + ) + @out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}" + servers_jobs_ids[server.id] = job_id + end + puts_and_flush "\n" + servers_jobs_ids + end +end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb new file mode 100644 index 0000000..3a30eab --- /dev/null +++ b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb @@ -0,0 +1,69 @@ +require 'workers/stack_bootstrap/chef_node_name_builder' + +class StackServersPersister + include PutsAndFlush + attr_reader :stack, :out + + def initialize(stack, out) + @stack, @out = stack, out + @project = mongo.project(stack.project) + @deploy_env = @project.deploy_env(stack.deploy_env) + @provider = stack.provider_instance + end + + # returns: { priority_as_integer => [Servers] } + def persist + puts_and_flush 'Start syncing stack servers with CID' + + stack_servers_with_priority = {} + stack_servers_info.each do |priority, info_array| + stack_servers_with_priority[priority] = info_array.map do |info_hash| + out.puts "Instance '#{info_hash["id"]}' has been launched with stack." + persist_stack_server(info_hash) + end + end + puts_and_flush "Stack servers have been synced with CID" + stack_servers_with_priority.each do |priority, servers| + out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" + end + out.flush + stack_servers_with_priority + end + + private + + # returns: {priority_as_integer => array_of_sersvers_info} + def stack_servers_info + stack_servers = @provider.stack_servers(stack) + stack_servers.each do |info| + info['tags']['cid:priority'] = info['tags']['cid:priority'].to_i + end + stack_servers.group_by{|info| info['tags']['cid:priority']} + end + + # takes a hash, returns Server model + def persist_stack_server(info_hash) + server_attrs = { + '_id' => info_hash['id'], + 'chef_node_name' => ChefNodeNameBuilder.new(info_hash, @project, @deploy_env).build_node_name, + 'created_by' => stack.owner, + 'deploy_env' => @deploy_env.identifier, + 'key' => info_hash['key_name'] || @provider.ssh_key, + 'project' => @project.id, + 'provider' => @provider.name, + 'remote_user' => mongo.image(@deploy_env.image).remote_user, + 'private_ip' => info_hash['private_ip'], + 'public_ip' => info_hash['public_ip'], + 'run_list' => stack.run_list || [], + 'stack' => stack.name + } + + server = ::Devops::Model::Server.new(server_attrs) + mongo.server_insert(server) + server + end + + def mongo + Devops::Db.connector + end +end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb new file mode 100644 index 0000000..7144d9f --- /dev/null +++ b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb @@ -0,0 +1,71 @@ +class StackSynchronizer + include PutsAndFlush + attr_reader :out, :stack + + def initialize(stack, out) + @stack, @out = stack, out + @printed_events = [] + end + + def sync + puts_and_flush "Syncing stack '#{stack.id}'..." + + # 5 tries each 5 seconds, then 200 tries each 10 seconds + sleep_times = [5]*5 + [10]*400 + + sleep_times.each do |sleep_time| + sleep sleep_time + stack.sync_details! + print_new_events + case stack.stack_status + when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS' + when 'CREATE_COMPLETE' + ::Devops::Db.connector.stack_update(stack) + puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}" + return 0 + when 'ROLLBACK_COMPLETE' + puts_and_flush "Stack '#{stack.id}' status is rolled back" + return error_code(:stack_rolled_back) + else + puts_and_flush "Unknown stack status: '#{stack.stack_status}'" + return error_code(:unkown_status) + end + end + puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds." + error_code(:timeout) + rescue StandardError => e + DevopsLogger.logger.error e.message + puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}" + error_code(:error) + end + + def reason_from_error_code(code) + error_codes.key(code) + end + + private + + def error_code(reason) + error_codes.fetch(reason) + end + + def error_codes + { + stack_rolled_back: 1, + unkown_status: 2, + timeout: 3, + error: 5 + } + end + + + def print_new_events + stack.events.each do |event| + unless @printed_events.include?(event["event_id"]) + @printed_events << event["event_id"] + out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}" + end + end + out.flush + end +end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index da7f712..51e8b0a 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -1,14 +1,13 @@ -require "commands/stack" require "db/mongo/models/stack/stack_factory" require "db/mongo/models/project" require "db/mongo/models/report" +require "workers/stack_bootstrap/stack_synchronizer" +require "workers/stack_bootstrap/stack_servers_bootstrapper" +require "workers/stack_bootstrap/stack_servers_persister" +require "workers/stack_bootstrap/errors" -class StackCreatingError < StandardError; end -class BootstrapingStackServerError < StandardError; end -class DeployingStackServerError < StandardError; end class StackBootstrapWorker < Worker - include StackCommands def perform(options) stack_attrs = options.fetch('stack_attributes') @@ -18,38 +17,33 @@ class StackBootstrapWorker < Worker without_bootstrap = stack_attrs.delete('without_bootstrap') @out.puts "Received 'without_bootstrap' option" if without_bootstrap - report = save_report(file, stack_attrs) + save_report(file, stack_attrs) begin - stack = create_stack(stack_attrs) + @stack = create_stack(stack_attrs) - #TODO: errors begin - servers_with_priority = persist_stack_servers!(stack) - unless without_bootstrap - sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x} - sorted_keys.each do |key| - @out.puts "Servers with priority '#{key}':" - bootstrap_servers!(servers_with_priority[key], report) - end - end - @out.puts "Done." + @servers_with_priority = stack_servers_persister.persist + bootstrap_in_priority_order unless without_bootstrap 0 - rescue BootstrapingStackServerError - @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback." - rollback_stack!(stack) + rescue StackServerBootstrapError + puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback." + rollback_stack!(@stack) 2 - rescue DeployingStackServerError => e - @out.puts "\nStack was launched, but an error occured during deploying stack servers." - @out.puts "You can redeploy stack after fixing the error." + rescue StackServerDeployError => e + out.puts "\nStack was launched, but an error occured during deploying stack servers." + puts_and_flush "You can redeploy stack after fixing the error." 3 + rescue StackServerBootstrapDeployTimeout + puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout." + 4 rescue StandardError => e - @out.puts "\nAn error occured. Initiating stack rollback." - rollback_stack!(stack) + puts_and_flush "\nAn error occured. Initiating stack rollback." + rollback_stack!(@stack) raise e end rescue StackCreatingError - @out.puts "Stack creating error" + puts_and_flush "Stack creating error" 1 end end @@ -57,97 +51,52 @@ class StackBootstrapWorker < Worker private - def rollback_stack!(stack) - @out.puts "\nStart rollback of a stack" - stack.delete_stack_in_cloud! - Devops::Db.connector.stack_servers_delete(stack.name) - Devops::Db.connector.stack_delete(stack.id) - @out.puts "Rollback has been completed" + def stack_synchronizer(stack) + StackSynchronizer.new(stack, out) end + def stack_servers_persister + @stack_servers_persister ||= StackServersPersister.new(@stack, out) + end + + def stack_servers_bootstrapper + @stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid) + end + + # builds and persist stack model, initiate stack creating in cloud def create_stack(stack_attrs) stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) mongo.stack_insert(stack) - operation_result = sync_stack_proc.call(@out, stack, mongo) + synchronizer = stack_synchronizer(stack) + operation_result = synchronizer.sync if operation_result == 0 - @out.puts "\nStack '#{stack.name}' has been created" - @out.flush + puts_and_flush "\nStack '#{stack.name}' has been created" stack else - human_readable_code = StackCommands.result_codes.key(operation_result) - @out.puts "An error ocurred during stack creating" - @out.puts "Stack creating operation result was #{human_readable_code}" + human_readable_code = synchronizer.reason_from_error_code(operation_result) + out.puts "An error ocurred during stack creating" + puts_and_flush "Stack creating operation result was #{human_readable_code}" raise StackCreatingError end end - def bootstrap_servers!(servers, report) - @out << "\nStart bootstraping stack servers\n" - - subreports = [] - data = {} - servers.each do |server| - sjid = Worker.start_async(BootstrapWorker, - server_attrs: server.to_mongo_hash, - bootstrap_template: 'omnibus', - owner: server.created_by - ) - subreports << sjid - @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}" - data[server.id] = sjid + # Bootstrap servers with high priorities first + def bootstrap_in_priority_order + sorted_priorities = @servers_with_priority.keys.sort.reverse + sorted_priorities.each do |priority| + @out.puts "Servers with priority '#{priority}':" + stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority]) end - @out.puts - @out.flush - mongo.add_report_subreports(jid, subreports) - results = [] - data.each do |server_id, subreport_id| - begin - sleep(5) - subreport = mongo.report(subreport_id) - status = subreport.status - if status == Worker::STATUS::COMPLETED - @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}" - break - elsif status == Worker::STATUS::FAILED - results << subreport.job_result_code - @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'" - break - end - end while(true) - end - @out.flush - results.empty? ? 0 : -5 + puts_and_flush "Done." end - def check_bootstrap_results!(results) - if results.values.all?(&:zero?) - # everything is OK - @out.puts "Stack servers have been bootstraped" - @out.flush - return 0 - end - - @out.puts - results.each do |chef_node_name, code| - human_readable_code = Devops::Executor::ServerExecutor.symbolic_error_code(code) - @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}" - end - - if errors_in_bootstrapping_present?(results.values) - raise BootstrapingStackServerError # will cause rollback of a stack - else - raise DeployingStackServerError #will not cause rollback of a stack - end - end - - def errors_in_bootstrapping_present?(result_codes) - bootstrap_error_codes = [] - [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code| - bootstrap_error_codes << Devops::Executor::ServerExecutor.error_code(symbolic_code) - end - - (bootstrap_error_codes & result_codes).size > 0 + def rollback_stack!(stack) + puts_and_flush "\nStart rollback of a stack" + stack.delete_stack_in_cloud! + Devops::Db.connector.stack_servers_delete(stack.name) + Devops::Db.connector.stack_delete(stack.id) + puts_and_flush "Rollback has been completed" end def save_report(file, stack_attrs) @@ -164,54 +113,4 @@ class StackBootstrapWorker < Worker mongo.save_report(report) report end - - # returns - # { - # "priority" => [Servers] - # } - def persist_stack_servers!(stack) - @out.puts "Start syncing stack servers with CID" - @out.flush - project = mongo.project(stack.project) - deploy_env = project.deploy_env(stack.deploy_env) - provider = stack.provider_instance - - stack_servers = provider.stack_servers(stack) - stack_servers.each do |info| - info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i - end - stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]} - stack_servers_with_priority = {} - stack_servers_info.each do |priority, info_array| - stack_servers_with_priority[priority] = info_array.map do |extended_info| - @out.puts "Instance '#{extended_info["id"]}' has been launched with stack." - server_attrs = { - 'provider' => provider.name, - 'project' => project.id, - 'deploy_env' => deploy_env.identifier, - 'remote_user' => mongo.image(deploy_env.image).remote_user, - 'key' => extended_info["key_name"] || provider.ssh_key, - '_id' => extended_info["id"], - 'chef_node_name' => extended_info["name"], - 'private_ip' => extended_info["private_ip"], - 'public_ip' => extended_info["public_ip"], - 'created_by' => stack.owner, - 'run_list' => stack.run_list || [], - 'stack' => stack.name - } - - server = ::Devops::Model::Server.new(server_attrs) - mongo.server_insert(server) - # server.chef_node_name = provider.create_default_chef_node_name(server) - server - end - end - @out.puts "Stack servers have been synced with CID" - stack_servers_with_priority.each do |priority, servers| - @out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}" - end - @out.flush - stack_servers_with_priority - end - end diff --git a/devops-service/workers/worker.rb b/devops-service/workers/worker.rb index eb5af3e..c7daada 100644 --- a/devops-service/workers/worker.rb +++ b/devops-service/workers/worker.rb @@ -12,11 +12,12 @@ require "core/devops-logger" require "core/devops-db" require "providers/provider_factory" require "lib/knife/knife_factory" - +require "lib/puts_and_flush" # All options keys MUST be a symbol!!! class Worker include Sidekiq::Worker + include PutsAndFlush attr_accessor :out