From 9ba09269c7d77fc07ef6da21678724225c9a5aa8 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Fri, 25 Mar 2016 14:50:22 +0300 Subject: [PATCH] introduce ResultObject --- devops-service/db/mongo/connectors/stack.rb | 1 - .../lib/executors/server_operation_result.rb | 47 +++++------- devops-service/lib/helpers/result_object.rb | 33 +++++++++ devops-service/lib/puts_and_flush.rb | 1 - .../stack_synchronizer_spec.rb | 19 ++--- .../workers/stack_bootstrap_worker_spec.rb | 9 ++- .../workers/stack_bootstrap/errors.rb | 1 - .../stack_servers_bootstrapper.rb | 2 +- .../stack_bootstrap/stack_synchronizer.rb | 68 +++++++++--------- .../workers/stack_bootstrap_worker.rb | 71 +++++++++---------- 10 files changed, 129 insertions(+), 123 deletions(-) create mode 100644 devops-service/lib/helpers/result_object.rb diff --git a/devops-service/db/mongo/connectors/stack.rb b/devops-service/db/mongo/connectors/stack.rb index feee768..e28b189 100644 --- a/devops-service/db/mongo/connectors/stack.rb +++ b/devops-service/db/mongo/connectors/stack.rb @@ -1,7 +1,6 @@ module Connectors class Stack < Base include Helpers::InsertCommand, -# Helpers::ShowCommand, Helpers::ListCommand, Helpers::DeleteCommand, Helpers::UpdateCommand diff --git a/devops-service/lib/executors/server_operation_result.rb b/devops-service/lib/executors/server_operation_result.rb index ede226f..b24e1ad 100644 --- a/devops-service/lib/executors/server_operation_result.rb +++ b/devops-service/lib/executors/server_operation_result.rb @@ -1,39 +1,26 @@ +require 'lib/helpers/result_object' + module Devops module Executor - class ServerOperationResult - RESULT_CODES = { - ok: 0, - server_bootstrap_fail: 2, - server_cannot_update_tags: 3, - server_bootstrap_private_ip_unset: 4, - server_not_in_chef_nodes: 5, - server_bootstrap_unknown_error: 7, - deploy_unknown_error: 6, - deploy_failed: 8, - creating_server_unknown_error: 9, - creating_server_in_cloud_failed: 10 - } + class ServerOperationResult < Helpers::ResultObject - attr_reader :code - - def initialize(error_code) - @code = error_code - end - - def ok? - @code == 0 - end - - def reason - RESULT_CODES.key(@code) || :unknown_error - end - - def occured_during_bootstrap?(code) + def occured_during_bootstrap? [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason) end - def self.code_of_reason(reason) - RESULT_CODES.fetch(reason) + def self.result_codes + { + ok: 0, + server_bootstrap_fail: 2, + server_cannot_update_tags: 3, + server_bootstrap_private_ip_unset: 4, + server_not_in_chef_nodes: 5, + server_bootstrap_unknown_error: 7, + deploy_unknown_error: 6, + deploy_failed: 8, + creating_server_unknown_error: 9, + creating_server_in_cloud_failed: 10 + } end end diff --git a/devops-service/lib/helpers/result_object.rb b/devops-service/lib/helpers/result_object.rb new file mode 100644 index 0000000..c197f9b --- /dev/null +++ b/devops-service/lib/helpers/result_object.rb @@ -0,0 +1,33 @@ +module Devops + module Helpers + class ResultObject + # this method should be overrided in descendents + def self.result_codes + {ok: 0} + end + + attr_reader :code + + def initialize(code) + @code = code + end + + def ok? + @code == 0 + end + + def reason + self.class.result_codes.key(@code) || :unknown_error + end + + def self.code_of_reason(reason) + result_codes.fetch(reason) + end + + def self.from_reason(reason) + new(code_of_reason(reason)) + end + + end + end +end \ No newline at end of file diff --git a/devops-service/lib/puts_and_flush.rb b/devops-service/lib/puts_and_flush.rb index a0af609..f01c910 100644 --- a/devops-service/lib/puts_and_flush.rb +++ b/devops-service/lib/puts_and_flush.rb @@ -1,7 +1,6 @@ module PutsAndFlush private - # out stream should be defined def puts_and_flush(message) out.puts message out.flush diff --git a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb index bbadaea..be82dc1 100644 --- a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb @@ -53,14 +53,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do it 'updates stack in DB when stack creating is finished and returns 0' do setup_statuses(['CREATE_COMPLETE']) expect(stubbed_connector).to receive(:stack_update).with(stack) - expect(syncer.sync).to eq 0 + expect(syncer.sync.code).to eq 0 end end context 'when stack was rollbacked' do it 'returns 1 (:stack_rolled_back)' do setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) - expect(syncer.sync).to eq 1 + expect(syncer.sync.code).to eq 1 end end @@ -68,14 +68,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do it 'returns 2 (:unkown_status)' do setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) - expect(syncer.sync).to eq 2 + expect(syncer.sync.code).to eq 2 end end context "when stack hasn't been synced in an hour" do it 'returns 3 (:timeout)' do allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} - expect(syncer.sync).to eq 3 + expect(syncer.sync.code).to eq 3 end end @@ -83,18 +83,9 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do it 'returns 5 (:error)' do setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) allow(stubbed_connector).to receive(:stack_update) { raise } - expect(syncer.sync).to eq 5 + expect(syncer.sync.code).to eq 5 end end end - describe '#reason_from_error_code' do - it 'returns reason as symbol for integer error_code' do - expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back - expect(syncer.reason_from_error_code(2)).to eq :unkown_status - expect(syncer.reason_from_error_code(3)).to eq :timeout - expect(syncer.reason_from_error_code(5)).to eq :error - end - end - end \ No newline at end of file diff --git a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb index fc18301..fd0c13c 100644 --- a/devops-service/spec/workers/stack_bootstrap_worker_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap_worker_spec.rb @@ -4,7 +4,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys } let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) } - let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) } + let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) } let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) } let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) } let(:worker) { described_class.new } @@ -17,8 +17,8 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper } allow(worker).to receive(:stack_servers_persister) { stack_servers_persister } - allow(stubbed_connector).to receive(:stack_insert) - allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) } + allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) } + allow(Devops::Model::StackEc2).to receive(:create) end @@ -115,8 +115,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do context "when stack creation wasn't successful" do it 'returns 1' do - allow(stack_synchronizer).to receive(:sync) { 5 } - allow(stack_synchronizer).to receive(:reason_from_error_code) { :error } + allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) } expect(perform_without_bootstrap).to eq 1 end end diff --git a/devops-service/workers/stack_bootstrap/errors.rb b/devops-service/workers/stack_bootstrap/errors.rb index e9d6035..0612377 100644 --- a/devops-service/workers/stack_bootstrap/errors.rb +++ b/devops-service/workers/stack_bootstrap/errors.rb @@ -1,4 +1,3 @@ -class StackCreatingError < StandardError; end class StackServerBootstrapError < StandardError; end class StackServerDeployError < StandardError; end class StackServerBootstrapDeployTimeout < StandardError; end \ No newline at end of file diff --git a/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb index bb0d3e9..26dacd2 100644 --- a/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb +++ b/devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb @@ -37,7 +37,7 @@ class StackServersBootstrapper puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}" - if operation_result.occured_during_bootstrap?(operation_result.code) + if operation_result.occured_during_bootstrap? raise StackServerBootstrapError # will cause rollback of a stack else raise StackServerDeployError # will not cause rollback of a stack diff --git a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb index 3a540a8..b16816d 100644 --- a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb +++ b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb @@ -2,6 +2,20 @@ class StackSynchronizer include PutsAndFlush attr_reader :out, :stack + class SyncResult < Devops::Helpers::ResultObject + def self.result_codes + { + ok: 0, + stack_rolled_back: 1, + unkown_status: 2, + timeout: 3, + error: 5, + stack_deleted: 6, + stack_not_found: 7 + } + end + end + def initialize(stack, out) @stack, @out = stack, out @printed_events = [] @@ -10,9 +24,6 @@ class StackSynchronizer def sync puts_and_flush "Syncing stack '#{stack.id}'..." - # 5 tries each 5 seconds, then 200 tries each 10 seconds - sleep_times = [5]*5 + [10]*400 - sleep_times.each do |sleep_time| sleep sleep_time stack.sync! @@ -24,33 +35,38 @@ class StackSynchronizer when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}" - return code_for_status(stack.stack_status) + return result_for_provider_status(stack.stack_status) else puts_and_flush "Unknown stack status: '#{stack.stack_status}'" - return error_code(:unkown_status) + return result(:unkown_status) end end puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds." - error_code(:timeout) + result(:timeout) rescue StandardError => e DevopsLogger.logger.error e.message puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" - error_code(:error) - end - - def reason_from_error_code(code) - error_codes.key(code) + result(:error) end private - def code_for_status(status) - { - 'CREATE_COMPLETE' => 0, - 'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back), - 'DELETE_COMPLETE' => error_code(:stack_deleted), - 'NOT_FOUND' => error_code(:stack_not_found) - }.fetch(status) + def sleep_times + [5]*5 + [10]*400 + end + + def result(reason) + SyncResult.from_reason(reason) + end + + def result_for_provider_status(status) + provider_status_mapping = { + 'CREATE_COMPLETE' => :ok, + 'ROLLBACK_COMPLETE' => :stack_rolled_back, + 'DELETE_COMPLETE' => :stack_deleted, + 'NOT_FOUND' => :stack_not_found + } + result(provider_status_mapping.fetch(status)) end def update_stack_status @@ -62,22 +78,6 @@ class StackSynchronizer @last_status != stack.stack_status end - def error_code(reason) - error_codes.fetch(reason) - end - - def error_codes - { - stack_rolled_back: 1, - unkown_status: 2, - timeout: 3, - error: 5, - stack_deleted: 6, - stack_not_found: 7 - } - end - - def print_new_events stack.events.each do |event| unless @printed_events.include?(event["event_id"]) diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index 3d1f3fb..9595a66 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -19,40 +19,39 @@ class StackBootstrapWorker < Worker save_report(stack_attrs) - begin - @stack = create_stack(stack_attrs) - - begin - @servers_with_priority = stack_servers_persister.persist - bootstrap_in_priority_order unless without_bootstrap - 0 - rescue StackServerBootstrapError - puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback." - rollback_stack!(@stack) - 2 - rescue StackServerDeployError => e - out.puts "\nStack was launched, but an error occured during deploying stack servers." - puts_and_flush "You can redeploy stack after fixing the error." - 3 - rescue StackServerBootstrapDeployTimeout - puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout." - 4 - rescue StandardError => e - puts_and_flush "\nAn error occured. Initiating stack rollback." - rollback_stack!(@stack) - raise e - end - rescue StackCreatingError + @stack = create_stack(stack_attrs) + if !sync_stack puts_and_flush "Stack creating error" - 1 + return 1 + end + + begin + @servers_with_priority = stack_servers_persister.persist + bootstrap_in_priority_order unless without_bootstrap + 0 + rescue StackServerBootstrapError + puts_and_flush "\nAn error occured during bootstraping stack servers." + rollback_stack!(@stack) + 2 + rescue StackServerDeployError + out.puts "\nStack was launched, but an error occured during deploying stack servers." + puts_and_flush "You can redeploy stack after fixing the error." + 3 + rescue StackServerBootstrapDeployTimeout + puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout." + 4 + rescue StandardError => e + puts_and_flush "\nAn error occured." + rollback_stack!(@stack) + raise e end end end private - def stack_synchronizer(stack) - StackSynchronizer.new(stack, out) + def stack_synchronizer + @stack_synchronizer ||= StackSynchronizer.new(@stack, out) end def stack_servers_persister @@ -67,17 +66,17 @@ class StackBootstrapWorker < Worker def create_stack(stack_attrs) stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) mongo.stack_insert(stack) - synchronizer = stack_synchronizer(stack) - operation_result = synchronizer.sync + end - if operation_result == 0 - puts_and_flush "\nStack '#{stack.name}' has been created" - stack + def sync_stack + sync_result = stack_synchronizer.sync + + if sync_result.ok? + puts_and_flush "\nStack '#{@stack.name}' has been created" + true else - human_readable_code = synchronizer.reason_from_error_code(operation_result) - out.puts "An error ocurred during stack creating" - puts_and_flush "Stack creating operation result was #{human_readable_code}" - raise StackCreatingError + puts_and_flush "An error ocurred during stack synchronization: #{sync_result.reason}" + false end end