diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index a6a882e..4f44613 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -90,6 +90,8 @@ module Devops def sync! self.stack_status = provider_instance.stack_details(self)[:stack_status] self.events = provider_instance.stack_events(self) + rescue Fog::AWS::CloudFormation::NotFound + self.stack_status = 'NOT_FOUND' end def resources diff --git a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb b/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb index a2acd2f..bbadaea 100644 --- a/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb +++ b/devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb @@ -10,12 +10,20 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do allow(syncer).to receive(:sleep) allow(stubbed_connector).to receive(:stack_update) - lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'] - allow(stack).to receive(:stack_status).and_return(*lots_of_statuses) + # lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'] + # allow(stack).to receive(:stack_status).and_return(*lots_of_statuses) end - describe '#sync' do + def setup_statuses(statuses_array) + statuses = statuses_array.to_enum + allow(stack).to receive(:sync!) { + stack.stack_status = statuses.next + } + end + + describe '#sync', stubbed_logger: true do it 'waits for stack creating to be finished' do + setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']) expect(syncer).to receive(:sleep).at_least(10).times expect(stack).to receive(:sync!).at_least(10).times syncer.sync @@ -27,14 +35,23 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do event3 = {'event_id' => 3, 'timestamp' => 't3'} allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3]) + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) syncer.sync expect(out).to have_received(:puts).with(/t1/).once.ordered expect(out).to have_received(:puts).with(/t2/).once.ordered expect(out).to have_received(:puts).with(/t3/).once.ordered end + it 'updates stack when status is changed' do + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) + + expect(stubbed_connector).to receive(:stack_update).exactly(3).times + syncer.sync + end + context 'when stack creating was successful' do it 'updates stack in DB when stack creating is finished and returns 0' do + setup_statuses(['CREATE_COMPLETE']) expect(stubbed_connector).to receive(:stack_update).with(stack) expect(syncer.sync).to eq 0 end @@ -42,14 +59,15 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do context 'when stack was rollbacked' do it 'returns 1 (:stack_rolled_back)' do - allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE') + setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) expect(syncer.sync).to eq 1 end end context 'when unkown stack status was found' do it 'returns 2 (:unkown_status)' do - allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown') + setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) + expect(syncer.sync).to eq 2 end end @@ -63,7 +81,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do context 'when an error occured during syncing', stubbed_logger: true do it 'returns 5 (:error)' do - allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE') + setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) allow(stubbed_connector).to receive(:stack_update) { raise } expect(syncer.sync).to eq 5 end diff --git a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb index d420954..3a540a8 100644 --- a/devops-service/workers/stack_bootstrap/stack_synchronizer.rb +++ b/devops-service/workers/stack_bootstrap/stack_synchronizer.rb @@ -17,18 +17,14 @@ class StackSynchronizer sleep sleep_time stack.sync! print_new_events + + update_stack_status if stack_status_changed? + case stack.stack_status when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' - when 'CREATE_COMPLETE' - ::Devops::Db.connector.stack_update(stack) + when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}" - return 0 - when 'ROLLBACK_COMPLETE' - puts_and_flush "Stack '#{stack.id}' status is rolled back" - return error_code(:stack_rolled_back) - when 'DELETE_COMPLETE' - puts_and_flush "Stack '#{stack.id}' status is deleted" - return error_code(:stack_deleted) + return code_for_status(stack.stack_status) else puts_and_flush "Unknown stack status: '#{stack.stack_status}'" return error_code(:unkown_status) @@ -38,7 +34,7 @@ class StackSynchronizer error_code(:timeout) rescue StandardError => e DevopsLogger.logger.error e.message - puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}" + puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" error_code(:error) end @@ -48,6 +44,24 @@ class StackSynchronizer private + def code_for_status(status) + { + 'CREATE_COMPLETE' => 0, + 'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back), + 'DELETE_COMPLETE' => error_code(:stack_deleted), + 'NOT_FOUND' => error_code(:stack_not_found) + }.fetch(status) + end + + def update_stack_status + ::Devops::Db.connector.stack_update(stack) + @last_status = stack.stack_status + end + + def stack_status_changed? + @last_status != stack.stack_status + end + def error_code(reason) error_codes.fetch(reason) end @@ -58,7 +72,8 @@ class StackSynchronizer unkown_status: 2, timeout: 3, error: 5, - stack_deleted: 6 + stack_deleted: 6, + stack_not_found: 7 } end