CID-444: persist stacks even in case of failures

devops-service/db/mongo/models/stack/stack_base.rb

@@ -90,6 +90,8 @@ module Devops
       def sync!
         self.stack_status = provider_instance.stack_details(self)[:stack_status]
         self.events = provider_instance.stack_events(self)
+      rescue Fog::AWS::CloudFormation::NotFound
+        self.stack_status = 'NOT_FOUND'
       end
 
       def resources

devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb

@@ -10,12 +10,20 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
     allow(syncer).to receive(:sleep)
     allow(stubbed_connector).to receive(:stack_update)
 
-    lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
-    allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
+    # lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
+    # allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
   end
 
-  describe '#sync' do
+  def setup_statuses(statuses_array)
+    statuses = statuses_array.to_enum
+    allow(stack).to receive(:sync!) {
+      stack.stack_status = statuses.next
+    }
+  end
+
+  describe '#sync', stubbed_logger: true do
     it 'waits for stack creating to be finished' do
+      setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
       expect(syncer).to receive(:sleep).at_least(10).times
       expect(stack).to receive(:sync!).at_least(10).times
       syncer.sync
@@ -27,14 +35,23 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
       event3 = {'event_id' => 3, 'timestamp' => 't3'}
 
       allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
+      setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
       syncer.sync
       expect(out).to have_received(:puts).with(/t1/).once.ordered
       expect(out).to have_received(:puts).with(/t2/).once.ordered
       expect(out).to have_received(:puts).with(/t3/).once.ordered
     end
 
+    it 'updates stack when status is changed' do
+      setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
+
+      expect(stubbed_connector).to receive(:stack_update).exactly(3).times
+      syncer.sync
+    end
+
     context 'when stack creating was successful' do
       it 'updates stack in DB when stack creating is finished and returns 0' do
+        setup_statuses(['CREATE_COMPLETE'])
         expect(stubbed_connector).to receive(:stack_update).with(stack)
         expect(syncer.sync).to eq 0
       end
@@ -42,14 +59,15 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
 
     context 'when stack was rollbacked' do
       it 'returns 1 (:stack_rolled_back)' do
-        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE')
+        setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
         expect(syncer.sync).to eq 1
       end
     end
 
     context 'when unkown stack status was found' do
       it 'returns 2 (:unkown_status)' do
-        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown')
+        setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
+
         expect(syncer.sync).to eq 2
       end
     end
@@ -63,7 +81,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
 
     context 'when an error occured during syncing', stubbed_logger: true do
       it 'returns 5 (:error)' do
-        allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+        setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
         allow(stubbed_connector).to receive(:stack_update) { raise }
         expect(syncer.sync).to eq 5
       end

devops-service/workers/stack_bootstrap/stack_synchronizer.rb

@@ -17,18 +17,14 @@ class StackSynchronizer
       sleep sleep_time
       stack.sync!
       print_new_events
+
+      update_stack_status if stack_status_changed?
+
       case stack.stack_status
       when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
-      when 'CREATE_COMPLETE'
-        ::Devops::Db.connector.stack_update(stack)
+      when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
         puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
-        return 0
-      when 'ROLLBACK_COMPLETE'
-        puts_and_flush "Stack '#{stack.id}' status is rolled back"
-        return error_code(:stack_rolled_back)
-      when 'DELETE_COMPLETE'
-        puts_and_flush "Stack '#{stack.id}' status is deleted"
-        return error_code(:stack_deleted)
+        return code_for_status(stack.stack_status)
       else
         puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
         return error_code(:unkown_status)
@@ -38,7 +34,7 @@ class StackSynchronizer
     error_code(:timeout)
   rescue StandardError => e
     DevopsLogger.logger.error e.message
-    puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}"
+    puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
     error_code(:error)
   end
 
@@ -48,6 +44,24 @@ class StackSynchronizer
 
   private
 
+  def code_for_status(status)
+    {
+      'CREATE_COMPLETE' => 0,
+      'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back),
+      'DELETE_COMPLETE' => error_code(:stack_deleted),
+      'NOT_FOUND' => error_code(:stack_not_found)
+    }.fetch(status)
+  end
+
+  def update_stack_status
+    ::Devops::Db.connector.stack_update(stack)
+    @last_status = stack.stack_status
+  end
+
+  def stack_status_changed?
+    @last_status != stack.stack_status
+  end
+
   def error_code(reason)
     error_codes.fetch(reason)
   end
@@ -58,7 +72,8 @@ class StackSynchronizer
       unkown_status: 2,
       timeout: 3,
       error: 5,
-      stack_deleted: 6
+      stack_deleted: 6,
+      stack_not_found: 7
     }
   end