refactore stack syncronizer
This commit is contained in:
parent
5b645ad971
commit
a365138164
@ -30,7 +30,7 @@ module Devops
|
|||||||
raise "Invalid message value with key '#{key}', it must be a string" unless msg.is_a?(String)
|
raise "Invalid message value with key '#{key}', it must be a string" unless msg.is_a?(String)
|
||||||
unless msg.nil?
|
unless msg.nil?
|
||||||
params.each do |k, v|
|
params.each do |k, v|
|
||||||
msg = msg.gsub("%{#{k}}", v)
|
msg = msg.gsub("%{#{k}}", v.to_s)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
msg
|
msg
|
||||||
|
|||||||
@ -1,10 +1,11 @@
|
|||||||
module Devops
|
module Devops
|
||||||
module Helpers
|
module Helpers
|
||||||
class ResultObject
|
class ResultObject
|
||||||
attr_reader :code
|
attr_reader :code, :reason
|
||||||
|
|
||||||
def initialize(code)
|
def initialize(code)
|
||||||
@code = code
|
@code = code
|
||||||
|
@reason = self.class.result_codes.key(@code) || :unknown_error
|
||||||
end
|
end
|
||||||
|
|
||||||
def ok?
|
def ok?
|
||||||
@ -15,10 +16,6 @@ module Devops
|
|||||||
!ok?
|
!ok?
|
||||||
end
|
end
|
||||||
|
|
||||||
def reason
|
|
||||||
self.class.result_codes.key(@code) || :unknown_error
|
|
||||||
end
|
|
||||||
|
|
||||||
class << self
|
class << self
|
||||||
def result_codes
|
def result_codes
|
||||||
@result_codes || {ok: 0}
|
@result_codes || {ok: 0}
|
||||||
|
|||||||
@ -19,3 +19,11 @@ en:
|
|||||||
timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
|
timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
|
||||||
bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
|
bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
|
||||||
deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."
|
deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."
|
||||||
|
stack_synchronizer:
|
||||||
|
result:
|
||||||
|
ok: "Stack '%{stack_id}' status is now %{status}"
|
||||||
|
stack_rolled_back: "Stack '%{stack_id}' status is now %{status}"
|
||||||
|
stack_deleted: "Stack '%{stack_id}' status is now %{status}"
|
||||||
|
stack_not_found: "Stack '%{stack_id}' status is now %{status}"
|
||||||
|
unkown_status: "Unknown stack status: '%{status}'"
|
||||||
|
timeout: "Stack hasn't been synced in %{seconds} seconds."
|
||||||
@ -1,5 +1,5 @@
|
|||||||
require 'workers/stack_bootstrap/stack_synchronizer'
|
require 'workers/stack_bootstrap/stack_synchronizer'
|
||||||
RSpec.describe StackSynchronizer, stubbed_connector: true do
|
RSpec.describe StackSynchronizer, stubbed_connector: true, init_messages: true do
|
||||||
let(:out) { double(:out, puts: nil, flush: nil) }
|
let(:out) { double(:out, puts: nil, flush: nil) }
|
||||||
let(:stack) { build(:stack) }
|
let(:stack) { build(:stack) }
|
||||||
let(:syncer) { described_class.new(stack, out) }
|
let(:syncer) { described_class.new(stack, out) }
|
||||||
@ -9,9 +9,6 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
|
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
|
||||||
allow(syncer).to receive(:sleep)
|
allow(syncer).to receive(:sleep)
|
||||||
allow(stubbed_connector).to receive(:stack_update)
|
allow(stubbed_connector).to receive(:stack_update)
|
||||||
|
|
||||||
# lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
|
|
||||||
# allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def setup_statuses(statuses_array)
|
def setup_statuses(statuses_array)
|
||||||
@ -36,15 +33,15 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
|
|
||||||
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
|
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
|
||||||
|
expect(syncer).to receive(:sleep).exactly(4).times
|
||||||
|
expect(out).to receive(:puts).with(/t1/).once.ordered
|
||||||
|
expect(out).to receive(:puts).with(/t2/).once.ordered
|
||||||
|
expect(out).to receive(:puts).with(/t3/).once.ordered
|
||||||
syncer.sync
|
syncer.sync
|
||||||
expect(out).to have_received(:puts).with(/t1/).once.ordered
|
|
||||||
expect(out).to have_received(:puts).with(/t2/).once.ordered
|
|
||||||
expect(out).to have_received(:puts).with(/t3/).once.ordered
|
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'updates stack when status is changed' do
|
it 'updates stack when status is changed' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
||||||
|
|
||||||
expect(stubbed_connector).to receive(:stack_update).exactly(3).times
|
expect(stubbed_connector).to receive(:stack_update).exactly(3).times
|
||||||
syncer.sync
|
syncer.sync
|
||||||
end
|
end
|
||||||
@ -53,6 +50,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
it 'updates stack in DB when stack creating is finished and returns 0' do
|
it 'updates stack in DB when stack creating is finished and returns 0' do
|
||||||
setup_statuses(['CREATE_COMPLETE'])
|
setup_statuses(['CREATE_COMPLETE'])
|
||||||
expect(stubbed_connector).to receive(:stack_update).with(stack)
|
expect(stubbed_connector).to receive(:stack_update).with(stack)
|
||||||
|
expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
|
||||||
expect(syncer.sync).to be_ok
|
expect(syncer.sync).to be_ok
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -60,6 +58,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
context 'when stack was rollbacked' do
|
context 'when stack was rollbacked' do
|
||||||
it 'returns 1 (:stack_rolled_back)' do
|
it 'returns 1 (:stack_rolled_back)' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
||||||
|
expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
|
||||||
expect(syncer.sync).to be_stack_rolled_back
|
expect(syncer.sync).to be_stack_rolled_back
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -67,6 +66,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
context 'when unkown stack status was found' do
|
context 'when unkown stack status was found' do
|
||||||
it 'returns 2 (:unkown_status)' do
|
it 'returns 2 (:unkown_status)' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
|
||||||
|
expect(out).to receive(:puts).with(/unknown/)
|
||||||
expect(syncer.sync).to be_unkown_status
|
expect(syncer.sync).to be_unkown_status
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -74,6 +74,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
context "when stack hasn't been synced in an hour" do
|
context "when stack hasn't been synced in an hour" do
|
||||||
it 'returns 3 (:timeout)' do
|
it 'returns 3 (:timeout)' do
|
||||||
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
|
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
|
||||||
|
expect(out).to receive(:puts).with(/hasn't been synced/)
|
||||||
expect(syncer.sync).to be_timeout
|
expect(syncer.sync).to be_timeout
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -18,30 +18,24 @@ class StackSynchronizer
|
|||||||
def initialize(stack, out)
|
def initialize(stack, out)
|
||||||
@stack, @out = stack, out
|
@stack, @out = stack, out
|
||||||
@printed_events = []
|
@printed_events = []
|
||||||
|
@sync_result = nil
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync
|
def sync
|
||||||
puts_and_flush "Syncing stack '#{stack.id}'..."
|
puts_and_flush "Syncing stack '#{stack.id}'..."
|
||||||
|
|
||||||
sleep_times.each do |sleep_time|
|
sleep_times.detect do |sleep_time|
|
||||||
sleep sleep_time
|
sleep sleep_time
|
||||||
stack.sync!
|
stack.sync!
|
||||||
print_new_events
|
print_new_events
|
||||||
|
|
||||||
update_stack_status if stack_status_changed?
|
update_stack_status if stack_status_changed?
|
||||||
|
stack_is_already_created_or_failed?
|
||||||
case stack.stack_status
|
|
||||||
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
|
|
||||||
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
|
|
||||||
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
|
|
||||||
return result_for_provider_status(stack.stack_status)
|
|
||||||
else
|
|
||||||
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
|
|
||||||
return result(:unkown_status)
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
|
|
||||||
result(:timeout)
|
@sync_result ||= result(:timeout)
|
||||||
|
print_result_message
|
||||||
|
@sync_result
|
||||||
rescue StandardError => e
|
rescue StandardError => e
|
||||||
DevopsLogger.logger.error e.message
|
DevopsLogger.logger.error e.message
|
||||||
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
|
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
|
||||||
@ -50,6 +44,17 @@ class StackSynchronizer
|
|||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
|
def stack_is_already_created_or_failed?
|
||||||
|
case stack.stack_status
|
||||||
|
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
|
||||||
|
false
|
||||||
|
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
|
||||||
|
@sync_result = result_for_provider_status(stack.stack_status)
|
||||||
|
else
|
||||||
|
@sync_result = result(:unkown_status)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def sleep_times
|
def sleep_times
|
||||||
[5]*5 + [10]*400
|
[5]*5 + [10]*400
|
||||||
end
|
end
|
||||||
@ -68,6 +73,11 @@ class StackSynchronizer
|
|||||||
result(provider_status_mapping.fetch(status))
|
result(provider_status_mapping.fetch(status))
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def print_result_message
|
||||||
|
puts_and_flush Devops::Messages.t("stack_synchronizer.result.#{@sync_result.reason}",
|
||||||
|
stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
|
||||||
|
end
|
||||||
|
|
||||||
def update_stack_status
|
def update_stack_status
|
||||||
::Devops::Db.connector.stack_update(stack)
|
::Devops::Db.connector.stack_update(stack)
|
||||||
@last_status = stack.stack_status
|
@last_status = stack.stack_status
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user