refactore stack syncronizer

This commit is contained in:
Anton Chuchkalov 2016-03-28 00:16:11 +03:00
parent 5b645ad971
commit a365138164
5 changed files with 44 additions and 28 deletions

View File

@ -30,7 +30,7 @@ module Devops
raise "Invalid message value with key '#{key}', it must be a string" unless msg.is_a?(String) raise "Invalid message value with key '#{key}', it must be a string" unless msg.is_a?(String)
unless msg.nil? unless msg.nil?
params.each do |k, v| params.each do |k, v|
msg = msg.gsub("%{#{k}}", v) msg = msg.gsub("%{#{k}}", v.to_s)
end end
end end
msg msg

View File

@ -1,10 +1,11 @@
module Devops module Devops
module Helpers module Helpers
class ResultObject class ResultObject
attr_reader :code attr_reader :code, :reason
def initialize(code) def initialize(code)
@code = code @code = code
@reason = self.class.result_codes.key(@code) || :unknown_error
end end
def ok? def ok?
@ -15,10 +16,6 @@ module Devops
!ok? !ok?
end end
def reason
self.class.result_codes.key(@code) || :unknown_error
end
class << self class << self
def result_codes def result_codes
@result_codes || {ok: 0} @result_codes || {ok: 0}

View File

@ -19,3 +19,11 @@ en:
timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached." timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})." bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})." deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."
stack_synchronizer:
result:
ok: "Stack '%{stack_id}' status is now %{status}"
stack_rolled_back: "Stack '%{stack_id}' status is now %{status}"
stack_deleted: "Stack '%{stack_id}' status is now %{status}"
stack_not_found: "Stack '%{stack_id}' status is now %{status}"
unkown_status: "Unknown stack status: '%{status}'"
timeout: "Stack hasn't been synced in %{seconds} seconds."

View File

@ -1,5 +1,5 @@
require 'workers/stack_bootstrap/stack_synchronizer' require 'workers/stack_bootstrap/stack_synchronizer'
RSpec.describe StackSynchronizer, stubbed_connector: true do RSpec.describe StackSynchronizer, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) } let(:out) { double(:out, puts: nil, flush: nil) }
let(:stack) { build(:stack) } let(:stack) { build(:stack) }
let(:syncer) { described_class.new(stack, out) } let(:syncer) { described_class.new(stack, out) }
@ -9,9 +9,6 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] ) allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
allow(syncer).to receive(:sleep) allow(syncer).to receive(:sleep)
allow(stubbed_connector).to receive(:stack_update) allow(stubbed_connector).to receive(:stack_update)
# lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
# allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
end end
def setup_statuses(statuses_array) def setup_statuses(statuses_array)
@ -36,15 +33,15 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3]) allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).exactly(4).times
expect(out).to receive(:puts).with(/t1/).once.ordered
expect(out).to receive(:puts).with(/t2/).once.ordered
expect(out).to receive(:puts).with(/t3/).once.ordered
syncer.sync syncer.sync
expect(out).to have_received(:puts).with(/t1/).once.ordered
expect(out).to have_received(:puts).with(/t2/).once.ordered
expect(out).to have_received(:puts).with(/t3/).once.ordered
end end
it 'updates stack when status is changed' do it 'updates stack when status is changed' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).exactly(3).times expect(stubbed_connector).to receive(:stack_update).exactly(3).times
syncer.sync syncer.sync
end end
@ -53,6 +50,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
it 'updates stack in DB when stack creating is finished and returns 0' do it 'updates stack in DB when stack creating is finished and returns 0' do
setup_statuses(['CREATE_COMPLETE']) setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack) expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
expect(syncer.sync).to be_ok expect(syncer.sync).to be_ok
end end
end end
@ -60,6 +58,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
context 'when stack was rollbacked' do context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
expect(syncer.sync).to be_stack_rolled_back expect(syncer.sync).to be_stack_rolled_back
end end
end end
@ -67,6 +66,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
context 'when unkown stack status was found' do context 'when unkown stack status was found' do
it 'returns 2 (:unkown_status)' do it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(out).to receive(:puts).with(/unknown/)
expect(syncer.sync).to be_unkown_status expect(syncer.sync).to be_unkown_status
end end
end end
@ -74,6 +74,7 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
context "when stack hasn't been synced in an hour" do context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(out).to receive(:puts).with(/hasn't been synced/)
expect(syncer.sync).to be_timeout expect(syncer.sync).to be_timeout
end end
end end

View File

@ -18,30 +18,24 @@ class StackSynchronizer
def initialize(stack, out) def initialize(stack, out)
@stack, @out = stack, out @stack, @out = stack, out
@printed_events = [] @printed_events = []
@sync_result = nil
end end
def sync def sync
puts_and_flush "Syncing stack '#{stack.id}'..." puts_and_flush "Syncing stack '#{stack.id}'..."
sleep_times.each do |sleep_time| sleep_times.detect do |sleep_time|
sleep sleep_time sleep sleep_time
stack.sync! stack.sync!
print_new_events print_new_events
update_stack_status if stack_status_changed? update_stack_status if stack_status_changed?
stack_is_already_created_or_failed?
end
case stack.stack_status @sync_result ||= result(:timeout)
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' print_result_message
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' @sync_result
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
return result_for_provider_status(stack.stack_status)
else
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
return result(:unkown_status)
end
end
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
result(:timeout)
rescue StandardError => e rescue StandardError => e
DevopsLogger.logger.error e.message DevopsLogger.logger.error e.message
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
@ -50,6 +44,17 @@ class StackSynchronizer
private private
def stack_is_already_created_or_failed?
case stack.stack_status
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
false
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
@sync_result = result_for_provider_status(stack.stack_status)
else
@sync_result = result(:unkown_status)
end
end
def sleep_times def sleep_times
[5]*5 + [10]*400 [5]*5 + [10]*400
end end
@ -68,6 +73,11 @@ class StackSynchronizer
result(provider_status_mapping.fetch(status)) result(provider_status_mapping.fetch(status))
end end
def print_result_message
puts_and_flush Devops::Messages.t("stack_synchronizer.result.#{@sync_result.reason}",
stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
end
def update_stack_status def update_stack_status
::Devops::Db.connector.stack_update(stack) ::Devops::Db.connector.stack_update(stack)
@last_status = stack.stack_status @last_status = stack.stack_status