introduce ResultObject

This commit is contained in:
Anton Chuchkalov 2016-03-25 14:50:22 +03:00
parent 267e4dba56
commit 9ba09269c7
10 changed files with 129 additions and 123 deletions

View File

@ -1,7 +1,6 @@
module Connectors module Connectors
class Stack < Base class Stack < Base
include Helpers::InsertCommand, include Helpers::InsertCommand,
# Helpers::ShowCommand,
Helpers::ListCommand, Helpers::ListCommand,
Helpers::DeleteCommand, Helpers::DeleteCommand,
Helpers::UpdateCommand Helpers::UpdateCommand

View File

@ -1,39 +1,26 @@
require 'lib/helpers/result_object'
module Devops module Devops
module Executor module Executor
class ServerOperationResult class ServerOperationResult < Helpers::ResultObject
RESULT_CODES = {
ok: 0,
server_bootstrap_fail: 2,
server_cannot_update_tags: 3,
server_bootstrap_private_ip_unset: 4,
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8,
creating_server_unknown_error: 9,
creating_server_in_cloud_failed: 10
}
attr_reader :code def occured_during_bootstrap?
def initialize(error_code)
@code = error_code
end
def ok?
@code == 0
end
def reason
RESULT_CODES.key(@code) || :unknown_error
end
def occured_during_bootstrap?(code)
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason) [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
end end
def self.code_of_reason(reason) def self.result_codes
RESULT_CODES.fetch(reason) {
ok: 0,
server_bootstrap_fail: 2,
server_cannot_update_tags: 3,
server_bootstrap_private_ip_unset: 4,
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8,
creating_server_unknown_error: 9,
creating_server_in_cloud_failed: 10
}
end end
end end

View File

@ -0,0 +1,33 @@
module Devops
module Helpers
class ResultObject
# this method should be overrided in descendents
def self.result_codes
{ok: 0}
end
attr_reader :code
def initialize(code)
@code = code
end
def ok?
@code == 0
end
def reason
self.class.result_codes.key(@code) || :unknown_error
end
def self.code_of_reason(reason)
result_codes.fetch(reason)
end
def self.from_reason(reason)
new(code_of_reason(reason))
end
end
end
end

View File

@ -1,7 +1,6 @@
module PutsAndFlush module PutsAndFlush
private private
# out stream should be defined
def puts_and_flush(message) def puts_and_flush(message)
out.puts message out.puts message
out.flush out.flush

View File

@ -53,14 +53,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
it 'updates stack in DB when stack creating is finished and returns 0' do it 'updates stack in DB when stack creating is finished and returns 0' do
setup_statuses(['CREATE_COMPLETE']) setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack) expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(syncer.sync).to eq 0 expect(syncer.sync.code).to eq 0
end end
end end
context 'when stack was rollbacked' do context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE']) setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(syncer.sync).to eq 1 expect(syncer.sync.code).to eq 1
end end
end end
@ -68,14 +68,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
it 'returns 2 (:unkown_status)' do it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown']) setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(syncer.sync).to eq 2 expect(syncer.sync.code).to eq 2
end end
end end
context "when stack hasn't been synced in an hour" do context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'} allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(syncer.sync).to eq 3 expect(syncer.sync.code).to eq 3
end end
end end
@ -83,18 +83,9 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
it 'returns 5 (:error)' do it 'returns 5 (:error)' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']) setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
allow(stubbed_connector).to receive(:stack_update) { raise } allow(stubbed_connector).to receive(:stack_update) { raise }
expect(syncer.sync).to eq 5 expect(syncer.sync.code).to eq 5
end end
end end
end end
describe '#reason_from_error_code' do
it 'returns reason as symbol for integer error_code' do
expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
expect(syncer.reason_from_error_code(2)).to eq :unkown_status
expect(syncer.reason_from_error_code(3)).to eq :timeout
expect(syncer.reason_from_error_code(5)).to eq :error
end
end
end end

View File

@ -4,7 +4,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys } let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) } let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) } let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) } let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) }
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) } let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) } let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
let(:worker) { described_class.new } let(:worker) { described_class.new }
@ -17,8 +17,8 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper } allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister } allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
allow(stubbed_connector).to receive(:stack_insert) allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) }
allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) } allow(Devops::Model::StackEc2).to receive(:create)
end end
@ -115,8 +115,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
context "when stack creation wasn't successful" do context "when stack creation wasn't successful" do
it 'returns 1' do it 'returns 1' do
allow(stack_synchronizer).to receive(:sync) { 5 } allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) }
allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
expect(perform_without_bootstrap).to eq 1 expect(perform_without_bootstrap).to eq 1
end end
end end

View File

@ -1,4 +1,3 @@
class StackCreatingError < StandardError; end
class StackServerBootstrapError < StandardError; end class StackServerBootstrapError < StandardError; end
class StackServerDeployError < StandardError; end class StackServerDeployError < StandardError; end
class StackServerBootstrapDeployTimeout < StandardError; end class StackServerBootstrapDeployTimeout < StandardError; end

View File

@ -37,7 +37,7 @@ class StackServersBootstrapper
puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}" puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
if operation_result.occured_during_bootstrap?(operation_result.code) if operation_result.occured_during_bootstrap?
raise StackServerBootstrapError # will cause rollback of a stack raise StackServerBootstrapError # will cause rollback of a stack
else else
raise StackServerDeployError # will not cause rollback of a stack raise StackServerDeployError # will not cause rollback of a stack

View File

@ -2,6 +2,20 @@ class StackSynchronizer
include PutsAndFlush include PutsAndFlush
attr_reader :out, :stack attr_reader :out, :stack
class SyncResult < Devops::Helpers::ResultObject
def self.result_codes
{
ok: 0,
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
}
end
end
def initialize(stack, out) def initialize(stack, out)
@stack, @out = stack, out @stack, @out = stack, out
@printed_events = [] @printed_events = []
@ -10,9 +24,6 @@ class StackSynchronizer
def sync def sync
puts_and_flush "Syncing stack '#{stack.id}'..." puts_and_flush "Syncing stack '#{stack.id}'..."
# 5 tries each 5 seconds, then 200 tries each 10 seconds
sleep_times = [5]*5 + [10]*400
sleep_times.each do |sleep_time| sleep_times.each do |sleep_time|
sleep sleep_time sleep sleep_time
stack.sync! stack.sync!
@ -24,33 +35,38 @@ class StackSynchronizer
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS' when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND' when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}" puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
return code_for_status(stack.stack_status) return result_for_provider_status(stack.stack_status)
else else
puts_and_flush "Unknown stack status: '#{stack.stack_status}'" puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
return error_code(:unkown_status) return result(:unkown_status)
end end
end end
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds." puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
error_code(:timeout) result(:timeout)
rescue StandardError => e rescue StandardError => e
DevopsLogger.logger.error e.message DevopsLogger.logger.error e.message
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
error_code(:error) result(:error)
end
def reason_from_error_code(code)
error_codes.key(code)
end end
private private
def code_for_status(status) def sleep_times
{ [5]*5 + [10]*400
'CREATE_COMPLETE' => 0, end
'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back),
'DELETE_COMPLETE' => error_code(:stack_deleted), def result(reason)
'NOT_FOUND' => error_code(:stack_not_found) SyncResult.from_reason(reason)
}.fetch(status) end
def result_for_provider_status(status)
provider_status_mapping = {
'CREATE_COMPLETE' => :ok,
'ROLLBACK_COMPLETE' => :stack_rolled_back,
'DELETE_COMPLETE' => :stack_deleted,
'NOT_FOUND' => :stack_not_found
}
result(provider_status_mapping.fetch(status))
end end
def update_stack_status def update_stack_status
@ -62,22 +78,6 @@ class StackSynchronizer
@last_status != stack.stack_status @last_status != stack.stack_status
end end
def error_code(reason)
error_codes.fetch(reason)
end
def error_codes
{
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
}
end
def print_new_events def print_new_events
stack.events.each do |event| stack.events.each do |event|
unless @printed_events.include?(event["event_id"]) unless @printed_events.include?(event["event_id"])

View File

@ -19,40 +19,39 @@ class StackBootstrapWorker < Worker
save_report(stack_attrs) save_report(stack_attrs)
begin @stack = create_stack(stack_attrs)
@stack = create_stack(stack_attrs) if !sync_stack
begin
@servers_with_priority = stack_servers_persister.persist
bootstrap_in_priority_order unless without_bootstrap
0
rescue StackServerBootstrapError
puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
rollback_stack!(@stack)
2
rescue StackServerDeployError => e
out.puts "\nStack was launched, but an error occured during deploying stack servers."
puts_and_flush "You can redeploy stack after fixing the error."
3
rescue StackServerBootstrapDeployTimeout
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
4
rescue StandardError => e
puts_and_flush "\nAn error occured. Initiating stack rollback."
rollback_stack!(@stack)
raise e
end
rescue StackCreatingError
puts_and_flush "Stack creating error" puts_and_flush "Stack creating error"
1 return 1
end
begin
@servers_with_priority = stack_servers_persister.persist
bootstrap_in_priority_order unless without_bootstrap
0
rescue StackServerBootstrapError
puts_and_flush "\nAn error occured during bootstraping stack servers."
rollback_stack!(@stack)
2
rescue StackServerDeployError
out.puts "\nStack was launched, but an error occured during deploying stack servers."
puts_and_flush "You can redeploy stack after fixing the error."
3
rescue StackServerBootstrapDeployTimeout
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
4
rescue StandardError => e
puts_and_flush "\nAn error occured."
rollback_stack!(@stack)
raise e
end end
end end
end end
private private
def stack_synchronizer(stack) def stack_synchronizer
StackSynchronizer.new(stack, out) @stack_synchronizer ||= StackSynchronizer.new(@stack, out)
end end
def stack_servers_persister def stack_servers_persister
@ -67,17 +66,17 @@ class StackBootstrapWorker < Worker
def create_stack(stack_attrs) def create_stack(stack_attrs)
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out) stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
mongo.stack_insert(stack) mongo.stack_insert(stack)
synchronizer = stack_synchronizer(stack) end
operation_result = synchronizer.sync
if operation_result == 0 def sync_stack
puts_and_flush "\nStack '#{stack.name}' has been created" sync_result = stack_synchronizer.sync
stack
if sync_result.ok?
puts_and_flush "\nStack '#{@stack.name}' has been created"
true
else else
human_readable_code = synchronizer.reason_from_error_code(operation_result) puts_and_flush "An error ocurred during stack synchronization: #{sync_result.reason}"
out.puts "An error ocurred during stack creating" false
puts_and_flush "Stack creating operation result was #{human_readable_code}"
raise StackCreatingError
end end
end end