introduce ResultObject
This commit is contained in:
parent
267e4dba56
commit
9ba09269c7
@ -1,7 +1,6 @@
|
||||
module Connectors
|
||||
class Stack < Base
|
||||
include Helpers::InsertCommand,
|
||||
# Helpers::ShowCommand,
|
||||
Helpers::ListCommand,
|
||||
Helpers::DeleteCommand,
|
||||
Helpers::UpdateCommand
|
||||
|
||||
@ -1,7 +1,15 @@
|
||||
require 'lib/helpers/result_object'
|
||||
|
||||
module Devops
|
||||
module Executor
|
||||
class ServerOperationResult
|
||||
RESULT_CODES = {
|
||||
class ServerOperationResult < Helpers::ResultObject
|
||||
|
||||
def occured_during_bootstrap?
|
||||
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
|
||||
end
|
||||
|
||||
def self.result_codes
|
||||
{
|
||||
ok: 0,
|
||||
server_bootstrap_fail: 2,
|
||||
server_cannot_update_tags: 3,
|
||||
@ -13,27 +21,6 @@ module Devops
|
||||
creating_server_unknown_error: 9,
|
||||
creating_server_in_cloud_failed: 10
|
||||
}
|
||||
|
||||
attr_reader :code
|
||||
|
||||
def initialize(error_code)
|
||||
@code = error_code
|
||||
end
|
||||
|
||||
def ok?
|
||||
@code == 0
|
||||
end
|
||||
|
||||
def reason
|
||||
RESULT_CODES.key(@code) || :unknown_error
|
||||
end
|
||||
|
||||
def occured_during_bootstrap?(code)
|
||||
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
|
||||
end
|
||||
|
||||
def self.code_of_reason(reason)
|
||||
RESULT_CODES.fetch(reason)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
33
devops-service/lib/helpers/result_object.rb
Normal file
33
devops-service/lib/helpers/result_object.rb
Normal file
@ -0,0 +1,33 @@
|
||||
module Devops
|
||||
module Helpers
|
||||
class ResultObject
|
||||
# this method should be overrided in descendents
|
||||
def self.result_codes
|
||||
{ok: 0}
|
||||
end
|
||||
|
||||
attr_reader :code
|
||||
|
||||
def initialize(code)
|
||||
@code = code
|
||||
end
|
||||
|
||||
def ok?
|
||||
@code == 0
|
||||
end
|
||||
|
||||
def reason
|
||||
self.class.result_codes.key(@code) || :unknown_error
|
||||
end
|
||||
|
||||
def self.code_of_reason(reason)
|
||||
result_codes.fetch(reason)
|
||||
end
|
||||
|
||||
def self.from_reason(reason)
|
||||
new(code_of_reason(reason))
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,7 +1,6 @@
|
||||
module PutsAndFlush
|
||||
private
|
||||
|
||||
# out stream should be defined
|
||||
def puts_and_flush(message)
|
||||
out.puts message
|
||||
out.flush
|
||||
|
||||
@ -53,14 +53,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
||||
it 'updates stack in DB when stack creating is finished and returns 0' do
|
||||
setup_statuses(['CREATE_COMPLETE'])
|
||||
expect(stubbed_connector).to receive(:stack_update).with(stack)
|
||||
expect(syncer.sync).to eq 0
|
||||
expect(syncer.sync.code).to eq 0
|
||||
end
|
||||
end
|
||||
|
||||
context 'when stack was rollbacked' do
|
||||
it 'returns 1 (:stack_rolled_back)' do
|
||||
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
||||
expect(syncer.sync).to eq 1
|
||||
expect(syncer.sync.code).to eq 1
|
||||
end
|
||||
end
|
||||
|
||||
@ -68,14 +68,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
||||
it 'returns 2 (:unkown_status)' do
|
||||
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
|
||||
|
||||
expect(syncer.sync).to eq 2
|
||||
expect(syncer.sync.code).to eq 2
|
||||
end
|
||||
end
|
||||
|
||||
context "when stack hasn't been synced in an hour" do
|
||||
it 'returns 3 (:timeout)' do
|
||||
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
|
||||
expect(syncer.sync).to eq 3
|
||||
expect(syncer.sync.code).to eq 3
|
||||
end
|
||||
end
|
||||
|
||||
@ -83,18 +83,9 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
||||
it 'returns 5 (:error)' do
|
||||
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
|
||||
allow(stubbed_connector).to receive(:stack_update) { raise }
|
||||
expect(syncer.sync).to eq 5
|
||||
expect(syncer.sync.code).to eq 5
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#reason_from_error_code' do
|
||||
it 'returns reason as symbol for integer error_code' do
|
||||
expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
|
||||
expect(syncer.reason_from_error_code(2)).to eq :unkown_status
|
||||
expect(syncer.reason_from_error_code(3)).to eq :timeout
|
||||
expect(syncer.reason_from_error_code(5)).to eq :error
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
@ -4,7 +4,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
||||
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
|
||||
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
|
||||
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
|
||||
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) }
|
||||
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) }
|
||||
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
|
||||
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
|
||||
let(:worker) { described_class.new }
|
||||
@ -17,8 +17,8 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
||||
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
|
||||
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
|
||||
|
||||
allow(stubbed_connector).to receive(:stack_insert)
|
||||
allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
|
||||
allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) }
|
||||
allow(Devops::Model::StackEc2).to receive(:create)
|
||||
end
|
||||
|
||||
|
||||
@ -115,8 +115,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
||||
|
||||
context "when stack creation wasn't successful" do
|
||||
it 'returns 1' do
|
||||
allow(stack_synchronizer).to receive(:sync) { 5 }
|
||||
allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
|
||||
allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) }
|
||||
expect(perform_without_bootstrap).to eq 1
|
||||
end
|
||||
end
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
class StackCreatingError < StandardError; end
|
||||
class StackServerBootstrapError < StandardError; end
|
||||
class StackServerDeployError < StandardError; end
|
||||
class StackServerBootstrapDeployTimeout < StandardError; end
|
||||
@ -37,7 +37,7 @@ class StackServersBootstrapper
|
||||
|
||||
puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
|
||||
|
||||
if operation_result.occured_during_bootstrap?(operation_result.code)
|
||||
if operation_result.occured_during_bootstrap?
|
||||
raise StackServerBootstrapError # will cause rollback of a stack
|
||||
else
|
||||
raise StackServerDeployError # will not cause rollback of a stack
|
||||
|
||||
@ -2,6 +2,20 @@ class StackSynchronizer
|
||||
include PutsAndFlush
|
||||
attr_reader :out, :stack
|
||||
|
||||
class SyncResult < Devops::Helpers::ResultObject
|
||||
def self.result_codes
|
||||
{
|
||||
ok: 0,
|
||||
stack_rolled_back: 1,
|
||||
unkown_status: 2,
|
||||
timeout: 3,
|
||||
error: 5,
|
||||
stack_deleted: 6,
|
||||
stack_not_found: 7
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(stack, out)
|
||||
@stack, @out = stack, out
|
||||
@printed_events = []
|
||||
@ -10,9 +24,6 @@ class StackSynchronizer
|
||||
def sync
|
||||
puts_and_flush "Syncing stack '#{stack.id}'..."
|
||||
|
||||
# 5 tries each 5 seconds, then 200 tries each 10 seconds
|
||||
sleep_times = [5]*5 + [10]*400
|
||||
|
||||
sleep_times.each do |sleep_time|
|
||||
sleep sleep_time
|
||||
stack.sync!
|
||||
@ -24,33 +35,38 @@ class StackSynchronizer
|
||||
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
|
||||
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
|
||||
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
|
||||
return code_for_status(stack.stack_status)
|
||||
return result_for_provider_status(stack.stack_status)
|
||||
else
|
||||
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
|
||||
return error_code(:unkown_status)
|
||||
return result(:unkown_status)
|
||||
end
|
||||
end
|
||||
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
|
||||
error_code(:timeout)
|
||||
result(:timeout)
|
||||
rescue StandardError => e
|
||||
DevopsLogger.logger.error e.message
|
||||
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
|
||||
error_code(:error)
|
||||
end
|
||||
|
||||
def reason_from_error_code(code)
|
||||
error_codes.key(code)
|
||||
result(:error)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def code_for_status(status)
|
||||
{
|
||||
'CREATE_COMPLETE' => 0,
|
||||
'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back),
|
||||
'DELETE_COMPLETE' => error_code(:stack_deleted),
|
||||
'NOT_FOUND' => error_code(:stack_not_found)
|
||||
}.fetch(status)
|
||||
def sleep_times
|
||||
[5]*5 + [10]*400
|
||||
end
|
||||
|
||||
def result(reason)
|
||||
SyncResult.from_reason(reason)
|
||||
end
|
||||
|
||||
def result_for_provider_status(status)
|
||||
provider_status_mapping = {
|
||||
'CREATE_COMPLETE' => :ok,
|
||||
'ROLLBACK_COMPLETE' => :stack_rolled_back,
|
||||
'DELETE_COMPLETE' => :stack_deleted,
|
||||
'NOT_FOUND' => :stack_not_found
|
||||
}
|
||||
result(provider_status_mapping.fetch(status))
|
||||
end
|
||||
|
||||
def update_stack_status
|
||||
@ -62,22 +78,6 @@ class StackSynchronizer
|
||||
@last_status != stack.stack_status
|
||||
end
|
||||
|
||||
def error_code(reason)
|
||||
error_codes.fetch(reason)
|
||||
end
|
||||
|
||||
def error_codes
|
||||
{
|
||||
stack_rolled_back: 1,
|
||||
unkown_status: 2,
|
||||
timeout: 3,
|
||||
error: 5,
|
||||
stack_deleted: 6,
|
||||
stack_not_found: 7
|
||||
}
|
||||
end
|
||||
|
||||
|
||||
def print_new_events
|
||||
stack.events.each do |event|
|
||||
unless @printed_events.include?(event["event_id"])
|
||||
|
||||
@ -19,18 +19,21 @@ class StackBootstrapWorker < Worker
|
||||
|
||||
save_report(stack_attrs)
|
||||
|
||||
begin
|
||||
@stack = create_stack(stack_attrs)
|
||||
if !sync_stack
|
||||
puts_and_flush "Stack creating error"
|
||||
return 1
|
||||
end
|
||||
|
||||
begin
|
||||
@servers_with_priority = stack_servers_persister.persist
|
||||
bootstrap_in_priority_order unless without_bootstrap
|
||||
0
|
||||
rescue StackServerBootstrapError
|
||||
puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
|
||||
puts_and_flush "\nAn error occured during bootstraping stack servers."
|
||||
rollback_stack!(@stack)
|
||||
2
|
||||
rescue StackServerDeployError => e
|
||||
rescue StackServerDeployError
|
||||
out.puts "\nStack was launched, but an error occured during deploying stack servers."
|
||||
puts_and_flush "You can redeploy stack after fixing the error."
|
||||
3
|
||||
@ -38,21 +41,17 @@ class StackBootstrapWorker < Worker
|
||||
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
|
||||
4
|
||||
rescue StandardError => e
|
||||
puts_and_flush "\nAn error occured. Initiating stack rollback."
|
||||
puts_and_flush "\nAn error occured."
|
||||
rollback_stack!(@stack)
|
||||
raise e
|
||||
end
|
||||
rescue StackCreatingError
|
||||
puts_and_flush "Stack creating error"
|
||||
1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def stack_synchronizer(stack)
|
||||
StackSynchronizer.new(stack, out)
|
||||
def stack_synchronizer
|
||||
@stack_synchronizer ||= StackSynchronizer.new(@stack, out)
|
||||
end
|
||||
|
||||
def stack_servers_persister
|
||||
@ -67,17 +66,17 @@ class StackBootstrapWorker < Worker
|
||||
def create_stack(stack_attrs)
|
||||
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
|
||||
mongo.stack_insert(stack)
|
||||
synchronizer = stack_synchronizer(stack)
|
||||
operation_result = synchronizer.sync
|
||||
end
|
||||
|
||||
if operation_result == 0
|
||||
puts_and_flush "\nStack '#{stack.name}' has been created"
|
||||
stack
|
||||
def sync_stack
|
||||
sync_result = stack_synchronizer.sync
|
||||
|
||||
if sync_result.ok?
|
||||
puts_and_flush "\nStack '#{@stack.name}' has been created"
|
||||
true
|
||||
else
|
||||
human_readable_code = synchronizer.reason_from_error_code(operation_result)
|
||||
out.puts "An error ocurred during stack creating"
|
||||
puts_and_flush "Stack creating operation result was #{human_readable_code}"
|
||||
raise StackCreatingError
|
||||
puts_and_flush "An error ocurred during stack synchronization: #{sync_result.reason}"
|
||||
false
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user