introduce ResultObject
This commit is contained in:
parent
267e4dba56
commit
9ba09269c7
@ -1,7 +1,6 @@
|
|||||||
module Connectors
|
module Connectors
|
||||||
class Stack < Base
|
class Stack < Base
|
||||||
include Helpers::InsertCommand,
|
include Helpers::InsertCommand,
|
||||||
# Helpers::ShowCommand,
|
|
||||||
Helpers::ListCommand,
|
Helpers::ListCommand,
|
||||||
Helpers::DeleteCommand,
|
Helpers::DeleteCommand,
|
||||||
Helpers::UpdateCommand
|
Helpers::UpdateCommand
|
||||||
|
|||||||
@ -1,39 +1,26 @@
|
|||||||
|
require 'lib/helpers/result_object'
|
||||||
|
|
||||||
module Devops
|
module Devops
|
||||||
module Executor
|
module Executor
|
||||||
class ServerOperationResult
|
class ServerOperationResult < Helpers::ResultObject
|
||||||
RESULT_CODES = {
|
|
||||||
ok: 0,
|
|
||||||
server_bootstrap_fail: 2,
|
|
||||||
server_cannot_update_tags: 3,
|
|
||||||
server_bootstrap_private_ip_unset: 4,
|
|
||||||
server_not_in_chef_nodes: 5,
|
|
||||||
server_bootstrap_unknown_error: 7,
|
|
||||||
deploy_unknown_error: 6,
|
|
||||||
deploy_failed: 8,
|
|
||||||
creating_server_unknown_error: 9,
|
|
||||||
creating_server_in_cloud_failed: 10
|
|
||||||
}
|
|
||||||
|
|
||||||
attr_reader :code
|
def occured_during_bootstrap?
|
||||||
|
|
||||||
def initialize(error_code)
|
|
||||||
@code = error_code
|
|
||||||
end
|
|
||||||
|
|
||||||
def ok?
|
|
||||||
@code == 0
|
|
||||||
end
|
|
||||||
|
|
||||||
def reason
|
|
||||||
RESULT_CODES.key(@code) || :unknown_error
|
|
||||||
end
|
|
||||||
|
|
||||||
def occured_during_bootstrap?(code)
|
|
||||||
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
|
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.code_of_reason(reason)
|
def self.result_codes
|
||||||
RESULT_CODES.fetch(reason)
|
{
|
||||||
|
ok: 0,
|
||||||
|
server_bootstrap_fail: 2,
|
||||||
|
server_cannot_update_tags: 3,
|
||||||
|
server_bootstrap_private_ip_unset: 4,
|
||||||
|
server_not_in_chef_nodes: 5,
|
||||||
|
server_bootstrap_unknown_error: 7,
|
||||||
|
deploy_unknown_error: 6,
|
||||||
|
deploy_failed: 8,
|
||||||
|
creating_server_unknown_error: 9,
|
||||||
|
creating_server_in_cloud_failed: 10
|
||||||
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|||||||
33
devops-service/lib/helpers/result_object.rb
Normal file
33
devops-service/lib/helpers/result_object.rb
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
module Devops
|
||||||
|
module Helpers
|
||||||
|
class ResultObject
|
||||||
|
# this method should be overrided in descendents
|
||||||
|
def self.result_codes
|
||||||
|
{ok: 0}
|
||||||
|
end
|
||||||
|
|
||||||
|
attr_reader :code
|
||||||
|
|
||||||
|
def initialize(code)
|
||||||
|
@code = code
|
||||||
|
end
|
||||||
|
|
||||||
|
def ok?
|
||||||
|
@code == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
def reason
|
||||||
|
self.class.result_codes.key(@code) || :unknown_error
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.code_of_reason(reason)
|
||||||
|
result_codes.fetch(reason)
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.from_reason(reason)
|
||||||
|
new(code_of_reason(reason))
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@ -1,7 +1,6 @@
|
|||||||
module PutsAndFlush
|
module PutsAndFlush
|
||||||
private
|
private
|
||||||
|
|
||||||
# out stream should be defined
|
|
||||||
def puts_and_flush(message)
|
def puts_and_flush(message)
|
||||||
out.puts message
|
out.puts message
|
||||||
out.flush
|
out.flush
|
||||||
|
|||||||
@ -53,14 +53,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
it 'updates stack in DB when stack creating is finished and returns 0' do
|
it 'updates stack in DB when stack creating is finished and returns 0' do
|
||||||
setup_statuses(['CREATE_COMPLETE'])
|
setup_statuses(['CREATE_COMPLETE'])
|
||||||
expect(stubbed_connector).to receive(:stack_update).with(stack)
|
expect(stubbed_connector).to receive(:stack_update).with(stack)
|
||||||
expect(syncer.sync).to eq 0
|
expect(syncer.sync.code).to eq 0
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'when stack was rollbacked' do
|
context 'when stack was rollbacked' do
|
||||||
it 'returns 1 (:stack_rolled_back)' do
|
it 'returns 1 (:stack_rolled_back)' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
|
||||||
expect(syncer.sync).to eq 1
|
expect(syncer.sync.code).to eq 1
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -68,14 +68,14 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
it 'returns 2 (:unkown_status)' do
|
it 'returns 2 (:unkown_status)' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
|
||||||
|
|
||||||
expect(syncer.sync).to eq 2
|
expect(syncer.sync.code).to eq 2
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context "when stack hasn't been synced in an hour" do
|
context "when stack hasn't been synced in an hour" do
|
||||||
it 'returns 3 (:timeout)' do
|
it 'returns 3 (:timeout)' do
|
||||||
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
|
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
|
||||||
expect(syncer.sync).to eq 3
|
expect(syncer.sync.code).to eq 3
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -83,18 +83,9 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
|
|||||||
it 'returns 5 (:error)' do
|
it 'returns 5 (:error)' do
|
||||||
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
|
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
|
||||||
allow(stubbed_connector).to receive(:stack_update) { raise }
|
allow(stubbed_connector).to receive(:stack_update) { raise }
|
||||||
expect(syncer.sync).to eq 5
|
expect(syncer.sync.code).to eq 5
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '#reason_from_error_code' do
|
|
||||||
it 'returns reason as symbol for integer error_code' do
|
|
||||||
expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
|
|
||||||
expect(syncer.reason_from_error_code(2)).to eq :unkown_status
|
|
||||||
expect(syncer.reason_from_error_code(3)).to eq :timeout
|
|
||||||
expect(syncer.reason_from_error_code(5)).to eq :error
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
end
|
||||||
@ -4,7 +4,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
|||||||
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
|
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
|
||||||
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
|
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
|
||||||
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
|
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
|
||||||
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) }
|
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) }
|
||||||
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
|
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
|
||||||
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
|
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
|
||||||
let(:worker) { described_class.new }
|
let(:worker) { described_class.new }
|
||||||
@ -17,8 +17,8 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
|||||||
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
|
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
|
||||||
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
|
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
|
||||||
|
|
||||||
allow(stubbed_connector).to receive(:stack_insert)
|
allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) }
|
||||||
allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
|
allow(Devops::Model::StackEc2).to receive(:create)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
@ -115,8 +115,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
|
|||||||
|
|
||||||
context "when stack creation wasn't successful" do
|
context "when stack creation wasn't successful" do
|
||||||
it 'returns 1' do
|
it 'returns 1' do
|
||||||
allow(stack_synchronizer).to receive(:sync) { 5 }
|
allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) }
|
||||||
allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
|
|
||||||
expect(perform_without_bootstrap).to eq 1
|
expect(perform_without_bootstrap).to eq 1
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -1,4 +1,3 @@
|
|||||||
class StackCreatingError < StandardError; end
|
|
||||||
class StackServerBootstrapError < StandardError; end
|
class StackServerBootstrapError < StandardError; end
|
||||||
class StackServerDeployError < StandardError; end
|
class StackServerDeployError < StandardError; end
|
||||||
class StackServerBootstrapDeployTimeout < StandardError; end
|
class StackServerBootstrapDeployTimeout < StandardError; end
|
||||||
@ -37,7 +37,7 @@ class StackServersBootstrapper
|
|||||||
|
|
||||||
puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
|
puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
|
||||||
|
|
||||||
if operation_result.occured_during_bootstrap?(operation_result.code)
|
if operation_result.occured_during_bootstrap?
|
||||||
raise StackServerBootstrapError # will cause rollback of a stack
|
raise StackServerBootstrapError # will cause rollback of a stack
|
||||||
else
|
else
|
||||||
raise StackServerDeployError # will not cause rollback of a stack
|
raise StackServerDeployError # will not cause rollback of a stack
|
||||||
|
|||||||
@ -2,6 +2,20 @@ class StackSynchronizer
|
|||||||
include PutsAndFlush
|
include PutsAndFlush
|
||||||
attr_reader :out, :stack
|
attr_reader :out, :stack
|
||||||
|
|
||||||
|
class SyncResult < Devops::Helpers::ResultObject
|
||||||
|
def self.result_codes
|
||||||
|
{
|
||||||
|
ok: 0,
|
||||||
|
stack_rolled_back: 1,
|
||||||
|
unkown_status: 2,
|
||||||
|
timeout: 3,
|
||||||
|
error: 5,
|
||||||
|
stack_deleted: 6,
|
||||||
|
stack_not_found: 7
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def initialize(stack, out)
|
def initialize(stack, out)
|
||||||
@stack, @out = stack, out
|
@stack, @out = stack, out
|
||||||
@printed_events = []
|
@printed_events = []
|
||||||
@ -10,9 +24,6 @@ class StackSynchronizer
|
|||||||
def sync
|
def sync
|
||||||
puts_and_flush "Syncing stack '#{stack.id}'..."
|
puts_and_flush "Syncing stack '#{stack.id}'..."
|
||||||
|
|
||||||
# 5 tries each 5 seconds, then 200 tries each 10 seconds
|
|
||||||
sleep_times = [5]*5 + [10]*400
|
|
||||||
|
|
||||||
sleep_times.each do |sleep_time|
|
sleep_times.each do |sleep_time|
|
||||||
sleep sleep_time
|
sleep sleep_time
|
||||||
stack.sync!
|
stack.sync!
|
||||||
@ -24,33 +35,38 @@ class StackSynchronizer
|
|||||||
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
|
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
|
||||||
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
|
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
|
||||||
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
|
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
|
||||||
return code_for_status(stack.stack_status)
|
return result_for_provider_status(stack.stack_status)
|
||||||
else
|
else
|
||||||
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
|
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
|
||||||
return error_code(:unkown_status)
|
return result(:unkown_status)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
|
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
|
||||||
error_code(:timeout)
|
result(:timeout)
|
||||||
rescue StandardError => e
|
rescue StandardError => e
|
||||||
DevopsLogger.logger.error e.message
|
DevopsLogger.logger.error e.message
|
||||||
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
|
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
|
||||||
error_code(:error)
|
result(:error)
|
||||||
end
|
|
||||||
|
|
||||||
def reason_from_error_code(code)
|
|
||||||
error_codes.key(code)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def code_for_status(status)
|
def sleep_times
|
||||||
{
|
[5]*5 + [10]*400
|
||||||
'CREATE_COMPLETE' => 0,
|
end
|
||||||
'ROLLBACK_COMPLETE' => error_code(:stack_rolled_back),
|
|
||||||
'DELETE_COMPLETE' => error_code(:stack_deleted),
|
def result(reason)
|
||||||
'NOT_FOUND' => error_code(:stack_not_found)
|
SyncResult.from_reason(reason)
|
||||||
}.fetch(status)
|
end
|
||||||
|
|
||||||
|
def result_for_provider_status(status)
|
||||||
|
provider_status_mapping = {
|
||||||
|
'CREATE_COMPLETE' => :ok,
|
||||||
|
'ROLLBACK_COMPLETE' => :stack_rolled_back,
|
||||||
|
'DELETE_COMPLETE' => :stack_deleted,
|
||||||
|
'NOT_FOUND' => :stack_not_found
|
||||||
|
}
|
||||||
|
result(provider_status_mapping.fetch(status))
|
||||||
end
|
end
|
||||||
|
|
||||||
def update_stack_status
|
def update_stack_status
|
||||||
@ -62,22 +78,6 @@ class StackSynchronizer
|
|||||||
@last_status != stack.stack_status
|
@last_status != stack.stack_status
|
||||||
end
|
end
|
||||||
|
|
||||||
def error_code(reason)
|
|
||||||
error_codes.fetch(reason)
|
|
||||||
end
|
|
||||||
|
|
||||||
def error_codes
|
|
||||||
{
|
|
||||||
stack_rolled_back: 1,
|
|
||||||
unkown_status: 2,
|
|
||||||
timeout: 3,
|
|
||||||
error: 5,
|
|
||||||
stack_deleted: 6,
|
|
||||||
stack_not_found: 7
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
def print_new_events
|
def print_new_events
|
||||||
stack.events.each do |event|
|
stack.events.each do |event|
|
||||||
unless @printed_events.include?(event["event_id"])
|
unless @printed_events.include?(event["event_id"])
|
||||||
|
|||||||
@ -19,40 +19,39 @@ class StackBootstrapWorker < Worker
|
|||||||
|
|
||||||
save_report(stack_attrs)
|
save_report(stack_attrs)
|
||||||
|
|
||||||
begin
|
@stack = create_stack(stack_attrs)
|
||||||
@stack = create_stack(stack_attrs)
|
if !sync_stack
|
||||||
|
|
||||||
begin
|
|
||||||
@servers_with_priority = stack_servers_persister.persist
|
|
||||||
bootstrap_in_priority_order unless without_bootstrap
|
|
||||||
0
|
|
||||||
rescue StackServerBootstrapError
|
|
||||||
puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
|
|
||||||
rollback_stack!(@stack)
|
|
||||||
2
|
|
||||||
rescue StackServerDeployError => e
|
|
||||||
out.puts "\nStack was launched, but an error occured during deploying stack servers."
|
|
||||||
puts_and_flush "You can redeploy stack after fixing the error."
|
|
||||||
3
|
|
||||||
rescue StackServerBootstrapDeployTimeout
|
|
||||||
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
|
|
||||||
4
|
|
||||||
rescue StandardError => e
|
|
||||||
puts_and_flush "\nAn error occured. Initiating stack rollback."
|
|
||||||
rollback_stack!(@stack)
|
|
||||||
raise e
|
|
||||||
end
|
|
||||||
rescue StackCreatingError
|
|
||||||
puts_and_flush "Stack creating error"
|
puts_and_flush "Stack creating error"
|
||||||
1
|
return 1
|
||||||
|
end
|
||||||
|
|
||||||
|
begin
|
||||||
|
@servers_with_priority = stack_servers_persister.persist
|
||||||
|
bootstrap_in_priority_order unless without_bootstrap
|
||||||
|
0
|
||||||
|
rescue StackServerBootstrapError
|
||||||
|
puts_and_flush "\nAn error occured during bootstraping stack servers."
|
||||||
|
rollback_stack!(@stack)
|
||||||
|
2
|
||||||
|
rescue StackServerDeployError
|
||||||
|
out.puts "\nStack was launched, but an error occured during deploying stack servers."
|
||||||
|
puts_and_flush "You can redeploy stack after fixing the error."
|
||||||
|
3
|
||||||
|
rescue StackServerBootstrapDeployTimeout
|
||||||
|
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
|
||||||
|
4
|
||||||
|
rescue StandardError => e
|
||||||
|
puts_and_flush "\nAn error occured."
|
||||||
|
rollback_stack!(@stack)
|
||||||
|
raise e
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def stack_synchronizer(stack)
|
def stack_synchronizer
|
||||||
StackSynchronizer.new(stack, out)
|
@stack_synchronizer ||= StackSynchronizer.new(@stack, out)
|
||||||
end
|
end
|
||||||
|
|
||||||
def stack_servers_persister
|
def stack_servers_persister
|
||||||
@ -67,17 +66,17 @@ class StackBootstrapWorker < Worker
|
|||||||
def create_stack(stack_attrs)
|
def create_stack(stack_attrs)
|
||||||
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
|
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
|
||||||
mongo.stack_insert(stack)
|
mongo.stack_insert(stack)
|
||||||
synchronizer = stack_synchronizer(stack)
|
end
|
||||||
operation_result = synchronizer.sync
|
|
||||||
|
|
||||||
if operation_result == 0
|
def sync_stack
|
||||||
puts_and_flush "\nStack '#{stack.name}' has been created"
|
sync_result = stack_synchronizer.sync
|
||||||
stack
|
|
||||||
|
if sync_result.ok?
|
||||||
|
puts_and_flush "\nStack '#{@stack.name}' has been created"
|
||||||
|
true
|
||||||
else
|
else
|
||||||
human_readable_code = synchronizer.reason_from_error_code(operation_result)
|
puts_and_flush "An error ocurred during stack synchronization: #{sync_result.reason}"
|
||||||
out.puts "An error ocurred during stack creating"
|
false
|
||||||
puts_and_flush "Stack creating operation result was #{human_readable_code}"
|
|
||||||
raise StackCreatingError
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user