tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactore ResultObject and ServersBootstrapper

Browse Source
This commit is contained in:
Anton Chuchkalov 2016-03-27 23:37:57 +03:00
parent 9ba09269c7
commit 5b645ad971
19 changed files with 359 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devops-service/Guardfile
View File

@ -43,6 +43,7 @@ guard :rspec, cmd: "rspec" do
# Devops files
watch(%r{db/.+\.rb}) { rspec.spec_dir }
watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
watch(%r{lib/helpers/.+\.rb}) { ["#{rspec.spec_dir}/executors", "#{rspec.spec_dir}/workers"] }
watch(%r{workers/.+\.rb}) { "#{rspec.spec_dir}/workers" }
watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" }
end

4
devops-service/core/devops-messages.rb
View File

@ -15,7 +15,6 @@ module Devops
end
def merge file
puts "Trying to merge messages with file '#{file}'"
lang = lang_key
messages = read_file(file)[lang]
raise "It is undefined main key '#{lang}' in file '#{file}'" if messages.nil?
@ -45,7 +44,8 @@ module Devops
end
def lang_key
DevopsConfig.config["messages.lang"] || "en"
locale = DevopsConfig.config && DevopsConfig.config['messages.lang']
locale || "en"
end
end

30
devops-service/lib/executors/server_operation_result.rb
View File

@ -4,23 +4,21 @@ module Devops
module Executor
class ServerOperationResult < Helpers::ResultObject
def occured_during_bootstrap?
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
end
set_result_codes(
ok: 0,
server_bootstrap_fail: 2,
server_cannot_update_tags: 3,
server_bootstrap_private_ip_unset: 4,
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8,
creating_server_unknown_error: 9,
creating_server_in_cloud_failed: 10
)
def self.result_codes
{
ok: 0,
server_bootstrap_fail: 2,
server_cannot_update_tags: 3,
server_bootstrap_private_ip_unset: 4,
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8,
creating_server_unknown_error: 9,
creating_server_in_cloud_failed: 10
}
def one_of_bootstrap_errors?
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].include?(reason)
end
end

38
devops-service/lib/helpers/result_object.rb
View File

@ -1,11 +1,6 @@
module Devops
module Helpers
class ResultObject
# this method should be overrided in descendents
def self.result_codes
{ok: 0}
end
attr_reader :code
def initialize(code)
@ -16,17 +11,40 @@ module Devops
@code == 0
end
def failed?
!ok?
end
def reason
self.class.result_codes.key(@code) || :unknown_error
end
def self.code_of_reason(reason)
result_codes.fetch(reason)
class << self
def result_codes
@result_codes || {ok: 0}
end
def code_of_reason(reason)
result_codes.fetch(reason)
end
def from_reason(reason)
new(code_of_reason(reason))
end
private
# defines methods like :bootstrap_error?
def set_result_codes(new_result_codes)
@result_codes = new_result_codes
@result_codes.each do |pretendent_reason, pretendent_code|
define_method "#{pretendent_reason}?" do
code == pretendent_code
end
end
end
end
def self.from_reason(reason)
new(code_of_reason(reason))
end
end
end

15
devops-service/messages/en.yml
View File

@ -4,3 +4,18 @@ en:
validation:
users:
not_exist: "These users are missing in mongo: '%{users}'"
worker:
stack_bootstrap:
bootstrap_result:
ok: "All servers have been successfully bootstrapped."
bootstrap_error: An error occured during bootstraping stack servers.
deploy_error: |
Stack was launched, but an error occured during deploying stack servers.
You can redeploy stack after fixing the error.
timeout_reached: Bootstrap or deploy wasn't completed due to timeout.
servers_bootstrapper:
bootstrap_servers:
ok: "Server '%{server_id}' has been bootstraped (job %{job_id})."
timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."

5
devops-service/spec/shared_contexts/init_messages.rb Normal file
View File

@ -0,0 +1,5 @@
RSpec.shared_context 'init messages', init_messages: true do
before(:all) do
Devops::Messages.init
end
end

48
devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb Normal file
View File

@ -0,0 +1,48 @@
require 'workers/stack_bootstrap/prioritized_groups_bootstrapper'
RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
before do
@array1 = []; @array2 = []; @array3 = []
@servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
end
describe '#bootstrap_servers_by_priority' do
subject { groups_bootstrapper.bootstrap_servers_by_priority }
it 'bootstraps servers in order by priorities, separately' do
allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
expect(subject).to be_ok
end
it 'it returns :bootstrap_error result if error occured during bootstrap' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
}
expect(subject).to be_bootstrap_error
end
it 'it returns :deploy_error result if error occured during deploy' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
expect(subject).to be_deploy_error
end
it "doesn't bootstrap group if previous one failed" do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
allow(ServersBootstrapper).to receive(:new).and_call_original
expect(ServersBootstrapper).to receive(:new).once
subject
end
end
end

40
devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb Normal file
View File

@ -0,0 +1,40 @@
require 'workers/stack_bootstrap/servers_bootstrapper'
RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
let(:bootstrapper) { described_class.new(out, jid, servers ) }
let(:bootstrap_job_ids) { %w(100 200) }
describe '#bootstrap_group' do
before do
allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
allow(stubbed_connector).to receive(:add_report_subreports)
allow(stubbed_connector).to receive(:report)
allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
end
it 'start bootstrap workers and add subreports' do
expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
bootstrapper.bootstrap_group
end
it 'returns :ok result if everything ok' do
expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
end
it 'returns proper error results' do
waiter = instance_double(JobWaiter)
allow(waiter).to receive(:wait).and_return(2, 8)
allow(JobWaiter).to receive(:new) { waiter }
expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
end
it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
end
end
end

58
devops-service/spec/workers/stack_bootstrap/stack_servers_bootstrapper_spec.rb
View File

@ -1,58 +0,0 @@
require 'workers/stack_bootstrap_worker'
RSpec.describe StackServersBootstrapper, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:bootstrapper) { described_class.new(out, jid) }
let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
let(:bootstrap_job_ids) { %w(100 200) }
let(:subreport1) { build(:report, id: bootstrap_job_ids.first) }
let(:subreport2) { build(:report, id: bootstrap_job_ids.last) }
describe '#bootstrap' do
let(:bootstrap!) { bootstrapper.bootstrap(servers) }
before do
allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
allow(stubbed_connector).to receive(:add_report_subreports)
allow(stubbed_connector).to receive(:report) do |subreport_id|
subreport_id == '100' ? subreport1 : subreport2
end
allow(bootstrapper).to receive(:get_bootstrap_result) { 0 }
end
it 'start bootstrap workers' do
expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
bootstrap!
end
it 'add subreports' do
expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
bootstrap!
end
it 'delegates waiting to JobWaiter' do
allow(bootstrapper).to receive(:get_bootstrap_result).and_call_original
allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
expect_any_instance_of(JobWaiter).to receive(:wait)
bootstrapper.bootstrap(build_list(:server, 1))
end
it 'raises StackServerBootstrapError if an error occured during bootstrap' do
allow(bootstrapper).to receive(:get_bootstrap_result) { 2 }
expect { bootstrap! }.to raise_error StackServerBootstrapError
end
it 'raises StackServerDeployError if an error occured during deploy' do
allow(bootstrapper).to receive(:get_bootstrap_result) { 8 }
expect { bootstrap! }.to raise_error StackServerDeployError
end
it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do
allow(bootstrapper).to receive(:get_bootstrap_result).and_call_original
allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout
end
end
end

9
devops-service/spec/workers/stack_bootstrap/stack_synchronizer_spec.rb
View File

@ -53,29 +53,28 @@ RSpec.describe StackSynchronizer, stubbed_connector: true do
it 'updates stack in DB when stack creating is finished and returns 0' do
setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(syncer.sync.code).to eq 0
expect(syncer.sync).to be_ok
end
end
context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(syncer.sync.code).to eq 1
expect(syncer.sync).to be_stack_rolled_back
end
end
context 'when unkown stack status was found' do
it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(syncer.sync.code).to eq 2
expect(syncer.sync).to be_unkown_status
end
end
context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(syncer.sync.code).to eq 3
expect(syncer.sync).to be_timeout
end
end

125
devops-service/spec/workers/stack_bootstrap_worker_spec.rb
View File

@ -1,21 +1,18 @@
require 'workers/stack_bootstrap_worker'
RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0)) }
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
let(:worker) { described_class.new }
before do
allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
allow(worker).to receive(:update_report)
allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
allow(worker).to receive(:sync_stack) { true }
allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} }
allow(worker).to receive(:bootstrap_servers_by_priority) { ServersBootstrapper::Result.new(0) }
allow(stubbed_connector).to receive(:stack_insert) { Devops::Model::StackEc2.new(stack_attrs) }
allow(Devops::Model::StackEc2).to receive(:create)
@ -35,13 +32,13 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
allow(worker).to receive(:create_stack).and_call_original
expect(worker).to receive(:update_report).ordered
expect(worker).to receive(:create_stack).ordered
expect(stack_servers_persister).to receive(:persist).ordered
expect(worker).to receive(:persist_stack_servers).ordered
perform_without_bootstrap
end
context 'if without_bootstrap is true' do
it "doesn't bootstrap servers" do
expect(stack_servers_bootstrapper).not_to receive(:bootstrap)
expect(worker).not_to receive(:bootstrap_servers_by_priority)
perform_without_bootstrap
end
@ -51,72 +48,72 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
end
context 'if without_bootstrap is false or not set' do
it 'bootstraps servers in order by priorities, separately' do
first_servers = build_list(:server, 2)
last_servers = build_list(:server, 3)
allow(stack_servers_persister).to receive(:persist) {
{3 => first_servers, 1 => last_servers}
it 'returns 0 when bootstraping servers was successful' do
expect(perform_with_bootstrap).to eq 0
end
it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do
allow(worker).to receive(:bootstrap_servers_by_priority) {
ServersBootstrapper::Result.from_reason(:bootstrap_error)
}
expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered
expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered
expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
expect(stubbed_connector).to receive(:stack_servers_delete)
expect(stubbed_connector).to receive(:stack_delete)
perform_with_bootstrap
expect(perform_with_bootstrap).to eq 2
end
context 'when bootstraping servers was successful' do
it 'returns 0' do
expect(perform_with_bootstrap).to eq 0
end
it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do
allow(worker).to receive(:bootstrap_servers_by_priority) {
ServersBootstrapper::Result.from_reason(:deploy_error)
}
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 3
end
context 'when a known error occured during servers bootstrap' do
before do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError }
end
it 'rollbacks stack and returns 2' do
expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
expect(stubbed_connector).to receive(:stack_servers_delete)
expect(stubbed_connector).to receive(:stack_delete)
perform_with_bootstrap
end
it 'returns 2' do
allow(worker).to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 2
end
it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do
allow(worker).to receive(:bootstrap_servers_by_priority) {
ServersBootstrapper::Result.from_reason(:timeout_reached)
}
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 4
end
context 'when a known error occured during servers deploy' do
it "doesn't rollback stack and returns 3" do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 3
end
end
context "when a servers bootstrap & deploy haven't been finished due to timeout" do
it "doesn't rollback stack and returns 3" do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 4
end
end
context 'when an unknown error occured during servers bootsrap and deploy' do
it 'rollbacks stack and reraises that error' do
error = StandardError.new
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error }
allow(worker).to receive(:rollback_stack!)
expect(worker).to receive(:rollback_stack!)
expect{perform_with_bootstrap}.to raise_error(error)
end
it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do
error = StandardError.new
allow(worker).to receive(:bootstrap_servers_by_priority) { raise error }
allow(worker).to receive(:rollback_stack!)
expect(worker).to receive(:rollback_stack!)
expect{perform_with_bootstrap}.to raise_error(error)
end
end
context "when stack creation wasn't successful" do
it 'returns 1' do
allow(stack_synchronizer).to receive(:sync) { StackSynchronizer::SyncResult.new(5) }
expect(perform_without_bootstrap).to eq 1
context "without stubbing methods", stubbed_connector: true do
before do
allow(worker).to receive(:sync_stack).and_call_original
allow(worker).to receive(:persist_stack_servers).and_call_original
allow(worker).to receive(:bootstrap_servers_by_priority).and_call_original
allow(StackServersPersister).to receive(:new) {
instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)})
}
allow(PrioritizedGroupsBootstrapper).to receive(:new) {
instance_double(PrioritizedGroupsBootstrapper, bootstrap_servers_by_priority: ServersBootstrapper::Result.new(0))
}
end
it "return 0 if syncer returns ok" do
allow(StackSynchronizer).to receive(:new) {
instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(0))
}
expect(StackServersPersister).to receive(:new).with(instance_of(Devops::Model::StackEc2), anything)
expect(perform_with_bootstrap).to eq 0
end
it 'returns 1 if syncer returns error' do
allow(StackSynchronizer).to receive(:new) {
instance_double(StackSynchronizer, sync: StackSynchronizer::SyncResult.new(5))
}
expect(perform_with_bootstrap).to eq 1
end
end
end

3
devops-service/workers/stack_bootstrap/errors.rb
View File

@ -1,3 +0,0 @@
class StackServerBootstrapError < StandardError; end
class StackServerDeployError < StandardError; end
class StackServerBootstrapDeployTimeout < StandardError; end

36
devops-service/workers/stack_bootstrap/prioritized_groups_bootstrapper.rb Normal file
View File

@ -0,0 +1,36 @@
require_relative 'servers_bootstrapper'
# Bootstrap groups of servers based on priorities: higher first.
# Doesn't start bootstrap of next group if bootstrap of previous group failed.
class PrioritizedGroupsBootstrapper
include PutsAndFlush
attr_reader :out
def initialize(out, jid, servers_with_priorities)
@out, @jid, @servers_with_priorities = out, jid, servers_with_priorities
end
# @param servers_with_priorities [Hash] is a Hash like
# {1 => [server1, server2]}
# Starts bootstrapping another group only after successful bootstrapping of previous.
def bootstrap_servers_by_priority
sorted_priorities.each do |priority|
puts_and_flush "Bootstrap servers with priority '#{priority}':"
bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority])
bootstrap_results = bootstrapper.bootstrap_group
error = most_critical_error(bootstrap_results)
return error if error
end
ServersBootstrapper::Result.from_reason(:ok)
end
private
def sorted_priorities
@servers_with_priorities.keys.sort.reverse
end
def most_critical_error(results)
results.detect(&:bootstrap_error?) || results.detect(&:failed?)
end
end

72
devops-service/workers/stack_bootstrap/servers_bootstrapper.rb Normal file
View File

@ -0,0 +1,72 @@
require 'workers/bootstrap_worker'
require 'workers/helpers/job_waiter'
# Starts bootstrap workers for each server in group and wait for them to end (synchroniously).
class ServersBootstrapper
include PutsAndFlush
attr_reader :out
class Result < Devops::Helpers::ResultObject
set_result_codes(
ok: 0,
bootstrap_error: 2,
deploy_error: 3,
timeout_reached: 4
)
end
def initialize(out, jid, servers)
@out, @jid, @servers = out, jid, servers
@server_bootstrap_jobs = {}
end
# returns array of Results
def bootstrap_group
start_workers!
::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values)
@server_bootstrap_jobs.map do |server_id, job_id|
result = wait_for_bootstrap_job(job_id)
puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
result
end
end
private
# returns hash: {server_id => worker_job_id}
def start_workers!
@servers.each do |server|
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
@out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
@server_bootstrap_jobs[server.id] = job_id
end
puts_and_flush "\n\n\n"
end
def result(reason)
Result.from_reason(reason)
end
def wait_for_bootstrap_job(job_id)
result_code = JobWaiter.new(job_id).wait
result_from_job_code(result_code)
rescue JobWaiter::TimeoutReached
result(:timeout_reached)
end
def result_from_job_code(result_code)
job_result = Devops::Executor::ServerOperationResult.new(result_code)
if job_result.ok?
result(:ok)
elsif job_result.one_of_bootstrap_errors?
result(:bootstrap_error)
else
result(:deploy_error)
end
end
end

69
devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb
View File

@ -1,69 +0,0 @@
require 'workers/bootstrap_worker'
require 'workers/stack_bootstrap/errors'
require 'workers/helpers/job_waiter'
class StackServersBootstrapper
include PutsAndFlush
attr_reader :out
def initialize(out, jid)
@out, @jid = out, jid
end
def bootstrap(servers)
@servers = servers
puts_and_flush "\nStart bootstraping stack servers"
servers_jobs_ids = start_workers
::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values)
out.puts
servers_jobs_ids.each do |server_id, job_id|
bootstrap_result_code = get_bootstrap_result(server_id, job_id)
check_bootstrap_result!(server_id, bootstrap_result_code, job_id)
end
puts_and_flush "Stack servers have been bootstraped"
end
private
def check_bootstrap_result!(server_id, result_code, job_id)
operation_result = Devops::Executor::ServerOperationResult.new(result_code)
if operation_result.ok?
puts_and_flush "Server '#{server_id}' has been bootstraped (job #{job_id})."
return
end
puts_and_flush "Server '#{server_id}' bootstraped failed (job #{job_id}). Reason: #{operation_result.reason}"
if operation_result.occured_during_bootstrap?
raise StackServerBootstrapError # will cause rollback of a stack
else
raise StackServerDeployError # will not cause rollback of a stack
end
end
def get_bootstrap_result(server_id, job_id)
JobWaiter.new(job_id).wait
rescue JobWaiter::TimeoutReached
puts_and_flush "Waiting for job #{job_id} halted: timeout reached."
raise StackServerBootstrapDeployTimeout
end
# returns hash: {server_id => worker_job_id}
def start_workers
servers_jobs_ids = {}
@servers.each do |server|
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
@out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}"
servers_jobs_ids[server.id] = job_id
end
puts_and_flush "\n"
servers_jobs_ids
end
end

1
devops-service/workers/stack_bootstrap/stack_servers_persister.rb
View File

@ -1,5 +1,6 @@
require 'workers/stack_bootstrap/chef_node_name_builder'
# Fetches info about stack servers from provider and then persist them in mongo.
class StackServersPersister
include PutsAndFlush
attr_reader :stack, :out

21
devops-service/workers/stack_bootstrap/stack_synchronizer.rb
View File

@ -1,19 +1,18 @@
# Polling stack status until it's completed or failed.
class StackSynchronizer
include PutsAndFlush
attr_reader :out, :stack
class SyncResult < Devops::Helpers::ResultObject
def self.result_codes
{
ok: 0,
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
}
end
set_result_codes(
ok: 0,
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
)
end
def initialize(stack, out)

62
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -1,11 +1,8 @@
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
require "db/mongo/models/report"
require "workers/stack_bootstrap/stack_synchronizer"
require "workers/stack_bootstrap/stack_servers_bootstrapper"
require "workers/stack_bootstrap/servers_bootstrapper"
require "workers/stack_bootstrap/stack_servers_persister"
require "workers/stack_bootstrap/errors"
class StackBootstrapWorker < Worker
@ -26,20 +23,13 @@ class StackBootstrapWorker < Worker
end
begin
@servers_with_priority = stack_servers_persister.persist
bootstrap_in_priority_order unless without_bootstrap
0
rescue StackServerBootstrapError
puts_and_flush "\nAn error occured during bootstraping stack servers."
rollback_stack!(@stack)
2
rescue StackServerDeployError
out.puts "\nStack was launched, but an error occured during deploying stack servers."
puts_and_flush "You can redeploy stack after fixing the error."
3
rescue StackServerBootstrapDeployTimeout
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
4
@servers_by_priorities = persist_stack_servers
return 0 if without_bootstrap
bootstrap_result = bootstrap_servers_by_priority
puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}")
rollback_stack!(@stack) if bootstrap_result.bootstrap_error?
bootstrap_result.code
rescue StandardError => e
puts_and_flush "\nAn error occured."
rollback_stack!(@stack)
@ -50,16 +40,12 @@ class StackBootstrapWorker < Worker
private
def stack_synchronizer
@stack_synchronizer ||= StackSynchronizer.new(@stack, out)
def persist_stack_servers
StackServersPersister.new(@stack, out).persist
end
def stack_servers_persister
@stack_servers_persister ||= StackServersPersister.new(@stack, out)
end
def stack_servers_bootstrapper
@stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid)
def bootstrap_servers_by_priority
PrioritizedGroupsBootstrapper.new(out, jid, @servers_by_priorities).bootstrap_servers_by_priority
end
# builds and persist stack model, initiate stack creating in cloud
@ -69,7 +55,7 @@ class StackBootstrapWorker < Worker
end
def sync_stack
sync_result = stack_synchronizer.sync
sync_result = StackSynchronizer.new(@stack, out).sync
if sync_result.ok?
puts_and_flush "\nStack '#{@stack.name}' has been created"
@ -80,22 +66,16 @@ class StackBootstrapWorker < Worker
end
end
# Bootstrap servers with high priorities first
def bootstrap_in_priority_order
sorted_priorities = @servers_with_priority.keys.sort.reverse
sorted_priorities.each do |priority|
@out.puts "Servers with priority '#{priority}':"
stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority])
end
puts_and_flush "Done."
end
def rollback_stack!(stack)
puts_and_flush "\nStart rollback of a stack"
stack.delete_stack_in_cloud!
Devops::Db.connector.stack_servers_delete(stack.name)
Devops::Db.connector.stack_delete(stack.id)
puts_and_flush "Rollback has been completed"
begin
stack.delete_stack_in_cloud!
Devops::Db.connector.stack_servers_delete(stack.name)
Devops::Db.connector.stack_delete(stack.id)
puts_and_flush "Stack rollback has been completed"
rescue StandardError, Sinatra::NotFound # Sinatra::NotFound often raised in tests
puts_and_flush "Stack rollback failed"
end
end
def save_report(stack_attrs)

1
devops-service/workers/worker.rb
View File

@ -10,6 +10,7 @@ require "core/devops-service"
require "core/devops-config"
require "core/devops-logger"
require "core/devops-db"
require "db/mongo/models/report"
require "providers/provider_factory"
require "lib/knife/knife_factory"
require "lib/puts_and_flush"