tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

put executor helper classes inside modules

Browse Source
This commit is contained in:
Anton Chuchkalov 2016-03-30 12:57:24 +03:00
parent 28b4099f27
commit d410d5db11
22 changed files with 877 additions and 847 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devops-service/lib/executors/stack_executor.rb
View File

@ -1,5 +1,6 @@
require "db/mongo/models/stack/stack_factory"
require "lib/puts_and_flush"
module Devops; module Executor; class StackExecutor; end; end; end # predeclare for visibility
require_relative "stack_executor/stack_creation_waiter"
require_relative "stack_executor/stack_servers_persister"
require_relative "stack_executor/prioritized_groups_bootstrapper"

107
devops-service/lib/executors/stack_executor/chef_node_name_builder.rb
View File

@ -1,67 +1,70 @@
# Builds node name from mask. Mask could be passed in server's +cid:node-name-mask+ tag,
# default mask is used otherwise. Mask is a string with possible several variables inserted, e.g.
# +':project/:env/:instanceid/:instancename/:increment-group1:'+
# ':project/:env/:instanceid/:instancename/:increment-group1:'
# Variables meanings:
# - +:project+ is replaced with project name (given in constructor)
# - +:env+ is replaced with env name (given in constructor)
# - +:time+ is replaced with current timestamp
# - +:instanceid+ is replaced with provider instance id (fetched from server info)
# - +:instancename+ is replaced with value of Name tag (fetched from server info)
# - +:increment-groupname:+ is replaced with incremented number tied to group name. There could be several groups in one stack.
# - :project is replaced with project name (given in constructor)
# - :env is replaced with env name (given in constructor)
# - :time is replaced with current timestamp
# - :instanceid is replaced with provider instance id (fetched from server info)
# - :instancename is replaced with value of Name tag (fetched from server info)
# - :increment-groupname: is replaced with incremented number tied to group name. There could be several groups in one stack.
# P.S. Colons are used instead of dollar signs, because stacks don't support dollar signs in tags (unlike EC2 instances),
# but it's convenient to set mask tag directly to a stack (not in template): you set tag once and it propagates to all instances.
class ChefNodeNameBuilder
# We need to ensure that node name is uniq by default. We can't use :time to provide uniqueness because
# several servers are persisting at once on stack creating at it's likely that Time.now.to_i will give similar values
# to different servers. So (by default) we should use :instanceid.
DEFAULT_MASK = ':project-:env-:instanceid'
class Devops::Executor::StackExecutor
class ChefNodeNameBuilder
# We need to ensure that node name is uniq by default. We can't use :time to provide uniqueness because
# several servers are persisting at once on stack creating at it's likely that Time.now.to_i will give similar values
# to different servers. So (by default) we should use :instanceid.
DEFAULT_MASK = ':project-:env-:instanceid'
# @param attrs [Hash] should contain
# +:provider_server_info+
# +:project_id+
# +:env_id+
def initialize(attrs)
@server_info = attrs[:provider_server_info]
@project, @env = attrs[:project_id], attrs[:env_id]
@mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK
end
# @param attrs [Hash] should contain
# :provider_server_info
# :project_id
# :env_id
def initialize(attrs)
@server_info = attrs[:provider_server_info]
@project, @env = attrs[:project_id], attrs[:env_id]
@mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK
end
# @param incrementers_values [Hash] is a hash in which key is name of a variable and value is last substituted number for that var.
# This method modifies +incrementers_values+, updating values for substituted variables.
#
# Examples (assume mask is set to +':project-master-:increment-group1:'+):
# incremeters_values = {}
# builder.build_node_name!(incremeters_values) # returns 'mpda-master-01'
# puts incremeters_values # {'group1' => 1}
# builder.build_node_name!(incremeters_values) # returns 'mpda-master-02'
# puts incremeters_values # {'group1' => 2}
def build_node_name!(incrementers_values)
result = @mask.dup
replace_variables!(result)
replace_incrementers!(result, incrementers_values)
result
end
# @param incrementers_values [Hash] is a hash in which key is name of a variable
# and value is last substituted number for that var.
# This method modifies incrementers_values, updating values for substituted variables.
#
# Examples (assume mask is set to +':project-master-:increment-group1:'+):
# incremeters_values = {}
# builder.build_node_name!(incremeters_values) # returns 'mpda-master-01'
# puts incremeters_values # {'group1' => 1}
# builder.build_node_name!(incremeters_values) # returns 'mpda-master-02'
# puts incremeters_values # {'group1' => 2}
def build_node_name!(incrementers_values)
result = @mask.dup
replace_variables!(result)
replace_incrementers!(result, incrementers_values)
result
end
private
private
def replace_variables!(result)
result.gsub!(':project', @project)
result.gsub!(':env', @env)
result.gsub!(':instanceid', @server_info['id'])
result.gsub!(':instancename', @server_info['tags']['Name'] || '')
result.gsub!(':time', Time.now.to_i.to_s)
end
def replace_variables!(result)
result.gsub!(':project', @project)
result.gsub!(':env', @env)
result.gsub!(':instanceid', @server_info['id'])
result.gsub!(':instancename', @server_info['tags']['Name'] || '')
result.gsub!(':time', Time.now.to_i.to_s)
end
def replace_incrementers!(result, incrementers_values)
groupname_regexp = /(?<=:increment-)\w+(?=:)/
result.gsub!(/:increment-\w+:/) do |incrementer|
group_name = groupname_regexp.match(incrementer)[0]
prev_value = incrementers_values[group_name] || 0
current_value = prev_value + 1
incrementers_values[group_name] = current_value
current_value.to_s.rjust(2, '0')
def replace_incrementers!(result, incrementers_values)
groupname_regexp = /(?<=:increment-)\w+(?=:)/
result.gsub!(/:increment-\w+:/) do |incrementer|
group_name = groupname_regexp.match(incrementer)[0]
prev_value = incrementers_values[group_name] || 0
current_value = prev_value + 1
incrementers_values[group_name] = current_value
current_value.to_s.rjust(2, '0')
end
end
end
end

52
devops-service/lib/executors/stack_executor/prioritized_groups_bootstrapper.rb
View File

@ -2,35 +2,37 @@ require_relative 'servers_bootstrapper'
# Bootstrap groups of servers based on priorities: higher first.
# Doesn't start bootstrap of next group if bootstrap of previous group failed.
class PrioritizedGroupsBootstrapper
include PutsAndFlush
attr_reader :out
class Devops::Executor::StackExecutor
class PrioritizedGroupsBootstrapper
include PutsAndFlush
attr_reader :out
def initialize(out, jid, servers_with_priorities)
@out, @jid, @servers_with_priorities = out, jid, servers_with_priorities
end
# @param servers_with_priorities [Hash] is a Hash like
# {1 => [server1, server2]}
# Starts bootstrapping another group only after successful bootstrapping of previous.
def bootstrap_servers_by_priority
sorted_priorities.each do |priority|
puts_and_flush "Bootstrap servers with priority '#{priority}':"
bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority])
bootstrap_results = bootstrapper.bootstrap_group
error = most_critical_error(bootstrap_results)
return error if error
def initialize(out, jid, servers_with_priorities)
@out, @jid, @servers_with_priorities = out, jid, servers_with_priorities
end
ServersBootstrapper::Result.from_reason(:ok)
end
private
# @param servers_with_priorities [Hash] is a Hash like
# {1 => [server1, server2]}
# Starts bootstrapping another group only after successful bootstrapping of previous.
def bootstrap_servers_by_priority
sorted_priorities.each do |priority|
puts_and_flush "Bootstrap servers with priority '#{priority}':"
bootstrapper = ServersBootstrapper.new(@out, @jid, @servers_with_priorities[priority])
bootstrap_results = bootstrapper.bootstrap_group
error = most_critical_error(bootstrap_results)
return error if error
end
ServersBootstrapper::Result.from_reason(:ok)
end
def sorted_priorities
@servers_with_priorities.keys.sort.reverse
end
private
def most_critical_error(results)
results.detect(&:bootstrap_error?) || results.detect(&:failed?)
def sorted_priorities
@servers_with_priorities.keys.sort.reverse
end
def most_critical_error(results)
results.detect(&:bootstrap_error?) || results.detect(&:failed?)
end
end
end

122
devops-service/lib/executors/stack_executor/servers_bootstrapper.rb
View File

@ -1,72 +1,74 @@
require 'lib/helpers/job_waiter'
require 'workers/bootstrap_worker'
require 'workers/helpers/job_waiter'
# Starts bootstrap workers for each server in group and wait for them to end (synchroniously).
class ServersBootstrapper
include PutsAndFlush
attr_reader :out
class Devops::Executor::StackExecutor
class ServersBootstrapper
include PutsAndFlush
attr_reader :out
class Result < Devops::Helpers::ResultObject
set_result_codes(
ok: 0,
bootstrap_error: 2,
deploy_error: 3,
timeout_reached: 4
)
end
def initialize(out, jid, servers)
@out, @jid, @servers = out, jid, servers
@server_bootstrap_jobs = {}
end
# returns array of Results
def bootstrap_group
start_workers!
::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values)
@server_bootstrap_jobs.map do |server_id, job_id|
result = wait_for_bootstrap_job(job_id)
puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
result
end
end
private
# returns hash: {server_id => worker_job_id}
def start_workers!
@servers.each do |server|
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
class Result < Devops::Helpers::ResultObject
set_result_codes(
ok: 0,
bootstrap_error: 2,
deploy_error: 3,
timeout_reached: 4
)
@out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
@server_bootstrap_jobs[server.id] = job_id
end
puts_and_flush "\n\n\n"
end
def result(reason)
Result.from_reason(reason)
end
def initialize(out, jid, servers)
@out, @jid, @servers = out, jid, servers
@server_bootstrap_jobs = {}
end
def wait_for_bootstrap_job(job_id)
result_code = JobWaiter.new(job_id).wait
result_from_job_code(result_code)
rescue JobWaiter::TimeoutReached
result(:timeout_reached)
end
# returns array of Results
def bootstrap_group
start_workers!
::Devops::Db.connector.add_report_subreports(@jid, @server_bootstrap_jobs.values)
def result_from_job_code(result_code)
job_result = Devops::Executor::ServerOperationResult.new(result_code)
if job_result.ok?
result(:ok)
elsif job_result.one_of_bootstrap_errors?
result(:bootstrap_error)
else
result(:deploy_error)
@server_bootstrap_jobs.map do |server_id, job_id|
result = wait_for_bootstrap_job(job_id)
puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
result
end
end
private
# returns hash: {server_id => worker_job_id}
def start_workers!
@servers.each do |server|
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
@out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
@server_bootstrap_jobs[server.id] = job_id
end
puts_and_flush "\n\n\n"
end
def result(reason)
Result.from_reason(reason)
end
def wait_for_bootstrap_job(job_id)
result_code = Devops::Helpers::JobWaiter.new(job_id).wait
result_from_job_code(result_code)
rescue Devops::Helpers::JobWaiter::TimeoutReached
result(:timeout_reached)
end
def result_from_job_code(result_code)
job_result = Devops::Executor::ServerOperationResult.new(result_code)
if job_result.ok?
result(:ok)
elsif job_result.one_of_bootstrap_errors?
result(:bootstrap_error)
else
result(:deploy_error)
end
end
end
end

170
devops-service/lib/executors/stack_executor/stack_creation_waiter.rb
View File

@ -1,99 +1,101 @@
# Polling stack status until it's completed or failed.
class StackCreationWaiter
include PutsAndFlush
attr_reader :out, :stack
class Devops::Executor::StackExecutor
class StackCreationWaiter
include PutsAndFlush
attr_reader :out, :stack
class SyncResult < Devops::Helpers::ResultObject
set_result_codes(
ok: 0,
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
)
end
def initialize(stack, out)
@stack, @out = stack, out
@printed_events = []
@sync_result = nil
end
def sync
puts_and_flush "Syncing stack '#{stack.id}'..."
sleep_times.detect do |sleep_time|
sleep sleep_time
stack.sync!
print_new_events
update_stack_status if stack_status_changed?
stack_is_already_created_or_failed?
class SyncResult < Devops::Helpers::ResultObject
set_result_codes(
ok: 0,
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5,
stack_deleted: 6,
stack_not_found: 7
)
end
@sync_result ||= result(:timeout)
print_result_message
@sync_result
rescue StandardError => e
DevopsLogger.logger.error e.message
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
result(:error)
end
private
def stack_is_already_created_or_failed?
case stack.stack_status
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
false
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
@sync_result = result_for_provider_status(stack.stack_status)
else
@sync_result = result(:unkown_status)
def initialize(stack, out)
@stack, @out = stack, out
@printed_events = []
@sync_result = nil
end
end
def sleep_times
[5]*5 + [10]*400
end
def sync
puts_and_flush "Syncing stack '#{stack.id}'..."
def result(reason)
SyncResult.from_reason(reason)
end
sleep_times.detect do |sleep_time|
sleep sleep_time
stack.sync!
print_new_events
def result_for_provider_status(status)
provider_status_mapping = {
'CREATE_COMPLETE' => :ok,
'ROLLBACK_COMPLETE' => :stack_rolled_back,
'DELETE_COMPLETE' => :stack_deleted,
'NOT_FOUND' => :stack_not_found
}
result(provider_status_mapping.fetch(status))
end
update_stack_status if stack_status_changed?
stack_is_already_created_or_failed?
end
def print_result_message
puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}",
stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
end
@sync_result ||= result(:timeout)
print_result_message
@sync_result
rescue StandardError => e
DevopsLogger.logger.error e.message
puts_and_flush "Error: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
result(:error)
end
def update_stack_status
::Devops::Db.connector.stack_update(stack)
@last_status = stack.stack_status
end
private
def stack_status_changed?
@last_status != stack.stack_status
end
def print_new_events
stack.events.each do |event|
unless @printed_events.include?(event["event_id"])
@printed_events << event["event_id"]
out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
def stack_is_already_created_or_failed?
case stack.stack_status
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'DELETE_IN_PROGRESS'
false
when 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'DELETE_COMPLETE', 'CREATE_FAILED', 'NOT_FOUND'
@sync_result = result_for_provider_status(stack.stack_status)
else
@sync_result = result(:unkown_status)
end
end
out.flush
def sleep_times
[5]*5 + [10]*400
end
def result(reason)
SyncResult.from_reason(reason)
end
def result_for_provider_status(status)
provider_status_mapping = {
'CREATE_COMPLETE' => :ok,
'ROLLBACK_COMPLETE' => :stack_rolled_back,
'DELETE_COMPLETE' => :stack_deleted,
'NOT_FOUND' => :stack_not_found
}
result(provider_status_mapping.fetch(status))
end
def print_result_message
puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}",
stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
end
def update_stack_status
::Devops::Db.connector.stack_update(stack)
@last_status = stack.stack_status
end
def stack_status_changed?
@last_status != stack.stack_status
end
def print_new_events
stack.events.each do |event|
unless @printed_events.include?(event["event_id"])
@printed_events << event["event_id"]
out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
end
end
out.flush
end
end
end

188
devops-service/lib/executors/stack_executor/stack_servers_persister.rb
View File

@ -1,111 +1,113 @@
require_relative 'chef_node_name_builder'
# Fetches info about stack servers from provider and then persist them in mongo.
class StackServersPersister
include PutsAndFlush
attr_reader :stack, :out, :deleted, :servers_info
class Devops::Executor::StackExecutor
class StackServersPersister
include PutsAndFlush
attr_reader :stack, :out, :deleted, :servers_info
NEW = 'new'
DELETED = 'deleted'
PERSISTED = 'persisted'
JUST_PERSISTED = 'just_persisted'
NEW = 'new'
DELETED = 'deleted'
PERSISTED = 'persisted'
JUST_PERSISTED = 'just_persisted'
def initialize(stack, out)
@stack, @out = stack, out
@project = mongo.project(stack.project)
@deploy_env = @project.deploy_env(stack.deploy_env)
fetch_provider_servers_info
set_servers_states
end
def persist_new_servers
with_state(NEW).each do |info|
info[:server] = persist_stack_server(info[:provider_info])
info[:state] = JUST_PERSISTED
def initialize(stack, out)
@stack, @out = stack, out
@project = mongo.project(stack.project)
@deploy_env = @project.deploy_env(stack.deploy_env)
fetch_provider_servers_info
set_servers_states
end
end
# returns: { priority_as_integer => [Servers] }
def just_persisted_by_priority
stack_servers_with_priority = {}
with_state(JUST_PERSISTED).each do |info|
stack_servers_with_priority[info[:priority]] ||= []
stack_servers_with_priority[info[:priority]] << info[:server]
end
stack_servers_with_priority
end
private
def fetch_provider_servers_info
@servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info|
{
id: provider_info['id'],
provider_info: provider_info,
priority: provider_info['tags']['cid:priority'].to_i
}
end
end
def set_servers_states
persisted = Devops::Db.connector.stack_servers(stack.id)
persisted_ids = persisted.map(&:id)
in_cloud_ids = @servers_info.map {|info| info[:id]}
new_ids = in_cloud_ids - persisted_ids
deleted_ids = persisted_ids - in_cloud_ids
@servers_info.each do |info|
if new_ids.include?(info[:id])
info[:state] = NEW
else
info[:state] = PERSISTED
def persist_new_servers
with_state(NEW).each do |info|
info[:server] = persist_stack_server(info[:provider_info])
info[:state] = JUST_PERSISTED
end
end
@deleted = persisted.select { |server| deleted_ids.include?(server.id) }
end
# returns: { priority_as_integer => [Servers] }
def just_persisted_by_priority
stack_servers_with_priority = {}
with_state(JUST_PERSISTED).each do |info|
stack_servers_with_priority[info[:priority]] ||= []
stack_servers_with_priority[info[:priority]] << info[:server]
end
stack_servers_with_priority
end
def with_state(state)
@servers_info.select { |info| info[:state] == state }
end
private
# takes a hash, returns Server model
def persist_stack_server(server_info)
server_attrs = {
'_id' => server_info['id'],
'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values),
'created_by' => stack.owner,
'deploy_env' => @deploy_env.identifier,
'key' => server_info['key_name'] || stack.provider_instance.ssh_key,
'project' => @project.id,
'provider' => @stack.provider,
'provider_account' => @stack.provider_account,
'remote_user' => mongo.image(@deploy_env.image).remote_user,
'private_ip' => server_info['private_ip'],
'public_ip' => server_info['public_ip'],
'run_list' => stack.run_list || [],
'stack' => stack.name
}
def fetch_provider_servers_info
@servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info|
{
id: provider_info['id'],
provider_info: provider_info,
priority: provider_info['tags']['cid:priority'].to_i
}
end
end
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
server
end
def set_servers_states
persisted = Devops::Db.connector.stack_servers(stack.id)
persisted_ids = persisted.map(&:id)
in_cloud_ids = @servers_info.map {|info| info[:id]}
new_ids = in_cloud_ids - persisted_ids
deleted_ids = persisted_ids - in_cloud_ids
def get_name_builder(server_info)
ChefNodeNameBuilder.new(
provider_server_info: server_info,
project_id: @project.id,
env_id: @deploy_env.identifier,
owner: stack.owner
)
end
@servers_info.each do |info|
if new_ids.include?(info[:id])
info[:state] = NEW
else
info[:state] = PERSISTED
end
end
def incrementers_values
@incrementers_values ||= {}
end
@deleted = persisted.select { |server| deleted_ids.include?(server.id) }
end
def mongo
Devops::Db.connector
def with_state(state)
@servers_info.select { |info| info[:state] == state }
end
# takes a hash, returns Server model
def persist_stack_server(server_info)
server_attrs = {
'_id' => server_info['id'],
'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values),
'created_by' => stack.owner,
'deploy_env' => @deploy_env.identifier,
'key' => server_info['key_name'] || stack.provider_instance.ssh_key,
'project' => @project.id,
'provider' => @stack.provider,
'provider_account' => @stack.provider_account,
'remote_user' => mongo.image(@deploy_env.image).remote_user,
'private_ip' => server_info['private_ip'],
'public_ip' => server_info['public_ip'],
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
server
end
def get_name_builder(server_info)
ChefNodeNameBuilder.new(
provider_server_info: server_info,
project_id: @project.id,
env_id: @deploy_env.identifier,
owner: stack.owner
)
end
def incrementers_values
@incrementers_values ||= {}
end
def mongo
Devops::Db.connector
end
end
end

27
devops-service/lib/helpers/job_waiter.rb Normal file
View File

@ -0,0 +1,27 @@
module Devops
module Helpers
class JobWaiter
class TimeoutReached < StandardError; end
INTERVAL = 5
def initialize(job_id, timeout=5000)
@job_id, @timeout = job_id, timeout
end
def wait
(@timeout / INTERVAL).times do
sleep(INTERVAL)
report = ::Devops::Db.connector.report(@job_id)
case report.status
when Worker::STATUS::COMPLETED
return 0
when Worker::STATUS::FAILED
return report.job_result_code
end
end
raise TimeoutReached
end
end
end
end

36
devops-service/spec/executors/helpers/job_waiter_spec.rb Normal file
View File

@ -0,0 +1,36 @@
require 'lib/helpers/job_waiter'
require "db/mongo/models/report"
module Devops::Helpers
RSpec.describe JobWaiter, stubbed_connector: true do
let(:job_waiter) { described_class.new('job_id') }
let(:report) { build(:report) }
before do
allow(stubbed_connector).to receive(:report) { report }
allow(job_waiter).to receive(:sleep)
end
it 'it returns 0 when job become completed' do
allow(report).to receive(:status) { 'completed' }
expect(job_waiter.wait).to eq 0
end
it 'returns error code when job failes' do
allow(report).to receive(:status) { 'failed' }
allow(report).to receive(:job_result_code) { 1 }
expect(job_waiter.wait).to eq 1
end
it 'sleeps until something happens' do
allow(report).to receive(:status).and_return('running', 'running', 'running', 'completed')
expect(job_waiter).to receive(:sleep).exactly(4).times
job_waiter.wait
end
it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do
allow(report).to receive(:status) { 'running' }
expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached)
end
end
end

74
devops-service/spec/executors/stack_executor/chef_node_name_builder_spec.rb Normal file
View File

@ -0,0 +1,74 @@
require 'lib/executors/stack_executor/chef_node_name_builder'
class Devops::Executor::StackExecutor
RSpec.describe ChefNodeNameBuilder do
# test with real response to ensure it is processed correctly
let(:server_info) do
{
"name"=>"stack-achuchkalov-aws-test-1455976199-master01",
"id"=>"i-fac32c7e",
"key_name"=>"achuchkalov",
"private_ip"=>"172.31.11.30",
"public_ip"=>"52.90.250.51",
"tags" => {
"Name"=>"master01",
"aws:cloudformation:logical-id"=>"EC2Instance1",
"aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199",
"StackTemplate"=>"1inst",
"aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6",
"cid:deployEnv"=>"test",
"cid:project"=>"aws",
"cid:user"=>"root",
"cid:priority"=>0
}
}
end
let(:node_name_builder) {
ChefNodeNameBuilder.new(
provider_server_info: server_info,
project_id: 'proj',
env_id: 'dev'
)
}
let(:build_node_name) { node_name_builder.build_node_name!({}) }
def set_mask(mask)
server_info['tags']['cid:node-name-mask'] = mask
end
describe '#build_node_name' do
it 'uses default mask (":project-:env-:instanceid")' do
expect(build_node_name).to eq 'proj-dev-i-fac32c7e'
end
it 'substitutes :project, :env, :instanceid and :instancename' do
set_mask(':project/:env/:instanceid/:instancename')
expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01'
end
it 'substitutes :time' do
set_mask(':project-:time')
expect(build_node_name).to match /proj-\d+/
end
describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do
it 'starts with 01 for empty hash' do
set_mask('node-:increment-slave:')
expect(node_name_builder.build_node_name!({})).to eq 'node-01'
end
it "continues with next values if hash isn't empty" do
set_mask('node-:increment-slave:')
expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01'
expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02'
expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51'
end
it 'could substitute different incrementers at once' do
set_mask('node-:increment-slave:-:increment-master:')
expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04'
end
end
end
end
end

50
devops-service/spec/executors/stack_executor/prioritized_groups_bootstrapper_spec.rb Normal file
View File

@ -0,0 +1,50 @@
require 'lib/executors/stack_executor/prioritized_groups_bootstrapper'
class Devops::Executor::StackExecutor
RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
before do
@array1 = []; @array2 = []; @array3 = []
@servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
end
describe '#bootstrap_servers_by_priority' do
subject { groups_bootstrapper.bootstrap_servers_by_priority }
it 'bootstraps servers in order by priorities, separately' do
allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
expect(subject).to be_ok
end
it 'it returns :bootstrap_error result if error occured during bootstrap' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
}
expect(subject).to be_bootstrap_error
end
it 'it returns :deploy_error result if error occured during deploy' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
expect(subject).to be_deploy_error
end
it "doesn't bootstrap group if previous one failed" do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
allow(ServersBootstrapper).to receive(:new).and_call_original
expect(ServersBootstrapper).to receive(:new).once
subject
end
end
end
end

43
devops-service/spec/executors/stack_executor/servers_bootstrapper_spec.rb Normal file
View File

@ -0,0 +1,43 @@
require 'lib/executors/stack_executor/servers_bootstrapper'
class Devops::Executor::StackExecutor
RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
let(:bootstrapper) { described_class.new(out, jid, servers ) }
let(:bootstrap_job_ids) { %w(100 200) }
describe '#bootstrap_group' do
before do
allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
allow(stubbed_connector).to receive(:add_report_subreports)
allow(stubbed_connector).to receive(:report)
allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) { 0 }
end
it 'start bootstrap workers and add subreports' do
expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
bootstrapper.bootstrap_group
end
it 'returns :ok result if everything ok' do
expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
end
it 'returns proper error results' do
waiter = instance_double(Devops::Helpers::JobWaiter)
allow(waiter).to receive(:wait).and_return(2, 8)
allow(Devops::Helpers::JobWaiter).to receive(:new) { waiter }
expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
end
it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
allow_any_instance_of(Devops::Helpers::JobWaiter).to receive(:wait) {
raise Devops::Helpers::JobWaiter::TimeoutReached
}
expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
end
end
end
end

94
devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb Normal file
View File

@ -0,0 +1,94 @@
require 'lib/executors/stack_executor/stack_creation_waiter'
class Devops::Executor::StackExecutor
RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:stack) { build(:stack) }
let(:syncer) { described_class.new(stack, out) }
before do
allow(stack).to receive(:sync!)
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
allow(syncer).to receive(:sleep)
allow(stubbed_connector).to receive(:stack_update)
end
def setup_statuses(statuses_array)
statuses = statuses_array.to_enum
allow(stack).to receive(:sync!) {
stack.stack_status = statuses.next
}
end
describe '#sync', stubbed_logger: true do
it 'waits for stack creating to be finished' do
setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).at_least(10).times
expect(stack).to receive(:sync!).at_least(10).times
syncer.sync
end
it 'prints each message only once' do
event1 = {'event_id' => 1, 'timestamp' => 't1'}
event2 = {'event_id' => 2, 'timestamp' => 't2'}
event3 = {'event_id' => 3, 'timestamp' => 't3'}
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).exactly(4).times
expect(out).to receive(:puts).with(/t1/).once.ordered
expect(out).to receive(:puts).with(/t2/).once.ordered
expect(out).to receive(:puts).with(/t3/).once.ordered
syncer.sync
end
it 'updates stack when status is changed' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).exactly(3).times
syncer.sync
end
context 'when stack creating was successful' do
it 'updates stack in DB when stack creating is finished and returns 0' do
setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
expect(syncer.sync).to be_ok
end
end
context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
expect(syncer.sync).to be_stack_rolled_back
end
end
context 'when unkown stack status was found' do
it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(out).to receive(:puts).with(/unknown/)
expect(syncer.sync).to be_unkown_status
end
end
context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(out).to receive(:puts).with(/hasn't been synced/)
expect(syncer.sync).to be_timeout
end
end
context 'when an error occured during syncing', stubbed_logger: true do
it 'returns 5 (:error)' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
allow(stubbed_connector).to receive(:stack_update) { raise }
expect(syncer.sync.code).to eq 5
end
end
end
end
end

167
devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb Normal file
View File

@ -0,0 +1,167 @@
require 'lib/executors/stack_executor/stack_servers_persister'
class Devops::Executor::StackExecutor
RSpec.describe StackServersPersister, stubbed_connector: true do
def info_hash_for_id(id)
{
'id' => id,
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2',
'tags' => {
'cid:priority' => '3',
'Name' => 'server1'
}
}
end
let(:out) { double(:out, puts: nil, flush: nil) }
let(:run_list) { ['role[asd]'] }
let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
let(:project) { build(:project, id: 'name') }
let(:persister) { described_class.new(stack, out) }
let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
let(:new_server_info) { info_hash_for_id('i-new') }
let(:deleted_server_info) { info_hash_for_id('i-deleted') }
let(:persisted_server_info) { info_hash_for_id('i-persisted') }
before do
allow(stubbed_connector).to receive(:project) { project }
allow(stubbed_connector).to receive(:image) {
instance_double(Devops::Model::Image, remote_user: 'user')
}
allow(stubbed_connector).to receive(:server_insert)
allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
allow(stack).to receive(:provider_instance) { provider }
allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
end
describe '#initialize' do
it 'fetches stack servers info' do
expect(provider).to receive(:stack_servers).with(stack)
described_class.new(stack, out)
end
it "doesn't raise error if cid:priority tag is absent" do
new_server_info['tags'].delete('cid:priority')
expect { described_class.new(stack, out) }.not_to raise_error
end
it 'sets proper statuses' do
infos = described_class.new(stack, out).servers_info
expect(infos.length).to eq 2
expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
end
it 'sets deleted servers' do
deleted = described_class.new(stack, out).deleted
expect(deleted.length).to eq 1
expect(deleted.first.id).to eq 'i-deleted'
end
end
describe '#persist_new_servers' do
it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.id).to eq 'i-new'
expect(server.key).to eq 'key'
expect(server.private_ip).to eq '127.0.0.1'
expect(server.public_ip).to eq '127.0.0.2'
end
persister.persist_new_servers
end
it 'takes created_by, run_list and stack attrs from stack' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.created_by).to eq 'root'
expect(server.run_list).to eq run_list
expect(server.stack).to eq 'iamstack'
end
persister.persist_new_servers
end
it 'takes remote_user from image user' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.remote_user).to eq 'user'
end
persister.persist_new_servers
end
it "takes deploy_env from project's deploy_env identifier" do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.deploy_env).to eq 'foo'
end
persister.persist_new_servers
end
it "takes default provider's ssh key if info doesn't contain it" do
allow(provider).to receive(:ssh_key) { 'default_key' }
new_server_info.delete('key_name')
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.key).to eq 'default_key'
end
persister.persist_new_servers
end
it "sets server's run list to empty array if stack's run_list is nil" do
stack.run_list = nil
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.run_list).to eq []
end
persister.persist_new_servers
end
it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-foo-i-new'
end
persister.persist_new_servers
end
it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-server1-123'
end
persister.persist_new_servers
end
it "sets provider and provider account from stack" do
stack.provider_account = 'foo'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.provider).to eq 'ec2'
expect(server.provider_account).to eq 'foo'
end
persister.persist_new_servers
end
describe 'incremented variables' do
it 'substitutes :increment-groupid: with incrementing numbers' do
allow(provider).to receive(:stack_servers) {[
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
]}
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-01-dev'
end.ordered
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-02-dev'
end
persister.persist_new_servers
end
end
end
describe '#just_persisted_by_priority' do
it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
persister.persist_new_servers
result = persister.just_persisted_by_priority
expect(result).to be_a(Hash)
expect(result.size).to eq 1
expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
end
end
end
end

116
devops-service/spec/executors/stack_executor_spec.rb
View File

@ -1,65 +1,67 @@
require 'lib/executors/stack_executor'
RSpec.describe Devops::Executor::StackExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do
let(:out) { double('out', puts: nil, flush: nil) }
let(:stack) { build(:stack) }
let(:executor_without_stack) { described_class.new(out: out) }
let(:executor_with_stack) { described_class.new(out: out, stack: stack) }
class Devops::Executor::StackExecutor
RSpec.describe self, type: :executor, stubbed_connector: true, stubbed_logger: true do
let(:out) { double('out', puts: nil, flush: nil) }
let(:stack) { build(:stack) }
let(:executor_without_stack) { described_class.new(out: out) }
let(:executor_with_stack) { described_class.new(out: out, stack: stack) }
describe '#wait_till_stack_is_created' do
it "return true if syncer returns ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) }
expect(executor_with_stack.wait_till_stack_is_created).to be true
describe '#wait_till_stack_is_created' do
it "return true if syncer returns ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) }
expect(executor_with_stack.wait_till_stack_is_created).to be true
end
it "return false if syncer returns not ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') }
expect(executor_with_stack.wait_till_stack_is_created).to be false
end
end
it "return false if syncer returns not ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') }
expect(executor_with_stack.wait_till_stack_is_created).to be false
describe '#create_stack', stubbed_connector: true do
it 'initiate creation in cloud and persists stack' do
expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out)
expect(stubbed_connector).to receive(:stack_insert)
executor_with_stack.create_stack({'provider' => 'ec2'})
end
end
describe '#persist_stack_servers' do
let(:persister) {
instance_double(StackServersPersister,
persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil
)
}
before { allow(StackServersPersister).to receive(:new).and_return(persister) }
it 'calls StackServersPersister#persist_new_servers' do
expect(persister).to receive(:persist_new_servers)
executor_with_stack.persist_stack_servers
end
it 'returns hash with :just_persisted_by_priority and :deleted keys' do
expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil)
end
end
describe '#bootstrap_servers_by_priority' do
it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do
result = double('bootstrap_result')
allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result }
expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority)
expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result
end
end
describe '#delete_stack', stubbed_connector: true do
it 'deletes stack from cloud, then deletes stack servers, and then deletes stack itself' do
expect(stack).to receive(:delete_stack_in_cloud!).ordered
expect(stubbed_connector).to receive(:stack_servers_delete).ordered
expect(stubbed_connector).to receive(:stack_delete).ordered
executor_with_stack.delete_stack
end
end
end
describe '#create_stack', stubbed_connector: true do
it 'initiate creation in cloud and persists stack' do
expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out)
expect(stubbed_connector).to receive(:stack_insert)
executor_with_stack.create_stack({'provider' => 'ec2'})
end
end
describe '#persist_stack_servers' do
let(:persister) {
instance_double(StackServersPersister, persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil)
}
before { allow(StackServersPersister).to receive(:new).and_return(persister) }
it 'calls StackServersPersister#persist_new_servers' do
expect(persister).to receive(:persist_new_servers)
executor_with_stack.persist_stack_servers
end
it 'returns hash with :just_persisted_by_priority and :deleted keys' do
expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil)
end
end
describe '#bootstrap_servers_by_priority' do
it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do
result = double('bootstrap_result')
allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result }
expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority)
expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result
end
end
describe '#delete_stack', stubbed_connector: true do
it 'deletes stack from cloud, then deletes stack servers, and then deletes stack itself' do
expect(stack).to receive(:delete_stack_in_cloud!).ordered
expect(stubbed_connector).to receive(:stack_servers_delete).ordered
expect(stubbed_connector).to receive(:stack_delete).ordered
executor_with_stack.delete_stack
end
end
end

35
devops-service/spec/workers/helpers/job_waiter_spec.rb
View File

@ -1,35 +0,0 @@
require 'workers/helpers/job_waiter'
require "db/mongo/models/report"
RSpec.describe JobWaiter, stubbed_connector: true do
let(:job_waiter) { described_class.new('job_id') }
before do
@report_double = instance_double(Devops::Model::Report)
allow(stubbed_connector).to receive(:report) { @report_double }
allow(job_waiter).to receive(:sleep)
end
it 'it returns 0 when job become completed' do
allow(@report_double).to receive(:status) { 'completed' }
expect(job_waiter.wait).to eq 0
end
it 'returns error code when job failes' do
allow(@report_double).to receive(:status) { 'failed' }
allow(@report_double).to receive(:job_result_code) { 1 }
expect(job_waiter.wait).to eq 1
end
it 'sleeps until something happens' do
allow(@report_double).to receive(:status).and_return('running', 'running', 'running', 'completed')
expect(job_waiter).to receive(:sleep).exactly(4).times
job_waiter.wait
end
it 'raises JobWaiter::TimeoutReached if nothing happens for too long' do
allow(@report_double).to receive(:status) { 'running' }
expect { job_waiter.wait }.to raise_error(JobWaiter::TimeoutReached)
end
end

72
devops-service/spec/workers/stack_bootstrap/chef_node_name_builder_spec.rb
View File

@ -1,72 +0,0 @@
require 'lib/executors/stack_executor/chef_node_name_builder'
RSpec.describe ChefNodeNameBuilder do
# test with real response to ensure it is processed correctly
let(:server_info) do
{
"name"=>"stack-achuchkalov-aws-test-1455976199-master01",
"id"=>"i-fac32c7e",
"key_name"=>"achuchkalov",
"private_ip"=>"172.31.11.30",
"public_ip"=>"52.90.250.51",
"tags" => {
"Name"=>"master01",
"aws:cloudformation:logical-id"=>"EC2Instance1",
"aws:cloudformation:stack-name"=>"stack-achuchkalov-aws-test-1455976199",
"StackTemplate"=>"1inst",
"aws:cloudformation:stack-id" => "arn:aws:cloudformation:us-east-1:736558555923:stack/stack-achuchkalov-aws-test-1455976199/d5f3ca60-d7d8-11e5-9ba1-50d5cd24fac6",
"cid:deployEnv"=>"test",
"cid:project"=>"aws",
"cid:user"=>"root",
"cid:priority"=>0
}
}
end
let(:node_name_builder) {
ChefNodeNameBuilder.new(
provider_server_info: server_info,
project_id: 'proj',
env_id: 'dev'
)
}
let(:build_node_name) { node_name_builder.build_node_name!({}) }
def set_mask(mask)
server_info['tags']['cid:node-name-mask'] = mask
end
describe '#build_node_name' do
it 'uses default mask (":project-:env-:instanceid")' do
expect(build_node_name).to eq 'proj-dev-i-fac32c7e'
end
it 'substitutes :project, :env, :instanceid and :instancename' do
set_mask(':project/:env/:instanceid/:instancename')
expect(build_node_name).to eq 'proj/dev/i-fac32c7e/master01'
end
it 'substitutes :time' do
set_mask(':project-:time')
expect(build_node_name).to match /proj-\d+/
end
describe 'substitutes incrementers variables :increment-groupname: with numbers depending on @incrementers_values param' do
it 'starts with 01 for empty hash' do
set_mask('node-:increment-slave:')
expect(node_name_builder.build_node_name!({})).to eq 'node-01'
end
it "continues with next values if hash isn't empty" do
set_mask('node-:increment-slave:')
expect(node_name_builder.build_node_name!({'slave' => nil})).to eq 'node-01'
expect(node_name_builder.build_node_name!({'slave' => 1})).to eq 'node-02'
expect(node_name_builder.build_node_name!({'slave' => 50})).to eq 'node-51'
end
it 'could substitute different incrementers at once' do
set_mask('node-:increment-slave:-:increment-master:')
expect(node_name_builder.build_node_name!({'slave' => 1, 'master' => 3})).to eq 'node-02-04'
end
end
end
end

48
devops-service/spec/workers/stack_bootstrap/prioritized_groups_bootstrapper_spec.rb
View File

@ -1,48 +0,0 @@
require 'lib/executors/stack_executor/prioritized_groups_bootstrapper'
RSpec.describe PrioritizedGroupsBootstrapper, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:groups_bootstrapper) { described_class.new(out, jid, @servers_by_priority) }
before do
@array1 = []; @array2 = []; @array3 = []
@servers_by_priority = {2 => @array2, 1 => @array1, 3 => @array3}
end
describe '#bootstrap_servers_by_priority' do
subject { groups_bootstrapper.bootstrap_servers_by_priority }
it 'bootstraps servers in order by priorities, separately' do
allow(ServersBootstrapper).to receive(:new) { instance_double(ServersBootstrapper, bootstrap_group: []) }
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array3).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array2).ordered
expect(ServersBootstrapper).to receive(:new).with(out, jid, @array1).ordered
expect(subject).to be_ok
end
it 'it returns :bootstrap_error result if error occured during bootstrap' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error), ServersBootstrapper::Result.from_reason(:bootstrap_error)]
}
expect(subject).to be_bootstrap_error
end
it 'it returns :deploy_error result if error occured during deploy' do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
expect(subject).to be_deploy_error
end
it "doesn't bootstrap group if previous one failed" do
allow_any_instance_of(ServersBootstrapper).to receive(:bootstrap_group) {
[ServersBootstrapper::Result.from_reason(:deploy_error)]
}
allow(ServersBootstrapper).to receive(:new).and_call_original
expect(ServersBootstrapper).to receive(:new).once
subject
end
end
end

40
devops-service/spec/workers/stack_bootstrap/servers_bootstrapper_spec.rb
View File

@ -1,40 +0,0 @@
require 'lib/executors/stack_executor/servers_bootstrapper'
RSpec.describe ServersBootstrapper, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
let(:bootstrapper) { described_class.new(out, jid, servers ) }
let(:bootstrap_job_ids) { %w(100 200) }
describe '#bootstrap_group' do
before do
allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
allow(stubbed_connector).to receive(:add_report_subreports)
allow(stubbed_connector).to receive(:report)
allow_any_instance_of(JobWaiter).to receive(:wait) { 0 }
end
it 'start bootstrap workers and add subreports' do
expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
bootstrapper.bootstrap_group
end
it 'returns :ok result if everything ok' do
expect( bootstrapper.bootstrap_group.first.reason ).to eq :ok
end
it 'returns proper error results' do
waiter = instance_double(JobWaiter)
allow(waiter).to receive(:wait).and_return(2, 8)
allow(JobWaiter).to receive(:new) { waiter }
expect( bootstrapper.bootstrap_group.map(&:reason) ).to eq [:bootstrap_error, :deploy_error]
end
it "returns :timeout_reached result if bootstrap and deploy hasn't been finished in 5000 seconds" do
allow_any_instance_of(JobWaiter).to receive(:wait) { raise JobWaiter::TimeoutReached }
expect( bootstrapper.bootstrap_group.first.reason ).to eq :timeout_reached
end
end
end

91
devops-service/spec/workers/stack_bootstrap/stack_creation_waiter_spec.rb
View File

@ -1,91 +0,0 @@
require 'lib/executors/stack_executor/stack_creation_waiter'
RSpec.describe StackCreationWaiter, stubbed_connector: true, init_messages: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:stack) { build(:stack) }
let(:syncer) { described_class.new(stack, out) }
before do
allow(stack).to receive(:sync!)
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
allow(syncer).to receive(:sleep)
allow(stubbed_connector).to receive(:stack_update)
end
def setup_statuses(statuses_array)
statuses = statuses_array.to_enum
allow(stack).to receive(:sync!) {
stack.stack_status = statuses.next
}
end
describe '#sync', stubbed_logger: true do
it 'waits for stack creating to be finished' do
setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).at_least(10).times
expect(stack).to receive(:sync!).at_least(10).times
syncer.sync
end
it 'prints each message only once' do
event1 = {'event_id' => 1, 'timestamp' => 't1'}
event2 = {'event_id' => 2, 'timestamp' => 't2'}
event3 = {'event_id' => 3, 'timestamp' => 't3'}
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).exactly(4).times
expect(out).to receive(:puts).with(/t1/).once.ordered
expect(out).to receive(:puts).with(/t2/).once.ordered
expect(out).to receive(:puts).with(/t3/).once.ordered
syncer.sync
end
it 'updates stack when status is changed' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).exactly(3).times
syncer.sync
end
context 'when stack creating was successful' do
it 'updates stack in DB when stack creating is finished and returns 0' do
setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
expect(syncer.sync).to be_ok
end
end
context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
expect(syncer.sync).to be_stack_rolled_back
end
end
context 'when unkown stack status was found' do
it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(out).to receive(:puts).with(/unknown/)
expect(syncer.sync).to be_unkown_status
end
end
context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(out).to receive(:puts).with(/hasn't been synced/)
expect(syncer.sync).to be_timeout
end
end
context 'when an error occured during syncing', stubbed_logger: true do
it 'returns 5 (:error)' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
allow(stubbed_connector).to receive(:stack_update) { raise }
expect(syncer.sync.code).to eq 5
end
end
end
end

166
devops-service/spec/workers/stack_bootstrap/stack_servers_persister_spec.rb
View File

@ -1,166 +0,0 @@
require 'lib/executors/stack_executor/stack_servers_persister'
RSpec.describe StackServersPersister, stubbed_connector: true do
def info_hash_for_id(id)
{
'id' => id,
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2',
'tags' => {
'cid:priority' => '3',
'Name' => 'server1'
}
}
end
let(:out) { double(:out, puts: nil, flush: nil) }
let(:run_list) { ['role[asd]'] }
let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
let(:project) { build(:project, id: 'name') }
let(:persister) { described_class.new(stack, out) }
let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
let(:new_server_info) { info_hash_for_id('i-new') }
let(:deleted_server_info) { info_hash_for_id('i-deleted') }
let(:persisted_server_info) { info_hash_for_id('i-persisted') }
before do
allow(stubbed_connector).to receive(:project) { project }
allow(stubbed_connector).to receive(:image) {
instance_double(Devops::Model::Image, remote_user: 'user')
}
allow(stubbed_connector).to receive(:server_insert)
allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
allow(stack).to receive(:provider_instance) { provider }
allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
end
describe '#initialize' do
it 'fetches stack servers info' do
expect(provider).to receive(:stack_servers).with(stack)
described_class.new(stack, out)
end
it "doesn't raise error if cid:priority tag is absent" do
new_server_info['tags'].delete('cid:priority')
expect { described_class.new(stack, out) }.not_to raise_error
end
it 'sets proper statuses' do
infos = described_class.new(stack, out).servers_info
expect(infos.length).to eq 2
expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
end
it 'sets deleted servers' do
deleted = described_class.new(stack, out).deleted
expect(deleted.length).to eq 1
expect(deleted.first.id).to eq 'i-deleted'
end
end
describe '#persist_new_servers' do
it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.id).to eq 'i-new'
expect(server.key).to eq 'key'
expect(server.private_ip).to eq '127.0.0.1'
expect(server.public_ip).to eq '127.0.0.2'
end
persister.persist_new_servers
end
it 'takes created_by, run_list and stack attrs from stack' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.created_by).to eq 'root'
expect(server.run_list).to eq run_list
expect(server.stack).to eq 'iamstack'
end
persister.persist_new_servers
end
it 'takes remote_user from image user' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.remote_user).to eq 'user'
end
persister.persist_new_servers
end
it "takes deploy_env from project's deploy_env identifier" do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.deploy_env).to eq 'foo'
end
persister.persist_new_servers
end
it "takes default provider's ssh key if info doesn't contain it" do
allow(provider).to receive(:ssh_key) { 'default_key' }
new_server_info.delete('key_name')
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.key).to eq 'default_key'
end
persister.persist_new_servers
end
it "sets server's run list to empty array if stack's run_list is nil" do
stack.run_list = nil
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.run_list).to eq []
end
persister.persist_new_servers
end
it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-foo-i-new'
end
persister.persist_new_servers
end
it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-server1-123'
end
persister.persist_new_servers
end
it "sets provider and provider account from stack" do
stack.provider_account = 'foo'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.provider).to eq 'ec2'
expect(server.provider_account).to eq 'foo'
end
persister.persist_new_servers
end
describe 'incremented variables' do
it 'substitutes :increment-groupid: with incrementing numbers' do
allow(provider).to receive(:stack_servers) {[
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
]}
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-01-dev'
end.ordered
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-02-dev'
end
persister.persist_new_servers
end
end
end
describe '#just_persisted_by_priority' do
it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
persister.persist_new_servers
result = persister.just_persisted_by_priority
expect(result).to be_a(Hash)
expect(result.size).to eq 1
expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
end
end
end

2
devops-service/spec/workers/stack_bootstrap_worker_spec.rb
View File

@ -16,7 +16,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
}
def bootstrap_result(reason)
ServersBootstrapper::Result.from_reason(reason)
Devops::Executor::StackExecutor::ServersBootstrapper::Result.from_reason(reason)
end
before do

23
devops-service/workers/helpers/job_waiter.rb
View File

@ -1,23 +0,0 @@
class JobWaiter
class TimeoutReached < StandardError; end
INTERVAL = 5
def initialize(job_id, timeout=5000)
@job_id, @timeout = job_id, timeout
end
def wait
(@timeout / INTERVAL).times do
sleep(INTERVAL)
report = ::Devops::Db.connector.report(@job_id)
case report.status
when Worker::STATUS::COMPLETED
return 0
when Worker::STATUS::FAILED
return report.job_result_code
end
end
raise TimeoutReached
end
end