tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

CID-472: sync groups

Browse Source
This commit is contained in:
Anton Chuchkalov 2016-04-04 13:17:07 +03:00
parent 1dfd6fbbcf
commit 497489dd77
33 changed files with 650 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
devops-client/lib/devops-client/handler/stack.rb
View File

@ -41,30 +41,31 @@ class Stack < Handler
end
def create_handler
q = {}
q[:without_bootstrap] = options[:without_bootstrap]
# q[:provider] = options[:provider] || resources_selector.select_available_provider
q[:project] = options[:project] || resources_selector.select_available_project
q[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(q[:project])
env = fetcher.fetch_project(q[:project])['deploy_envs'].detect {|env| env['identifier'] == q[:deploy_env]}
q[:provider] = env['provider']
attrs = {}
attrs[:project] = options[:project] || resources_selector.select_available_project
attrs[:deploy_env] = options[:deploy_env] || resources_selector.select_available_env(attrs[:project])
env = fetcher.fetch_project(attrs[:project])['deploy_envs'].detect {|env| env['identifier'] == attrs[:deploy_env]}
attrs[:provider] = env['provider']
params_filepath = options[:parameters_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.parameters_file'))
if params_filepath.empty?
q[:parameters] = {}
attrs[:parameters] = {}
else
q[:parameters] = JSON.parse(File.read(params_filepath))
attrs[:parameters] = JSON.parse(File.read(params_filepath))
end
tags_filepath = options[:tags_file] || enter_parameter_or_empty(I18n.t('handler.stack.create.tags_file'))
if tags_filepath.empty?
q[:tags] = {}
attrs[:tags] = {}
else
q[:tags] = JSON.parse(File.read(tags_filepath))
attrs[:tags] = JSON.parse(File.read(tags_filepath))
end
json = JSON.pretty_generate(q)
json = JSON.pretty_generate(
without_bootstrap: options[:without_bootstrap] || false,
skip_rollback: options[:skip_rollback] || false,
stack_attributes: attrs
)
if question(I18n.t("handler.stack.question.create")) {puts json}
job_ids = post_body "/stack", json
reports_urls(job_ids)

1
devops-client/lib/devops-client/options/stack_options.rb
View File

@ -31,6 +31,7 @@ class StackOptions < CommonOptions
parser.recognize_option_value(:tags_file)
parser.recognize_option_value(:run_list)
parser.recognize_option_value(:without_bootstrap, type: :switch, switch_value: true)
parser.recognize_option_value(:skip_rollback, type: :switch, switch_value: true)
end
end

22
devops-service/app/api2/handlers/provider_notification.rb Normal file
View File

@ -0,0 +1,22 @@
require 'workers/stack_sync_worker'
require_relative "request_handler"
module Devops
module API2_0
module Handler
class ProviderNotification < RequestHandler
def autoscaling_groups_change(group_id, provider_account)
provider = ::Provider::ProviderFactory.get('ec2', provider_account)
stack_id = provider.stack_id_of_autoscaling_group(group_id)
stack = ::Devops::Db.connector.stack_by_id(stack_id)
jid = Worker.start_async(StackSyncWorker, stack_name: stack.name)
puts jid
jid
end
end
end
end
end

25
devops-service/app/api2/handlers/stack.rb
View File

@ -21,16 +21,16 @@ module Devops
def create_stack
object = parser.create
project = Devops::Db.connector.project(object["project"])
env = project.deploy_env(object["deploy_env"])
stack_attrs = object['stack_attributes']
project = Devops::Db.connector.project(stack_attrs['project'])
env = project.deploy_env(stack_attrs['deploy_env'])
raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil?
object["stack_template"] = env.stack_template
object["owner"] = parser.current_user
object["provider"] = env.provider
object["provider_account"] = env.provider_account
add_stack_attributes(stack_attrs, env, parser)
jid = Worker.start_async(StackBootstrapWorker,
stack_attributes: object
stack_attributes: stack_attrs,
without_bootstrap: object['without_bootstrap'],
skip_rollback: object['skip_rollback']
)
[jid]
end
@ -48,7 +48,7 @@ module Devops
def sync id
stack = self.stack(id)
stack.sync!
stack.sync_status_and_events!
Devops::Db.connector.stack_update(stack)
stack
@ -201,6 +201,15 @@ module Devops
stack.change_stack_template!(stack_template_id)
end
private
def add_stack_attributes(stack_attrs, env, parser)
stack_attrs['stack_template'] = env.stack_template
stack_attrs['owner'] = parser.current_user
stack_attrs['provider'] = env.provider
stack_attrs['provider_account'] = env.provider_account
end
end
end
end

9
devops-service/app/api2/parsers/stack.rb
View File

@ -7,10 +7,11 @@ module Devops
def create
@body ||= create_object_from_json_body
project_name = check_string(@body["project"], "Parameter 'project' must be a not empty string")
env_name = check_string(@body["deploy_env"], "Parameter 'deploy_env' must be a not empty string")
check_string(@body["name"], "Parameter 'name' must be a not empty string", true, false)
list = check_array(@body["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true)
stack_attributes = @body.fetch('stack_attributes')
project_name = check_string(stack_attributes["project"], "Parameter 'project' must be a not empty string")
env_name = check_string(stack_attributes["deploy_env"], "Parameter 'deploy_env' must be a not empty string")
check_string(stack_attributes["name"], "Parameter 'name' must be a not empty string", true, false)
list = check_array(stack_attributes["run_list"], "Parameter 'run_list' is invalid, it should be not empty array of strings", String, true, true)
Validators::Helpers::RunList.new(list).validate! unless list.nil?
@body
end

24
devops-service/app/api2/routes/provider_notification.rb Normal file
View File

@ -0,0 +1,24 @@
module Devops
module API2_0
module Routes
module ProviderNotificationRoutes
def self.registered(app)
# * *Request*
# - method : POST
# Checks if given autoscaling group is launched within stack that is handled by CID.
# If so, starts syncing that stack. Otherwise returns 404 error.
#
# * *Returns* :
# report_id
app.post_with_headers '/provider_notifications/aws/:provider_account/autoscaling_groups/:id/changes', :headers => [:accept] do |provider_account, group_id|
check_privileges("stack", "r")
json Devops::API2_0::Handler::ProviderNotification.new(request).autoscaling_groups_change(group_id, provider_account)
end
end
end
end
end
end

21
devops-service/app/api2/routes/stack.rb
View File

@ -30,16 +30,19 @@ module Devops
# - Content-Type: application/json
# - body :
# {
# "without_bootstrap": null,
# "project": "project_name",
# "deploy_env": "test",
# "provider": "ec2",
# "tags": {
# "tagName": "tagValue"
# },
# "parameters": {
# "KeyName": "Value"
# "stack_attributes": {
# "project": "project_name",
# "deploy_env": "test",
# "provider": "ec2",
# "tags": {
# "tagName": "tagValue"
# },
# "parameters": {
# "KeyName": "Value"
# }
# }
# "without_bootstrap": false,
# "skip_rollback": false
# }
#
# * *Returns* :

3
devops-service/app/devops-api2.rb
View File

@ -19,9 +19,9 @@ module Devops
require_relative "api2/handlers/server"
require_relative "api2/handlers/image"
require_relative "api2/handlers/project"
require_relative "api2/handlers/stack"
require_relative "api2/handlers/stack_template"
require_relative "api2/handlers/provider_notification"
require 'lib/stubber'
end
@ -70,6 +70,7 @@ module Devops
require_relative "api2/routes/stack_template"
require_relative "api2/routes/statistic"
require_relative "api2/routes/report"
require_relative "api2/routes/provider_notification"
routes = Devops::API2_0::Routes.constants.collect{|s| Devops::API2_0::Routes.const_get(s)}.select {|const| const.class == Module}
routes.each do |r|

15
devops-service/db/mongo/connectors/stack.rb
View File

@ -21,6 +21,14 @@ module Connectors
collection.update({"_id" => id}, {"$set" => {"run_list" => run_list}})
end
def lock_persisting_stack(id)
collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => true}})
end
def unlock_persisting_stack(id)
collection.update({'_id' => id}, {'$set' => {'persisting_is_locked' => false}})
end
def stack(name, options={})
query = {'name' => name}.merge(options)
bson = collection.find(query).to_a.first
@ -28,6 +36,13 @@ module Connectors
model_from_bson(bson)
end
def stack_by_id(id)
query = {'_id' => id}
bson = collection.find(query).to_a.first
raise RecordNotFound.new("'#{id}' not found") unless bson
model_from_bson(bson)
end
private
def model_from_bson(bson)

4
devops-service/db/mongo/models/report.rb
View File

@ -12,6 +12,10 @@ module Devops
DEPLOY_STACK_TYPE = 6
DELETE_SERVER_TYPE = 7
EXPIRE_SERVER_TYPE = 8
SYNC_STACK_TYPE = 9
SYSTEM_OWNER = 'SYSTEM'
attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code

20
devops-service/db/mongo/models/stack/stack_base.rb
View File

@ -7,7 +7,7 @@ module Devops
include ModelWithProvider
attr_accessor :id, :name, :project, :deploy_env, :stack_template, :parameters, :events, :owner, :run_list, :stack_status
attr_accessor :parameters, :events, :stack_status, :persisting_is_locked
set_field_validators :id, [::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
@ -56,6 +56,7 @@ module Devops
self.run_list = attrs['run_list'] || []
self.tags = attrs['tags'] || {}
self.stack_status = attrs['stack_status']
self.persisting_is_locked = attrs['persisting_is_locked']
self
end
@ -70,7 +71,8 @@ module Devops
stack_status: stack_status,
owner: owner,
run_list: run_list,
tags: tags
tags: tags,
persisting_is_locked: persisting_is_locked
}.merge(provider_hash)
end
@ -87,7 +89,7 @@ module Devops
provider_instance.delete_stack(self)
end
def sync!
def sync_status_and_events!
self.stack_status = provider_instance.stack_details(self)[:stack_status]
self.events = provider_instance.stack_events(self)
rescue Fog::AWS::CloudFormation::NotFound
@ -111,6 +113,16 @@ module Devops
Devops::Db.connector.stack_template(stack_template)
end
def lock_persisting!
self.persisting_is_locked = true
Devops::Db.connector.lock_persisting_stack(id)
end
def unlock_persisting!
self.persisting_is_locked = false
Devops::Db.connector.unlock_persisting_stack(id)
end
class << self
# attrs should include:
@ -122,7 +134,7 @@ module Devops
def create(attrs, out)
model = new(attrs)
model.create_stack_in_cloud!(out)
model.sync!
model.sync_status_and_events!
model
end

3
devops-service/db/mongo/mongo_connector.rb
View File

@ -17,7 +17,8 @@ class MongoConnector
delegate(
[:images, :image, :image_insert, :image_delete, :image_update] => :images_connector,
[:stack_templates, :stack_template, :stack_template_insert, :stack_template_delete, :stack_template_update] => :stack_templates_connector,
[:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list] => :stacks_connector,
[:stacks, :stack, :stack_insert, :stack_delete, :stack_update, :set_stack_run_list,
:stack_by_id, :lock_persisting_stack, :unlock_persisting_stack] => :stacks_connector,
[:available_images, :add_available_images, :delete_available_images] => :filters_connector,
[:project, :projects_all, :projects, :project_names_with_envs,
:projects_by_image, :projects_by_user, :project_insert, :project_update,

8
devops-service/lib/executors/server_executor.rb
View File

@ -131,7 +131,7 @@ module Devops
res = self.run_hook(:before_bootstrap, @out)
@out << "Done\n"
if @server.private_ip.nil?
@out << "Error: Private IP is null"
@out.puts "Error: Private IP is null"
return error_code(:server_bootstrap_private_ip_unset)
end
ja = {
@ -238,13 +238,15 @@ module Devops
@out << "Server #{@server.chef_node_name} is created"
else
@out.puts "Can not find client or node on chef-server"
roll_back
@out.puts "Skip rollback because :skip_rollback option is set"
roll_back unless options[:skip_rollback]
@out.flush
mongo.server_delete @server.id
return error_code(:server_not_in_chef_nodes)
end
else
roll_back
@out.puts "Skip rollback because :skip_rollback option is set"
roll_back unless options[:skip_rollback]
mongo.server_delete @server.id
msg = "Failed while bootstraping server with id '#{@server.id}'\n"
msg << "Bootstraping operation result was #{bootstrap_status}"

79
devops-service/lib/executors/stack_executor.rb
View File

@ -4,6 +4,7 @@ module Devops; module Executor; class StackExecutor; end; end; end # predeclare
require_relative "stack_executor/stack_creation_waiter"
require_relative "stack_executor/stack_servers_persister"
require_relative "stack_executor/prioritized_groups_bootstrapper"
require_relative "stack_executor/stack_servers_fetcher"
module Devops
module Executor
@ -14,38 +15,52 @@ module Devops
def initialize(options)
@out = options.fetch(:out)
@stack = options[:stack]
end
def wait_till_stack_is_created
wait_result = StackCreationWaiter.new(stack, out).sync
if wait_result.ok?
puts_and_flush "\nStack '#{stack.name}' has been created"
true
else
puts_and_flush "An error ocurred during stack creation: #{wait_result.reason}"
false
end
self.just_persisted_by_priority = {}
end
def create_stack(stack_attrs)
stack_attrs.merge!('persisting_is_locked' => true)
@stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, out)
mongo.stack_insert(@stack)
end
def persist_stack_servers
puts_and_flush 'Start saving stack servers into CID database.'
persister = StackServersPersister.new(stack, out)
persister.persist_new_servers
puts_and_flush "Stack servers have been saved."
{
just_persisted_by_priority: persister.just_persisted_by_priority,
deleted: persister.deleted
}
def wait_till_stack_is_created
wait_result = waiter.wait
stack.unlock_persisting!
wait_result.ok?
end
def bootstrap_servers_by_priority(servers_by_priorities, jid)
PrioritizedGroupsBootstrapper.new(out, jid, servers_by_priorities).bootstrap_servers_by_priority
def persist_new_servers
reload_stack
raise 'It seems that stack is synchronizing at the moment' if stack.persisting_is_locked
begin
stack.lock_persisting!
fetcher.new_servers_by_priorities.each do |priority, provider_infos|
servers = provider_infos.map {|info| persister.persist(info) }
just_persisted_by_priority[priority] = servers
end
just_persisted_by_priority.values.flatten.each do |server|
puts_and_flush "\n\nPersisted server #{server.id}: #{JSON.pretty_generate(server.to_hash)}"
end
ensure
stack.unlock_persisting!
end
end
def bootstrap_just_persisted(jid)
puts_and_flush "Bootstrapping just persisted servers" if just_persisted_by_priority.values.flatten.any?
just_persisted_by_priority.each do |priority, servers|
puts_and_flush "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
PrioritizedGroupsBootstrapper.new(out, jid, just_persisted_by_priority).bootstrap_servers_by_priority
end
def delete_stale_servers
fetcher.stale_servers.each do |server|
server_executor = Devops::Executor::ServerExecutor.new(server, out)
server_executor.delete_server
end
end
def delete_stack
@ -56,10 +71,28 @@ module Devops
private
attr_accessor :just_persisted_by_priority
def mongo
Devops::Db.connector
end
def reload_stack
@stack = mongo.stack(@stack.name)
end
def fetcher
@fetcher ||= StackServersFetcher.new(stack, out)
end
def persister
@persister ||= StackServersPersister.new(@stack)
end
def waiter
StackCreationWaiter.new(stack, out)
end
end
end
end

3
devops-service/lib/executors/stack_executor/chef_node_name_builder.rb
View File

@ -25,7 +25,8 @@ class Devops::Executor::StackExecutor
def initialize(attrs)
@server_info = attrs[:provider_server_info]
@project, @env = attrs[:project_id], attrs[:env_id]
@mask = @server_info['tags']['cid:node-name-mask'] || DEFAULT_MASK
@mask = @server_info['tags']['cid:node-name-mask'] if @server_info['tags']
@mask ||= DEFAULT_MASK
end

5
devops-service/lib/executors/stack_executor/servers_bootstrapper.rb
View File

@ -28,7 +28,7 @@ class Devops::Executor::StackExecutor
@server_bootstrap_jobs.map do |server_id, job_id|
result = wait_for_bootstrap_job(job_id)
puts_and_flush Devops::Messages.t("worker.servers_bootstrapper.bootstrap_servers.#{result.reason}", server_id: server_id, job_id: job_id)
puts_and_flush Devops::Messages.t("stack_executor.servers_bootstrapper.result.#{result.reason}", server_id: server_id, job_id: job_id)
result
end
end
@ -41,7 +41,8 @@ class Devops::Executor::StackExecutor
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
owner: server.created_by,
skip_rollback: true
)
@out.puts "Start bootstraping server '#{server.id}' job (job id: #{job_id})."
@server_bootstrap_jobs[server.id] = job_id

6
devops-service/lib/executors/stack_executor/stack_creation_waiter.rb
View File

@ -22,12 +22,12 @@ class Devops::Executor::StackExecutor
@sync_result = nil
end
def sync
def wait
puts_and_flush "Syncing stack '#{stack.id}'..."
sleep_times.detect do |sleep_time|
sleep sleep_time
stack.sync!
stack.sync_status_and_events!
print_new_events
update_stack_status if stack_status_changed?
@ -75,7 +75,7 @@ class Devops::Executor::StackExecutor
end
def print_result_message
puts_and_flush Devops::Messages.t("stack_creation_waiter.result.#{@sync_result.reason}",
puts_and_flush Devops::Messages.t("stack_executor.stack_creation_waiter.result.#{@sync_result.reason}",
stack_id: stack.id, status: stack.stack_status, seconds: sleep_times.inject(&:+))
end

97
devops-service/lib/executors/stack_executor/stack_servers_fetcher.rb Normal file
View File

@ -0,0 +1,97 @@
class Devops::Executor::StackExecutor
class StackServersFetcher
include PutsAndFlush
attr_reader :out
NEW = :new
PERSISTED = :persisted
STALE = :stale
def initialize(stack, out)
@stack, @out = stack, out
@by_state = {NEW => [], PERSISTED => [], STALE => []}
fetch
end
def new_servers_by_priorities
servers_with_priority = {}
@by_state[NEW].each do |provider_info|
priority = priority_from_info(provider_info)
servers_with_priority[priority] ||= []
servers_with_priority[priority] << provider_info
end
servers_with_priority
end
def servers_to_persist
@by_state[NEW]
end
def already_persisted_servers
@by_state[PERSISTED]
end
def stale_servers
@by_state[STALE]
end
private
def fetch
@servers_info = @stack.provider_instance.stack_servers(@stack)
divide_into_states
output_fetched_servers
end
def priority_from_info(provider_info)
if provider_info['tags']
priority = provider_info['tags']['cid:priority'].to_i
else
0
end
end
def divide_into_states
persisted = Devops::Db.connector.stack_servers(@stack.name)
persisted_ids = persisted.map(&:id)
in_cloud_ids = @servers_info.map {|info| info['id']}
new_ids = in_cloud_ids - persisted_ids
deleted_ids = persisted_ids - in_cloud_ids
@servers_info.each do |info|
if new_ids.include?(info['id'])
@by_state[NEW] << info
else
@by_state[PERSISTED] << info
end
end
@by_state[STALE] = persisted.select { |server| deleted_ids.include?(server.id) }
end
# Do not move it to stack executor till set events handling properly.
# For now there may be already persisted servers on creation because of too early events from aws.
def output_fetched_servers
if servers_to_persist.any?
out.puts 'Servers to persist:'
servers_to_persist.each { |info| out.puts JSON.pretty_generate(info) }
else
out.puts 'There are no servers to persist.'
end
if already_persisted_servers.any?
out.puts "\nAlready persisted servers:"
already_persisted_servers.each { |info| out.puts JSON.pretty_generate(info) }
end
if stale_servers.any?
out.puts "\nStale servers:"
stale_servers.each { |server| out.puts JSON.pretty_generate(server.to_hash) }
else
out.puts "\nThere are no stale servers."
end
out.flush
end
end
end

87
devops-service/lib/executors/stack_executor/stack_servers_persister.rb
View File

@ -1,102 +1,43 @@
require_relative 'chef_node_name_builder'
# Fetches info about stack servers from provider and then persist them in mongo.
class Devops::Executor::StackExecutor
class StackServersPersister
include PutsAndFlush
attr_reader :stack, :out, :deleted, :servers_info
attr_reader :stack
NEW = 'new'
DELETED = 'deleted'
PERSISTED = 'persisted'
JUST_PERSISTED = 'just_persisted'
def initialize(stack, out)
@stack, @out = stack, out
def initialize(stack)
@stack = stack
@project = mongo.project(stack.project)
@deploy_env = @project.deploy_env(stack.deploy_env)
fetch_provider_servers_info
set_servers_states
end
def persist_new_servers
with_state(NEW).each do |info|
info[:server] = persist_stack_server(info[:provider_info])
info[:state] = JUST_PERSISTED
puts_and_flush "Persisted server with id '#{info[:id]}' and name '#{info[:server].chef_node_name}'"
end
end
# returns: { priority_as_integer => [Servers] }
def just_persisted_by_priority
stack_servers_with_priority = {}
with_state(JUST_PERSISTED).each do |info|
stack_servers_with_priority[info[:priority]] ||= []
stack_servers_with_priority[info[:priority]] << info[:server]
end
stack_servers_with_priority
end
private
def fetch_provider_servers_info
@servers_info = stack.provider_instance.stack_servers(stack).map do |provider_info|
{
id: provider_info['id'],
provider_info: provider_info,
priority: provider_info['tags']['cid:priority'].to_i
}
end
end
def set_servers_states
persisted = Devops::Db.connector.stack_servers(stack.id)
persisted_ids = persisted.map(&:id)
in_cloud_ids = @servers_info.map {|info| info[:id]}
new_ids = in_cloud_ids - persisted_ids
deleted_ids = persisted_ids - in_cloud_ids
@servers_info.each do |info|
if new_ids.include?(info[:id])
info[:state] = NEW
else
info[:state] = PERSISTED
end
end
@deleted = persisted.select { |server| deleted_ids.include?(server.id) }
end
def with_state(state)
@servers_info.select { |info| info[:state] == state }
end
# takes a hash, returns Server model
def persist_stack_server(server_info)
def persist(provider_info)
server_attrs = {
'_id' => server_info['id'],
'chef_node_name' => get_name_builder(server_info).build_node_name!(incrementers_values),
'_id' => provider_info['id'],
'chef_node_name' => get_name_builder(provider_info).build_node_name!(incrementers_values),
'created_by' => stack.owner,
'deploy_env' => @deploy_env.identifier,
'key' => server_info['key_name'] || stack.provider_instance.ssh_key,
'key' => provider_info['key_name'] || stack.provider_instance.ssh_key,
'project' => @project.id,
'provider' => @stack.provider,
'provider_account' => @stack.provider_account,
'remote_user' => mongo.image(@deploy_env.image).remote_user,
'private_ip' => server_info['private_ip'],
'public_ip' => server_info['public_ip'],
'private_ip' => provider_info['private_ip'],
'public_ip' => provider_info['public_ip'],
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# here custom insert method is used and it doesn't return server model
server
end
def get_name_builder(server_info)
private
def get_name_builder(provider_info)
ChefNodeNameBuilder.new(
provider_server_info: server_info,
provider_server_info: provider_info,
project_id: @project.id,
env_id: @deploy_env.identifier,
owner: stack.owner

9
devops-service/lib/sync_stack_lambda.py
View File

@ -8,10 +8,11 @@ import urllib2
def lambda_handler(event, context):
print("Received event: " + json.dumps(event, indent=2))
stack_id = event['Records'][0]['Sns']['Message']
url = 'http://example.com/v2.0/stack/%s/sync' % stack_id
username = 'root'
password = 'pass'
host = 'http://CHANGE_ME'
group_name = event['detail']['AutoScalingGroupName']
url = '%s/v2.0/provider_notifications/aws/first/autoscaling_groups/%s/changes' % (host, group_name)
username = 'CHANGE_ME'
password = 'CHANGE_ME'
request = urllib2.Request(url, '')
base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')

27
devops-service/messages/en.yml
View File

@ -13,17 +13,26 @@ en:
Stack was launched, but an error occured during deploying stack servers.
You can redeploy stack after fixing the error.
timeout_reached: Bootstrap or deploy wasn't completed due to timeout.
stack_sync:
bootstrap_result:
ok: "Stack has been successfully synced."
bootstrap_error: An error occured during new stack servers bootstrapp.
deploy_error: An error occured during new stack servers deploy.
timeout_reached: Bootstrap or deploy of new servers wasn't completed due to timeout.
stack_executor:
servers_bootstrapper:
bootstrap_servers:
result:
ok: "Server '%{server_id}' has been bootstraped (job %{job_id})."
timeout_reached: "Waiting for bootstrapping '%{server_id}' (job %{job_id}) halted: timeout reached."
bootstrap_error: "Server '%{server_id}' bootstrapping failed (job %{job_id})."
deploy_error: "Server '%{server_id}' deploy failed (job %{job_id})."
stack_creation_waiter:
result:
ok: "Stack '%{stack_id}' status is now %{status}"
stack_rolled_back: "Stack '%{stack_id}' status is now %{status}"
stack_deleted: "Stack '%{stack_id}' status is now %{status}"
stack_not_found: "Stack '%{stack_id}' status is now %{status}"
unkown_status: "Unknown stack status: '%{status}'"
timeout: "Stack hasn't been synced in %{seconds} seconds."
stack_creation_waiter:
result:
ok: |
Stack '%{stack_id}' status is now %{status}
Stack has been created
stack_rolled_back: "Stack '%{stack_id}' status is now %{status}"
stack_deleted: "Stack '%{stack_id}' status is now %{status}"
stack_not_found: "Stack '%{stack_id}' status is now %{status}"
unkown_status: "Unknown stack status: '%{status}'"
timeout: "Stack hasn't been synced in %{seconds} seconds."

42
devops-service/providers/ec2.rb
View File

@ -211,10 +211,6 @@ module Provider
connection_compute(connection_options)
end
def cloud_formation
@cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options)
end
def create_stack(stack, out)
begin
out << "Creating stack for project '#{stack.project}' and environment '#{stack.deploy_env}'...\n"
@ -291,26 +287,20 @@ module Provider
cloud_formation.describe_stack_events(stack.name).body['StackEvents'].map{|se| {"timestamp" => se["Timestamp"], "stack_name" => se["StackName"], "stack_id" => se["StackId"], "event_id" => se["EventId"], "reason" => se["ResourceStatusReason"], "status" => se["ResourceStatus"]}}.sort{|se1, se2| se1["timestamp"] <=> se2["timestamp"]}
end
# не работает, не используется
# def stack_resource(stack, resource_id)
# physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id
# compute.servers.get(physical_id)
# end
def stack_servers(stack)
# orchestration.describe_stack_resources возвращает мало информации
resources = compute.describe_instances(
'tag-key' => 'aws:cloudformation:stack-id',
'tag-value' => stack.id
).body["reservationSet"]
# В ресурсах могут лежать не только конкретные инстансы, но и MasterNodesGroup, которые управляют
# несколькими инстансами. Обрабатываем эту ситуацию.
# There may be not only instances but autoscaling groups with several instances.
# Handle such situations.
instances = resources.map { |resource| resource["instancesSet"] }.flatten
instances.delete_if {|i| i['instanceState']['name'] == 'terminated'}
instances.map do |instance|
{
# 'name' => instance["tagSet"]["Name"],
'name' => [stack.name, instance_name(instance)].join('-'),
'id' => instance["instanceId"],
'key_name' => instance["keyName"],
@ -325,6 +315,18 @@ module Provider
"stack-#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}".gsub('_', '-')
end
def stack_id_of_autoscaling_group(id)
response = auto_scaling.describe_auto_scaling_groups('AutoScalingGroupNames' => [id])
tags = response.body['DescribeAutoScalingGroupsResult']['AutoScalingGroups'].first['Tags']
stack_id_tag = tags.find {|t| t['Key'] == 'aws:cloudformation:stack-id'}
if stack_id_tag
stack_id_tag['Value']
end
rescue StandardError => e
puts "An error occured while analyzing group '#{id}': #{[e.name, e.message, e.backtrace].join("\n")}"
nil
end
def describe_vpcs
self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } }
end
@ -391,14 +393,18 @@ module Provider
r
end
def orchestration
@orchestration ||= Fog::AWS::CloudFormation.new(connection_options)
end
def storage
@storage ||= Fog::Storage.new(connection_options)
end
def cloud_formation
@cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options)
end
def auto_scaling
@auto_scaling ||= Fog::AWS::AutoScaling.new(connection_options)
end
def stack_templates_bucket
bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplatesnibrdev'
bucket = storage.directories.get(bucket_name)

24
devops-service/spec/executors/stack_executor/stack_creation_waiter_spec.rb
View File

@ -7,7 +7,7 @@ class Devops::Executor::StackExecutor
let(:syncer) { described_class.new(stack, out) }
before do
allow(stack).to receive(:sync!)
allow(stack).to receive(:sync_status_and_events!)
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
allow(syncer).to receive(:sleep)
allow(stubbed_connector).to receive(:stack_update)
@ -15,17 +15,17 @@ class Devops::Executor::StackExecutor
def setup_statuses(statuses_array)
statuses = statuses_array.to_enum
allow(stack).to receive(:sync!) {
allow(stack).to receive(:sync_status_and_events!) {
stack.stack_status = statuses.next
}
end
describe '#sync', stubbed_logger: true do
describe '#wait', stubbed_logger: true do
it 'waits for stack creating to be finished' do
setup_statuses(['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE'])
expect(syncer).to receive(:sleep).at_least(10).times
expect(stack).to receive(:sync!).at_least(10).times
syncer.sync
expect(stack).to receive(:sync_status_and_events!).at_least(10).times
syncer.wait
end
it 'prints each message only once' do
@ -39,13 +39,13 @@ class Devops::Executor::StackExecutor
expect(out).to receive(:puts).with(/t1/).once.ordered
expect(out).to receive(:puts).with(/t2/).once.ordered
expect(out).to receive(:puts).with(/t3/).once.ordered
syncer.sync
syncer.wait
end
it 'updates stack when status is changed' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).exactly(3).times
syncer.sync
syncer.wait
end
context 'when stack creating was successful' do
@ -53,7 +53,7 @@ class Devops::Executor::StackExecutor
setup_statuses(['CREATE_COMPLETE'])
expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(out).to receive(:puts).with(/CREATE_COMPLETE/)
expect(syncer.sync).to be_ok
expect(syncer.wait).to be_ok
end
end
@ -61,7 +61,7 @@ class Devops::Executor::StackExecutor
it 'returns 1 (:stack_rolled_back)' do
setup_statuses(['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'])
expect(out).to receive(:puts).with(/ROLLBACK_COMPLETE/)
expect(syncer.sync).to be_stack_rolled_back
expect(syncer.wait).to be_stack_rolled_back
end
end
@ -69,7 +69,7 @@ class Devops::Executor::StackExecutor
it 'returns 2 (:unkown_status)' do
setup_statuses(['CREATE_IN_PROGRESS', 'unknown'])
expect(out).to receive(:puts).with(/unknown/)
expect(syncer.sync).to be_unkown_status
expect(syncer.wait).to be_unkown_status
end
end
@ -77,7 +77,7 @@ class Devops::Executor::StackExecutor
it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(out).to receive(:puts).with(/hasn't been synced/)
expect(syncer.sync).to be_timeout
expect(syncer.wait).to be_timeout
end
end
@ -85,7 +85,7 @@ class Devops::Executor::StackExecutor
it 'returns 5 (:error)' do
setup_statuses(['CREATE_IN_PROGRESS', 'CREATE_COMPLETE'])
allow(stubbed_connector).to receive(:stack_update) { raise }
expect(syncer.sync.code).to eq 5
expect(syncer.wait.code).to eq 5
end
end
end

55
devops-service/spec/executors/stack_executor/stack_servers_fetcher_spec.rb Normal file
View File

@ -0,0 +1,55 @@
require 'lib/executors/stack_executor/stack_servers_fetcher'
class Devops::Executor::StackExecutor
RSpec.describe StackServersFetcher, stubbed_connector: true do
def info_hash_for_id(id)
{
'id' => id,
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2',
'tags' => {
'cid:priority' => '3',
'Name' => 'server1'
}
}
end
let(:out) { double('out', puts: nil, flush: nil) }
let(:stack) { build(:stack_ec2) }
let(:fetcher) { described_class.new(stack, out) }
let(:new_server_info) { info_hash_for_id('i-new') }
let(:persisted_server_info) { info_hash_for_id('i-persisted') }
let(:stale_server) { build(:server, id: 'i-deleted') }
before do
allow(stubbed_connector).to receive(:stack_servers) { [stale_server, build(:server, id: 'i-persisted')] }
provider = instance_double(Provider::Ec2, name: 'ec2')
allow(stack).to receive(:provider_instance) { provider }
allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
end
describe '#new_servers_by_priorities' do
let(:new_servers_by_priorities) { fetcher.new_servers_by_priorities }
it 'returns new servers info by :new key divided by priorities' do
expect(new_servers_by_priorities).to be_a(Hash)
expect(new_servers_by_priorities.length).to eq 1
expect(new_servers_by_priorities[3].length).to eq 1
expect(new_servers_by_priorities[3].first).to eq new_server_info
end
it "sets priority to 0 if it's absent" do
new_server_info['tags'].delete('cid:priority')
expect(new_servers_by_priorities[0].first).to eq new_server_info
end
end
describe '#stale_servers' do
it 'returns array of stale servers' do
expect(fetcher.stale_servers).to match_array [stale_server]
end
end
end
end

136
devops-service/spec/executors/stack_executor/stack_servers_persister_spec.rb
View File

@ -2,9 +2,9 @@ require 'lib/executors/stack_executor/stack_servers_persister'
class Devops::Executor::StackExecutor
RSpec.describe StackServersPersister, stubbed_connector: true do
def info_hash_for_id(id)
let(:provider_info) {
{
'id' => id,
'id' => 'i-new',
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
@ -14,154 +14,88 @@ class Devops::Executor::StackExecutor
'Name' => 'server1'
}
}
end
}
let(:out) { double(:out, puts: nil, flush: nil) }
let(:run_list) { ['role[asd]'] }
let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) }
let(:project) { build(:project, id: 'name') }
let(:persister) { described_class.new(stack, out) }
let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
let(:new_server_info) { info_hash_for_id('i-new') }
let(:deleted_server_info) { info_hash_for_id('i-deleted') }
let(:persisted_server_info) { info_hash_for_id('i-persisted') }
let(:persister) { described_class.new(stack) }
before do
allow(stubbed_connector).to receive(:project) { project }
allow(stubbed_connector).to receive(:image) {
instance_double(Devops::Model::Image, remote_user: 'user')
}
allow(stubbed_connector).to receive(:server_insert)
allow(stubbed_connector).to receive(:stack_servers) { [build(:server, id: 'i-deleted'), build(:server, id: 'i-persisted')] }
allow(stubbed_connector).to receive(:server_insert) {|server| server}
allow(stack).to receive(:provider_instance) { provider }
allow(provider).to receive(:stack_servers) {[new_server_info, persisted_server_info]}
end
describe '#initialize' do
it 'fetches stack servers info' do
expect(provider).to receive(:stack_servers).with(stack)
described_class.new(stack, out)
describe '#persist' do
let(:persist) { persister.persist(provider_info) }
it 'persist record to mongo' do
expect(stubbed_connector).to receive(:server_insert)
persist
end
it "doesn't raise error if cid:priority tag is absent" do
new_server_info['tags'].delete('cid:priority')
expect { described_class.new(stack, out) }.not_to raise_error
end
it 'sets proper statuses' do
infos = described_class.new(stack, out).servers_info
expect(infos.length).to eq 2
expect(infos.find{|t| t[:id] == 'i-persisted'}[:state]).to eq 'persisted'
expect(infos.find{|t| t[:id] == 'i-new'}[:state]).to eq 'new'
end
it 'sets deleted servers' do
deleted = described_class.new(stack, out).deleted
expect(deleted.length).to eq 1
expect(deleted.first.id).to eq 'i-deleted'
end
end
describe '#persist_new_servers' do
it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.id).to eq 'i-new'
expect(server.key).to eq 'key'
expect(server.private_ip).to eq '127.0.0.1'
expect(server.public_ip).to eq '127.0.0.2'
end
persister.persist_new_servers
expect(persist.to_hash).to include(
'id' => 'i-new',
'key' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2'
)
end
it 'takes created_by, run_list and stack attrs from stack' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.created_by).to eq 'root'
expect(server.run_list).to eq run_list
expect(server.stack).to eq 'iamstack'
end
persister.persist_new_servers
expect(persist.to_hash).to include(
'created_by' => 'root',
'run_list' => run_list,
'stack' => 'iamstack'
)
end
it 'takes remote_user from image user' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.remote_user).to eq 'user'
end
persister.persist_new_servers
expect(persist.remote_user).to eq 'user'
end
it "takes deploy_env from project's deploy_env identifier" do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.deploy_env).to eq 'foo'
end
persister.persist_new_servers
expect(persist.deploy_env).to eq 'foo'
end
it "takes default provider's ssh key if info doesn't contain it" do
allow(provider).to receive(:ssh_key) { 'default_key' }
new_server_info.delete('key_name')
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.key).to eq 'default_key'
end
persister.persist_new_servers
provider_info.delete('key_name')
expect(persist.key).to eq 'default_key'
end
it "sets server's run list to empty array if stack's run_list is nil" do
stack.run_list = nil
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.run_list).to eq []
end
persister.persist_new_servers
expect(persist.run_list).to eq []
end
it 'build chef_node_name with default mask ":project-:env-:instanceid"' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-foo-i-new'
end
persister.persist_new_servers
expect(persist.chef_node_name).to eq 'name-foo-i-new'
end
it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
new_server_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-server1-123'
end
persister.persist_new_servers
provider_info['tags']['cid:node-name-mask'] = ':project-:instancename-123'
expect(persist.chef_node_name).to eq 'name-server1-123'
end
it "sets provider and provider account from stack" do
stack.provider_account = 'foo'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.provider).to eq 'ec2'
expect(server.provider_account).to eq 'foo'
end
persister.persist_new_servers
expect(persist.provider_account).to eq 'foo'
end
describe 'incremented variables' do
it 'substitutes :increment-groupid: with incrementing numbers' do
allow(provider).to receive(:stack_servers) {[
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'},
{'id' => 'server1', 'tags' => {'cid:node-name-mask' => 'node-:increment-group1:-dev'}, 'key_name' => 'key'}
]}
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-01-dev'
end.ordered
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'node-02-dev'
end
persister.persist_new_servers
provider_info['tags']['cid:node-name-mask'] = 'node-:increment-group1:-dev'
expect(persister.persist(provider_info).chef_node_name).to eq 'node-01-dev'
expect(persister.persist(provider_info).chef_node_name).to eq 'node-02-dev'
end
end
end
describe '#just_persisted_by_priority' do
it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
persister.persist_new_servers
result = persister.just_persisted_by_priority
expect(result).to be_a(Hash)
expect(result.size).to eq 1
expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
end
end
end
end

105
devops-service/spec/executors/stack_executor_spec.rb
View File

@ -6,51 +6,119 @@ class Devops::Executor::StackExecutor
let(:stack) { build(:stack) }
let(:executor_without_stack) { described_class.new(out: out) }
let(:executor_with_stack) { described_class.new(out: out, stack: stack) }
let(:new_servers_by_priorities) {
{
0 => [{id: 1}],
2 => [{id: 2}]
}
}
let(:just_persisted_by_priority) {
{
0 => [double('info 1', id: 1)],
2 => [double('info 2', id: 2)]
}
}
let(:fetcher) {
instance_double(StackServersFetcher,
new_servers_by_priorities: new_servers_by_priorities,
stale_servers: build_list(:server, 2),
fetch: nil
)
}
before do
allow(executor_with_stack).to receive(:fetcher) { fetcher }
end
describe '#wait_till_stack_is_created' do
let(:waiter) { instance_double(StackCreationWaiter, wait: double("creation_result", ok?: true)) }
before do
allow(executor_with_stack).to receive(:waiter) { waiter }
allow(stack).to receive(:unlock_persisting!)
end
it 'waites till stack is created, then fetches stack servers and unlocks stack persisting' do
expect(waiter).to receive(:wait).ordered
expect(fetcher).to receive(:fetch).ordered
expect(stack).to receive(:unlock_persisting!).ordered
executor_with_stack.wait_till_stack_is_created
end
it "return true if syncer returns ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: true) }
expect(executor_with_stack.wait_till_stack_is_created).to be true
end
it "return false if syncer returns not ok" do
allow_any_instance_of(StackCreationWaiter).to receive(:sync) { double("creation_result", ok?: false, reason: '') }
allow(waiter).to receive(:wait) { double("creation_result", ok?: false, reason: '') }
expect(executor_with_stack.wait_till_stack_is_created).to be false
end
end
describe '#create_stack', stubbed_connector: true do
describe '#create_stack' do
before { expect(stubbed_connector).to receive(:stack_insert) }
it 'initiate creation in cloud and persists stack' do
expect(Devops::Model::StackFactory).to receive(:create).with('ec2', instance_of(Hash), out)
expect(stubbed_connector).to receive(:stack_insert)
executor_with_stack.create_stack({'provider' => 'ec2'})
end
it 'locks persisting on create' do
expect(Devops::Model::StackFactory).to receive(:create) do |_, params|
expect(params).to include('persisting_is_locked' => true)
end
executor_with_stack.create_stack({'provider' => 'ec2'})
end
end
describe '#persist_stack_servers' do
let(:persister) {
instance_double(StackServersPersister,
persist_new_servers: nil, just_persisted_by_priority: nil, deleted: nil
)
}
before { allow(StackServersPersister).to receive(:new).and_return(persister) }
describe '#persist_new_servers' do
let(:persister) { instance_double(StackServersPersister, persist: build(:server)) }
it 'calls StackServersPersister#persist_new_servers' do
expect(persister).to receive(:persist_new_servers)
executor_with_stack.persist_stack_servers
before do
allow(executor_with_stack).to receive(:persister) { persister }
allow(stack).to receive(:lock_persisting!)
allow(stack).to receive(:unlock_persisting!)
allow(stubbed_connector).to receive(:stack) { stack }
end
it 'returns hash with :just_persisted_by_priority and :deleted keys' do
expect(executor_with_stack.persist_stack_servers).to include(just_persisted_by_priority: nil, deleted: nil)
it 'calls StackServersPersister#persist for each server' do
expect(persister).to receive(:persist).exactly(2).times
executor_with_stack.persist_new_servers
end
it 'locks persisting of a stack before start and unlocks after finish' do
expect(stack).to receive(:lock_persisting!).ordered
expect(persister).to receive(:persist).ordered
expect(stack).to receive(:unlock_persisting!).ordered
executor_with_stack.persist_new_servers
end
it 'unlocks persisting even in case of failures' do
allow(persister).to receive(:persist) { raise }
expect(stack).to receive(:unlock_persisting!)
expect { executor_with_stack.persist_new_servers }.to raise_error StandardError
end
end
describe '#bootstrap_servers_by_priority' do
describe '#bootstrap_just_persisted' do
it 'calls PrioritizedGroupsBootstrapper#bootstrap_servers_by_priority' do
result = double('bootstrap_result')
allow(executor_with_stack).to receive(:just_persisted_by_priority) { just_persisted_by_priority }
allow_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority) { result }
expect_any_instance_of(PrioritizedGroupsBootstrapper).to receive(:bootstrap_servers_by_priority)
expect(executor_with_stack.bootstrap_servers_by_priority({}, 1000)).to eq result
expect(executor_with_stack.bootstrap_just_persisted(1000)).to eq result
end
end
describe '#delete_stale_servers' do
it 'builds server executor per stale server and properly delete them' do
executor1 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil)
executor2 = instance_double(Devops::Executor::ServerExecutor, delete_server: nil)
allow(Devops::Executor::ServerExecutor).to receive(:new).and_return(executor1, executor2)
expect(executor1).to receive(:delete_server).ordered
expect(executor2).to receive(:delete_server).ordered
executor_with_stack.delete_stale_servers
end
end
@ -61,7 +129,6 @@ class Devops::Executor::StackExecutor
expect(stubbed_connector).to receive(:stack_delete).ordered
executor_with_stack.delete_stack
end
end
end
end

10
devops-service/spec/models/stack/stack_ec2_spec.rb
View File

@ -80,7 +80,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
end
describe '#sync!' do
describe '#sync_status_and_events!' do
let(:fresh_events) { double('fresh_events') }
let(:provider) {
instance_double('Provider::Ec2',
@ -95,12 +95,12 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
it "get fresh stack details and updates stack status" do
expect(provider).to receive(:stack_details)
expect {stack.sync!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE')
expect {stack.sync_status_and_events!}.to change {stack.stack_status}.from(nil).to('CREATE_COMPLETE')
end
it "get fresh stack events and stores it in @events" do
expect(stack).to receive_message_chain('provider_instance.stack_events')
expect {stack.sync!}.to change {stack.events}.from(nil).to(fresh_events)
expect {stack.sync_status_and_events!}.to change {stack.events}.from(nil).to(fresh_events)
end
end
@ -140,7 +140,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
before do
allow_any_instance_of(described_class).to receive(:create_stack_in_cloud!)
allow_any_instance_of(described_class).to receive(:sync!)
allow_any_instance_of(described_class).to receive(:sync_status_and_events!)
end
it "returns instance of #{described_class.name}" do
@ -153,7 +153,7 @@ RSpec.describe Devops::Model::StackEc2, type: :model do
end
it 'synchronizes details' do
expect_any_instance_of(described_class).to receive(:sync!)
expect_any_instance_of(described_class).to receive(:sync_status_and_events!)
subject
end
end

23
devops-service/spec/workers/stack_bootstrap_worker_spec.rb
View File

@ -3,15 +3,16 @@ require 'lib/executors/stack_executor'
RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, init_messages: true do
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
let(:worker) { described_class.new }
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs, 'without_bootstrap' => true) }
let(:executor) {
instance_double(Devops::Executor::StackExecutor,
wait_till_stack_is_created: true,
create_stack: Devops::Model::StackEc2.new(stack_attrs),
persist_stack_servers: nil,
delete_stack: nil
persist_new_servers: nil,
delete_stack: nil,
bootstrap_just_persisted: bootstrap_result(:ok)
)
}
@ -22,8 +23,6 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
before do
allow(worker).to receive(:update_report)
allow(worker).to receive(:executor) { executor }
allow(worker).to receive(:persist_stack_servers) { {1 => build_list(:server, 2)} }
allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:ok) }
end
@ -39,7 +38,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
it 'updates report about operation, creates stack and persists stack servers' do
expect(worker).to receive(:update_report).ordered
expect(executor).to receive(:create_stack).ordered
expect(worker).to receive(:persist_stack_servers).ordered
expect(executor).to receive(:persist_new_servers).ordered
perform_without_bootstrap
end
@ -55,7 +54,7 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
context 'if without_bootstrap is true' do
it "doesn't bootstrap servers" do
expect(worker).not_to receive(:bootstrap_servers_by_priority)
expect(worker).not_to receive(:bootstrap_or_rollback_if_failed)
perform_without_bootstrap
end
@ -70,27 +69,27 @@ RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true, ini
end
it 'rollbacks stack and returns 2 when a known error occured during servers bootstrap' do
allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:bootstrap_error) }
allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:bootstrap_error) }
expect(executor).to receive(:delete_stack)
perform_with_bootstrap
expect(perform_with_bootstrap).to eq 2
end
it "doesn't rollback stack and returns 3 when a known error occured during servers deploy" do
allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:deploy_error) }
allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:deploy_error) }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 3
end
it "doesn't rollback stack and returns 3 when a servers bootstrap & deploy haven't been finished due to timeout" do
allow(worker).to receive(:bootstrap_servers_by_priority) { bootstrap_result(:timeout_reached) }
allow(executor).to receive(:bootstrap_just_persisted) { bootstrap_result(:timeout_reached) }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 4
end
it 'rollbacks stack and reraises that error when an unknown error occured during servers bootsrap and deploy' do
error = StandardError.new
allow(worker).to receive(:bootstrap_servers_by_priority) { raise error }
allow(executor).to receive(:bootstrap_just_persisted) { raise error }
expect(worker).to receive(:rollback_stack!)
expect{perform_with_bootstrap}.to raise_error(error)
end

11
devops-service/workers/bootstrap_worker.rb
View File

@ -6,10 +6,19 @@ require "db/mongo/models/report"
class BootstrapWorker < Worker
# options must contain 'server_attrs', 'owner'
# @options
# 'server_attrs': required
# 'owner': required
# 'skip_rollback': optional
# 'deploy_info': optional
# 'deployers': optional
# 'bootstrap_template': optional
# 'chef_environment': optional
# 'config': optional, whatever this parameter really means
def perform(options)
call do
owner = options.fetch('owner')
skip_rollback = options['skip_rollback']
converted_options = convert_config(options)
server = Devops::Model::Server.new(options.fetch('server_attrs'))

2
devops-service/workers/delete_expired_server_worker.rb
View File

@ -22,7 +22,7 @@ class DeleteExpiredServerWorker < Worker
def save_report(server)
update_report(
"created_by" => 'SYSTEM',
"created_by" => Devops::Model::Report::SYSTEM_OWNER,
"project" => server.project,
"deploy_env" => server.deploy_env,
"type" => Devops::Model::Report::EXPIRE_SERVER_TYPE

3
devops-service/workers/run_workers.rb
View File

@ -7,6 +7,7 @@ require File.join(root, "project_test_worker")
require File.join(root, "stack_bootstrap_worker")
require File.join(root, "delete_server_worker")
require File.join(root, "delete_expired_server_worker")
require File.join(root, "stack_sync_worker")
require 'byebug'
config = {}
#require File.join(root, "../proxy")

30
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -2,14 +2,16 @@ require 'lib/executors/stack_executor'
class StackBootstrapWorker < Worker
# options must contain 'stack_attributes'
# @options:
# 'stack_attributes', required
# 'without_bootstrap', optional. false by default
# 'skip_rollback', optional. false by default
def perform(options)
call do
puts_and_flush JSON.pretty_generate(options)
stack_attrs = options.fetch('stack_attributes')
without_bootstrap = stack_attrs.delete('without_bootstrap')
skip_rollback = false # take it from options in future
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
without_bootstrap = options['without_bootstrap'] || false
skip_rollback = options['skip_rollback'] || false
save_report(stack_attrs)
@ -20,7 +22,7 @@ class StackBootstrapWorker < Worker
end
begin
persist_stack_servers
executor.persist_new_servers
if without_bootstrap
0
else
@ -40,17 +42,13 @@ class StackBootstrapWorker < Worker
@executor ||= Devops::Executor::StackExecutor.new(out: out)
end
# let it stay inside method to improve readability of #perform method
def create_stack(stack_attrs)
@stack = executor.create_stack(stack_attrs)
end
def persist_stack_servers
@servers_by_priorities = executor.persist_stack_servers[:just_persisted_by_priority]
end
# options should contain :skip_rollback
def bootstrap_or_rollback_if_failed(options)
bootstrap_result = bootstrap_servers_by_priority
bootstrap_result = executor.bootstrap_just_persisted(jid)
puts_and_flush Devops::Messages.t("worker.stack_bootstrap.bootstrap_result.#{bootstrap_result.reason}")
if bootstrap_result.bootstrap_error? && !options[:skip_rollback]
rollback_stack!
@ -58,14 +56,6 @@ class StackBootstrapWorker < Worker
bootstrap_result.code
end
def bootstrap_servers_by_priority
out.puts "Bootstrapping just persisted servers"
@servers_by_priorities.each do |priority, servers|
out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
out.flush
executor.bootstrap_servers_by_priority(@servers_by_priorities, jid)
end
def rollback_stack!
puts_and_flush "\nStart rollback of a stack"

62
devops-service/workers/stack_sync_worker.rb Normal file
View File

@ -0,0 +1,62 @@
require 'lib/executors/stack_executor'
class StackSyncWorker < Worker
MAX_LOCK_WAITING_TIME = 15000
# @options:
# 'stack_name' required
# 'created_by' optional
def perform(options)
call do
stack_name = options.fetch('stack_name')
created_by = options['created_by'] || ::Devops::Model::Report::SYSTEM_OWNER
@stack = mongo.stack(stack_name)
save_report(created_by)
wait_until_stack_is_unlocked
puts_and_flush 'Persisting new servers.'
executor.persist_new_servers
puts_and_flush "\n\nDeleting stale servers."
executor.delete_stale_servers
puts_and_flush "\n\nBootstrapping just persisted servers."
bootstrap_result = executor.bootstrap_just_persisted(jid)
puts_and_flush Devops::Messages.t("worker.stack_sync.bootstrap_result.#{bootstrap_result.reason}")
bootstrap_result.code
end
end
private
def executor
@executor ||= Devops::Executor::StackExecutor.new(out: out, stack: @stack)
end
def wait_until_stack_is_unlocked
return unless @stack.persisting_is_locked
puts_and_flush 'Stack is locked, waiting...'
waiting_time = 0
sleep_time = 10
loop do
sleep(sleep_time)
@stack = mongo.stack(@stack.name)
return unless @stack.persisting_is_locked
waiting_time += sleep_time
raise "Stack has been locked for too long" if waiting_time > MAX_LOCK_WAITING_TIME
end
end
def save_report(created_by)
update_report(
"created_by" => created_by,
"project" => @stack.project,
"deploy_env" => @stack.deploy_env,
"type" => ::Devops::Model::Report::SYNC_STACK_TYPE,
"subreports" => [],
"stack" => @stack.name
)
end
end