tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'cid-381-improve_chef_node_name_setting' into temp_merge

Browse Source
This commit is contained in:
Anton Chuchkalov 2016-02-10 20:54:15 +03:00
commit fd8a14fdb7
74 changed files with 2032 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,3 +2,5 @@ devops-service/tests/features/support/config.yml
.devops_files/
devops-service/plugins
devops-service/spec/examples.txt
devops-service/coverage
devops-service/tmp

2
devops-service/Gemfile
View File

@ -36,4 +36,6 @@ end
group :devepoment do
gem 'byebug'
gem 'guard-rspec', require: false
gem 'simplecov', require: false
gem 'simplecov-rcov', require: false
end

13
devops-service/Gemfile.lock
View File

@ -60,6 +60,7 @@ GEM
gherkin (~> 2.12.0)
daemons (1.2.3)
diff-lcs (1.2.5)
docile (1.1.5)
em-websocket (0.3.8)
addressable (>= 2.1.1)
eventmachine (>= 0.12.9)
@ -291,6 +292,13 @@ GEM
json
redis (>= 3.0.6)
redis-namespace (>= 1.3.1)
simplecov (0.11.1)
docile (~> 1.1.0)
json (~> 1.8)
simplecov-html (~> 0.10.0)
simplecov-html (0.10.0)
simplecov-rcov (0.2.3)
simplecov (>= 0.4.1)
sinatra (1.4.5)
rack (~> 1.4)
rack-protection (~> 1.4)
@ -352,9 +360,14 @@ DEPENDENCIES
rspec (~> 3.3)
rspec_junit_formatter
sidekiq (= 3.2.6)
simplecov
simplecov-rcov
sinatra (= 1.4.5)
sinatra-contrib
sinatra-websocket
test-unit
thin (~> 1.5.1)
wisper
BUNDLED WITH
1.11.2

2
devops-service/Guardfile
View File

@ -42,4 +42,6 @@ guard :rspec, cmd: "rspec" do
# Devops files
watch(%r{db/.+\.rb}) { rspec.spec_dir }
watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" }
watch(%r{workers/stack_bootstrap/.+\.rb}) { "#{rspec.spec_dir}/workers" }
end

4
devops-service/app/api2/handlers/provider.rb
View File

@ -27,14 +27,14 @@ module Devops
def add_account provider
account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account)
account.validate_fields!
Devops::Db.connector.provider_accounts_insert(account)
Devops::Db.connector.provider_account_insert(account)
::Provider::ProviderFactory.add_account(provider, account)
account.to_hash
end
def delete_account name, provider
account = Devops::Db.connector.provider_account(provider, name)
Devops::Db.connector.provider_accounts_delete(name)
Devops::Db.connector.provider_account_delete(name)
::Provider::ProviderFactory.delete_account(provider, account)
account.to_hash
end

7
devops-service/commands/knife_commands.rb
View File

@ -38,6 +38,13 @@ class KnifeCommands
knife("tag delete #{name} #{tagsStr}")
end
# extracted from server_executor.rb
def swap_tags(name, from, to)
tags_delete(name, from)
result = tags_create(name, to)
!!result[1]
end
def create_role role_name, project, env
file = "/tmp/new_role.json"
File.open(file, "w") do |f|

62
devops-service/commands/stack.rb
View File

@ -1,62 +0,0 @@
module StackCommands
extend self
RESULT_CODES = {
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5
}
def self.result_codes
RESULT_CODES
end
def self.result_code(code)
result_codes.fetch(code)
end
def sync_stack_proc
lambda do |out, stack, mongo|
# 5 tries each 5 seconds, then 200 tries each 10 seconds
sleep_times = [5]*5 + [10]*200
begin
out << "Syncing stack '#{stack.id}'...\n"
events_keys = []
sleep_times.each do |sleep_time|
sleep sleep_time
stack.sync_details!
stack.events.each do |event|
unless events_keys.include?(event["event_id"])
events_keys << event["event_id"]
out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
end
end
case stack.stack_status
when 'CREATE_IN_PROGRESS'
out.flush
when 'CREATE_COMPLETE'
mongo.stack_update(stack)
out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n"
out.flush
return 0
when 'ROLLBACK_COMPLETE'
out << "\nStack '#{stack.id}' status is rolled back\n"
return StackCommands.result_code(:stack_rolled_back)
else
out.puts "\nUnknown stack status: '#{stack.stack_status}'"
return StackCommands.result_code(:unkown_status)
end
end
out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds."
return StackCommands.result_code(:timeout)
rescue StandardError => e
logger.error e.message
out << "Error: #{e.message}\n"
return StackCommands.result_code(:error)
end
end
end
end

6
devops-service/core/devops-service.rb
View File

@ -1,7 +1,7 @@
require "wisper"
require "lib/hash_ext"
require "lib/nil_class_ext"
require "lib/string_ext"
require "lib/core_ext/hash"
require "lib/core_ext/nil_class"
require "lib/core_ext/string"
require_relative "devops-loader"
require_relative "devops-application"

4
devops-service/db/mongo/connectors/helpers/delete_command.rb
View File

@ -1,5 +1,3 @@
require 'lib/string_helper'
module Connectors
module Helpers
module DeleteCommand
@ -9,7 +7,7 @@ module Connectors
# We need this alias to forward methods from MongoConnector to resources connectors.
def self.included(base)
resource_name = StringHelper.underscore_class(base)
resource_name = base.to_s.underscore_class
method_name = "#{resource_name}_delete".to_sym
alias_method method_name, :delete
end

6
devops-service/db/mongo/connectors/helpers/insert_command.rb
View File

@ -1,5 +1,3 @@
require 'lib/string_helper'
module Connectors
module Helpers
module InsertCommand
@ -9,7 +7,7 @@ module Connectors
# We need this alias to forward methods from MongoConnector to resources connectors.
def self.included(base)
resource_name = StringHelper.underscore_class(base)
resource_name = base.to_s.underscore_class
method_name = "#{resource_name}_insert".to_sym
alias_method method_name, :insert
end
@ -24,7 +22,7 @@ module Connectors
rescue Mongo::OperationFailure => e
# exception's message doesn't always start from error code
if e.message =~ /11000/
resource_name = StringHelper.underscore_class(record.class)
resource_name = record.class.to_s.underscore_class
raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
end
end

6
devops-service/db/mongo/connectors/helpers/list_command.rb
View File

@ -1,5 +1,3 @@
require 'lib/string_helper'
module Connectors
module Helpers
module ListCommand
@ -9,9 +7,7 @@ module Connectors
# We need this alias to forward methods from MongoConnector to resources connectors.
def self.included(base)
resource_name = StringHelper.underscore_class(base).to_sym
method_name = StringHelper.pluralize(resource_name)
alias_method method_name, :list
alias_method base.to_s.underscore_class.pluralize, :list
end
# query options is needed, for example, for fields limiting

4
devops-service/db/mongo/connectors/helpers/show_command.rb
View File

@ -1,5 +1,3 @@
require 'lib/string_helper'
module Connectors
module Helpers
module ShowCommand
@ -9,7 +7,7 @@ module Connectors
# We need this alias to forward methods from MongoConnector to resources connectors.
def self.included(base)
method_name = StringHelper.underscore_class(base).to_sym
method_name = base.to_s.underscore_class.to_sym
alias_method method_name, :show
end

5
devops-service/db/mongo/connectors/helpers/update_command.rb
View File

@ -1,5 +1,3 @@
require 'lib/string_helper'
module Connectors
module Helpers
module UpdateCommand
@ -8,8 +6,7 @@ module Connectors
# We need second method name to forward methods from MongoConnector to resources connectors.
def self.included(base)
resource_name = StringHelper.underscore_class(base)
method_name = "#{resource_name}_update".to_sym
method_name = "#{base.to_s.underscore_class}_update".to_sym
alias_method method_name, :update
end

4
devops-service/db/mongo/connectors/provider_accounts.rb → devops-service/db/mongo/connectors/provider_account.rb
View File

@ -1,5 +1,5 @@
module Connectors
class ProviderAccounts < Base
class ProviderAccount < Base
include Helpers::InsertCommand,
Helpers::DeleteCommand
@ -16,7 +16,7 @@ module Connectors
def provider_account provider, account
c = Provider::ProviderFactory.get_account_class(provider)
bson = collection.find({provider: provider, _id: account}).to_a.first
raise RecordNotFound.new("'Account #{account}' for provider '#{provider}' not found") unless bson
raise RecordNotFound.new("Account '#{account}' for provider '#{provider}' not found") unless bson
c.build_from_bson(bson)
end

2
devops-service/db/mongo/models/deploy_env/deploy_env_ec2.rb
View File

@ -53,7 +53,7 @@ module Devops
def subnets_filter
networks = provider_instance.networks
unless self.subnets.empty?
if subnets && !subnets.empty?
network = networks.detect {|n| n["name"] == self.subnets[0]}
if network
{"vpc-id" => network["vpcId"] }

4
devops-service/db/mongo/models/image.rb
View File

@ -73,10 +73,6 @@ module Devops
}
end
def self.create_from_json! json
Image.new( JSON.parse(json) )
end
end
end
end

4
devops-service/db/mongo/models/key.rb
View File

@ -30,10 +30,6 @@ module Devops
key
end
def self.create_from_json json
Key.new( JSON.parse(json) )
end
def filename
File.basename(self.path)
end

1
devops-service/db/mongo/models/project.rb
View File

@ -22,6 +22,7 @@ module Devops
#define_hook :after_add_deploy_env
attr_accessor :id, :deploy_envs, :type, :archived, :description, :run_list
attr_accessor :components
MULTI_TYPE = "multi"

6
devops-service/db/mongo/models/provider_accounts/provider_account.rb
View File

@ -61,6 +61,12 @@ module Devops
}
end
# absent of "id" attribute can cause some inconviniences.
# for example, we have "record.id" call in InsertCommand
def id
account_name
end
end
end
end

4
devops-service/db/mongo/models/stack_template/stack_template_ec2.rb
View File

@ -16,10 +16,6 @@ module Devops
super.merge(template_url: template_url)
end
def delete_template_file_from_storage
raise 'Implement me'
end
def update_template_url
self.template_url = generate_template_file_and_upload_to_storage(id, template_body)
end

7
devops-service/db/mongo/models/user.rb
View File

@ -75,17 +75,12 @@ module Devops
user
end
def self.create_from_json json
User.new( JSON.parse(json) )
end
def to_hash_without_id
o = {
{
"email" => self.email,
"password" => self.password,
"privileges" => self.privileges
}
o
end
def check_privileges cmd, required_privelege

4
devops-service/db/mongo/mongo_connector.rb
View File

@ -33,7 +33,7 @@ class MongoConnector
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector,
[:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_account] => :provider_accounts_connector
[:provider_accounts, :provider_account_insert, :provider_account_delete, :provider_account] => :provider_accounts_connector
)
def initialize(db, host, port=27017, user=nil, password=nil)
@ -48,7 +48,7 @@ class MongoConnector
private
def provider_accounts_connector
@provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db)
@provider_accounts_connector ||= Connectors::ProviderAccount.new(@db)
end
def images_connector

8
devops-service/db/validators/deploy_env/flavor.rb
View File

@ -3,7 +3,7 @@ module Validators
def valid?
return true unless @model.flavor
available_flavors.detect do |flavor|
@model.provider_instance.flavors.detect do |flavor|
flavor['id'] == @model.flavor
end
end
@ -11,11 +11,5 @@ module Validators
def message
"Invalid flavor '#{@model.flavor}'."
end
private
def available_flavors
@model.provider_instance.flavors
end
end
end

10
devops-service/db/validators/deploy_env/groups.rb
View File

@ -3,6 +3,9 @@ module Validators
def valid?
return true if @model.groups.nil?
subnets_filter = @model.subnets_filter
available_groups = @model.provider_instance.groups(subnets_filter).keys
@invalid_groups = @model.groups - available_groups
@invalid_groups.empty?
end
@ -10,12 +13,5 @@ module Validators
def message
"Invalid groups '#{@invalid_groups.join("', '")}'."
end
private
def available_groups
subnets_filter = @model.subnets_filter
@model.provider_instance.groups(subnets_filter).keys
end
end
end

8
devops-service/db/validators/deploy_env/image.rb
View File

@ -6,7 +6,7 @@ module Validators
def valid?
return true unless @model.image
available_images.detect do |image|
get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
image["id"] == @model.image
end
end
@ -14,11 +14,5 @@ module Validators
def message
"Invalid image '#{@model.image}'."
end
private
def available_images
get_available_provider_images(::Devops::Db.connector, @model.provider)
end
end
end

12
devops-service/db/validators/deploy_env/stack_template.rb
View File

@ -4,21 +4,13 @@ module Validators
def valid?
return true unless @model.stack_template
available_stack_templates.detect do |template|
template['id'] == @model.stack_template
Devops::Db.connector.stack_templates.detect do |template|
template.id == @model.stack_template
end
end
def message
"Invalid stack template '#{@model.stack_template}'."
end
private
def available_stack_templates
# map to hash to simplify mocks. Later replace this method with something more suitable
Devops::Db.connector.stack_templates.map(&:to_hash)
end
end
end

8
devops-service/db/validators/field_validators/flavor.rb
View File

@ -4,7 +4,7 @@ module Validators
class Flavor < Base
def valid?
available_flavors.detect do |flavor|
@model.provider_instance.flavors.detect do |flavor|
flavor['id'] == @value
end
end
@ -12,12 +12,6 @@ module Validators
def message
"Invalid flavor '#{@value}'."
end
private
def available_flavors
@model.provider_instance.flavors
end
end
end
end

8
devops-service/db/validators/field_validators/image.rb
View File

@ -7,7 +7,7 @@ module Validators
include ::ImageCommands
def valid?
available_images.detect do |image|
get_available_provider_images(::Devops::Db.connector, @model.provider).detect do |image|
image["id"] == @value
end
end
@ -15,12 +15,6 @@ module Validators
def message
"Invalid image '#{@value}'."
end
private
def available_images
get_available_provider_images(::Devops::Db.connector, @model.provider)
end
end
end
end

7
devops-service/db/validators/helpers/users.rb
View File

@ -2,6 +2,7 @@ module Validators
class Helpers::Users < Base
def valid?
available_users = ::Devops::Db.connector.users_names(@model)
@nonexistent_users = (@model || []) - available_users
@nonexistent_users.empty?
end
@ -9,11 +10,5 @@ module Validators
def message
Devops::Messages.t("project.deploy_env.validation.users.not_exist", users: @nonexistent_users.join("', '"))
end
private
def available_users
::Devops::Db.connector.users_names(@model)
end
end
end

12
devops-service/db/validators/image/bootstrap_template.rb
View File

@ -8,22 +8,12 @@ module Validators
include BootstrapTemplatesCommands
def valid?
if @model.bootstrap_template
available_templates.include?(@model.bootstrap_template)
else
true
end
get_templates.include?(@model.bootstrap_template)
end
def message
"Invalid bootstrap template '#{@model.bootstrap_template}' for image '#{@model.id}'"
end
private
def available_templates
get_templates
end
end
end
end

0
devops-service/lib/hash_ext.rb → devops-service/lib/core_ext/hash.rb
View File

0
devops-service/lib/nil_class_ext.rb → devops-service/lib/core_ext/nil_class.rb
View File

27
devops-service/lib/core_ext/string.rb Normal file
View File

@ -0,0 +1,27 @@
class String
def present?
!empty?
end
def blank?
empty?
end
# from ActiveSupport
def underscore
gsub(/::/, '/').
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
gsub(/([a-z\d])([A-Z])/,'\1_\2').
tr("-", "_").
downcase
end
def underscore_class
split('::').last.underscore
end
# rough simplification
def pluralize
"#{self}s"
end
end

35
devops-service/lib/executors/expiration_scheduler.rb Normal file
View File

@ -0,0 +1,35 @@
require "workers/delete_server_worker"
module Devops
module Executor
class ExpirationScheduler
def initialize(expires, server)
@expires, @server = expires, server
end
def schedule_expiration!
return unless @expires
DeleteServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name)
end
def interval_in_seconds
interval = @expires.to_i
measure_unit = @expires.chars.last
case measure_unit
when 's'
interval
when 'm'
interval * 60
when 'h'
interval * 60 * 60
when 'd'
interval * 60 * 60 * 24
when 'w'
interval * 60 * 60 * 24 * 7
else
raise 'Wrong interval format'
end
end
end
end
end

239
devops-service/lib/executors/server_executor.rb
View File

@ -1,6 +1,5 @@
require "lib/knife/knife_factory"
require "workers/worker"
require "workers/delete_server_worker"
require "lib/executors/expiration_scheduler"
require "hooks"
require 'net/ssh'
@ -9,12 +8,16 @@ module Devops
class ServerExecutor
include Hooks
RESULT_CODES = {
ERROR_CODES = {
server_bootstrap_fail: 2,
server_cannot_update_tags: 3,
server_bootstrap_private_ip_unset: 4,
server_not_in_chef_nodes: 5,
server_bootstrap_unknown_error: 7,
deploy_unknown_error: 6,
deploy_failed: 8
deploy_failed: 8,
creating_server_unknown_error: 9,
creating_server_in_cloud_failed: 10
}
# waiting for 5*60 seconds (5 min)
@ -34,46 +37,36 @@ module Devops
define_hook :before_bootstrap
define_hook :after_bootstrap
before_deploy :create_run_list
before_deploy :add_run_list_to_deploy_info
attr_accessor :server, :deploy_env, :report, :project
def initialize server, out, options={}
if server
@project = Devops::Db.connector.project(server.project)
@deploy_env = @project.deploy_env(server.deploy_env)
end
@knife_instance = KnifeFactory.instance
@server = server
@out = out
@out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
@current_user = options[:current_user]
end
def self.result_code(symbolic_code)
RESULT_CODES.fetch(symbolic_code)
def self.error_code(reason)
ERROR_CODES.fetch(reason)
end
def self.symbolic_result_code(integer_code)
RESULT_CODES.key(integer_code) || :unknown_error
def self.reason_from_error_code(integer_code)
ERROR_CODES.key(integer_code) || :unknown_error
end
def result_code(symbolic_code)
self.class.result_code(symbolic_code)
def self.bootstrap_errors_reasons
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error]
end
def report= r
@report = r
end
def project= p
@project = p
end
def deploy_env= e
@deploy_env = e
end
def server
@server
def error_code(reason)
self.class.error_code(reason)
end
def create_server_object options
@ -109,7 +102,9 @@ module Devops
res[:before] = self.run_hook :before_create
@out << "Done\n"
return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
return error_code(:creating_server_in_cloud_failed)
end
mongo.server_insert @server
@out << "\nAfter create hooks...\n"
@ -118,7 +113,7 @@ module Devops
@out.flush
DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"
schedule_expiration(@server)
schedule_expiration()
unless options["without_bootstrap"]
bootstrap_options = {
@ -135,11 +130,15 @@ module Devops
DevopsLogger.logger.error e.message
roll_back
mongo.server_delete @server.id
# return 5
return result_code(:server_not_in_chef_nodes)
error_code(:creating_server_unknown_error)
end
end
# options:
# :run_list (optional)
# :bootstrap_template (optional)
# :chef_environment (optional)
# :config (optional)
def bootstrap options
@out << "\n\nBootstrap...\n"
@out.flush
@ -151,7 +150,7 @@ module Devops
@out << "Done\n"
if @server.private_ip.nil?
@out << "Error: Private IP is null"
return false
return error_code(:server_bootstrap_private_ip_unset)
end
ja = {
:provider => @server.provider,
@ -166,13 +165,7 @@ module Devops
address = "#{@server.remote_user}@#{ip}"
cmd = 'ssh '
cmd << "-i #{cert_path} "
cmd << '-q '
cmd << '-o StrictHostKeyChecking=no '
cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
cmd << "#{address} 'exit'"
cmd << " 2>&1"
cmd = check_ssh_command(cert_path, address)
@out << "\nWaiting for SSH..."
@out << "\nTest command: '#{cmd}'\n"
@ -181,16 +174,16 @@ module Devops
retries_amount = 0
begin
sleep(5)
res = `#{cmd}`
res = execute_system_command(cmd)
retries_amount += 1
if retries_amount > MAX_SSH_RETRIES_AMOUNT
if retries_amount >= MAX_SSH_RETRIES_AMOUNT
@out.puts "Can not connect to #{address}"
@out.puts res
@out.flush
DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
return result_code(:server_bootstrap_fail)
return error_code(:server_bootstrap_fail)
end
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless last_command_successful?
rescue ArgumentError => e
@out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
@out.flush
@ -200,7 +193,7 @@ module Devops
provider = @server.provider_instance
@server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?
r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))
if r == 0
@out << "Chef node name: #{@server.chef_node_name}\n"
@ -217,10 +210,16 @@ module Devops
else
@out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
@out.flush
result_code(:server_bootstrap_fail)
error_code(:server_bootstrap_fail)
end
end
# options:
# :cert_path (required)
# :run_list (optional)
# :bootstrap_template (optional)
# :chef_environment (optional)
# :config (optional)
def bootstrap_options attributes, options
bootstrap_options = [
"-x #{@server.remote_user}",
@ -242,6 +241,7 @@ module Devops
@out << "Done\n"
end
# essentially, it just bootstrap and then deploy
def two_phase_bootstrap options
prepare_two_phase_bootstrap(options)
# bootstrap phase
@ -252,14 +252,14 @@ module Devops
bootstrap_status = bootstrap(options)
if bootstrap_status == 0
if check_server
if check_server_on_chef_server
@out << "Server #{@server.chef_node_name} is created"
else
@out.puts "Can not find client or node on chef-server"
roll_back
@out.flush
mongo.server_delete @server.id
return result_code(:server_not_in_chef_nodes)
return error_code(:server_not_in_chef_nodes)
end
else
# @out << roll_back
@ -269,22 +269,20 @@ module Devops
DevopsLogger.logger.error msg
@out.puts msg
@out.flush
return result_code(:server_bootstrap_fail)
return error_code(:server_bootstrap_fail)
end
rescue => e
@out << "\nError: #{e.message}\n"
@out.flush
return result_code(:server_bootstrap_unknown_error)
return error_code(:server_bootstrap_unknown_error)
end
# deploy phase. Assume that all servers are bootstraped successfully here.
begin
#raise "hello"
@out << "\n"
run_list = compute_run_list
@out << "\nComputed run list: #{run_list.join(", ")}"
@out << "\n\nComputed run list: #{run_list.join(", ")}"
@out.flush
@knife_instance.set_run_list(@server.chef_node_name, run_list)
knife_instance.set_run_list(@server.chef_node_name, run_list)
deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
deploy_status = deploy_server(deploy_info)
if deploy_status == 0
@ -294,19 +292,20 @@ module Devops
msg << "\nDeploing server operation status was #{deploy_status}"
DevopsLogger.logger.error msg
@out << "\n" + msg + "\n"
result_code(:deploy_failed)
error_code(:deploy_failed)
end
rescue => e
@out << "\nError: #{e.message}\n"
DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
result_code(:deploy_unknown_error)
error_code(:deploy_unknown_error)
end
end
def check_server
@knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
def check_server_on_chef_server
knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name)
end
# returns a hash with :chef_node, :chef_client and :server keys
def unbootstrap
k = Devops::Db.connector.key(@server.key)
cert_path = k.path
@ -346,35 +345,28 @@ module Devops
end
def deploy_server_with_tags tags, deploy_info
old_tags_str = nil
new_tags_str = nil
unless tags.empty?
old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
@out << "Server tags: #{old_tags_str}\n"
@knife_instance.tags_delete(@server.chef_node_name, old_tags_str)
return deploy_server(deploy_info) if tags.empty?
new_tags_str = tags.join(" ")
@out << "Server new tags: #{new_tags_str}\n"
cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
unless cmd[1]
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
DevopsLogger.logger.error(m)
@out << m + "\n"
return 3
end
DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ")
new_tags_str = tags.join(" ")
@out.puts "Temporarily changing tags (#{old_tags_str}) to (#{new_tags_str})"
unless knife_instance.swap_tags(@server.chef_node_name, old_tags_str, new_tags_str)
m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
DevopsLogger.logger.error(m)
@out.puts m
return error_code(:server_cannot_update_tags)
end
DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
r = deploy_server deploy_info
unless tags.empty?
@out << "Restore tags\n"
cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
begin
deploy_result = deploy_server deploy_info
ensure
@out.puts "Restoring tags"
knife_instance.swap_tags(@server.chef_node_name, new_tags_str, old_tags_str)
DevopsLogger.logger.info("Restoring tags for #{@server.chef_node_name}: from #{new_tags_str} back to (#{old_tags_str})")
end
return r
deploy_result
end
def deploy_server deploy_info
@ -397,7 +389,7 @@ module Devops
f.write json
end
end
@out << "Deploy Input Parameters:\n"
@out.puts "Deploy Input Parameters:"
@out.puts json
@out.flush
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
@ -412,7 +404,7 @@ module Devops
end
@out.flush
k = Devops::Db.connector.key(@server.key)
lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
r = /Chef\sClient\sfinished/i
if lline && lline[r]
@ -432,21 +424,11 @@ module Devops
def delete_from_chef_server node_name
{
:chef_node => @knife_instance.chef_node_delete(node_name),
:chef_client => @knife_instance.chef_client_delete(node_name)
:chef_node => knife_instance.chef_node_delete(node_name),
:chef_client => knife_instance.chef_client_delete(node_name)
}
end
=begin
def delete_etc_chef s, cert_path
cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
cmd += " sudo " unless s.remote_user == "root"
cmd += "rm -Rf /etc/chef"
r = `#{cmd}`
raise(r) unless $?.success?
end
=end
def delete_server
mongo = ::Devops::Db.connector
if @server.static?
@ -487,7 +469,7 @@ module Devops
end
end
def create_run_list out, deploy_info
def add_run_list_to_deploy_info out, deploy_info
out << "\nGenerate run list hook...\n"
if deploy_info["run_list"]
out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
@ -496,14 +478,6 @@ module Devops
out << "Project run list: #{@project.run_list.join(", ")}\n"
out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
out << "Server run list: #{@server.run_list.join(", ")}\n"
=begin
rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
if @server.stack
stack = Devops::Db.connector.stack(@server.stack)
out << "Stack run list: #{stack.run_list.join(", ")}\n"
rlist.merge(stack.run_list)
end
=end
deploy_info["run_list"] = compute_run_list
out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
end
@ -511,45 +485,46 @@ module Devops
def compute_run_list
rlist = []
[@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
rlist += sub_run_list if sub_run_list.is_a?(Array)
rlist += sub_run_list if sub_run_list
end
rlist = Set.new(rlist)
if @server.stack
stack = Devops::Db.connector.stack(@server.stack)
# out << "Stack run list: #{stack.run_list.join(", ")}\n"
srl = stack.run_list
rlist.merge(srl) if srl.is_a?(Array)
rlist += srl if srl
end
rlist.to_a
rlist.uniq
end
private
def schedule_expiration(server)
expires = @deploy_env.expires
return unless expires
interval = interval_in_seconds(expires)
@out << "Planning expiration in #{expires}"
DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name)
def schedule_expiration
if @deploy_env.expires
@out << "Planning expiration in #{@deploy_env.expires}"
ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration!
end
end
def interval_in_seconds(interval_as_string)
interval = interval_as_string.to_i
measure_unit = interval_as_string.chars.last
case measure_unit
when 's'
interval
when 'm'
interval * 60
when 'h'
interval * 60 * 60
when 'd'
interval * 60 * 60 * 24
when 'w'
interval * 60 * 60 * 24 * 7
else
raise 'Wrong interval format'
end
def check_ssh_command(cert_path, address)
cmd = 'ssh '
cmd << "-i #{cert_path} "
cmd << '-q '
cmd << '-o StrictHostKeyChecking=no '
cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
cmd << "#{address} 'exit'"
cmd << " 2>&1"
cmd
end
def execute_system_command(cmd)
`#{cmd}`
end
def last_command_successful?
$?.success?
end
def knife_instance
@knife_instance ||= KnifeFactory.instance
end
end

9
devops-service/lib/puts_and_flush.rb Normal file
View File

@ -0,0 +1,9 @@
module PutsAndFlush
private
# out stream should be defined
def puts_and_flush(message)
out.puts message
out.flush
end
end

9
devops-service/lib/string_ext.rb
View File

@ -1,9 +0,0 @@
class String
def present?
!empty?
end
def blank?
empty?
end
end

35
devops-service/lib/string_helper.rb
View File

@ -1,35 +0,0 @@
module StringHelper
extend self
# from Rails' ActiveSupport
def underscore(string)
string.gsub(/::/, '/').
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
gsub(/([a-z\d])([A-Z])/,'\1_\2').
tr("-", "_").
downcase
end
def underscore_class(klass, without_ancestors=true)
class_name = if without_ancestors
klass.to_s.split('::').last
else
klass.to_s
end
StringHelper.underscore(class_name)
end
# from Rails' ActiveSupport
def camelize(term)
string = term.to_s
string = string.sub(/^[a-z\d]*/) { $&.capitalize }
string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
string.gsub!(/\//, '::')
string
end
# rough simplification
def pluralize(string)
"#{string}s"
end
end

0
devops-service/lib/users_permissions_updater.rb → devops-service/migrations/users_permissions_updater.rb
View File

58
devops-service/spec/connectors/provider_accounts_connector_spec.rb Normal file
View File

@ -0,0 +1,58 @@
require 'db/mongo/models/provider_accounts/ec2_provider_account'
require 'db/mongo/models/provider_accounts/openstack_provider_account'
require 'db/mongo/connectors/provider_account'
require 'spec/connectors/tester_connector/provider_account'
RSpec.describe Connectors::ProviderAccount, type: :connector do
set_tester_connector TesterConnector::ProviderAccount
include_examples 'mongo connector', {
model_name: :provider_account,
factory_name: :ec2_provider_account,
only: [:insert, :delete],
field_to_update: :description
}
describe '#provider_accounts', cleanup_after: :all do
before(:all) do
@tester_connector.create(id: 'foo', provider: 'ec2')
@tester_connector.create(id: 'bar', provider: 'openstack')
end
it 'returns array of Ec2ProviderAccount if @provider is ec2' do
expect(
@connector.provider_accounts('ec2')
).to be_an_array_of(Devops::Model::Ec2ProviderAccount).and have_size(1)
end
it 'returns array of Ec2ProviderAccount if @provider is openstack' do
expect(
@connector.provider_accounts('openstack')
).to be_an_array_of(Devops::Model::OpenstackProviderAccount).and have_size(1)
end
end
describe '#provider_account', cleanup_after: :all do
before(:all) do
@tester_connector.create(id: 'foo', provider: 'ec2')
@tester_connector.create(id: 'bar', provider: 'openstack')
end
it 'returns ec2 provider account' do
acc = @connector.provider_account('ec2', 'foo')
expect(acc).to be_a(Devops::Model::Ec2ProviderAccount)
expect(acc.account_name).to eq 'foo'
end
it 'returns openstack provider account' do
acc = @connector.provider_account('openstack', 'bar')
expect(acc).to be_a(Devops::Model::OpenstackProviderAccount)
expect(acc.account_name).to eq 'bar'
end
it 'raises error if account is missing' do
expect{@connector.provider_account('ec2', 'missing')}.to raise_error(RecordNotFound)
end
end
end

6
devops-service/spec/connectors/tester_connector/provider_account.rb Normal file
View File

@ -0,0 +1,6 @@
require_relative 'base'
module TesterConnector
class ProviderAccount < Base
end
end

47
devops-service/spec/executors/expiration_scheduler_spec.rb Normal file
View File

@ -0,0 +1,47 @@
require 'lib/executors/expiration_scheduler'
RSpec.describe Devops::Executor::ExpirationScheduler do
let(:server) { build(:server) }
describe '#schedule_expiration!' do
it 'schedules server deleting at given time' do
expect(DeleteServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name')
described_class.new('2m', server).schedule_expiration!
end
it "doesn't schedule job if expires is nil" do
expect(DeleteServerWorker).not_to receive(:perform_in)
described_class.new(nil, server).schedule_expiration!
end
end
describe '#interval_in_seconds' do
def interval_in_seconds(expires)
described_class.new(expires, server).interval_in_seconds
end
it 'recognizes seconds' do
expect(interval_in_seconds('2s')).to eq 2
end
it 'recognizes minutes' do
expect(interval_in_seconds('3m')).to eq 180
end
it 'recognizes hours' do
expect(interval_in_seconds('1h')).to eq 3600
end
it 'recognizes days' do
expect(interval_in_seconds('1d')).to eq 86400
end
it 'recognizes weeks' do
expect(interval_in_seconds('1w')).to eq 604800
end
it 'raises on wrong format' do
expect { interval_in_seconds('wrong') }.to raise_error(StandardError)
end
end
end

792
devops-service/spec/executors/server_executor_spec.rb Normal file
View File

@ -0,0 +1,792 @@
require 'lib/executors/server_executor'
RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connector: true, stubbed_logger: true do
let(:project) { build(:project) }
let(:deploy_env) { project.deploy_env('foo') }
let(:server) { build(:server, project: project.id, deploy_env: 'foo') }
let(:output) { File.open(File::NULL, "w") }
let(:provider) { double('Provider instance') }
let(:executor) { described_class.new(server, output) }
before do
allow(stubbed_connector).to receive(:project) { project }
allow(executor.deploy_env).to receive(:provider_instance) { provider }
allow(server).to receive(:provider_instance) { provider }
end
describe '#initialize' do
it 'sets server, project, deploy_env, out instance variables' do
expect(executor.server).to eq server
expect(executor.deploy_env).to eq deploy_env
expect(executor).to have_instance_variable_value(:project, project)
expect(executor).to have_instance_variable_value(:out, output)
end
it 'defines :flush method on @out if it is absent' do
out = Class.new.new
expect(out).not_to respond_to(:flush)
described_class.new(server, out)
expect(out).to respond_to(:flush)
end
it 'sets current_user from options' do
user = double
executor = described_class.new(server, '', {current_user: user})
expect(executor).to have_instance_variable_value(:current_user, user)
end
end
describe '#report=' do
it 'sets report instance variable' do
executor.report= 'foo'
expect(executor).to have_instance_variable_value(:report, 'foo')
end
end
describe '#project=' do
it 'sets project instance variable' do
executor.project= 'foo'
expect(executor).to have_instance_variable_value(:project, 'foo')
end
end
describe '.reason_from_error_code' do
it 'returns symbol given an integer' do
expect(described_class.reason_from_error_code(2)).to eq :server_bootstrap_fail
end
it "returns :unknown_error if can't recognize error code" do
expect(described_class.reason_from_error_code(123)).to eq :unknown_error
end
end
describe '#create_server_object' do
it 'builds Server object' do
server = executor.create_server_object('created_by' => 'me')
expect(server).to be_a(Devops::Model::Server)
expect(server.project).to eq 'my_project'
expect(server.deploy_env).to eq 'foo'
expect(server.created_by).to eq 'me'
end
end
describe '#create_server' do
let(:image) { double('Image instance', remote_user: 'remote_user') }
let(:create_server) {
executor.create_server(
'created_by' => 'user',
'run_list' => @run_list,
'name' => 'node_name',
'key' => @key,
'without_bootstrap' => @without_bootstrap
)
}
before do
allow(provider).to receive(:create_server) { true }
allow(stubbed_connector).to receive(:image) { image }
allow(stubbed_connector).to receive(:server_insert)
@without_bootstrap = true
@run_list = %w(role[asd])
@key = 'key'
end
it 'builds server model from given options' do
create_server
expect(executor.server.created_by).to eq 'user'
expect(executor.server.chef_node_name).to eq 'node_name'
expect(executor.server.key).to eq @key
expect(executor.server.run_list).to eq @run_list
end
it 'sets run list to an empty array by default' do
@run_list = nil
create_server
expect(executor.server.run_list).to eq []
end
it 'sets key to default provider ssh key by default' do
@key = nil
allow(provider).to receive(:ssh_key) { 'default_key' }
create_server
expect(executor.server.key).to eq 'default_key'
end
it 'runs hooks' do
expect(executor).to receive(:run_hook).with(:before_create).ordered
expect(executor).to receive(:run_hook).with(:after_create).ordered
create_server
end
it 'creates server in cloud' do
expect(provider).to receive(:create_server).with(
an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output
)
create_server
end
it 'inserts built server into mongo' do
expect(stubbed_connector).to receive(:server_insert)
create_server
end
it 'schedules expiration for server' do
deploy_env.expires = '2m'
allow(DeleteServerWorker).to receive(:perform_in)
expect(DeleteServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'})
create_server
end
it "doesn't schedule expiration if deploy_env.expires is nil" do
deploy_env.expires = nil
expect(DeleteServerWorker).not_to receive(:perform_in)
create_server
end
context 'without_bootstrap option is false' do
it 'launches bootstrap' do
@without_bootstrap = false
allow(image).to receive(:bootstrap_template) { 'template' }
allow(executor).to receive(:two_phase_bootstrap)
expect(executor).to receive(:two_phase_bootstrap)
create_server
end
end
context 'without_bootstrap option is nil' do
it 'launches bootstrap' do
@without_bootstrap = nil
allow(image).to receive(:bootstrap_template) { 'template' }
allow(executor).to receive(:two_phase_bootstrap)
expect(executor).to receive(:two_phase_bootstrap)
create_server
end
end
context 'without_bootstrap option is true' do
it "doesn't launch bootstrap" do
@without_bootstrap = true
expect(executor).not_to receive(:two_phase_bootstrap)
create_server
end
end
context 'if error has been raised during execution' do
before do
allow(stubbed_connector).to receive(:server_delete)
allow(provider).to receive(:create_server) { raise }
end
it 'rollbacks server creating' do
expect(executor).to receive(:roll_back)
create_server
end
it 'deletes server from mongo' do
expect(stubbed_connector).to receive(:server_delete)
create_server
end
end
context "if creating server in cloud wasn't successful" do
it 'returns creating_server_in_cloud_failed error code' do
allow(provider).to receive(:create_server) { false }
expect(create_server).to eq 10
end
end
end
describe '#bootstrap', stubbed_knife: true do
let(:image) { double('Key instance', path: 'path') }
let(:bootstrap) { executor.bootstrap({}) }
before do
allow(executor).to receive(:sleep)
allow(executor).to receive(:last_command_successful?).and_return(true)
allow(executor).to receive(:execute_system_command)
allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node')
allow(stubbed_connector).to receive(:key).and_return(image)
allow(stubbed_connector).to receive(:server_set_chef_node_name)
allow(stubbed_knife).to receive(:knife_bootstrap).and_return(0)
end
it 'run before hook' do
expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered
expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered
bootstrap
end
context "when server's private ip is unset" do
it 'returns server_bootstrap_private_ip_unset error code' do
server.private_ip = nil
expect(bootstrap).to eq 4
end
end
it 'tries to ssh to server' do
expect(executor).to receive(:execute_system_command).with(/ssh/)
bootstrap
end
context "couldn't ssh to server" do
before { allow(executor).to receive(:last_command_successful?) { false } }
it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do
max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT
expect(executor).to receive(:execute_system_command).exactly(max_retries).times
bootstrap
end
it 'returns server_bootstrap_fail error code' do
expect(bootstrap).to eq 2
end
end
context 'after successful ssh check' do
before { allow(executor).to receive(:last_command_successful?).and_return(false, true) }
it "sets default chef node name if it's nil" do
executor.server.chef_node_name = nil
expect {bootstrap}.to change {executor.server.chef_node_name}.to 'chef_node'
end
it 'executes knife bootstrap' do
expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array))
bootstrap
end
it "bootstraps to public ip if it's set" do
server.public_ip = '8.8.8.8'
expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array))
bootstrap
end
context 'after successful bootstrap' do
it "updates server's chef node name in db" do
expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server))
bootstrap
end
end
context "if bootstraping wasn't successful" do
before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) }
it 'returns :server_bootstrap_fail code' do
expect(bootstrap).to eq 2
end
it "doesn't run after hook" do
expect(executor).to receive(:run_hook).with(:before_bootstrap, output)
bootstrap
end
end
end
end
describe '#two_phase_bootstrap', stubbed_knife: true do
let(:two_phase_bootstrap) { executor.two_phase_bootstrap({}) }
before do
allow(provider).to receive(:run_list) {[]}
allow(stubbed_connector).to receive(:server_delete)
end
context 'when bootstrap was successful' do
before do
allow(executor).to receive(:bootstrap) { 0 }
allow(executor).to receive(:check_server_on_chef_server) { false }
end
context 'if node presents on chef server' do
before do
allow(executor).to receive(:check_server_on_chef_server) { true }
allow(executor).to receive(:deploy_server)
allow(stubbed_knife).to receive(:set_run_list)
end
it 'builds run list' do
expect(executor).to receive(:compute_run_list)
two_phase_bootstrap
end
it 'sets run list to chef node' do
expect(stubbed_knife).to receive(:set_run_list)
two_phase_bootstrap
end
it 'deploys server' do
expect(executor).to receive(:deploy_server)
two_phase_bootstrap
end
context 'if deploy was successful' do
it 'returns 0' do
allow(executor).to receive(:deploy_server) { 0 }
expect(two_phase_bootstrap).to eq 0
end
end
context "if deploy wasn't successful" do
it 'returns :deploy_failed code' do
allow(executor).to receive(:deploy_server) { 1 }
expect(two_phase_bootstrap).to eq 8
end
end
context 'when an error occured during deploy' do
it 'returns :deploy_unknown_error code' do
allow(executor).to receive(:deploy_server) { raise }
expect(two_phase_bootstrap).to eq 6
end
end
end
context "if node doesn't present on chef server" do
it 'roll backs and then deletes server from mongo' do
allow(executor).to receive(:check_server_on_chef_server) { false }
allow(executor).to receive(:roll_back)
allow(stubbed_connector).to receive(:server_delete)
expect(executor).to receive(:roll_back).ordered
expect(stubbed_connector).to receive(:server_delete).ordered
two_phase_bootstrap
end
end
end
context "when bootstrap wasn't successful" do
it 'returns :server_bootstrap_fail error code' do
allow(executor).to receive(:bootstrap) { 1 }
expect(two_phase_bootstrap).to eq 2
end
end
context 'when an error occured during bootstrap' do
it 'returns :server_bootstrap_unknown_error error code' do
allow(executor).to receive(:bootstrap) { raise }
expect(two_phase_bootstrap).to eq 7
end
end
end
describe '#check_server_on_chef_server', stubbed_knife: true do
before do
server.chef_node_name = 'a'
allow(stubbed_knife).to receive(:chef_node_list) { @node_list }
allow(stubbed_knife).to receive(:chef_client_list) { @client_list }
end
it 'returns true when node_name in node list and in client list' do
@node_list = %w(a); @client_list = %w(a)
expect(executor.check_server_on_chef_server).to be true
end
it "returns false if node name isn't in node list" do
@node_list = []; @client_list = %w(a)
expect(executor.check_server_on_chef_server).to be false
end
it "returns false if node name isn't in node list" do
@node_list = %w(a); @client_list = []
expect(executor.check_server_on_chef_server).to be false
end
end
describe '#unbootstrap', stubbed_knife: true do
before do
allow(stubbed_connector).to receive_message_chain('key.path') { 'path_to_key' }
allow(stubbed_knife).to receive(:chef_node_delete)
allow(stubbed_knife).to receive(:chef_client_delete)
allow(executor).to receive(:execute_system_command) { '' }
allow(executor).to receive(:last_command_successful?) { true }
allow(executor).to receive(:sleep)
allow(Net::SSH).to receive(:start)
end
it 'connects by ssh' do
expect(Net::SSH).to receive(:start)
executor.unbootstrap
end
it 'returns hash with error after 5 unsuccessful retries' do
allow(Net::SSH).to receive(:start) { raise }
expect(Net::SSH).to receive(:start).exactly(5).times
expect(executor.unbootstrap).to be_a(Hash).and include(:error)
end
end
describe '#deploy_server_with_tags', stubbed_knife: true do
let(:current_tags) { @current_tags }
let(:initial_tags) { %w(a b) }
let(:joined_initial_tags) { initial_tags.join(' ') }
let(:given_tags) { %w(c d) }
let(:joined_given_tags) { given_tags.join(' ') }
let(:deploy_server_with_tags) { executor.deploy_server_with_tags(given_tags, {}) }
before do
@current_tags = initial_tags.dup
allow(stubbed_knife).to receive(:tags_list) { @current_tags }
allow(stubbed_knife).to receive(:swap_tags) do |_, tags_to_delete, tags_to_add|
@current_tags -= tags_to_delete.split
@current_tags += tags_to_add.split
end
allow(executor).to receive(:deploy_server)
end
context 'when tags are empty' do
it 'just deploys server' do
expect(executor).to receive(:deploy_server)
expect(stubbed_knife).not_to receive(:swap_tags)
executor.deploy_server_with_tags([], {})
end
end
context 'when tags are not empty' do
it 'temporarily swaps current_tags with given ones, deploys server and then restores tags' do
expect(stubbed_knife).to receive(:tags_list).ordered
expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_initial_tags, joined_given_tags).ordered
expect(executor).to receive(:deploy_server).ordered
expect(stubbed_knife).to receive(:swap_tags).with(instance_of(String), joined_given_tags, joined_initial_tags).ordered
deploy_server_with_tags
end
end
context 'if error occures during deploy' do
it 'restores tags anyway' do
allow(executor).to receive(:deploy_server) { raise }
expect {
deploy_server_with_tags
}.to raise_error StandardError
expect(current_tags).to eq initial_tags
end
end
context 'if cannot add tags to server' do
it 'returns :server_cannot_update_tags code' do
allow(stubbed_knife).to receive(:swap_tags) { false }
expect(deploy_server_with_tags).to eq 3
end
end
end
describe '#deploy_server', stubbed_knife: true do
let(:deploy_info) { @deploy_info }
let(:json_file_name) { 'json.json' }
let(:json_file_path) { File.join(SpecSupport.tmp_dir, json_file_name) }
let(:deploy_server) { executor.deploy_server(deploy_info) }
before do
allow(executor).to receive(:run_hook).with(:before_deploy, any_args)
allow(executor).to receive(:run_hook).with(:after_deploy, any_args)
allow(stubbed_knife).to receive(:ssh_stream) { 'Chef Client finished'}
allow(stubbed_connector).to receive(:key) { double('Key', path: 'path_to_key') }
allow(stubbed_connector).to receive(:server_update)
@deploy_info = {}
end
it 'runs before_deploy and after_deploy hooks' do
expect(executor).to receive(:run_hook).with(:before_deploy, any_args).ordered
expect(executor).to receive(:run_hook).with(:after_deploy, any_args).ordered
deploy_server
end
context 'when uses json file' do
before(:all) do
@tmp_files_at_start = Dir.entries(SpecSupport.tmp_dir)
end
before do
allow(DevopsConfig).to receive(:config).and_return({
project_info_dir: SpecSupport.tmp_dir,
address: 'host.com',
port: '8080',
url_prefix: 'api'
})
deploy_info['use_json_file'] = true
end
after(:all) do
diff = Dir.entries(SpecSupport.tmp_dir) - @tmp_files_at_start
diff.each do |file|
FileUtils.rm(File.join(SpecSupport.tmp_dir, file))
end
end
it 'writes deploy_info to json file if it not exists' do
expect { deploy_server }.to change { Dir.entries(SpecSupport.tmp_dir)}
end
it "writes deploy_info to given json file name if it doesn't exist" do
FileUtils.rm(json_file_path) if File.exists?(json_file_path)
deploy_info['json_file'] = json_file_name
expect { deploy_server }.to change {
Dir.entries(SpecSupport.tmp_dir)
}
FileUtils.rm(json_file_path)
end
it 'reads json from file if it exists' do
deploy_info['json_file'] = json_file_name
File.open(json_file_path, 'w') { |file| file.puts '{"foo": "bar"'}
expect { deploy_server }.not_to change {
Dir.entries(SpecSupport.tmp_dir)
}
FileUtils.rm(json_file_path)
end
it 'adds link to json to deploy command' do
deploy_info['json_file'] = json_file_name
regexp = %r(-j http://host.com:8080/api/v2.0/deploy/data/#{json_file_name})
expect(stubbed_knife).to receive(:ssh_stream).with(anything, regexp, any_args)
deploy_server
end
end
context "doesn't use json file" do
before do
deploy_info['use_json_file'] = false
deploy_info['run_list'] = %w(foo bar)
end
it "adds run list to command if server's stack is set" do
server.stack = 'stack'
expect(stubbed_knife).to receive(:ssh_stream).with(anything, %r(-r foo,bar), any_args)
deploy_server
end
it "doesn't add run list to command if server's stack is unset" do
expect(stubbed_knife).to receive(:ssh_stream).with(anything, 'chef-client --no-color', any_args)
deploy_server
end
end
it "uses server's key" do
expect(stubbed_connector).to receive(:key).with('key_id')
expect(stubbed_knife).to receive(:ssh_stream).with(any_args, 'path_to_key')
deploy_server
end
it "uses public ip if it's set" do
server.public_ip = '127.0.0.1'
expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, '127.0.0.1', any_args)
deploy_server
end
it "uses private_ip if public_ip isn't set" do
expect(stubbed_knife).to receive(:ssh_stream).with(anything, anything, server.private_ip, any_args)
deploy_server
end
context 'if deploy was successful' do
it "updates server's last operation" do
expect(server).to receive(:set_last_operation).with('deploy', anything)
expect(stubbed_connector).to receive(:server_update).with(server)
deploy_server
end
it 'returns 0' do
expect(deploy_server).to eq 0
end
end
context "when deploy wasn't successful" do
before { allow(stubbed_knife).to receive(:ssh_stream) { 'fail'} }
it "doesn't run after_deploy hook" do
expect(executor).to receive(:run_hook).with(:before_deploy, any_args)
expect(executor).not_to receive(:run_hook).with(:after_deploy, any_args)
deploy_server
end
it 'returns 1' do
expect(deploy_server).to eq 1
end
end
end
describe '#delete_from_chef_server', stubbed_knife: true do
let(:delete_from_chef_server) { executor.delete_from_chef_server('foo') }
before do
allow(stubbed_knife).to receive(:chef_client_delete)
allow(stubbed_knife).to receive(:chef_node_delete)
delete_from_chef_server
end
it 'returns hash with :chef_node and :chef_client keys' do
expect(delete_from_chef_server).to be_a(Hash).and include(:chef_node, :chef_client)
end
it 'calls to :chef_node_delete and :chef_client_delete' do
expect(stubbed_knife).to have_received(:chef_client_delete)
expect(stubbed_knife).to have_received(:chef_node_delete)
end
end
describe '#delete_server' do
let(:delete_server) { executor.delete_server }
context 'when server is static' do
before do
server.provider = 'static'
allow(stubbed_connector).to receive(:server_delete).with(server.id)
allow(executor).to receive(:unbootstrap)
end
it 'performs unbootstrap' do
expect(executor).to receive(:unbootstrap)
delete_server
end
it 'deletes server from mongo' do
expect(stubbed_connector).to receive(:server_delete).with(server.id)
delete_server
end
it 'returns message and nil' do
expect(delete_server.first).to be_a(String)
expect(delete_server.last).to be nil
end
it "doesn't try to remove it from cloud" do
expect{delete_server}.not_to raise_error
end
end
context "when server isn't static", stubbed_knife: true do
before do
allow(server).to receive_message_chain('provider_instance.delete_server')
allow(stubbed_connector).to receive(:server_delete).with(server.id)
allow(stubbed_knife).to receive(:chef_node_delete)
allow(stubbed_knife).to receive(:chef_client_delete)
end
it 'deletes from info about note chef server' do
allow(executor).to receive(:delete_from_chef_server).and_call_original
expect(executor).to receive(:delete_from_chef_server)
delete_server
end
it "doesn't unbootstrap server" do
expect(executor).not_to receive(:unbootstrap)
delete_server
end
it 'deletes server from cloud' do
expect(server).to receive_message_chain('provider_instance.delete_server').with(server)
delete_server
end
it "doesn't raise error if server wasn't found in cloud" do
allow(server).to receive_message_chain('provider_instance.name')
allow(server).to receive_message_chain('provider_instance.delete_server') {
raise Fog::Compute::OpenStack::NotFound
}
expect { delete_server }.not_to raise_error
end
it 'deletes server from mongo' do
expect(stubbed_connector).to receive(:server_delete).with(server.id)
delete_server
end
it 'returns message and hash with :chef_node, :chef_client and :server keys' do
expect(delete_server.first).to be_a(String)
expect(delete_server.last).to be_a(Hash).and include(:chef_client, :chef_node, :server)
end
end
end
describe '#rollback' do
before do
allow(executor).to receive(:delete_from_chef_server) { {} }
allow(server).to receive_message_chain('provider_instance.delete_server')
end
it "does nothing if server.id is nil" do
server.id = nil
expect(executor).not_to receive(:delete_from_chef_server)
expect(server).not_to receive(:provider_instance)
executor.roll_back
end
it 'deletes node from chef server and instance from cloud' do
expect(executor).to receive(:delete_from_chef_server)
expect(server).to receive_message_chain('provider_instance.delete_server')
executor.roll_back
end
it "doesn't raise if deleting server in cloud raises an error" do
allow(server).to receive_message_chain('provider_instance.delete_server') { raise }
expect { executor.roll_back }.not_to raise_error
end
end
describe '#add_run_list_to_deploy_info' do
it "doesn't change deploy info if it already includes run list" do
deploy_info = {'run_list' => %w(foo)}
expect {
executor.add_run_list_to_deploy_info(output, deploy_info)
}.not_to change { deploy_info }
end
it 'computes and adds run_list to deploy_info' do
deploy_info = {}
allow(executor).to receive(:compute_run_list) { %w(foo) }
expect(executor).to receive(:compute_run_list)
executor.add_run_list_to_deploy_info(output, deploy_info)
expect(deploy_info['run_list']).to eq %w(foo)
end
end
describe '#compute_run_list' do
before do
allow(deploy_env).to receive_message_chain('provider_instance.run_list') { %w(a) }
project.run_list = %w(b)
deploy_env.run_list = %w(c)
server.run_list = %w(d)
end
it "returns array with run list merged from provider's, project's, env's and server's run lists" do
expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d))
end
it "includes stack's run list if stack is set", stubbed_connector: true do
server.stack = 'stack'
allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: %w(e)) }
expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c d e))
end
it "doesn't contain nils" do
server.run_list = nil
server.stack = 'stack'
allow(stubbed_connector).to receive(:stack) { instance_double(Devops::Model::StackEc2, run_list: nil) }
expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a b c))
end
it 'returns uniq elements' do
project.run_list = %w(a)
deploy_env.run_list = %w(a)
expect(executor.compute_run_list).to be_an(Array).and contain_exactly(*%w(a d))
end
end
end

2
devops-service/spec/factories/key.rb
View File

@ -3,7 +3,7 @@ require 'db/mongo/models/key'
FactoryGirl.define do
factory :key, class: Devops::Model::Key do
id 'user_key'
path SpecSupport::BLANK_FILE
path SpecSupport.blank_file
scope 'user'
end
end

16
devops-service/spec/models/deploy_env/deploy_env_ec2_spec.rb
View File

@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
RSpec.describe Devops::Model::DeployEnvEc2, type: :model do
let(:env) { build(:deploy_env_ec2) }
describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do
describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
before do
provider_double = instance_double('Provider::Ec2',
flavors: [{'id' => 'flavor'}],
networks: [{'default' => {'vpcId' => 'foo'}}],
groups: {'default' => nil},
images: [{'id' => 'image'}]
)
allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
allow(Provider::ProviderFactory).to receive(:get) { provider_double }
allow(stubbed_connector).to receive(:users_names) { %w(root) }
allow(stubbed_connector).to receive(:available_images) { %w(image) }
allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
end
it_behaves_like 'deploy env'
it_behaves_like 'cloud deploy env'
end

16
devops-service/spec/models/deploy_env/deploy_env_openstack_spec.rb
View File

@ -5,7 +5,21 @@ require_relative 'shared_cloud_deploy_env_specs'
RSpec.describe Devops::Model::DeployEnvOpenstack, type: :model do
let(:env) { build(:deploy_env_openstack) }
describe 'it inherits from cloud deploy_env', stubbed_env_validators: true, stubbed_logger: true do
describe 'it inherits from cloud deploy_env', stubbed_connector: true, stubbed_logger: true do
before do
provider_double = instance_double('Provider::Openstack',
flavors: [{'id' => 'flavor'}],
networks: [{'default' => {'vpcId' => 'foo'}}],
groups: {'default' => nil},
images: [{'id' => 'image'}]
)
allow(Provider::ProviderFactory).to receive(:providers) { %w(openstack) }
allow(Provider::ProviderFactory).to receive(:get) { provider_double }
allow(stubbed_connector).to receive(:users_names) { %w(root) }
allow(stubbed_connector).to receive(:available_images) { %w(image) }
allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_openstack, id: 'template')] }
end
it_behaves_like 'deploy env'
it_behaves_like 'cloud deploy env'
end

4
devops-service/spec/models/deploy_env/deploy_env_static_spec.rb
View File

@ -5,10 +5,10 @@ RSpec.describe Devops::Model::DeployEnvStatic, type: :model do
let(:env) { build(:deploy_env_static) }
describe 'it inherits from deploy env', stubbed_logger: true do
describe 'it inherits from deploy env', stubbed_logger: true, stubbed_connector: true do
before do
allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(static))
allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
allow(stubbed_connector).to receive(:users_names) { %w(root) }
end
it_behaves_like 'deploy env'

33
devops-service/spec/models/project_spec.rb
View File

@ -3,7 +3,21 @@ require 'db/mongo/models/project'
RSpec.describe Devops::Model::Project, type: :model do
let(:project) { build(:project) }
describe 'validation rules:', stubbed_env_validators: true, stubbed_logger: true do
describe 'validation rules:', stubbed_connector: true, stubbed_logger: true do
before do
provider_double = instance_double('Provider::Ec2',
flavors: [{'id' => 'flavor'}],
networks: [{'default' => {'vpcId' => 'foo'}}],
groups: {'default' => nil},
images: [{'id' => 'image'}]
)
allow(Provider::ProviderFactory).to receive(:providers) { %w(ec2) }
allow(Provider::ProviderFactory).to receive(:get) { provider_double }
allow(stubbed_connector).to receive(:users_names) { %w(root) }
allow(stubbed_connector).to receive(:available_images) { %w(image) }
allow(stubbed_connector).to receive(:stack_templates) { [build(:stack_template_ec2, id: 'template')] }
end
include_examples 'field type validation', :id, :not_nil, :non_empty_string
include_examples 'field type validation', :deploy_envs, :not_nil, :non_empty_array
include_examples 'field type validation', :description, :maybe_nil, :maybe_empty_string
@ -23,6 +37,23 @@ RSpec.describe Devops::Model::Project, type: :model do
project = build(:project, with_deploy_env_identifiers: ['foo', nil])
expect(project).not_to be_valid
end
describe 'components validation' do
it 'is valid with components with filenames' do
project.components = {'foo' => {'filename' => 'bar'}}
expect{project.validate_components}.not_to raise_error
end
it "isn't valid if components isn't a hash" do
project.components = []
expect{project.validate_components}.to raise_error InvalidRecord
end
it "raises InvalidRecord if one of componentsц hasn't filename" do
project.components = {'foo' => {}}
expect{project.validate_components}.to raise_error InvalidRecord
end
end
end
describe '.fields' do

1
devops-service/spec/models/provider_account/openstack_provider_account_spec.rb
View File

@ -1,6 +1,5 @@
require 'spec_helper'
# не пытайся выделить в shared_specs, фигня выйдет
RSpec.describe Devops::Model::OpenstackProviderAccount, type: :model do
let(:provider_account) { build(:openstack_provider_account) }

8
devops-service/spec/models/server_spec.rb
View File

@ -44,6 +44,14 @@ RSpec.describe Devops::Model::Server, type: :model do
end
end
describe '.build_from_bson' do
it 'takes a hash and returns instance of Server model' do
model = described_class.build_from_bson('id' => 'foo')
expect(model).to be_an_instance_of(described_class)
expect(model.id).to eq 'foo'
end
end
it '#to_hash_without_id returns not nil fields' do
server = described_class.new('run_list' => [], 'project' => 'asd')
expect(server.to_hash_without_id.keys).to match_array(%w(run_list project))

16
devops-service/spec/models/stack_template/stack_template_ec2_spec.rb
View File

@ -6,21 +6,23 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do
before do
allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true }
allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} }
provider_double = instance_double('Provider::Ec2',
validate_stack_template: true,
store_stack_template: {'url' => 'template_url'}
)
allow(Provider::ProviderFactory).to receive(:get) { provider_double }
end
it_behaves_like 'stack template'
it 'uploads file to S3' do
expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template')
params = {
'id' => 'foo',
result = described_class.create('id' => 'foo',
'template_body' => '{}',
'owner' => 'root',
'provider' => 'ec2'
}
expect(described_class.create(params)).to be_an_instance_of(described_class)
)
expect(result).to be_an_instance_of(described_class)
expect(result.template_url).to eq 'template_url'
end
end

14
devops-service/spec/models/user_spec.rb
View File

@ -79,13 +79,25 @@ RSpec.describe Devops::Model::User, type: :model do
end
end
describe '.build_from_bson' do
it 'builds User model from given hash and assigns id' do
model = described_class.build_from_bson('_id' => 'foo', 'username' => 'not shown', 'email' => 'baz')
expect(model.id).to eq 'foo'
expect(model.email).to eq 'baz'
end
end
describe '#check_privileges' do
it "raises InvalidPrivileges if user hasn't specified privilege" do
expect { user.check_privileges('key', 'w') }.to raise_error(InvalidPrivileges)
end
it 'does nothing is user has specified privilege' do
user.check_privileges('key', 'r')
expect{user.check_privileges('key', 'r')}.not_to raise_error
end
it 'raises InvalidPrivileges if given privelege is wrong' do
expect{user.check_privileges('key', 't')}.to raise_error InvalidPrivileges
end
end

2
devops-service/spec/shared_contexts/stubbed_connector.rb
View File

@ -1,5 +1,5 @@
RSpec.shared_context 'stubbed calls to connector', stubbed_connector: true do
let(:stubbed_connector) { double() }
let(:stubbed_connector) { instance_double(MongoConnector) }
before do
allow(Devops::Db).to receive(:connector) { stubbed_connector }
end

5
devops-service/spec/shared_contexts/stubbed_env_validators.rb
View File

@ -1,9 +1,10 @@
RSpec.shared_context 'stubbed calls to connector in env validators', stubbed_env_validators: true do
before do
allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2 openstack))
allow_any_instance_of(env_class).to receive_message_chain('provider_instance.flavors').and_return [{'id' => 'flavor'}]
allow_any_instance_of(Validators::Helpers::Users).to receive(:available_users).and_return(['root'])
allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
# allow_any_instance_of(Validators::DeployEnv::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
# allow_any_instance_of(Validators::FieldValidator::Flavor).to receive(:available_flavors).and_return([{'id' => 'flavor'}])
allow_any_instance_of(Validators::DeployEnv::Groups).to receive(:available_groups).and_return(['default'])
allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])
allow_any_instance_of(Validators::DeployEnv::Image).to receive(:available_images).and_return([{'id' => 'image'}])

6
devops-service/spec/shared_contexts/stubbed_knife.rb Normal file
View File

@ -0,0 +1,6 @@
RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do
let(:stubbed_knife) { instance_double(KnifeCommands) }
before do
allow(KnifeFactory).to receive(:instance) { stubbed_knife }
end
end

5
devops-service/spec/shared_contexts/stubbed_logger.rb
View File

@ -1,7 +1,6 @@
RSpec.shared_context 'stubbed calls to logger', stubbed_logger: true do
before do
allow(DevopsLogger).to receive_message_chain('logger.debug')
allow(DevopsLogger).to receive_message_chain('logger.info')
allow(DevopsLogger).to receive_message_chain('logger.error')
logger = double('logger', debug: nil, info: nil, error: nil, warn: nil)
allow(DevopsLogger).to receive(:logger) { logger }
end
end

65
devops-service/spec/spec_helper.rb
View File

@ -4,32 +4,53 @@ require 'factory_girl'
require 'active_support/core_ext/hash/indifferent_access'
require 'active_support/inflector'
# setup load_path and require support files
root = File.join(File.dirname(__FILE__), "..")
$LOAD_PATH.push root unless $LOAD_PATH.include? root
# suppress output
original_stdout = $stdout
$stdout = File.open(File::NULL, "w")
Dir[("./spec/support/**/*.rb")].each { |f| require f }
Dir[("./spec/shared_contexts/**/*.rb")].each { |f| require f }
# Factory girl configuration
FactoryGirl.define do
# do not try to persist, but raise validation errors
to_create { |model| model.validate! }
def suppress_output!
original_stdout = $stdout
$stdout = File.open(File::NULL, "w")
RSpec.configure do |config|
config.after(:all) do
$stdout = original_stdout
end
end
end
FactoryGirl.find_definitions
def check_coverage
require 'simplecov'
if ENV['JENKINS']
require 'simplecov-rcov'
SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
end
SimpleCov.start do
add_filter { |src| src.filename =~ /spec\// }
end
end
def require_support_files
root = File.join(File.dirname(__FILE__), "..")
$LOAD_PATH.push root unless $LOAD_PATH.include?(root)
Dir[("#{root}/spec/support/**/*.rb")].each { |f| require f }
Dir[("#{root}/spec/shared_contexts/**/*.rb")].each { |f| require f }
end
def setup_factory_girl
FactoryGirl.define do
# do not persist, but raise validation errors
to_create { |model| model.validate! }
end
FactoryGirl.find_definitions
RSpec.configure { |config| config.include FactoryGirl::Syntax::Methods }
end
# extra configuration
suppress_output!
check_coverage if ENV['COVERAGE']
require_support_files
setup_factory_girl
# RSpec configuration
RSpec.configure do |config|
config.include FactoryGirl::Syntax::Methods
config.after(:all) do
$stdout = original_stdout
end
# rspec-expectations config goes here. You can use an alternate
# assertion/expectation library such as wrong or the stdlib/minitest
# assertions if you prefer.

6
devops-service/spec/support/instance_variable_matcher.rb Normal file
View File

@ -0,0 +1,6 @@
RSpec::Matchers.define :have_instance_variable_value do |name, value|
match do |actual|
actual.instance_variable_defined?("@#{name}") &&
actual.instance_variable_get("@#{name}") == value
end
end

14
devops-service/spec/support/spec_support.rb
View File

@ -2,7 +2,15 @@ require 'yaml'
module SpecSupport
ROOT = File.join(__dir__, '../../')
BLANK_FILE = File.join(ROOT, 'spec/support/blank_file')
def self.blank_file
File.join(ROOT, 'spec/support/templates/blank_file')
end
# for specs which write files
def self.tmp_dir
File.join(ROOT, 'spec/support/tmp/')
end
def self.db_params
@db_params ||= begin
@ -32,4 +40,8 @@ module SpecSupport
end
end
end
def self.root
File.join(__dir__, '../../')
end
end

0
devops-service/spec/support/blank_file → devops-service/spec/support/templates/blank_file
View File

0
devops-service/spec/support/tmp/.gitkeep Normal file
View File

45
devops-service/spec/workers/chef_node_name_builder_spec.rb Normal file
View File

@ -0,0 +1,45 @@
require 'workers/stack_bootstrap/chef_node_name_builder'
RSpec.describe ChefNodeNameBuilder do
let(:server_info) do
{
'id' => 'server1',
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2',
'tags' => {
'cid:priority' => '3'
}
}
end
let(:project) { build(:project, id: 'proj', with_deploy_env_identifiers: %w(dev)) }
let(:env) { project.deploy_env('dev') }
let(:name_builder) { described_class.new(server_info, project, env) }
let(:build_name) { name_builder.build_node_name }
def set_mask(mask)
server_info['tags']['cid:node-name-mask'] = mask
end
describe '#build_node_name' do
it 'uses default mask ("$project-$nodename-$env")' do
expect(build_name).to eq 'proj-server1-dev'
end
it 'substitutes project, env and nodename' do
set_mask('$project/$env/$nodename')
expect(build_name).to eq 'proj/dev/server1'
end
it 'substitutes $time' do
set_mask('$nodename-$time')
expect(build_name).to match /server1-\d+/
end
it 'substitutes underscores to dashes' do
server_info['id'] = 'server_1'
expect(build_name).to match 'proj-server-1-dev'
end
end
end

128
devops-service/spec/workers/stack_bootstrap_worker_spec.rb Normal file
View File

@ -0,0 +1,128 @@
require 'workers/stack_bootstrap_worker'
RSpec.describe StackBootstrapWorker, type: :worker, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:file) { 'temp.txt' }
let(:stack_attrs) { attributes_for(:stack_ec2).stringify_keys }
let(:perform_with_bootstrap) { worker.perform('stack_attributes' => stack_attrs) }
let(:perform_without_bootstrap) { worker.perform('stack_attributes' => stack_attrs.merge('without_bootstrap' => true)) }
let(:stack_synchronizer) { instance_double(StackSynchronizer, sync: 0) }
let(:stack_servers_bootstrapper) { instance_double(StackServersBootstrapper, bootstrap: true) }
let(:stack_servers_persister) { instance_double(StackServersPersister, persist: {1 => build_list(:server, 2)}) }
let(:worker) { described_class.new }
before do
allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2))
allow(stubbed_connector).to receive(:save_report)
allow(stubbed_connector).to receive(:stack_insert)
allow(worker).to receive(:stack_synchronizer) { stack_synchronizer }
allow(worker).to receive(:stack_servers_bootstrapper) { stack_servers_bootstrapper }
allow(worker).to receive(:stack_servers_persister) { stack_servers_persister }
allow(worker).to receive(:call).and_yield(out, file)
allow(Devops::Model::StackEc2).to receive(:create) { Devops::Model::StackEc2.new(stack_attrs) }
end
it 'requires "stack_attributes" in options' do
expect{
worker.perform({})
}.to raise_error KeyError
end
it 'saves report about operation' do
expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report))
perform_without_bootstrap
end
it 'saves report about operation, creates stack and persists stack servers' do
allow(worker).to receive(:create_stack).and_call_original
expect(stubbed_connector).to receive(:save_report).with(instance_of(Devops::Model::Report)).ordered
expect(worker).to receive(:create_stack).ordered
expect(stack_servers_persister).to receive(:persist).ordered
perform_without_bootstrap
end
context 'if without_bootstrap is true' do
it "doesn't bootstrap servers" do
expect(stack_servers_bootstrapper).not_to receive(:bootstrap)
perform_without_bootstrap
end
it 'returns 0' do
expect(perform_without_bootstrap).to eq 0
end
end
context 'if without_bootstrap is false or not set' do
it 'bootstraps servers in order by priorities, separately' do
first_servers = build_list(:server, 2)
last_servers = build_list(:server, 3)
allow(stack_servers_persister).to receive(:persist) {
{3 => first_servers, 1 => last_servers}
}
expect(stack_servers_bootstrapper).to receive(:bootstrap).with(first_servers).ordered
expect(stack_servers_bootstrapper).to receive(:bootstrap).with(last_servers).ordered
perform_with_bootstrap
end
context 'when bootstraping servers was successful' do
it 'returns 0' do
expect(perform_with_bootstrap).to eq 0
end
end
context 'when a known error occured during servers bootstrap' do
before do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapError }
end
it 'rollbacks stack and returns 2' do
expect_any_instance_of(Devops::Model::StackEc2).to receive(:delete_stack_in_cloud!)
expect(stubbed_connector).to receive(:stack_servers_delete)
expect(stubbed_connector).to receive(:stack_delete)
perform_with_bootstrap
end
it 'returns 2' do
allow(worker).to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 2
end
end
context 'when a known error occured during servers deploy' do
it "doesn't rollback stack and returns 3" do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerDeployError }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 3
end
end
context "when a servers bootstrap & deploy haven't been finished due to timeout" do
it "doesn't rollback stack and returns 3" do
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise StackServerBootstrapDeployTimeout }
expect(worker).not_to receive(:rollback_stack!)
expect(perform_with_bootstrap).to eq 4
end
end
context 'when an unknown error occured during servers bootsrap and deploy' do
it 'rollbacks stack and reraises that error' do
error = StandardError.new
allow(stack_servers_bootstrapper).to receive(:bootstrap) { raise error }
allow(worker).to receive(:rollback_stack!)
expect(worker).to receive(:rollback_stack!)
expect{perform_with_bootstrap}.to raise_error(error)
end
end
end
context "when stack creation wasn't successful" do
it 'returns 1' do
allow(stack_synchronizer).to receive(:sync) { 5 }
allow(stack_synchronizer).to receive(:reason_from_error_code) { :error }
expect(perform_without_bootstrap).to eq 1
end
end
end

59
devops-service/spec/workers/stack_servers_bootstrapper_spec.rb Normal file
View File

@ -0,0 +1,59 @@
require 'workers/stack_bootstrap_worker'
RSpec.describe StackServersBootstrapper, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:jid) { 1000 }
let(:bootstrapper) { described_class.new(out, jid) }
let(:servers) { [build(:server, id: 'a'), build(:server, id: 'b')] }
let(:bootstrap_job_ids) { %w(100 200) }
let(:subreport1) { build(:report, id: bootstrap_job_ids.first) }
let(:subreport2) { build(:report, id: bootstrap_job_ids.last) }
describe '#bootstrap' do
let(:bootstrap!) { bootstrapper.bootstrap(servers) }
before do
allow(Worker).to receive(:start_async).and_return(*bootstrap_job_ids)
allow(stubbed_connector).to receive(:add_report_subreports)
allow(stubbed_connector).to receive(:report) do |subreport_id|
subreport_id == '100' ? subreport1 : subreport2
end
allow(bootstrapper).to receive(:sleep)
end
it 'start bootstrap workers' do
expect(Worker).to receive(:start_async).with(BootstrapWorker, hash_including(:server_attrs, :bootstrap_template, :owner))
bootstrap!
end
it 'add subreports' do
expect(stubbed_connector).to receive(:add_report_subreports).with(jid, bootstrap_job_ids)
bootstrap!
end
it 'waits for job to end' do
allow(subreport1).to receive(:status).and_return('running', 'running', 'running', 'completed')
allow(subreport2).to receive(:status).and_return('running', 'running', 'running', 'completed')
expect(bootstrapper).to receive(:sleep).exactly(2*4).times
bootstrap!
end
it 'raises StackServerBootstrapError if an error occured during bootstrap' do
allow(subreport1).to receive(:status) {'failed'}
allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:server_bootstrap_fail) }
expect { bootstrap! }.to raise_error StackServerBootstrapError
end
it 'raises StackServerDeployError if an error occured during deploy' do
allow(subreport1).to receive(:status) {'failed'}
allow(subreport1).to receive(:job_result_code) { Devops::Executor::ServerExecutor.error_code(:deploy_failed) }
expect { bootstrap! }.to raise_error StackServerDeployError
end
it "raises StackServerBootstrapDeployTimeout if bootstrap and deploy hasn't been finished in 5000 seconds" do
allow(subreport1).to receive(:status) {'running'}
expect { bootstrap! }.to raise_error StackServerBootstrapDeployTimeout
end
end
end

116
devops-service/spec/workers/stack_servers_persister_spec.rb Normal file
View File

@ -0,0 +1,116 @@
require 'workers/stack_bootstrap/stack_servers_persister'
RSpec.describe StackServersPersister, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:run_list) { ['role[asd]'] }
let(:stack) { build(:stack, deploy_env: 'foo', run_list: run_list) }
let(:project) { build(:project, id: 'name') }
let(:persister) { described_class.new(stack, out) }
let(:provider) { instance_double(Provider::Ec2, name: 'ec2') }
let(:server_info_hash) do
{
'id' => 'server1',
'name' => 'server_name',
'key_name' => 'key',
'private_ip' => '127.0.0.1',
'public_ip' => '127.0.0.2',
'tags' => {
'cid:priority' => '3'
}
}
end
before do
allow(stubbed_connector).to receive(:project) { project }
allow(stubbed_connector).to receive(:image) {
instance_double(Devops::Model::Image, remote_user: 'user')
}
allow(stubbed_connector).to receive(:server_insert)
allow(stack).to receive(:provider_instance) { provider }
allow(provider).to receive(:stack_servers) {[server_info_hash]}
end
describe '#persist' do
it 'fetches stack servers info' do
expect(provider).to receive(:stack_servers).with(stack)
persister.persist
end
it "doesn't raise error if cid:priority tag is absent" do
server_info_hash['tags'].delete('cid:priority')
expect {persister.persist}.not_to raise_error
end
it 'returns hash {priority_as_integer => array of Devops::Model::Server}' do
result = persister.persist
expect(result).to be_a(Hash)
expect(result[3]).to be_an_array_of(Devops::Model::Server).and have_size(1)
end
it 'takes id, key_name, private_ip and public_ip attrs from info hash' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.id).to eq 'server1'
expect(server.key).to eq 'key'
expect(server.private_ip).to eq '127.0.0.1'
expect(server.public_ip).to eq '127.0.0.2'
end
persister.persist
end
it 'takes created_by, run_list and stack attrs from stack' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.created_by).to eq 'root'
expect(server.run_list).to eq run_list
expect(server.stack).to eq 'iamstack'
end
persister.persist
end
it 'takes remote_user from image user' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.remote_user).to eq 'user'
end
persister.persist
end
it "takes deploy_env from project's deploy_env identifier" do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.deploy_env).to eq 'foo'
end
persister.persist
end
it "takes default provider's ssh key if info doesn't contain it" do
allow(provider).to receive(:ssh_key) { 'default_key' }
server_info_hash.delete('key_name')
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.key).to eq 'default_key'
end
persister.persist
end
it "sets server's run list to empty array if stack's run_list is nil" do
stack.run_list = nil
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.run_list).to eq []
end
persister.persist
end
it 'build chef_node_name with default mask "$project-$nodename-$env"' do
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-server1-foo'
end
persister.persist
end
it "builds chef_node_name with custom mask if info['tags']['cid:node-name-mask'] exists" do
server_info_hash['tags']['cid:node-name-mask'] = '$project-$nodename-123'
expect(stubbed_connector).to receive(:server_insert) do |server|
expect(server.chef_node_name).to eq 'name-server1-123'
end
persister.persist
end
end
end

82
devops-service/spec/workers/stack_synchronizer_spec.rb Normal file
View File

@ -0,0 +1,82 @@
require 'workers/stack_bootstrap/stack_synchronizer'
RSpec.describe StackSynchronizer, stubbed_connector: true do
let(:out) { double(:out, puts: nil, flush: nil) }
let(:stack) { build(:stack) }
let(:syncer) { described_class.new(stack, out) }
before do
allow(stack).to receive(:sync_details!)
allow(stack).to receive(:events).and_return( [{'event_id' => 1}] )
allow(syncer).to receive(:sleep)
allow(stubbed_connector).to receive(:stack_update)
lots_of_statuses = ['CREATE_IN_PROGRESS'] * 10 + ['CREATE_COMPLETE']
allow(stack).to receive(:stack_status).and_return(*lots_of_statuses)
end
describe '#sync' do
it 'waits for stack creating to be finished' do
expect(syncer).to receive(:sleep).at_least(10).times
expect(stack).to receive(:sync_details!).at_least(10).times
syncer.sync
end
it 'prints each message only once' do
event1 = {'event_id' => 1, 'timestamp' => 't1'}
event2 = {'event_id' => 2, 'timestamp' => 't2'}
event3 = {'event_id' => 3, 'timestamp' => 't3'}
allow(stack).to receive(:events).and_return([event1], [event1, event2], [event1, event2, event3])
syncer.sync
expect(out).to have_received(:puts).with(/t1/).once.ordered
expect(out).to have_received(:puts).with(/t2/).once.ordered
expect(out).to have_received(:puts).with(/t3/).once.ordered
end
context 'when stack creating was successful' do
it 'updates stack in DB when stack creating is finished and returns 0' do
expect(stubbed_connector).to receive(:stack_update).with(stack)
expect(syncer.sync).to eq 0
end
end
context 'when stack was rollbacked' do
it 'returns 1 (:stack_rolled_back)' do
allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE')
expect(syncer.sync).to eq 1
end
end
context 'when unkown stack status was found' do
it 'returns 2 (:unkown_status)' do
allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'unknown')
expect(syncer.sync).to eq 2
end
end
context "when stack hasn't been synced in an hour" do
it 'returns 3 (:timeout)' do
allow(stack).to receive(:stack_status) {'CREATE_IN_PROGRESS'}
expect(syncer.sync).to eq 3
end
end
context 'when an error occured during syncing', stubbed_logger: true do
it 'returns 5 (:error)' do
allow(stack).to receive(:stack_status).and_return('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
allow(stubbed_connector).to receive(:stack_update) { raise }
expect(syncer.sync).to eq 5
end
end
end
describe '#reason_from_error_code' do
it 'returns reason as symbol for integer error_code' do
expect(syncer.reason_from_error_code(1)).to eq :stack_rolled_back
expect(syncer.reason_from_error_code(2)).to eq :unkown_status
expect(syncer.reason_from_error_code(3)).to eq :timeout
expect(syncer.reason_from_error_code(5)).to eq :error
end
end
end

1
devops-service/workers/delete_server_worker.rb
View File

@ -1,6 +1,7 @@
require "db/mongo/models/server"
require "db/mongo/models/report"
require "lib/executors/server_executor"
require "workers/worker"
class DeleteServerWorker < Worker

15
devops-service/workers/stack_bootstrap/chef_node_name_builder.rb Normal file
View File

@ -0,0 +1,15 @@
class ChefNodeNameBuilder
def initialize(server_info, project, env)
@server_info, @project, @env = server_info, project, env
@mask = server_info['tags']['cid:node-name-mask'] || '$project-$nodename-$env'
end
def build_node_name
@mask.gsub!('$project', @project.id)
@mask.gsub!('$env', @env.identifier)
@mask.gsub!('$nodename', @server_info['id'])
@mask.gsub!('$time', Time.now.to_i.to_s)
@mask.gsub!('_', '-')
@mask
end
end

4
devops-service/workers/stack_bootstrap/errors.rb Normal file
View File

@ -0,0 +1,4 @@
class StackCreatingError < StandardError; end
class StackServerBootstrapError < StandardError; end
class StackServerDeployError < StandardError; end
class StackServerBootstrapDeployTimeout < StandardError; end

78
devops-service/workers/stack_bootstrap/stack_servers_bootstrapper.rb Normal file
View File

@ -0,0 +1,78 @@
require 'workers/bootstrap_worker'
require "workers/stack_bootstrap/errors"
class StackServersBootstrapper
include PutsAndFlush
attr_reader :out
def initialize(out, jid)
@out, @jid = out, jid
end
def bootstrap(servers)
@servers = servers
puts_and_flush "\nStart bootstraping stack servers"
servers_jobs_ids = start_workers
::Devops::Db.connector.add_report_subreports(@jid, servers_jobs_ids.values)
out.puts
servers_jobs_ids.each do |server_id, subreport_id|
job_result_code = wait_for_job(server_id, subreport_id)
check_job_result!(server_id, job_result_code)
end
puts_and_flush "Stack servers have been bootstraped"
end
private
def check_job_result!(server_id, job_result_code)
return if job_result_code == 0
reason = Devops::Executor::ServerExecutor.reason_from_error_code(job_result_code)
puts_and_flush "Operation result for #{server_id}: #{reason}"
if error_occured_during_bootstrap?(reason)
raise StackServerBootstrapError # will cause rollback of a stack
else
raise StackServerDeployError #will not cause rollback of a stack
end
end
def error_occured_during_bootstrap?(reason)
Devops::Executor::ServerExecutor.bootstrap_errors_reasons.include?(reason)
end
def wait_for_job(server_id, subreport_id)
1000.times do
sleep(5)
subreport = ::Devops::Db.connector.report(subreport_id)
case subreport.status
when Worker::STATUS::COMPLETED
puts_and_flush "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
return 0
when Worker::STATUS::FAILED
puts_and_flush "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
return subreport.job_result_code
end
end
puts_and_flush "Waiting for job #{subreport_id} halted: timeout reached."
raise StackServerBootstrapDeployTimeout
end
# returns hash: {server_id => worker_job_id}
def start_workers
servers_jobs_ids = {}
@servers.each do |server|
job_id = Worker.start_async(::BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
@out.puts "Bootstraping server '#{server.id}'... job id: #{job_id}"
servers_jobs_ids[server.id] = job_id
end
puts_and_flush "\n"
servers_jobs_ids
end
end

69
devops-service/workers/stack_bootstrap/stack_servers_persister.rb Normal file
View File

@ -0,0 +1,69 @@
require 'workers/stack_bootstrap/chef_node_name_builder'
class StackServersPersister
include PutsAndFlush
attr_reader :stack, :out
def initialize(stack, out)
@stack, @out = stack, out
@project = mongo.project(stack.project)
@deploy_env = @project.deploy_env(stack.deploy_env)
@provider = stack.provider_instance
end
# returns: { priority_as_integer => [Servers] }
def persist
puts_and_flush 'Start syncing stack servers with CID'
stack_servers_with_priority = {}
stack_servers_info.each do |priority, info_array|
stack_servers_with_priority[priority] = info_array.map do |info_hash|
out.puts "Instance '#{info_hash["id"]}' has been launched with stack."
persist_stack_server(info_hash)
end
end
puts_and_flush "Stack servers have been synced with CID"
stack_servers_with_priority.each do |priority, servers|
out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
out.flush
stack_servers_with_priority
end
private
# returns: {priority_as_integer => array_of_sersvers_info}
def stack_servers_info
stack_servers = @provider.stack_servers(stack)
stack_servers.each do |info|
info['tags']['cid:priority'] = info['tags']['cid:priority'].to_i
end
stack_servers.group_by{|info| info['tags']['cid:priority']}
end
# takes a hash, returns Server model
def persist_stack_server(info_hash)
server_attrs = {
'_id' => info_hash['id'],
'chef_node_name' => ChefNodeNameBuilder.new(info_hash, @project, @deploy_env).build_node_name,
'created_by' => stack.owner,
'deploy_env' => @deploy_env.identifier,
'key' => info_hash['key_name'] || @provider.ssh_key,
'project' => @project.id,
'provider' => @provider.name,
'remote_user' => mongo.image(@deploy_env.image).remote_user,
'private_ip' => info_hash['private_ip'],
'public_ip' => info_hash['public_ip'],
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
server
end
def mongo
Devops::Db.connector
end
end

71
devops-service/workers/stack_bootstrap/stack_synchronizer.rb Normal file
View File

@ -0,0 +1,71 @@
class StackSynchronizer
include PutsAndFlush
attr_reader :out, :stack
def initialize(stack, out)
@stack, @out = stack, out
@printed_events = []
end
def sync
puts_and_flush "Syncing stack '#{stack.id}'..."
# 5 tries each 5 seconds, then 200 tries each 10 seconds
sleep_times = [5]*5 + [10]*400
sleep_times.each do |sleep_time|
sleep sleep_time
stack.sync_details!
print_new_events
case stack.stack_status
when 'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS'
when 'CREATE_COMPLETE'
::Devops::Db.connector.stack_update(stack)
puts_and_flush "Stack '#{stack.id}' status is now #{stack.stack_status}"
return 0
when 'ROLLBACK_COMPLETE'
puts_and_flush "Stack '#{stack.id}' status is rolled back"
return error_code(:stack_rolled_back)
else
puts_and_flush "Unknown stack status: '#{stack.stack_status}'"
return error_code(:unkown_status)
end
end
puts_and_flush "Stack hasn't been synced in #{sleep_times.inject(&:+)} seconds."
error_code(:timeout)
rescue StandardError => e
DevopsLogger.logger.error e.message
puts_and_flush "Error: #{e.message}\n#{e.backtrace.join("\n")}"
error_code(:error)
end
def reason_from_error_code(code)
error_codes.key(code)
end
private
def error_code(reason)
error_codes.fetch(reason)
end
def error_codes
{
stack_rolled_back: 1,
unkown_status: 2,
timeout: 3,
error: 5
}
end
def print_new_events
stack.events.each do |event|
unless @printed_events.include?(event["event_id"])
@printed_events << event["event_id"]
out.puts "#{event["timestamp"]} - #{event["status"]}: #{event["reason"]}"
end
end
out.flush
end
end

201
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -1,14 +1,13 @@
require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
require "db/mongo/models/report"
require "workers/stack_bootstrap/stack_synchronizer"
require "workers/stack_bootstrap/stack_servers_bootstrapper"
require "workers/stack_bootstrap/stack_servers_persister"
require "workers/stack_bootstrap/errors"
class StackCreatingError < StandardError; end
class BootstrapingStackServerError < StandardError; end
class DeployingStackServerError < StandardError; end
class StackBootstrapWorker < Worker
include StackCommands
def perform(options)
stack_attrs = options.fetch('stack_attributes')
@ -18,38 +17,33 @@ class StackBootstrapWorker < Worker
without_bootstrap = stack_attrs.delete('without_bootstrap')
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
report = save_report(file, stack_attrs)
save_report(file, stack_attrs)
begin
stack = create_stack(stack_attrs)
@stack = create_stack(stack_attrs)
#TODO: errors
begin
servers_with_priority = persist_stack_servers!(stack)
unless without_bootstrap
sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
sorted_keys.each do |key|
@out.puts "Servers with priority '#{key}':"
bootstrap_servers!(servers_with_priority[key], report)
end
end
@out.puts "Done."
@servers_with_priority = stack_servers_persister.persist
bootstrap_in_priority_order unless without_bootstrap
0
rescue BootstrapingStackServerError
@out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
rollback_stack!(stack)
rescue StackServerBootstrapError
puts_and_flush "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
rollback_stack!(@stack)
2
rescue DeployingStackServerError => e
@out.puts "\nStack was launched, but an error occured during deploying stack servers."
@out.puts "You can redeploy stack after fixing the error."
rescue StackServerDeployError => e
out.puts "\nStack was launched, but an error occured during deploying stack servers."
puts_and_flush "You can redeploy stack after fixing the error."
3
rescue StackServerBootstrapDeployTimeout
puts_and_flush "\nBootstrap or deploy wasn't completed due to timeout."
4
rescue StandardError => e
@out.puts "\nAn error occured. Initiating stack rollback."
rollback_stack!(stack)
puts_and_flush "\nAn error occured. Initiating stack rollback."
rollback_stack!(@stack)
raise e
end
rescue StackCreatingError
@out.puts "Stack creating error"
puts_and_flush "Stack creating error"
1
end
end
@ -57,97 +51,52 @@ class StackBootstrapWorker < Worker
private
def rollback_stack!(stack)
@out.puts "\nStart rollback of a stack"
stack.delete_stack_in_cloud!
Devops::Db.connector.stack_servers_delete(stack.name)
Devops::Db.connector.stack_delete(stack.id)
@out.puts "Rollback has been completed"
def stack_synchronizer(stack)
StackSynchronizer.new(stack, out)
end
def stack_servers_persister
@stack_servers_persister ||= StackServersPersister.new(@stack, out)
end
def stack_servers_bootstrapper
@stack_servers_bootstrapper ||= StackServersBootstrapper.new(out, jid)
end
# builds and persist stack model, initiate stack creating in cloud
def create_stack(stack_attrs)
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
mongo.stack_insert(stack)
operation_result = sync_stack_proc.call(@out, stack, mongo)
synchronizer = stack_synchronizer(stack)
operation_result = synchronizer.sync
if operation_result == 0
@out.puts "\nStack '#{stack.name}' has been created"
@out.flush
puts_and_flush "\nStack '#{stack.name}' has been created"
stack
else
human_readable_code = StackCommands.result_codes.key(operation_result)
@out.puts "An error ocurred during stack creating"
@out.puts "Stack creating operation result was #{human_readable_code}"
human_readable_code = synchronizer.reason_from_error_code(operation_result)
out.puts "An error ocurred during stack creating"
puts_and_flush "Stack creating operation result was #{human_readable_code}"
raise StackCreatingError
end
end
def bootstrap_servers!(servers, report)
@out << "\nStart bootstraping stack servers\n"
subreports = []
data = {}
servers.each do |server|
sjid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
subreports << sjid
@out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
data[server.id] = sjid
# Bootstrap servers with high priorities first
def bootstrap_in_priority_order
sorted_priorities = @servers_with_priority.keys.sort.reverse
sorted_priorities.each do |priority|
@out.puts "Servers with priority '#{priority}':"
stack_servers_bootstrapper.bootstrap(@servers_with_priority[priority])
end
@out.puts
@out.flush
mongo.add_report_subreports(jid, subreports)
results = []
data.each do |server_id, subreport_id|
begin
sleep(5)
subreport = mongo.report(subreport_id)
status = subreport.status
if status == Worker::STATUS::COMPLETED
@out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
break
elsif status == Worker::STATUS::FAILED
results << subreport.job_result_code
@out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
break
end
end while(true)
end
@out.flush
results.empty? ? 0 : -5
puts_and_flush "Done."
end
def check_bootstrap_results!(results)
if results.values.all?(&:zero?)
# everything is OK
@out.puts "Stack servers have been bootstraped"
@out.flush
return 0
end
@out.puts
results.each do |chef_node_name, code|
human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code)
@out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
end
if errors_in_bootstrapping_present?(results.values)
raise BootstrapingStackServerError # will cause rollback of a stack
else
raise DeployingStackServerError #will not cause rollback of a stack
end
end
def errors_in_bootstrapping_present?(result_codes)
bootstrap_error_codes = []
[:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
end
(bootstrap_error_codes & result_codes).size > 0
def rollback_stack!(stack)
puts_and_flush "\nStart rollback of a stack"
stack.delete_stack_in_cloud!
Devops::Db.connector.stack_servers_delete(stack.name)
Devops::Db.connector.stack_delete(stack.id)
puts_and_flush "Rollback has been completed"
end
def save_report(file, stack_attrs)
@ -164,54 +113,4 @@ class StackBootstrapWorker < Worker
mongo.save_report(report)
report
end
# returns
# {
# "priority" => [Servers]
# }
def persist_stack_servers!(stack)
@out.puts "Start syncing stack servers with CID"
@out.flush
project = mongo.project(stack.project)
deploy_env = project.deploy_env(stack.deploy_env)
provider = stack.provider_instance
stack_servers = provider.stack_servers(stack)
stack_servers.each do |info|
info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
end
stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
stack_servers_with_priority = {}
stack_servers_info.each do |priority, info_array|
stack_servers_with_priority[priority] = info_array.map do |extended_info|
@out.puts "Instance '#{extended_info["id"]}' has been launched with stack."
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info["key_name"] || provider.ssh_key,
'_id' => extended_info["id"],
'chef_node_name' => extended_info["name"],
'private_ip' => extended_info["private_ip"],
'public_ip' => extended_info["public_ip"],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# server.chef_node_name = provider.create_default_chef_node_name(server)
server
end
end
@out.puts "Stack servers have been synced with CID"
stack_servers_with_priority.each do |priority, servers|
@out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
@out.flush
stack_servers_with_priority
end
end

3
devops-service/workers/worker.rb
View File

@ -12,11 +12,12 @@ require "core/devops-logger"
require "core/devops-db"
require "providers/provider_factory"
require "lib/knife/knife_factory"
require "lib/puts_and_flush"
# All options keys MUST be a symbol!!!
class Worker
include Sidekiq::Worker
include PutsAndFlush
attr_accessor :out