tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'features' of /home/git/repositories/cloudtechlab/devops-service into qa

Browse Source
This commit is contained in:
Tim Lianov 2015-11-10 15:08:47 +04:00
commit be2831dcd6
40 changed files with 541 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
devops-service/app/api2/handlers/deploy.rb
View File

@ -19,10 +19,8 @@ module Devops
names = body["names"]
tags = body["tags"] || []
run_list = body["run_list"]
dir = DevopsConfig.config[:report_dir_v2]
files = []
jid = nil
uri = URI.parse(parser.request.url)
owner = parser.current_user
@deploy_info_buf = {}
servers(names).each do |s|
@ -36,7 +34,7 @@ module Devops
deploy_info = create_deploy_info(s, project, body["build_number"])
deploy_info["run_list"] = run_list if run_list
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -63,7 +61,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push(uri)
files.push(jid)
end
files
end

8
devops-service/app/api2/handlers/project.rb
View File

@ -251,13 +251,13 @@ module Devops
deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil)
end
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: parser.current_user,
tags: [],
deploy_info: deploy_info
)
files.push uri
files.push jid
end
files
end
@ -288,14 +288,14 @@ module Devops
raise InvalidRecord.new(msg)
end
uri = Worker.start_async(ProjectTestWorker, @request,
jid = Worker.start_async(ProjectTestWorker,
project: project.id,
deploy_env: env.identifier,
user: @request.env['REMOTE_USER']
)
sleep 1
return [uri]
return [jid]
end
end

11
devops-service/app/api2/handlers/provider.rb
View File

@ -17,12 +17,16 @@ module Devops
end
def accounts provider
::Provider::ProviderFactory.get(provider).accounts
::Provider::ProviderFactory.get_accounts_factory(provider).accounts
end
def account_fields provider
::Provider::ProviderFactory.get_account_class(provider).account_fields
end
def add_account provider
account = ::Provider::ProviderFactory.get(provider).create_account(parser.account)
key = Devops::Db.connector.key account.ssh_key
account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account)
account.validate_fields!
Devops::Db.connector.provider_accounts_insert(account)
::Provider::ProviderFactory.add_account(provider, account)
account.to_hash
@ -31,6 +35,7 @@ module Devops
def delete_account name, provider
account = Devops::Db.connector.provider_accounts_show(name)
Devops::Db.connector.provider_accounts_delete(name)
::Provider::ProviderFactory.delete_account(provider, account)
account.to_hash
end

81
devops-service/app/api2/handlers/server.rb
View File

@ -71,20 +71,57 @@ module Devops
user = parser.current_user
check_if_server_attrs_are_valid(body, user)
uri = Worker.start_async(CreateServerWorker, @request,
jid = Worker.start_async(CreateServerWorker,
server_attrs: body,
owner: user
)
[uri]
[jid]
end
def deploy node_name
call_deploy(node_name) do |options|
[ Worker.start_async(DeployWorker, @request, options) ]
end
end
def deploy_stream node_name, out
call_deploy(node_name) do |options|
Worker.start_sync(DeployWorker, @request, options, out)
end
rescue RecordNotFound => e
out << e.message
-10
end
def call_deploy node_name
body = parser.deploy
names = body["names"]
tags = body["tags"] || []
run_list = body["run_list"]
dir = DevopsConfig.config[:report_dir_v2]
owner = parser.current_user
s = Server.get_server_by_key(node_name, parser.instance_key)
project = Devops::Db.connector.check_project_auth s.project, s.deploy_env, owner
deploy_info = create_deploy_info(s, project, body["build_number"])
deploy_info["run_list"] = run_list if run_list
yield({
server_attrs: s.to_hash,
owner: owner,
tags: tags,
deploy_info: deploy_info
})
end
def pause_server node_name
s = Server.get_server_by_key(node_name, parser.instance_key)
s = get_server_by_key(node_name, parser.instance_key)
## Authorization
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
provider = s.provider_instance
r = provider.pause_server s
if r.nil?
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::PAUSE)
"Server with instance ID '#{s.id}' and node name '#{node_name}' is paused"
else
raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be paused, It in state '#{r}'", 409)
@ -92,12 +129,14 @@ module Devops
end
def unpause_server node_name
s = Server.get_server_by_key(node_name, parser.instance_key)
s = get_server_by_key(node_name, parser.instance_key)
## Authorization
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
provider = s.provider_instance
r = provider.unpause_server s
if r.nil?
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNPAUSE)
"Server with instance ID '#{s.id}' and node name '#{node_name}' is unpaused"
else
raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be unpaused, It in state '#{r}'")
@ -110,7 +149,7 @@ module Devops
Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
raise ConflictException.new("Server '#{node_name}' already reserved") unless s.reserved_by.nil?
s.reserved_by = user
Devops::Db.connector.server_update(s)
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::RESERVE)
end
def unreserve_server node_name
@ -118,22 +157,20 @@ module Devops
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
raise ConflictException.new("Server '#{node_name}' is not reserved") if s.reserved_by.nil?
s.reserved_by = nil
Devops::Db.connector.server_update(s)
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNRESERVE)
end
# TODO: check bootstrap template name
def bootstrap_server_stream out
s, rl, bt = prepare_bootstrap_server
options = prepare_bootstrap_server
s = options.delete(:server)
status = []
cert = Devops::Db.connector.key s.key
DevopsLogger.logger.debug "Bootstrap certificate path: #{cert.path}"
#bootstrap s, out, cert.path, logger
options = {
:bootstrap_template => bt,
:cert_path => cert.path,
:run_list => rl
}
r = two_phase_bootstrap s, options, out
options[:cert_path] = cert.path
executor = Devops::Executor::ServerExecutor.new(s, out)
r = executor.two_phase_bootstrap(options)
str = nil
r = if check_server(s)
Devops::Db.connector.server_set_chef_node_name s
@ -153,19 +190,15 @@ module Devops
def bootstrap_server
server, rl, bootstrap_template = prepare_bootstrap_server
dir = DevopsConfig[:report_dir_v2]
files = []
uri = URI.parse(@request.url)
uri = Worker.start_async(BootstrapWorker, @request,
provider_name: server.provider,
jid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: bootstrap_template,
owner: parser.current_user
)
sleep 1
[uri]
[jid]
end
def prepare_bootstrap_server
@ -175,6 +208,7 @@ module Devops
rl = body["run_list"]
t = body["bootstrap_template"]
s = Devops::Db.connector.server_by_instance_id(id)
res = {server: s}
p = Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
d = p.deploy_env(s.deploy_env)
@ -185,9 +219,11 @@ module Devops
unless t.nil?
templates = get_templates
halt_response("Invalid bootstrap template '#{t}', available values: #{templates.join(", ")}", 400) unless templates.include?(t)
res[:bootstrap_template] = t
end
s.chef_node_name = name || provider.create_default_chef_node_name(s)
return s, rl || d.run_list, t
res[:run_list] = rl || d.run_list
return res
end
def unbootstrap_server id
@ -284,6 +320,11 @@ module Devops
end
end
def set_last_operation_type_and_save(server, operation_type)
server.set_last_operation(operation_type)
Devops::Db.connector.server_update(server)
end
end
end
end

12
devops-service/app/api2/handlers/stack.rb
View File

@ -28,13 +28,10 @@ module Devops
object["provider"] = env.provider
object["provider_account"] = env.provider_account
# TODO: without provider_name
uri = Worker.start_async(StackBootstrapWorker, @request,
provider_name: env.provider,
jid = Worker.start_async(StackBootstrapWorker,
stack_attributes: object
)
puts "Syncing report is located here: #{uri}"
[uri]
[jid]
end
def stack id
@ -102,7 +99,6 @@ module Devops
def deploy id
stack = self.stack(id)
owner = parser.current_user
status = []
project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner)
deploy_env_model = project.deploy_env(stack.deploy_env)
body = parser.deploy
@ -114,7 +110,7 @@ module Devops
servers.each do |s|
begin
deploy_info = project.deploy_info(deploy_env_model, nil)
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -140,7 +136,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push uri
files.push jid
end
files
end

2
devops-service/app/api2/helpers/version_2.rb
View File

@ -30,7 +30,7 @@ module Devops
def check_provider provider
list = ::Provider::ProviderFactory.providers
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider)
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 400) unless list.include?(provider)
end
# Save information about requests with methods POST, PUT, DELETE

16
devops-service/app/api2/routes/provider.rb
View File

@ -26,6 +26,22 @@ module Devops
json Devops::API2_0::Handler::Provider.new(request).providers
end
# Get list of provider account fields
#
# * *Request*
# - method : GET
# - headers :
# - Accept: application/json
#
# * *Returns* : hash
# key - field
# value - description
app.get_with_headers "/provider/:provider/account/fields", :headers => [:accept] do |provider|
check_privileges("provider", "r")
check_provider(provider)
json Devops::API2_0::Handler::Provider.new(request).account_fields(provider)
end
# Get list of provider accounts
#
# * *Request*

47
devops-service/app/api2/routes/server.rb
View File

@ -132,6 +132,53 @@ module Devops
}
app.multi_routes "/server/:id", {:headers => [:accept, :content_type]}, hash
# Run deploy command on reserved server
#
# * *Request*
# - method : POST
# - headers :
# - Content-Type: application/json
# - body :
# {
# "names": [], -> array of servers names to run chef-client
# "tags": [], -> array of tags to apply on each server before running chef-client
# "build_number": "", -> string, build number to deploy
# "run_list": [], -> array of strings to set run_list for chef-client
# }
#
# * *Returns* : text stream
app.post_with_headers "/server/:node_name/deploy", :headers => [:content_type, :accept] do |node_name|
check_privileges("server", "x")
if request["HTTP_X_STREAM"]
stream() do |out|
status = []
begin
status = Devops::API2_0::Handler::Server.new(request).deploy_stream(node_name, out)
out << create_status(status)
rescue DeployInfoError => e
msg = "Can not get deploy info: " + e.message
DevopsLogger.logger.error "msg:\n#{e.backtrace.join('\n')}"
out.puts msg
rescue IOError => e
logger.error e.message
break
end
end # stream
else
files = begin
Devops::API2_0::Handler::Server.new(request).deploy(node_name)
rescue DeployInfoError => e
msg = "Can not get deploy info: " + e.message
DevopsLogger.logger.error "#{msg}:\n#{e.backtrace.join("\n")}"
out << "\nError - "
out.puts msg
end
sleep 1
json files
end
end
# Create devops server
#
# * *Request*

14
devops-service/app/api2/routes/v2.0.rb
View File

@ -61,13 +61,23 @@ module Devops
DevopsLogger.logger = @@logger
begin
res = super(env)
rescue DevopsError => e
rescue ::Devops::Exception::DevopsError => e
return [e.code, {}, e.message]
rescue InvalidRecord => e
return [e.code, {}, e.message]
end
@@access_logger.info(env["REQUEST_METHOD"] + " " + env["REQUEST_URI"] + " - from #{env["HTTP_USER_AGENT"]} (#{env["REMOTE_USER"]}) / #{res[0]}")
res
end
def handle_exception!(boom)
if boom.is_a?(::Devops::Exception::DevopsError)
boom.http_response
else
super(boom)
end
end
error Devops::ValidationError do
e = env["sinatra.error"]
#logger.warn e.message
@ -92,12 +102,14 @@ module Devops
halt_response(e.message, 404)
end
=begin
error InvalidRecord do
e = env["sinatra.error"]
logger.warn e.message
logger.warn "Request body: #{request.body.read}"
halt_response(e.message, 400)
end
=end
error InvalidCommand do
e = env["sinatra.error"]

2
devops-service/app/devops-api2.rb
View File

@ -78,7 +78,7 @@ module Devops
Devops::Api2.register r
end
Routes.route "/v2.0", Devops::Api2
Devops::Api2.routes_list
#Devops::Api2.routes_list
end
private

2
devops-service/app/version/version.rb
View File

@ -2,7 +2,7 @@ require 'sinatra/base'
class DevopsVersion < Sinatra::Base
VERSION = "2.3.0"
VERSION = "2.3.2"
get "/" do
VERSION

14
devops-service/db/mongo/connectors/report.rb
View File

@ -51,8 +51,18 @@ module Connectors
collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)}
end
def set_report_status(jid, status)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}})
def add_report_subreports(jid, subreports)
collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}})
end
def set_report_status(jid, status, job_result_code)
set = {"status" => status, "updated_at" => Time.new}
set["job_result_code"] = job_result_code unless job_result_code.nil?
collection.update({"_id" => jid}, {"$set" => set})
end
def set_report_status(jid, status, job_result_code)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}})
end
def set_report_server_data id, chef_node_name, host

3
devops-service/db/mongo/connectors/server.rb
View File

@ -63,17 +63,14 @@ module Connectors
end
def server_insert(server)
#server.validate!
server.created_at = Time.now
collection.insert(server.to_mongo_hash)
end
# somewhy servers are not validated in previous version of code. I leave this until I know, why.
def server_update(server)
collection.update({'_id' => server.id}, server.to_hash_without_id)
end
# somewhy servers are not validated in previous version of code. I leave this until I know, why.
def server_set_chef_node_name(server)
collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}})
end

26
devops-service/db/mongo/models/mongo_model.rb
View File

@ -66,6 +66,21 @@ module Devops
end
end
def validate_fields!
result = []
self.class.field_validators.each do |field, validation_method|
begin
self.send(validation_method)
rescue InvalidRecord => e
result << {key: field, message: e.message}
end
end
unless result.empty?
raise InvalidRecord.new(error_data: result)
end
true
end
def build_error_message(message)
# overrided in descendants
message
@ -123,17 +138,16 @@ module Devops
end
@validators = []
# @field_validators = []
@field_validators = {}
class << self
attr_accessor :validators
# attr_accessor :field_validators
attr_accessor :field_validators
def inherited(subclass)
subclass.validators = []
subclass.validators += self.validators
# subclass.field_validators = []
# subclass.field_validators += self.field_validators
subclass.field_validators = self.field_validators.clone
end
# all exceptions are handled in @validate! method
@ -146,11 +160,13 @@ module Devops
# validate field value
# if method validate! returns false, then stop validation without error
def set_field_validators field, *validators
define_method("validate_" + field.to_s + "!") do
method_name = "validate_" + field.to_s + "!"
define_method(method_name) do
validators.each do |validator|
break unless validator.new(self, send(field)).validate!
end
end
self.field_validators[field] = method_name
end
# private class methods

15
devops-service/db/mongo/models/provider_accounts/ec2_provider_account.rb
View File

@ -6,6 +6,13 @@ module Devops
attr_accessor :access_key_id, :availability_zone, :secret_access_key
set_field_validators :access_key_id, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty
set_field_validators :secret_access_key, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty
def initialize a={}
super(a)
self.provider = Provider::Ec2::PROVIDER
@ -34,6 +41,14 @@ module Devops
Ec2ProviderAccount.new a
end
def self.account_fields
{
access_key_id: "AWS account access key",
secret_access_key: "AWS account secret key",
availability_zone: "Availability zone, todo: remove field?"
}.merge(ProviderAccount::ACCOUNT_FIELDS)
end
# TODO: remove
def validate_fields_types

9
devops-service/db/mongo/models/provider_accounts/openstack_provider_account.rb
View File

@ -6,6 +6,15 @@ module Devops
attr_accessor :username, :auth_url, :tenant, :api_key
def self.account_fields
{
username: "Openstack user name",
auth_url: "Identity API endpoint",
tenant: "Tenant to access",
api_key: "Openstack user password"
}.merge(ProviderAccount::ACCOUNT_FIELDS)
end
def initialize a={}
super(a)
self.username = a["username"]

28
devops-service/db/mongo/models/provider_accounts/provider_account.rb
View File

@ -7,14 +7,34 @@ module Devops
include ModelWithProvider
attr_accessor :account_name, :description, :ssh_key, :certificate
attr_accessor :account_name, :description, :ssh_key
set_field_validators :account_name, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::Name
set_field_validators :description, ::Validators::FieldValidator::Nil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::Description
set_field_validators :ssh_key, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::SshKey
ACCOUNT_FIELDS = {
account_name: "Account name (id)",
description: "Account description",
ssh_key: "Ssh key id"
}
def initialize a={}
self.account_name = a["account_name"]
self.description = a["description"]
self.ssh_key = a["ssh_key"]
self.provider = a["provider"]
self.certificate = a["certificate"]
self.created_at = a["created_at"]
end
@ -28,7 +48,6 @@ module Devops
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider,
"certificate" => self.certificate,
"created_at" => self.created_at
}
end
@ -38,8 +57,7 @@ module Devops
"_id" => self.account_name,
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider,
"certificate" => self.certificate
"provider" => self.provider
}
end

3
devops-service/db/mongo/models/provider_accounts/static_provider_account.rb
View File

@ -9,6 +9,9 @@ module Devops
StaticProviderAccount.new a
end
def self.account_fields
ProviderAccount::ACCOUNT_FIELDS
end
end
end
end

10
devops-service/db/mongo/models/report.rb
View File

@ -12,7 +12,7 @@ module Devops
DEPLOY_STACK_TYPE = 6
DELETE_SERVER_TYPE = 7
attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack
attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
def initialize r
self.id = r["_id"]
@ -21,13 +21,15 @@ module Devops
self.project = r["project"]
self.deploy_env = r["deploy_env"]
self.type = r["type"]
self.created_at = r["created_at"]
# self.created_at = r["created_at"]
self.chef_node_name = r["chef_node_name"]
self.host = r["host"]
self.stack = r["stack"]
self.created_at = r["created_at"].localtime unless r["created_at"].nil?
self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil?
self.status = r["status"]
self.subreports = r["subreports"]
self.job_result_code = r["job_result_code"]
end
def to_hash_without_id
@ -42,7 +44,9 @@ module Devops
"chef_node_name" => self.chef_node_name,
"host" => self.host,
"status" => self.status,
"stack" => self.stack
"stack" => self.stack,
"subreports" => self.subreports,
"job_result_code" => self.job_result_code
}
end

25
devops-service/db/mongo/models/server.rb
View File

@ -9,8 +9,17 @@ module Devops
include ModelWithProvider
module OperationType
CREATION = :creation
DEPLOY = :deploy
RESERVE = :reserve
UNRESERVE = :unreserve
PAUSE = :pause
UNPAUSE = :unpause
end
attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list
attr_accessor :key
attr_accessor :key, :last_operation_at, :last_operation_type
types :id => {:type => String, :empty => false},
:provider => {:type => String, :empty => false},
@ -29,7 +38,8 @@ module Devops
set_validators ::Validators::DeployEnv::RunList
def self.fields
["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack"]
["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack",
"last_operation_type", "last_operation_at"]
end
def initialize s={}
@ -47,6 +57,8 @@ module Devops
self.reserved_by = s["reserved_by"]
self.stack = s["stack"]
self.run_list = s["run_list"] || []
self.last_operation_at = s["last_operation_at"]
self.last_operation_type = s["last_operation_type"]
self
end
@ -77,7 +89,9 @@ module Devops
"key" => self.key,
"reserved_by" => self.reserved_by,
"stack" => stack,
"run_list" => self.run_list
"run_list" => self.run_list,
"last_operation_at" => self.last_operation_at,
"last_operation_type" => self.last_operation_type
}.merge(provider_hash).delete_if { |k,v| v.nil? }
end
@ -100,6 +114,11 @@ module Devops
self.provider == Provider::Static::PROVIDER
end
def set_last_operation(operation_type)
self.last_operation_type = operation_type
self.last_operation_at = Time.now
end
end
end
end

2
devops-service/db/mongo/mongo_connector.rb
View File

@ -31,7 +31,7 @@ class MongoConnector
[:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
:user_update, :create_root_user, :check_user_privileges] => :users_connector,
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector,
[:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector
)

2
devops-service/db/validators/all.rb
View File

@ -15,5 +15,5 @@ require "db/validators/base"
'db/validators/image/*.rb',
'db/validators/field_validators/*.rb'
].each do |files_regexp|
Dir[files_regexp].each {|file| require file }
Dir[File.join(Devops::Application.root, files_regexp)].each {|file| require file }
end

17
devops-service/db/validators/field_validators/description.rb Normal file
View File

@ -0,0 +1,17 @@
require_relative "base"
module Validators
module FieldValidator
class Description < Base
MAX_LEN = 500
def valid?
@value.size <= 500
end
def message
"Invalid value '#{@value}': it should be less or equals then #{MAX_LEN} symbols"
end
end
end
end

2
devops-service/db/validators/field_validators/name.rb
View File

@ -11,7 +11,7 @@ module Validators
end
def message
"Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN}"
"Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN} symbols"
end
end
end

20
devops-service/db/validators/field_validators/ssh_key.rb Normal file
View File

@ -0,0 +1,20 @@
require_relative "base"
module Validators
module FieldValidator
class SshKey < Base
MAX_LEN = 500
def valid?
Devops::Db.connector.key @value
true
rescue RecordNotFound
false
end
def message
"Invalid value '#{@value}': ssh key '#{@value}' not found"
end
end
end
end

15
devops-service/exceptions/devops_error.rb
View File

@ -1,12 +1,25 @@
require "json"
module Devops
module Exception
class DevopsError < StandardError
def code
def http_status
500
end
def http_response
[self.http_status, self.http_headers, self.http_body]
end
def http_body
JSON.pretty_generate(message: self.message)
end
def http_headers
{"Content-Type" => "application/json"}
end
end
end

22
devops-service/exceptions/invalid_record.rb
View File

@ -1,3 +1,23 @@
class InvalidRecord < StandardError
require_relative "devops_error"
class InvalidRecord < ::Devops::Exception::DevopsError
def initialize msg
if msg.is_a?(String)
super(msg)
else
@object = msg
end
end
def http_status
400
end
def http_body
if @object.nil?
super
else
JSON.pretty_generate(@object)
end
end
end

48
devops-service/lib/executors/server_executor.rb
View File

@ -76,6 +76,7 @@ module Devops
def create_server options
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider, "provider_account" => @deploy_env.provider_account})
@server.set_last_operation(Devops::Model::Server::OperationType::CREATION)
provider = @deploy_env.provider_instance
mongo = ::Devops::Db.connector
begin
@ -193,8 +194,10 @@ module Devops
@out << "After bootstrap hooks...\n"
res = self.run_hook(:after_bootstrap, @out)
@out << "Done\n"
msg = "Server with id '#{@server.id}' is bootstraped"
@out.puts msg
@out.flush
DevopsLogger.logger.info "Server with id '#{@server.id}' is bootstraped"
DevopsLogger.logger.info msg
r
else
@out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
@ -218,7 +221,13 @@ module Devops
bootstrap_options
end
def prepare_two_phase_bootstrap options
@out << "Prepare bootstrap...\n"
@out << "Done\n"
end
def two_phase_bootstrap options
prepare_two_phase_bootstrap(options)
# bootstrap phase
begin
provider = @server.provider_instance
@ -230,7 +239,8 @@ module Devops
if check_server
@out << "Server #{@server.chef_node_name} is created"
else
@out << roll_back
@out.puts "Can not find client or node on chef-server"
roll_back
@out.flush
mongo.server_delete @server.id
return result_code(:server_not_in_chef_nodes)
@ -259,7 +269,7 @@ module Devops
@out << "\nComputed run list: #{run_list.join(", ")}"
@out.flush
@knife_instance.set_run_list(@server.chef_node_name, run_list)
deploy_info = @project.deploy_info(@deploy_env)
deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
deploy_status = deploy_server(deploy_info)
if deploy_status == 0
0
@ -348,15 +358,20 @@ module Devops
cmd = "chef-client --no-color"
if deploy_info["use_json_file"]
deploy_info.delete("use_json_file")
@out << "Deploy Input Parameters:\n"
json = JSON.pretty_generate(deploy_info)
@out << json
@out << "\n"
file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
json = nil
dir = DevopsConfig.config[:project_info_dir]
File.open(File.join(dir, file), "w") do |f|
f.write json
file = deploy_info["json_file"] || "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
path = File.join(dir, file)
if File.exists?(path)
json = File.read(path)
else
json = JSON.pretty_generate(deploy_info)
File.open(File.join(dir, file), "w") do |f|
f.write json
end
end
@out << "Deploy Input Parameters:\n"
@out.puts json
@out.flush
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
else
@ -378,6 +393,10 @@ module Devops
@out << "\nAfter deploy hooks...\n"
res = self.run_hook(:after_deploy, @out, deploy_info)
@out << "Done\n"
@server.set_last_operation(Devops::Model::Server::OperationType::DEPLOY)
Devops::Db.connector.server_update(@server)
0
end
end
@ -426,13 +445,14 @@ module Devops
end
def roll_back
@out.puts "Trying to roll back..."
unless @server.id.nil?
@out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n"
@out << delete_from_chef_server(@server.chef_node_name).values.join("\n")
@out.puts "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created"
@out.puts delete_from_chef_server(@server.chef_node_name).values.join("\n")
begin
@out << @server.provider_instance.delete_server(@server)
@out.puts @server.provider_instance.delete_server(@server)
rescue => e
@out << e.message
@out.puts e.message
end
@out << "\nRolled back\n"
end

12
devops-service/providers/accounts_factory.rb
View File

@ -14,8 +14,20 @@ module Provider
@connections[name] = conn
end
def delete_connection name
@connections.delete(name)
end
def create_connection_from_account config, account
end
def accounts
Devops::Db.connector.provider_accounts(provider_name)
end
def create_account hash
raise "override me"
end
end
end

8
devops-service/providers/base_provider.rb
View File

@ -6,14 +6,6 @@ module Provider
attr_accessor :ssh_key, :certificate_path, :connection_options, :run_list
def accounts
Devops::Db.connector.provider_accounts(self.name)
end
def create_account hash
raise "override me"
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end

5
devops-service/providers/ec2.rb
View File

@ -35,10 +35,6 @@ module Provider
super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
end
def create_account hash
Devops::Model::Ec2ProviderAccount.new(hash)
end
def name
PROVIDER
end
@ -323,6 +319,7 @@ module Provider
'key_name' => instance["keyName"],
'private_ip' => instance["privateIpAddress"],
'public_ip' => instance["ipAddress"],
'tags' => instance["tagSet"]
}
end
end

12
devops-service/providers/ec2_accounts_factory.rb
View File

@ -4,15 +4,19 @@ module Provider
def init config
@connections = {}
Devops::Db.connector.provider_accounts(Ec2::PROVIDER).each do |account|
accounts.each do |account|
create_connection_from_account(config, account)
puts "\tFound ec2 account '#{account.account_name}'"
end
ProviderFactory.add_provider Ec2::PROVIDER unless @connections.empty?
end
def provider_name
Ec2::PROVIDER
end
def create_connection_from_account config, account
options = {
aws_certificate: account.certificate,
aws_ssh_key: account.ssh_key,
aws_access_key_id: account.access_key_id,
aws_secret_access_key: account.secret_access_key,
@ -25,5 +29,9 @@ module Provider
add_connection(account.account_name, Ec2.new(options))
end
def create_account hash
Devops::Model::Ec2ProviderAccount.new(hash)
end
end
end

19
devops-service/providers/provider_factory.rb
View File

@ -3,11 +3,12 @@ require "sinatra"
module Provider
module ProviderFactory
@@available_providers = []
@@providers = {}
@@providers_with_accounts_factories = {}
def self.providers
@@providers.keys
@@available_providers
end
def self.get provider, account=nil
@ -25,6 +26,10 @@ module Provider
@@providers.values
end
def self.add_provider provider
@@available_providers << provider unless @@available_providers.include?(provider)
end
def self.init conf
# require providers here to get access to debug properties
require_all
@ -34,6 +39,7 @@ module Provider
o = Provider.const_get(p.capitalize).new(conf)
if o.configured?
@@providers[p] = o
@@available_providers << p
puts "Provider '#{p}' has been loaded"
end
factory = Provider.const_get(p.capitalize + "AccountsFactory").new
@ -44,6 +50,7 @@ module Provider
next
end
end
puts "Available providers: #{@@available_providers}"
end
def self.add_account provider, account
@ -52,6 +59,12 @@ module Provider
DevopsLogger.logger.info("Added #{provider} account '#{account.account_name}'")
end
def self.delete_account provider, account
factory = @@providers_with_accounts_factories[provider]
factory.delete_connection(account.account_name)
DevopsLogger.logger.info("Removed #{provider} account '#{account.account_name}'")
end
def self.require_all
["ec2", "openstack", "static"].each do |provider|
begin
@ -63,6 +76,10 @@ module Provider
end
end
def self.get_accounts_factory provider
@@providers_with_accounts_factories[provider]
end
def self.get_account_class provider
case(provider)
when ::Provider::Static::PROVIDER

9
devops-service/workers/bootstrap_worker.rb
View File

@ -1,25 +1,26 @@
require File.join(File.dirname(__FILE__), "worker")
require "lib/executors/server_executor"
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
class BootstrapWorker < Worker
def perform(options)
provider_name = options.fetch('provider_name')
server_attrs = options.fetch('server_attrs')
bootstrap_template = options.fetch('bootstrap_template')
# bootstrap_template = options.fetch('bootstrap_template')
owner = options.fetch('owner')
options = convert_config(options)
call(provider_name) do |provider, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)
=begin
options = {
bootstrap_template: bootstrap_template
}
=end
executor = Devops::Executor::ServerExecutor.new(server, out)
executor.report = report
status = executor.two_phase_bootstrap(options)

3
devops-service/workers/create_server_worker.rb
View File

@ -1,6 +1,5 @@
require File.join(File.dirname(__FILE__), "worker")
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
require "lib/executors/server_executor"
@ -11,7 +10,7 @@ class CreateServerWorker < Worker
server_attrs = options.fetch('server_attrs')
owner = options.fetch('owner')
call(nil) do |not_used, out, file|
call() do |out, file|
project = mongo.project(server_attrs["project"])
env = project.deploy_env(server_attrs["deploy_env"])
report = save_report(file, project, env, owner)

2
devops-service/workers/delete_server_worker.rb
View File

@ -8,7 +8,7 @@ class DeleteServerWorker < Worker
chef_node_name = options.fetch('server_chef_node_name')
puts "Expire server '#{chef_node_name}'."
call(nil) do |not_used, out, file|
call() do |out, file|
server = mongo.server_by_chef_node_name(chef_node_name)
report = save_report(file, server)

2
devops-service/workers/deploy_worker.rb
View File

@ -12,7 +12,7 @@ class DeployWorker < Worker
tags = options.fetch('tags')
deploy_info = options.fetch('deploy_info')
call(nil) do |not_used, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)

2
devops-service/workers/project_test_worker.rb
View File

@ -15,7 +15,7 @@ class ProjectTestWorker < Worker
project_name = params.fetch('project')
deploy_env_name = params.fetch('deploy_env')
call(nil) do |not_used, out, file|
call() do |out, file|
DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})"
project = mongo.project(project_name)
env = project.deploy_env(deploy_env_name)

124
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -1,4 +1,3 @@
require "providers/provider_factory"
require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker
include StackCommands
def perform(options)
provider_name = options.fetch('provider_name')
stack_attrs = options.fetch('stack_attributes')
call(provider_name) do |provider, out, file|
call() do |out, file|
@out = out
without_bootstrap = stack_attrs.delete('without_bootstrap')
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
@ -23,14 +21,23 @@ class StackBootstrapWorker < Worker
report = save_report(file, stack_attrs)
begin
stack = create_stack(provider_name, stack_attrs)
stack = create_stack(stack_attrs)
rescue StackCreatingError
return 1
end
#TODO: errors
begin
servers = persist_stack_servers!(stack, provider)
bootstrap_servers!(servers, report) unless without_bootstrap
servers_with_priority = persist_stack_servers!(stack)
unless without_bootstrap
sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
sorted_keys.each do |key|
@out.puts "Servers with priority '#{key}':"
bootstrap_servers!(servers_with_priority[key], report)
end
end
@out.puts "Done."
0
rescue BootstrapingStackServerError
@out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
rollback_stack!(stack)
@ -57,8 +64,8 @@ class StackBootstrapWorker < Worker
@out.puts "Rollback has been completed"
end
def create_stack(provider_name, stack_attrs)
stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out)
def create_stack(stack_attrs)
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
mongo.stack_insert(stack)
operation_result = sync_stack_proc.call(@out, stack, mongo)
@ -76,15 +83,40 @@ class StackBootstrapWorker < Worker
def bootstrap_servers!(servers, report)
@out << "\nStart bootstraping stack servers\n"
bootstraping_results = {}
subreports = []
data = {}
servers.each do |server|
executor = Devops::Executor::ServerExecutor.new(server, @out)
executor.report = report
#TODO: can stack choose bootstrap template?
bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'})
@out.flush
sjid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
subreports << sjid
@out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
data[server.id] = sjid
end
check_bootstrap_results!(bootstraping_results)
@out.puts
@out.flush
mongo.add_report_subreports(jid, subreports)
results = []
data.each do |server_id, subreport_id|
begin
sleep(5)
subreport = mongo.report(subreport_id)
status = subreport.status
if status == Worker::STATUS::COMPLETED
@out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
break
elsif status == Worker::STATUS::FAILED
results << subreport.job_result_code
@out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
break
end
end while(true)
end
@out.flush
results.empty? ? 0 : -5
end
def check_bootstrap_results!(results)
@ -124,42 +156,60 @@ class StackBootstrapWorker < Worker
"created_by" => stack_attrs['owner'],
"project" => stack_attrs["project"],
"deploy_env" => stack_attrs["deploy_env"],
"type" => ::Devops::Model::Report::STACK_TYPE
"type" => ::Devops::Model::Report::STACK_TYPE,
"subreports" => [],
"stack" => stack_attrs['name']
)
mongo.save_report(report)
report
end
def persist_stack_servers!(stack, provider)
# returns
# {
# "priority" => [Servers]
# }
def persist_stack_servers!(stack)
@out.puts "Start syncing stack servers with CID"
@out.flush
project = mongo.project(stack.project)
deploy_env = project.deploy_env(stack.deploy_env)
provider = stack.provider_instance
servers = provider.stack_servers(stack).map do |extended_info|
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info["key_name"] || provider.ssh_key,
'_id' => extended_info["id"],
'chef_node_name' => extended_info["name"],
'private_ip' => extended_info["private_ip"],
'public_ip' => extended_info["public_ip"],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.name
}
stack_servers = provider.stack_servers(stack)
stack_servers.each do |info|
info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
end
stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
stack_servers_with_priority = {}
stack_servers_info.each do |priority, info_array|
stack_servers_with_priority[priority] = info_array.map do |extended_info|
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info["key_name"] || provider.ssh_key,
'_id' => extended_info["id"],
'chef_node_name' => extended_info["name"],
'private_ip' => extended_info["private_ip"],
'public_ip' => extended_info["public_ip"],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# server.chef_node_name = provider.create_default_chef_node_name(server)
server
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# server.chef_node_name = provider.create_default_chef_node_name(server)
server
end
end
@out.puts "Stack servers have been synced with CID"
stack_servers_with_priority.each do |priority, servers|
@out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
@out.flush
servers
stack_servers_with_priority
end
end

52
devops-service/workers/worker.rb
View File

@ -13,6 +13,8 @@ require "core/devops-db"
require "providers/provider_factory"
require "lib/knife/knife_factory"
# All options keys MUST be a symbol!!!
class Worker
include Sidekiq::Worker
@ -26,21 +28,25 @@ class Worker
IN_QUEUE = "queued"
end
def self.start_async(worker_class, request, job_options)
def self.start_async(worker_class, job_options)
jid = worker_class.perform_async(job_options.dup)
Worker.set_status jid, Worker::STATUS::IN_QUEUE
DevopsLogger.logger.info "Job '#{jid}' has been queued"
jid
=begin
uri = URI.parse(request.url)
uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}"
uri.to_s
=end
end
def self.start_sync(worker_class, request, job_options, out)
stringified_options = {}
stringified_options = job_options
=begin
job_options.each do |key, value|
stringified_options[key.to_s] = value
end
=end
w = worker_class.new
w.out = out
w.perform(stringified_options)
@ -50,25 +56,22 @@ class Worker
Sidekiq.redis {|con| con.hset "devops", id, status}
end
def call provider_name, &block
def call &block
begin
initialize_devops(provider_name)
provider = ::Provider::ProviderFactory.get(provider_name) if provider_name
initialize_devops()
if jid
call_async(provider, &block)
call_async(&block)
else
call_sync(provider, &block)
call_sync(&block)
end
rescue StandardError => e
puts e.message
puts e.backtrace.join("\n")
DevopsLogger.logger.error e.message
DevopsLogger.logger.error "#{e.message}:\n#{e.backtrace.join("\n")}"
end
end
private
def initialize_devops(provider_name)
def initialize_devops()
DevopsLogger.logger = logger
DevopsConfig.read
DevopsService.init
@ -76,33 +79,33 @@ class Worker
end
# outputs to file
def call_async(provider)
def call_async()
dir = DevopsConfig[:report_dir_v2]
# directory is created on server start in config.ru
file = File.join(dir, jid)
update_job_status(STATUS::INIT)
update_job_status(STATUS::INIT, nil)
File.open(file, "w") do |out|
begin
update_job_status(STATUS::RUNNING)
update_job_status(STATUS::RUNNING, nil)
self.out = out
job_result = yield(provider, out, file)
job_result = yield(out, file)
canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED)
update_job_status(canonical_status)
update_job_status(canonical_status, job_result)
rescue StandardError, RecordNotFound => e
out << "\n #{e.class}\n #{e.message}\n"
out << e.backtrace.join("\n")
update_job_status(STATUS::FAILED)
update_job_status(STATUS::FAILED, -100)
end
end
end
# outputs to STDOUT
def call_sync(provider)
def call_sync()
out = STDOUT
begin
yield(provider, out, '')
yield(out, '')
rescue StandardError, RecordNotFound => e
out << "\n"
out << e.message
@ -115,17 +118,12 @@ class Worker
::Devops::Db.connector
end
def update_job_status(status)
def update_job_status(status, job_result_code)
set_status(jid, status)
mongo.set_report_status(jid, status)
mongo.set_report_status(jid, status, job_result_code)
status
end
def init_provider(provider_name)
::Provider::ProviderFactory.init(DevopsConfig.config)
::Provider::ProviderFactory.get(provider_name) if provider_name
end
def convert_config conf
config = {}
conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}