tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bug_fix' of git.stu.neva.ru:cloudtechlab/devops-service into bug_fix

Browse Source
This commit is contained in:
Anton Chuchkalov 2015-11-12 19:43:22 +03:00
commit c0d2912fb0
58 changed files with 1062 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
devops-service/app/api2/handlers/deploy.rb
View File

@ -19,10 +19,8 @@ module Devops
names = body["names"]
tags = body["tags"] || []
run_list = body["run_list"]
dir = DevopsConfig.config[:report_dir_v2]
files = []
jid = nil
uri = URI.parse(parser.request.url)
owner = parser.current_user
@deploy_info_buf = {}
servers(names).each do |s|
@ -36,7 +34,7 @@ module Devops
deploy_info = create_deploy_info(s, project, body["build_number"])
deploy_info["run_list"] = run_list if run_list
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -63,7 +61,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push(uri)
files.push(jid)
end
files
end

5
devops-service/app/api2/handlers/flavor.rb
View File

@ -7,9 +7,12 @@ module Devops
class Flavor < RequestHandler
def flavors provider
::Provider::ProviderFactory.get(provider).flavors
flavors_with_account(provider, nil)
end
def flavors_with_account provider, account
::Provider::ProviderFactory.get(provider, account).flavors
end
end
end
end

6
devops-service/app/api2/handlers/group.rb
View File

@ -8,7 +8,11 @@ module Devops
# TODO: vpc support for ec2
def groups provider
::Provider::ProviderFactory.get(provider).groups()#params
groups_with_account(provider, nil)
end
def groups_with_account provider, account
::Provider::ProviderFactory.get(provider, account).groups()#params
end
end
end

6
devops-service/app/api2/handlers/network.rb
View File

@ -7,7 +7,11 @@ module Devops
class Network < RequestHandler
def networks provider
p = ::Provider::ProviderFactory.get provider
networks_with_account provider, nil
end
def networks_with_account provider, account
p = ::Provider::ProviderFactory.get(provider, account)
p.networks_detail
end
end

8
devops-service/app/api2/handlers/project.rb
View File

@ -251,13 +251,13 @@ module Devops
deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil)
end
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: parser.current_user,
tags: [],
deploy_info: deploy_info
)
files.push uri
files.push jid
end
files
end
@ -288,14 +288,14 @@ module Devops
raise InvalidRecord.new(msg)
end
uri = Worker.start_async(ProjectTestWorker, @request,
jid = Worker.start_async(ProjectTestWorker,
project: project.id,
deploy_env: env.identifier,
user: @request.env['REMOTE_USER']
)
sleep 1
return [uri]
return [jid]
end
end

29
devops-service/app/api2/handlers/provider.rb
View File

@ -1,15 +1,44 @@
require "app/api2/parsers/provider"
require "providers/provider_factory"
require_relative "request_handler"
require "db/mongo/models/provider_accounts/static_provider_account"
require "db/mongo/models/provider_accounts/openstack_provider_account"
require "db/mongo/models/provider_accounts/ec2_provider_account"
module Devops
module API2_0
module Handler
class Provider < RequestHandler
set_parser Devops::API2_0::Parser::ProviderParser
def providers
::Provider::ProviderFactory.providers
end
def accounts provider
::Provider::ProviderFactory.get_accounts_factory(provider).accounts
end
def account_fields provider
::Provider::ProviderFactory.get_account_class(provider).account_fields
end
def add_account provider
account = ::Provider::ProviderFactory.get_accounts_factory(provider).create_account(parser.account)
account.validate_fields!
Devops::Db.connector.provider_accounts_insert(account)
::Provider::ProviderFactory.add_account(provider, account)
account.to_hash
end
def delete_account name, provider
account = Devops::Db.connector.provider_accounts_show(name)
Devops::Db.connector.provider_accounts_delete(name)
::Provider::ProviderFactory.delete_account(provider, account)
account.to_hash
end
end
end
end

94
devops-service/app/api2/handlers/server.rb
View File

@ -35,7 +35,11 @@ module Devops
end
def provider_servers provider
::Provider::ProviderFactory.get(provider).servers
provider_servers_with_account provider, nil
end
def provider_servers_with_account provider, account
::Provider::ProviderFactory.get(provider, account).servers
end
def server id
@ -67,20 +71,57 @@ module Devops
user = parser.current_user
check_if_server_attrs_are_valid(body, user)
uri = Worker.start_async(CreateServerWorker, @request,
jid = Worker.start_async(CreateServerWorker,
server_attrs: body,
owner: user
)
[uri]
[jid]
end
def deploy node_name
call_deploy(node_name) do |options|
[ Worker.start_async(DeployWorker, @request, options) ]
end
end
def deploy_stream node_name, out
call_deploy(node_name) do |options|
Worker.start_sync(DeployWorker, @request, options, out)
end
rescue RecordNotFound => e
out << e.message
-10
end
def call_deploy node_name
body = parser.deploy
names = body["names"]
tags = body["tags"] || []
run_list = body["run_list"]
dir = DevopsConfig.config[:report_dir_v2]
owner = parser.current_user
s = Server.get_server_by_key(node_name, parser.instance_key)
project = Devops::Db.connector.check_project_auth s.project, s.deploy_env, owner
deploy_info = create_deploy_info(s, project, body["build_number"])
deploy_info["run_list"] = run_list if run_list
yield({
server_attrs: s.to_hash,
owner: owner,
tags: tags,
deploy_info: deploy_info
})
end
def pause_server node_name
s = Server.get_server_by_key(node_name, parser.instance_key)
s = get_server_by_key(node_name, parser.instance_key)
## Authorization
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
provider = s.provider_instance
r = provider.pause_server s
if r.nil?
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::PAUSE)
"Server with instance ID '#{s.id}' and node name '#{node_name}' is paused"
else
raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be paused, It in state '#{r}'", 409)
@ -88,12 +129,14 @@ module Devops
end
def unpause_server node_name
s = Server.get_server_by_key(node_name, parser.instance_key)
s = get_server_by_key(node_name, parser.instance_key)
## Authorization
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
provider = s.provider_instance
r = provider.unpause_server s
if r.nil?
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNPAUSE)
"Server with instance ID '#{s.id}' and node name '#{node_name}' is unpaused"
else
raise ConflictException.new("Server with instance ID '#{s.id}' and node name '#{node_name}' can not be unpaused, It in state '#{r}'")
@ -106,7 +149,7 @@ module Devops
Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
raise ConflictException.new("Server '#{node_name}' already reserved") unless s.reserved_by.nil?
s.reserved_by = user
Devops::Db.connector.server_update(s)
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::RESERVE)
end
def unreserve_server node_name
@ -114,22 +157,20 @@ module Devops
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
raise ConflictException.new("Server '#{node_name}' is not reserved") if s.reserved_by.nil?
s.reserved_by = nil
Devops::Db.connector.server_update(s)
set_last_operation_type_and_save(s, Devops::Model::Server::OperationType::UNRESERVE)
end
# TODO: check bootstrap template name
def bootstrap_server_stream out
s, rl, bt = prepare_bootstrap_server
options = prepare_bootstrap_server
s = options.delete(:server)
status = []
cert = Devops::Db.connector.key s.key
DevopsLogger.logger.debug "Bootstrap certificate path: #{cert.path}"
#bootstrap s, out, cert.path, logger
options = {
:bootstrap_template => bt,
:cert_path => cert.path,
:run_list => rl
}
r = two_phase_bootstrap s, options, out
options[:cert_path] = cert.path
executor = Devops::Executor::ServerExecutor.new(s, out)
r = executor.two_phase_bootstrap(options)
str = nil
r = if check_server(s)
Devops::Db.connector.server_set_chef_node_name s
@ -149,19 +190,15 @@ module Devops
def bootstrap_server
server, rl, bootstrap_template = prepare_bootstrap_server
dir = DevopsConfig[:report_dir_v2]
files = []
uri = URI.parse(@request.url)
uri = Worker.start_async(BootstrapWorker, @request,
provider_name: server.provider,
jid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: bootstrap_template,
owner: parser.current_user
)
sleep 1
[uri]
[jid]
end
def prepare_bootstrap_server
@ -171,6 +208,7 @@ module Devops
rl = body["run_list"]
t = body["bootstrap_template"]
s = Devops::Db.connector.server_by_instance_id(id)
res = {server: s}
p = Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
d = p.deploy_env(s.deploy_env)
@ -181,9 +219,18 @@ module Devops
unless t.nil?
templates = get_templates
halt_response("Invalid bootstrap template '#{t}', available values: #{templates.join(", ")}", 400) unless templates.include?(t)
res[:bootstrap_template] = t
end
s.chef_node_name = name || provider.create_default_chef_node_name(s)
return s, rl || d.run_list, t
res[:run_list] = rl || d.run_list
return res
end
def unbootstrap_server id
s = get_server_by_key(id, parser.instance_key)
### Authorization
Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user
Devops::Executor::ServerExecutor.new(s, "").unbootstrap
end
def add_server
@ -273,6 +320,11 @@ module Devops
end
end
def set_last_operation_type_and_save(server, operation_type)
server.set_last_operation(operation_type)
Devops::Db.connector.server_update(server)
end
end
end
end

13
devops-service/app/api2/handlers/stack.rb
View File

@ -25,13 +25,13 @@ module Devops
raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil?
object["stack_template"] = env.stack_template
object["owner"] = parser.current_user
object["provider"] = env.provider
object["provider_account"] = env.provider_account
uri = Worker.start_async(StackBootstrapWorker, @request,
provider_name: env.provider,
jid = Worker.start_async(StackBootstrapWorker,
stack_attributes: object
)
puts "Syncing report is located here: #{uri}"
[uri]
[jid]
end
def stack id
@ -99,7 +99,6 @@ module Devops
def deploy id
stack = self.stack(id)
owner = parser.current_user
status = []
project = Devops::Db.connector.check_project_auth(stack.project, stack.deploy_env, owner)
deploy_env_model = project.deploy_env(stack.deploy_env)
body = parser.deploy
@ -111,7 +110,7 @@ module Devops
servers.each do |s|
begin
deploy_info = project.deploy_info(deploy_env_model, nil)
uri = Worker.start_async(DeployWorker, @request,
jid = Worker.start_async(DeployWorker,
server_attrs: s.to_hash,
owner: owner,
tags: tags,
@ -137,7 +136,7 @@ module Devops
}
Devops::Db.connector.save_report(Report.new(o))
end
files.push uri
files.push jid
end
files
end

2
devops-service/app/api2/helpers/version_2.rb
View File

@ -30,7 +30,7 @@ module Devops
def check_provider provider
list = ::Provider::ProviderFactory.providers
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider)
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 400) unless list.include?(provider)
end
# Save information about requests with methods POST, PUT, DELETE

4
devops-service/app/api2/parsers/project.rb
View File

@ -23,11 +23,11 @@ module Devops
end
def project_servers
@params[:deploy_env]
@params["deploy_env"]
end
def project_stacks
@params[:deploy_env]
@params["deploy_env"]
end
def create_project

16
devops-service/app/api2/parsers/provider.rb Normal file
View File

@ -0,0 +1,16 @@
require_relative "request_parser"
module Devops
module API2_0
module Parser
class ProviderParser < RequestParser
def account
create_object_from_json_body
end
end
end
end
end

7
devops-service/app/api2/routes/flavor.rb
View File

@ -37,6 +37,13 @@ module Devops
json Devops::API2_0::Handler::Flavor.new(request).flavors(provider)
end
# TODO: check account
app.get_with_headers "/flavors/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("flavor", "r")
check_provider(provider)
json Devops::API2_0::Handler::Flavor.new(request).flavors_with_account(provider, account)
end
puts "Flavor routes initialized"
end

6
devops-service/app/api2/routes/group.rb
View File

@ -50,6 +50,12 @@ module Devops
json Devops::API2_0::Handler::Group.new(request).groups(provider)
end
app.get_with_headers "/groups/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("group", "r")
check_provider(provider)
json Devops::API2_0::Handler::Group.new(request).groups_with_account(provider, account)
end
puts "Group routes initialized"
end

6
devops-service/app/api2/routes/network.rb
View File

@ -38,6 +38,12 @@ module Devops
json Devops::API2_0::Handler::Network.new(request).networks(provider)
end
app.get_with_headers "/networks/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("network", "r")
check_provider(provider)
json Devops::API2_0::Handler::Network.new(request).networks_with_account(provider, account)
end
puts "Network routes initialized"
end

92
devops-service/app/api2/routes/provider.rb
View File

@ -26,6 +26,98 @@ module Devops
json Devops::API2_0::Handler::Provider.new(request).providers
end
# Get list of provider account fields
#
# * *Request*
# - method : GET
# - headers :
# - Accept: application/json
#
# * *Returns* : hash
# key - field
# value - description
app.get_with_headers "/provider/:provider/account/fields", :headers => [:accept] do |provider|
check_privileges("provider", "r")
check_provider(provider)
json Devops::API2_0::Handler::Provider.new(request).account_fields(provider)
end
# Get list of provider accounts
#
# * *Request*
# - method : GET
# - headers :
# - Accept: application/json
#
# * *Returns* : array of strings
# - ec2:
# {
# "account_name": "",
# "description": "",
# "access_key_id" : "",
# "ssh_key": "",
# "certificate" : "path to file",
# "availability_zone": ""
#
# }
# - openstack:
# {
# "account_name": "",
# "description": "",
# "username": "",
# "auth_url": "",
# "tenant": "",
# "ssh_key": "",
# "certificate" : "path to file"
# }
# - static:
# {
# "account_name": "",
# "description": "",
# "ssh_key": "",
# "certificate" : "path to file"
# }
app.get_with_headers "/provider/:provider/accounts", :headers => [:accept] do |provider|
check_privileges("provider", "r")
check_provider(provider)
json Devops::API2_0::Handler::Provider.new(request).accounts(provider)
end
# Create provider account for :provider
#
# * *Request*
# - method : POST
# - headers :
# - Accept: application/json
# - Content-Type: application/json
# - body :
# {
#
#
# }
#
# * *Returns* : 201
app.post_with_headers "/provider/:provider/account", :headers => [:accept, :content_type] do |provider|
check_privileges("provider", "w")
check_provider(provider)
create_response("Created", {:account => Devops::API2_0::Handler::Provider.new(request).add_account(provider)}, 201)
end
# Delete account with name :account_name for :provider
#
# * *Request*
# - method : DELETE
# - headers :
# - Accept: application/json
# - Content-Type: application/json
#
# * *Returns* : 200
app.delete_with_headers "/provider/:provider/account/:account_name", :headers => [:accept, :content_type] do |provider, account_name|
check_privileges("provider", "w")
check_provider(provider)
create_response("Deleted", {:account => Devops::API2_0::Handler::Provider.new(request).delete_account(provider)})
end
puts "Provider routes initialized"
end
end

71
devops-service/app/api2/routes/server.rb
View File

@ -85,6 +85,11 @@ module Devops
json Devops::API2_0::Handler::Server.new(request).provider_servers(provider)
end
app.get_with_headers "/servers/provider/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("server", "r")
json Devops::API2_0::Handler::Server.new(request).provider_servers_with_account(provider, account)
end
# Get server info by :name
#
# * *Request*
@ -127,6 +132,53 @@ module Devops
}
app.multi_routes "/server/:id", {:headers => [:accept, :content_type]}, hash
# Run deploy command on reserved server
#
# * *Request*
# - method : POST
# - headers :
# - Content-Type: application/json
# - body :
# {
# "names": [], -> array of servers names to run chef-client
# "tags": [], -> array of tags to apply on each server before running chef-client
# "build_number": "", -> string, build number to deploy
# "run_list": [], -> array of strings to set run_list for chef-client
# }
#
# * *Returns* : text stream
app.post_with_headers "/server/:node_name/deploy", :headers => [:content_type, :accept] do |node_name|
check_privileges("server", "x")
if request["HTTP_X_STREAM"]
stream() do |out|
status = []
begin
status = Devops::API2_0::Handler::Server.new(request).deploy_stream(node_name, out)
out << create_status(status)
rescue DeployInfoError => e
msg = "Can not get deploy info: " + e.message
DevopsLogger.logger.error "msg:\n#{e.backtrace.join('\n')}"
out.puts msg
rescue IOError => e
logger.error e.message
break
end
end # stream
else
files = begin
Devops::API2_0::Handler::Server.new(request).deploy(node_name)
rescue DeployInfoError => e
msg = "Can not get deploy info: " + e.message
DevopsLogger.logger.error "#{msg}:\n#{e.backtrace.join("\n")}"
out << "\nError - "
out.puts msg
end
sleep 1
json files
end
end
# Create devops server
#
# * *Request*
@ -279,6 +331,25 @@ module Devops
end
end
# Unbootstrap devops server
#
# * *Request*
# - method : POST
# - headers :
# - Accept: application/json
# - Content-Type: application/json
# - body :
# {
# "key": "instance", -> search server by instance_id rather then chef_node_name
# }
#
# * *Returns* : 200
app.post_with_headers "/server/:id/unbootstrap", :headers => [:accept, :content_type] do |id|
check_privileges("server", "w")
info = Devops::API2_0::Handler::Server.new(request).unbootstrap_server(id)
create_response("Unbootstrap", info)
end
# Add external server to devops
#
# * *Request*

16
devops-service/app/api2/routes/v2.0.rb
View File

@ -61,13 +61,23 @@ module Devops
DevopsLogger.logger = @@logger
begin
res = super(env)
rescue DevopsError => e
rescue ::Devops::Exception::DevopsError => e
return [e.code, {}, e.message]
rescue InvalidRecord => e
return [e.code, {}, e.message]
end
@@access_logger.info(env["REQUEST_METHOD"] + " " + env["REQUEST_URI"] + " - from #{env["HTTP_USER_AGENT"]} (#{env["REMOTE_USER"]}) / #{res.inspect}")
@@access_logger.info(env["REQUEST_METHOD"] + " " + env["REQUEST_URI"] + " - from #{env["HTTP_USER_AGENT"]} (#{env["REMOTE_USER"]}) / #{res[0]}")
res
end
def handle_exception!(boom)
if boom.is_a?(::Devops::Exception::DevopsError)
boom.http_response
else
super(boom)
end
end
error Devops::ValidationError do
e = env["sinatra.error"]
#logger.warn e.message
@ -92,12 +102,14 @@ module Devops
halt_response(e.message, 404)
end
=begin
error InvalidRecord do
e = env["sinatra.error"]
logger.warn e.message
logger.warn "Request body: #{request.body.read}"
halt_response(e.message, 400)
end
=end
error InvalidCommand do
e = env["sinatra.error"]

2
devops-service/app/devops-api2.rb
View File

@ -78,7 +78,7 @@ module Devops
Devops::Api2.register r
end
Routes.route "/v2.0", Devops::Api2
Devops::Api2.routes_list
#Devops::Api2.routes_list
end
private

2
devops-service/app/version/version.rb
View File

@ -2,7 +2,7 @@ require 'sinatra/base'
class DevopsVersion < Sinatra::Base
VERSION = "2.3.0"
VERSION = "2.3.2"
get "/" do
VERSION

4
devops-service/commands/knife_commands.rb
View File

@ -73,7 +73,9 @@ EOH
end
def knife cmd
o = `bundle exec knife #{cmd} -c #{self.config} 2>&1`
cmd = "bundle exec knife #{cmd} -c #{self.config} 2>&1"
DevopsLogger.logger.info("Going to invoke command: #{cmd}")
o = `#{cmd}`
return o, $?.success?
end

7
devops-service/db/mongo/connectors/helpers/insert_command.rb
View File

@ -17,10 +17,13 @@ module Connectors
def insert(record)
begin
record.validate!
collection.insert(record.to_mongo_hash)
hash = record.to_mongo_hash
hash["created_at"] = Time.now.to_i
collection.insert(hash)
record
rescue Mongo::OperationFailure => e
if e.message =~ /^11000/
# exception's message doesn't always start from error code
if e.message =~ /11000/
resource_name = StringHelper.underscore_class(record.class)
raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
end

2
devops-service/db/mongo/connectors/project.rb
View File

@ -94,7 +94,7 @@ module Connectors
def check_project_auth(project_id, env, user_id)
project = show(project_id)
raise InvalidPrivileges.new("User '#{user_id}' unauthorized to work with project '#{project_id}'") unless project.check_authorization(user_id, env)
raise InvalidPrivileges.new("User '#{user_id}' is unauthorized to work with project '#{project_id}' and environment '#{env}'") unless project.check_authorization(user_id, env)
project
end

30
devops-service/db/mongo/connectors/provider_accounts.rb Normal file
View File

@ -0,0 +1,30 @@
module Connectors
class ProviderAccounts < Base
include Helpers::InsertCommand,
Helpers::DeleteCommand
def initialize(db)
super(db)
end
def provider_accounts provider
c = Provider::ProviderFactory.get_account_class(provider)
collection.find({provider: provider}).to_a.map{|bson| c.build_from_bson(bson)}
end
def collection_name
'provider_accounts'
end
private
def model_from_bson(bson)
c = Provider::ProviderFactory.get_account_class(bson["provider"])
raise InvalidRecord.new "Invalid provider '#{bson["provider"]}' for account '#{bson["_id"]}'" if c.nil?
c.build_from_bson(bson)
end
end
end

14
devops-service/db/mongo/connectors/report.rb
View File

@ -51,8 +51,18 @@ module Connectors
collection.find(options).sort(sort).to_a.map {|bson| model_from_bson(bson)}
end
def set_report_status(jid, status)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new}})
def add_report_subreports(jid, subreports)
collection.update({"_id" => jid}, {"$push" => {"subreports" => {"$each" => subreports}}})
end
def set_report_status(jid, status, job_result_code)
set = {"status" => status, "updated_at" => Time.new}
set["job_result_code"] = job_result_code unless job_result_code.nil?
collection.update({"_id" => jid}, {"$set" => set})
end
def set_report_status(jid, status, job_result_code)
collection.update({"_id" => jid}, {"$set" => {"status" => status, "updated_at" => Time.new, "job_result_code" => job_result_code}})
end
def set_report_server_data id, chef_node_name, host

3
devops-service/db/mongo/connectors/server.rb
View File

@ -63,17 +63,14 @@ module Connectors
end
def server_insert(server)
#server.validate!
server.created_at = Time.now
collection.insert(server.to_mongo_hash)
end
# somewhy servers are not validated in previous version of code. I leave this until I know, why.
def server_update(server)
collection.update({'_id' => server.id}, server.to_hash_without_id)
end
# somewhy servers are not validated in previous version of code. I leave this until I know, why.
def server_set_chef_node_name(server)
collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}})
end

1
devops-service/db/mongo/models/deploy_env/cloud_deploy_env.rb
View File

@ -26,6 +26,7 @@ module Devops
:stack_template => {:type => String, :empty => false, :nil => true}
=end
#TODO: account validator
set_validators ::Validators::DeployEnv::Flavor,
::Validators::DeployEnv::Image,
::Validators::DeployEnv::Groups,

5
devops-service/db/mongo/models/deploy_env/deploy_env_base.rb
View File

@ -32,10 +32,10 @@ module Devops
def initialize d={}
self.identifier = d["identifier"]
set_provider(d)
b = d["run_list"] || []
self.run_list = b.uniq
self.expires = d["expires"]
self.provider = d["provider"]
b = d["users"] || []
self.users = b.uniq
end
@ -45,9 +45,8 @@ module Devops
"identifier" => self.identifier,
"run_list" => self.run_list,
"expires" => self.expires,
"provider" => self.provider,
"users" => self.users
}
}.merge(provider_hash)
end
def add_users users

17
devops-service/db/mongo/models/model_with_provider.rb
View File

@ -4,7 +4,7 @@ module Devops
module Model
module ModelWithProvider
attr_accessor :provider
attr_accessor :provider, :provider_account
def ModelWithProvider.included(mod)
@ -15,8 +15,21 @@ module Devops
end
def provider_instance
@provider_instance ||= Provider::ProviderFactory.get(self.provider)
@provider_instance ||= Provider::ProviderFactory.get(self.provider, self.provider_account)
end
def set_provider hash
self.provider = hash["provider"]
self.provider_account = hash["provider_account"]
end
def provider_hash
{
"provider" => self.provider,
"provider_account" => self.provider_account
}
end
end
end
end

31
devops-service/db/mongo/models/mongo_model.rb
View File

@ -1,4 +1,4 @@
require "providers/provider_factory"
#require "providers/provider_factory"
require "exceptions/invalid_record"
require "json"
require 'db/validators/all'
@ -7,6 +7,8 @@ module Devops
module Model
class MongoModel
attr_accessor :created_at
# multi_json sends argument to 'to_json' method
def to_json arg=nil
JSON.pretty_generate self.to_hash
@ -64,6 +66,21 @@ module Devops
end
end
def validate_fields!
result = []
self.class.field_validators.each do |field, validation_method|
begin
self.send(validation_method)
rescue InvalidRecord => e
result << {key: field, message: e.message}
end
end
unless result.empty?
raise InvalidRecord.new(error_data: result)
end
true
end
def build_error_message(message)
# overrided in descendants
message
@ -76,6 +93,7 @@ module Devops
# :empty - can param be empty? (false)
# :nil - can param be nil? (false)
# :value_type - type of array element (String)
# TODO: @deprecated
def self.types types
define_method :validate_fields_types do
t = types.keys
@ -120,17 +138,16 @@ module Devops
end
@validators = []
# @field_validators = []
@field_validators = {}
class << self
attr_accessor :validators
# attr_accessor :field_validators
attr_accessor :field_validators
def inherited(subclass)
subclass.validators = []
subclass.validators += self.validators
# subclass.field_validators = []
# subclass.field_validators += self.field_validators
subclass.field_validators = self.field_validators.clone
end
# all exceptions are handled in @validate! method
@ -143,11 +160,13 @@ module Devops
# validate field value
# if method validate! returns false, then stop validation without error
def set_field_validators field, *validators
define_method("validate_" + field.to_s + "!") do
method_name = "validate_" + field.to_s + "!"
define_method(method_name) do
validators.each do |validator|
break unless validator.new(self, send(field)).validate!
end
end
self.field_validators[field] = method_name
end
# private class methods

2
devops-service/db/mongo/models/project.rb
View File

@ -148,7 +148,7 @@ module Devops
}
end
def deploy_info deploy_env, build_number
def deploy_info deploy_env, build_number=nil
{
"use_json_file" => true,
# "run_list" => Set.new.merge(self.run_list).merge(deploy_env.run_list).to_a,

59
devops-service/db/mongo/models/provider_accounts/ec2_provider_account.rb Normal file
View File

@ -0,0 +1,59 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class Ec2ProviderAccount < ProviderAccount
attr_accessor :access_key_id, :availability_zone, :secret_access_key
set_field_validators :access_key_id, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty
set_field_validators :secret_access_key, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty
def initialize a={}
super(a)
self.provider = Provider::Ec2::PROVIDER
self.availability_zone = a["availability_zone"]
self.access_key_id = a["access_key_id"]
self.secret_access_key = a["secret_access_key"]
end
def to_list_hash
s = super
s["availability_zone"] = self.availability_zone
s["access_key_id"] = self.access_key_id
s
end
def to_mongo_hash
s = super
s["availability_zone"] = self.availability_zone
s["access_key_id"] = self.access_key_id
s["secret_access_key"] = self.secret_access_key
s
end
def self.build_from_bson a
a["account_name"] = a["_id"]
Ec2ProviderAccount.new a
end
def self.account_fields
{
access_key_id: "AWS account access key",
secret_access_key: "AWS account secret key",
availability_zone: "Availability zone, todo: remove field?"
}.merge(ProviderAccount::ACCOUNT_FIELDS)
end
# TODO: remove
def validate_fields_types
end
end
end
end

49
devops-service/db/mongo/models/provider_accounts/openstack_provider_account.rb Normal file
View File

@ -0,0 +1,49 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class OpenstackProviderAccount < ProviderAccount
attr_accessor :username, :auth_url, :tenant, :api_key
def self.account_fields
{
username: "Openstack user name",
auth_url: "Identity API endpoint",
tenant: "Tenant to access",
api_key: "Openstack user password"
}.merge(ProviderAccount::ACCOUNT_FIELDS)
end
def initialize a={}
super(a)
self.username = a["username"]
self.auth_url = a["auth_url"]
self.tenant = a["tenant"]
self.api_key = a["api_key"]
end
def to_list_hash
s = super
s["username"] = self.username
s["auth_url"] = self.auth_url
s["tenant"] = self.tenant
end
def to_mongo_hash
s = super
s["username"] = self.username
s["auth_url"] = self.auth_url
s["tenant"] = self.tenant
s["api_key"] = self.api_key
end
def self.build_from_bson a
a["account_name"] = a["_id"]
OpenstackProviderAccount.new a
end
end
end
end

66
devops-service/db/mongo/models/provider_accounts/provider_account.rb Normal file
View File

@ -0,0 +1,66 @@
require "db/mongo/models/mongo_model"
require "db/mongo/models/model_with_provider"
module Devops
module Model
class ProviderAccount < MongoModel
include ModelWithProvider
attr_accessor :account_name, :description, :ssh_key
set_field_validators :account_name, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::Name
set_field_validators :description, ::Validators::FieldValidator::Nil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::Description
set_field_validators :ssh_key, ::Validators::FieldValidator::NotNil,
::Validators::FieldValidator::FieldType::String,
::Validators::FieldValidator::NotEmpty,
::Validators::FieldValidator::SshKey
ACCOUNT_FIELDS = {
account_name: "Account name (id)",
description: "Account description",
ssh_key: "Ssh key id"
}
def initialize a={}
self.account_name = a["account_name"]
self.description = a["description"]
self.ssh_key = a["ssh_key"]
self.provider = a["provider"]
self.created_at = a["created_at"]
end
def to_list_hash
to_hash
end
def to_hash
{
"account_name" => self.account_name,
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider,
"created_at" => self.created_at
}
end
def to_mongo_hash
{
"_id" => self.account_name,
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider
}
end
end
end
end

18
devops-service/db/mongo/models/provider_accounts/static_provider_account.rb Normal file
View File

@ -0,0 +1,18 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class StaticProviderAccount < ProviderAccount
def self.build_from_bson a
a["account_name"] = a["_id"]
StaticProviderAccount.new a
end
def self.account_fields
ProviderAccount::ACCOUNT_FIELDS
end
end
end
end

10
devops-service/db/mongo/models/report.rb
View File

@ -12,7 +12,7 @@ module Devops
DEPLOY_STACK_TYPE = 6
DELETE_SERVER_TYPE = 7
attr_accessor :id, :file, :created_at, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack
attr_accessor :id, :file, :updated_at, :created_by, :project, :deploy_env, :type, :chef_node_name, :host, :status, :stack, :subreports, :job_result_code
def initialize r
self.id = r["_id"]
@ -21,13 +21,15 @@ module Devops
self.project = r["project"]
self.deploy_env = r["deploy_env"]
self.type = r["type"]
self.created_at = r["created_at"]
# self.created_at = r["created_at"]
self.chef_node_name = r["chef_node_name"]
self.host = r["host"]
self.stack = r["stack"]
self.created_at = r["created_at"].localtime unless r["created_at"].nil?
self.updated_at = r["updated_at"].localtime unless r["updated_at"].nil?
self.status = r["status"]
self.subreports = r["subreports"]
self.job_result_code = r["job_result_code"]
end
def to_hash_without_id
@ -42,7 +44,9 @@ module Devops
"chef_node_name" => self.chef_node_name,
"host" => self.host,
"status" => self.status,
"stack" => self.stack
"stack" => self.stack,
"subreports" => self.subreports,
"job_result_code" => self.job_result_code
}
end

32
devops-service/db/mongo/models/server.rb
View File

@ -9,8 +9,17 @@ module Devops
include ModelWithProvider
module OperationType
CREATION = :creation
DEPLOY = :deploy
RESERVE = :reserve
UNRESERVE = :unreserve
PAUSE = :pause
UNPAUSE = :unpause
end
attr_accessor :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack, :run_list
attr_accessor :key
attr_accessor :key, :last_operation_at, :last_operation_type
types :id => {:type => String, :empty => false},
:provider => {:type => String, :empty => false},
@ -29,11 +38,12 @@ module Devops
set_validators ::Validators::DeployEnv::RunList
def self.fields
["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack"]
["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "key", "reserved_by", "run_list", "stack",
"last_operation_type", "last_operation_at"]
end
def initialize s={}
self.provider = s["provider"]
self.set_provider(s)
self.chef_node_name = s["chef_node_name"]
self.id = s["_id"]
self.remote_user = s["remote_user"]
@ -47,6 +57,8 @@ module Devops
self.reserved_by = s["reserved_by"]
self.stack = s["stack"]
self.run_list = s["run_list"] || []
self.last_operation_at = s["last_operation_at"]
self.last_operation_type = s["last_operation_type"]
self
end
@ -66,7 +78,6 @@ module Devops
def to_hash_without_id
{
"provider" => self.provider,
"chef_node_name" => self.chef_node_name,
"remote_user" => self.remote_user,
"project" => self.project,
@ -78,8 +89,10 @@ module Devops
"key" => self.key,
"reserved_by" => self.reserved_by,
"stack" => stack,
"run_list" => self.run_list
}.delete_if { |k,v| v.nil? }
"run_list" => self.run_list,
"last_operation_at" => self.last_operation_at,
"last_operation_type" => self.last_operation_type
}.merge(provider_hash).delete_if { |k,v| v.nil? }
end
def self.build_from_bson s
@ -98,7 +111,12 @@ module Devops
end
def static?
(self.provider == Provider::Static::PROVIDER) || false
self.provider == Provider::Static::PROVIDER
end
def set_last_operation(operation_type)
self.last_operation_type = operation_type
self.last_operation_at = Time.now
end
end

7
devops-service/db/mongo/models/stack/stack_base.rb
View File

@ -21,7 +21,7 @@ module Devops
def initialize attrs={}
# self.provider = self.class.provider
self.set_provider(attrs)
self.id = attrs['id']
self.project = attrs['project']
self.deploy_env = attrs['deploy_env']
@ -36,7 +36,6 @@ module Devops
def to_hash_without_id
{
provider: provider,
project: project,
deploy_env: deploy_env,
stack_template: stack_template,
@ -46,7 +45,7 @@ module Devops
details: bson_safe_details,
stack_status: stack_status,
owner: owner
}
}.merge(provider_hash)
end
# overrided in ec2
@ -106,6 +105,8 @@ module Devops
# - id (String)
# - deploy_env (String)
# - stack_template (String)
# - provider (String)
# - provider_account (String)
def create(attrs, out)
model = new(attrs)
model.create_stack_in_cloud!(out)

9
devops-service/db/mongo/mongo_connector.rb
View File

@ -31,8 +31,9 @@ class MongoConnector
[:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
:user_update, :create_root_user, :check_user_privileges] => :users_connector,
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector
[:save_report, :report, :reports, :set_report_status, :set_report_server_data, :add_report_subreports] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector,
[:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector
)
def initialize(db, host, port=27017, user=nil, password=nil)
@ -46,6 +47,10 @@ class MongoConnector
private
def provider_accounts_connector
@provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db)
end
def images_connector
@image_connector ||= Connectors::Image.new(@db)
end

2
devops-service/db/validators/all.rb
View File

@ -15,5 +15,5 @@ require "db/validators/base"
'db/validators/image/*.rb',
'db/validators/field_validators/*.rb'
].each do |files_regexp|
Dir[files_regexp].each {|file| require file }
Dir[File.join(Devops::Application.root, files_regexp)].each {|file| require file }
end

17
devops-service/db/validators/field_validators/description.rb Normal file
View File

@ -0,0 +1,17 @@
require_relative "base"
module Validators
module FieldValidator
class Description < Base
MAX_LEN = 500
def valid?
@value.size <= 500
end
def message
"Invalid value '#{@value}': it should be less or equals then #{MAX_LEN} symbols"
end
end
end
end

2
devops-service/db/validators/field_validators/name.rb
View File

@ -11,7 +11,7 @@ module Validators
end
def message
"Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN}"
"Invalid value '#{@value}': it should contains symbols 'a-zA-Z0-9_' and length should be more then 1 and less or equals then #{MAX_NAME_LEN} symbols"
end
end
end

20
devops-service/db/validators/field_validators/ssh_key.rb Normal file
View File

@ -0,0 +1,20 @@
require_relative "base"
module Validators
module FieldValidator
class SshKey < Base
MAX_LEN = 500
def valid?
Devops::Db.connector.key @value
true
rescue RecordNotFound
false
end
def message
"Invalid value '#{@value}': ssh key '#{@value}' not found"
end
end
end
end

15
devops-service/exceptions/devops_error.rb
View File

@ -1,12 +1,25 @@
require "json"
module Devops
module Exception
class DevopsError < StandardError
def code
def http_status
500
end
def http_response
[self.http_status, self.http_headers, self.http_body]
end
def http_body
JSON.pretty_generate(message: self.message)
end
def http_headers
{"Content-Type" => "application/json"}
end
end
end

22
devops-service/exceptions/invalid_record.rb
View File

@ -1,3 +1,23 @@
class InvalidRecord < StandardError
require_relative "devops_error"
class InvalidRecord < ::Devops::Exception::DevopsError
def initialize msg
if msg.is_a?(String)
super(msg)
else
@object = msg
end
end
def http_status
400
end
def http_body
if @object.nil?
super
else
JSON.pretty_generate(@object)
end
end
end

80
devops-service/lib/executors/server_executor.rb
View File

@ -16,7 +16,8 @@ module Devops
deploy_failed: 8
}
MAX_SSH_RETRIES_AMOUNT = 20
# waiting for 5*60 seconds (5 min)
MAX_SSH_RETRIES_AMOUNT = 60
#params:
# out - container for output data
@ -74,11 +75,13 @@ module Devops
end
def create_server options
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider})
provider = @server.provider_instance
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider, "provider_account" => @deploy_env.provider_account})
@server.set_last_operation(Devops::Model::Server::OperationType::CREATION)
provider = @deploy_env.provider_instance
mongo = ::Devops::Db.connector
begin
@out << "Create server...\n"
@out.puts "Using '#{@deploy_env.provider}' account '#{@deploy_env.provider_account}'\n" if @deploy_env.provider_account
@out.puts "Create server..."
@out.flush
@server.run_list = options["run_list"] || []
@ -137,6 +140,7 @@ module Devops
end
ja = {
:provider => @server.provider,
:provider_account => @server.provider_account,
:devops_host => `hostname`.strip
}
ip = @server.private_ip
@ -190,8 +194,10 @@ module Devops
@out << "After bootstrap hooks...\n"
res = self.run_hook(:after_bootstrap, @out)
@out << "Done\n"
msg = "Server with id '#{@server.id}' is bootstraped"
@out.puts msg
@out.flush
DevopsLogger.logger.info "Server with id '#{@server.id}' is bootstraped"
DevopsLogger.logger.info msg
r
else
@out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
@ -210,12 +216,18 @@ module Devops
bootstrap_options.push "--sudo" unless @server.remote_user == "root"
bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
rl = options[:run_list]
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil?# rl.empty?
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil? or rl.empty?
bootstrap_options.push "-c #{options[:config]}" if options[:config]
bootstrap_options
end
def prepare_two_phase_bootstrap options
@out << "Prepare bootstrap...\n"
@out << "Done\n"
end
def two_phase_bootstrap options
prepare_two_phase_bootstrap(options)
# bootstrap phase
begin
provider = @server.provider_instance
@ -227,7 +239,8 @@ module Devops
if check_server
@out << "Server #{@server.chef_node_name} is created"
else
@out << roll_back
@out.puts "Can not find client or node on chef-server"
roll_back
@out.flush
mongo.server_delete @server.id
return result_code(:server_not_in_chef_nodes)
@ -256,7 +269,7 @@ module Devops
@out << "\nComputed run list: #{run_list.join(", ")}"
@out.flush
@knife_instance.set_run_list(@server.chef_node_name, run_list)
deploy_info = @project.deploy_info(@deploy_env)
deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env)
deploy_status = deploy_server(deploy_info)
if deploy_status == 0
0
@ -269,6 +282,7 @@ module Devops
end
rescue => e
@out << "\nError: #{e.message}\n"
DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
result_code(:deploy_unknown_error)
end
end
@ -281,17 +295,27 @@ module Devops
k = Devops::Db.connector.key(@server.key)
cert_path = k.path
i = 0
res = delete_from_chef_server(@server.chef_node_name)
begin
r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
raise(r) unless $?.success?
new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}"
# r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} \"/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name}; else echo not found; fi'\""
DevopsLogger.logger.info("Trying to run command '#{cmd}'")
r = `#{cmd}`.strip
if r == 'not found'
res[:server] = "Directory '/etc/chef' does not exists"
else
raise(r) unless $?.success?
res[:server] = "'/etc/chef' renamed to '#{new_name}'"
end
rescue => e
DevopsLogger.logger.error "Unbootstrap error: " + e.message
i += 1
sleep(1)
retry unless i == 5
return e.message
return {error: e.message}
end
nil
res
end
def deploy_server_with_tags tags, deploy_info
@ -334,15 +358,20 @@ module Devops
cmd = "chef-client --no-color"
if deploy_info["use_json_file"]
deploy_info.delete("use_json_file")
@out << "Deploy Input Parameters:\n"
json = JSON.pretty_generate(deploy_info)
@out << json
@out << "\n"
file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
json = nil
dir = DevopsConfig.config[:project_info_dir]
File.open(File.join(dir, file), "w") do |f|
f.write json
file = deploy_info["json_file"] || "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
path = File.join(dir, file)
if File.exists?(path)
json = File.read(path)
else
json = JSON.pretty_generate(deploy_info)
File.open(File.join(dir, file), "w") do |f|
f.write json
end
end
@out << "Deploy Input Parameters:\n"
@out.puts json
@out.flush
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
else
@ -364,6 +393,10 @@ module Devops
@out << "\nAfter deploy hooks...\n"
res = self.run_hook(:after_deploy, @out, deploy_info)
@out << "Done\n"
@server.set_last_operation(Devops::Model::Server::OperationType::DEPLOY)
Devops::Db.connector.server_update(@server)
0
end
end
@ -412,13 +445,14 @@ module Devops
end
def roll_back
@out.puts "Trying to roll back..."
unless @server.id.nil?
@out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n"
@out << delete_from_chef_server(@server.chef_node_name).values.join("\n")
@out.puts "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created"
@out.puts delete_from_chef_server(@server.chef_node_name).values.join("\n")
begin
@out << @server.provider_instance.delete_server(@server)
@out.puts @server.provider_instance.delete_server(@server)
rescue => e
@out << e.message
@out.puts e.message
end
@out << "\nRolled back\n"
end

33
devops-service/providers/accounts_factory.rb Normal file
View File

@ -0,0 +1,33 @@
module Provider
class AccountsFactory
def init config
end
# providers instances
def connection account_name
@connections[account_name]
end
def add_connection name, conn
@connections[name] = conn
end
def delete_connection name
@connections.delete(name)
end
def create_connection_from_account config, account
end
def accounts
Devops::Db.connector.provider_accounts(provider_name)
end
def create_account hash
raise "override me"
end
end
end

5
devops-service/providers/ec2.rb
View File

@ -1,5 +1,7 @@
require "exceptions/conflict_exception"
require "providers/base_provider"
require "db/mongo/models/provider_accounts/ec2_provider_account"
require_relative "ec2_accounts_factory"
module Provider
# Provider for Amazon EC2
@ -88,7 +90,7 @@ module Provider
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s == "running"}.map do |server|
convert_server server["instancesSet"][0]
end
end
@ -317,6 +319,7 @@ module Provider
'key_name' => instance["keyName"],
'private_ip' => instance["privateIpAddress"],
'public_ip' => instance["ipAddress"],
'tags' => instance["tagSet"]
}
end
end

37
devops-service/providers/ec2_accounts_factory.rb Normal file
View File

@ -0,0 +1,37 @@
require_relative "accounts_factory"
module Provider
class Ec2AccountsFactory < AccountsFactory
def init config
@connections = {}
accounts.each do |account|
create_connection_from_account(config, account)
puts "\tFound ec2 account '#{account.account_name}'"
end
ProviderFactory.add_provider Ec2::PROVIDER unless @connections.empty?
end
def provider_name
Ec2::PROVIDER
end
def create_connection_from_account config, account
options = {
aws_ssh_key: account.ssh_key,
aws_access_key_id: account.access_key_id,
aws_secret_access_key: account.secret_access_key,
aws_availability_zone: account.availability_zone,
aws_proxy: config[:aws_proxy],
aws_no_proxy: config[:aws_no_proxy],
aws_integration_run_list: config[:aws_integration_run_list]
}
add_connection(account.account_name, Ec2.new(options))
end
def create_account hash
Devops::Model::Ec2ProviderAccount.new(hash)
end
end
end

6
devops-service/providers/openstack_accounts_factory.rb Normal file
View File

@ -0,0 +1,6 @@
require_relative "accounts_factory"
module Provider
class OpenstackAccountsFactory < AccountsFactory
end
end

52
devops-service/providers/provider_factory.rb
View File

@ -3,14 +3,21 @@ require "sinatra"
module Provider
module ProviderFactory
@@available_providers = []
@@providers = {}
@@providers_with_accounts_factories = {}
def self.providers
@@providers.keys
@@available_providers
end
def self.get provider
p = @@providers[provider]
def self.get provider, account=nil
p = if account.nil?
@@providers[provider]
else
@@providers_with_accounts_factories[provider].connection(account)
end
# TODO: new exception
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
@ -19,6 +26,10 @@ module Provider
@@providers.values
end
def self.add_provider provider
@@available_providers << provider unless @@available_providers.include?(provider)
end
def self.init conf
# require providers here to get access to debug properties
require_all
@ -28,24 +39,59 @@ module Provider
o = Provider.const_get(p.capitalize).new(conf)
if o.configured?
@@providers[p] = o
@@available_providers << p
puts "Provider '#{p}' has been loaded"
end
factory = Provider.const_get(p.capitalize + "AccountsFactory").new
factory.init(conf)
@@providers_with_accounts_factories[p] = factory
rescue => e
puts "Error while loading provider '#{p}': " + e.message
next
end
end
puts "Available providers: #{@@available_providers}"
end
def self.add_account provider, account
factory = @@providers_with_accounts_factories[provider]
factory.create_connection_from_account(DevopsConfig.config, account)
DevopsLogger.logger.info("Added #{provider} account '#{account.account_name}'")
end
def self.delete_account provider, account
factory = @@providers_with_accounts_factories[provider]
factory.delete_connection(account.account_name)
DevopsLogger.logger.info("Removed #{provider} account '#{account.account_name}'")
end
def self.require_all
["ec2", "openstack", "static"].each do |provider|
begin
require_relative provider
require_relative provider + "_accounts_factory"
rescue LoadError => e
puts "Can not load provider '#{provider}': " + e.message
end
end
end
def self.get_accounts_factory provider
@@providers_with_accounts_factories[provider]
end
def self.get_account_class provider
case(provider)
when ::Provider::Static::PROVIDER
::Devops::Model::StaticProviderAccount
when ::Provider::Ec2::PROVIDER
::Devops::Model::Ec2ProviderAccount
when ::Provider::Openstack::PROVIDER
::Devops::Model::OpenstackProviderAccount
else
nil
end
end
end
end

6
devops-service/providers/static_accounts_factory.rb Normal file
View File

@ -0,0 +1,6 @@
require_relative "accounts_factory"
module Provider
class StaticAccountsFactory < AccountsFactory
end
end

9
devops-service/workers/bootstrap_worker.rb
View File

@ -1,25 +1,26 @@
require File.join(File.dirname(__FILE__), "worker")
require "lib/executors/server_executor"
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
class BootstrapWorker < Worker
def perform(options)
provider_name = options.fetch('provider_name')
server_attrs = options.fetch('server_attrs')
bootstrap_template = options.fetch('bootstrap_template')
# bootstrap_template = options.fetch('bootstrap_template')
owner = options.fetch('owner')
options = convert_config(options)
call(provider_name) do |provider, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)
=begin
options = {
bootstrap_template: bootstrap_template
}
=end
executor = Devops::Executor::ServerExecutor.new(server, out)
executor.report = report
status = executor.two_phase_bootstrap(options)

3
devops-service/workers/create_server_worker.rb
View File

@ -1,6 +1,5 @@
require File.join(File.dirname(__FILE__), "worker")
require "providers/provider_factory"
require "db/mongo/models/server"
require "db/mongo/models/report"
require "lib/executors/server_executor"
@ -11,7 +10,7 @@ class CreateServerWorker < Worker
server_attrs = options.fetch('server_attrs')
owner = options.fetch('owner')
call(nil) do |not_used, out, file|
call() do |out, file|
project = mongo.project(server_attrs["project"])
env = project.deploy_env(server_attrs["deploy_env"])
report = save_report(file, project, env, owner)

2
devops-service/workers/delete_server_worker.rb
View File

@ -8,7 +8,7 @@ class DeleteServerWorker < Worker
chef_node_name = options.fetch('server_chef_node_name')
puts "Expire server '#{chef_node_name}'."
call(nil) do |not_used, out, file|
call() do |out, file|
server = mongo.server_by_chef_node_name(chef_node_name)
report = save_report(file, server)

2
devops-service/workers/deploy_worker.rb
View File

@ -12,7 +12,7 @@ class DeployWorker < Worker
tags = options.fetch('tags')
deploy_info = options.fetch('deploy_info')
call(nil) do |not_used, out, file|
call() do |out, file|
server = Devops::Model::Server.new(server_attrs)
report = save_report(file, owner, server)

2
devops-service/workers/project_test_worker.rb
View File

@ -15,7 +15,7 @@ class ProjectTestWorker < Worker
project_name = params.fetch('project')
deploy_env_name = params.fetch('deploy_env')
call(nil) do |not_used, out, file|
call() do |out, file|
DevopsLogger.logger.info "Test project '#{project_name}' and env '#{deploy_env_name}' (user - #{user})"
project = mongo.project(project_name)
env = project.deploy_env(deploy_env_name)

124
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -1,4 +1,3 @@
require "providers/provider_factory"
require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
@ -12,10 +11,9 @@ class StackBootstrapWorker < Worker
include StackCommands
def perform(options)
provider_name = options.fetch('provider_name')
stack_attrs = options.fetch('stack_attributes')
call(provider_name) do |provider, out, file|
call() do |out, file|
@out = out
without_bootstrap = stack_attrs.delete('without_bootstrap')
@out.puts "Received 'without_bootstrap' option" if without_bootstrap
@ -23,14 +21,23 @@ class StackBootstrapWorker < Worker
report = save_report(file, stack_attrs)
begin
stack = create_stack(provider_name, stack_attrs)
stack = create_stack(stack_attrs)
rescue StackCreatingError
return 1
end
#TODO: errors
begin
servers = persist_stack_servers!(stack, provider)
bootstrap_servers!(servers, report) unless without_bootstrap
servers_with_priority = persist_stack_servers!(stack)
unless without_bootstrap
sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
sorted_keys.each do |key|
@out.puts "Servers with priority '#{key}':"
bootstrap_servers!(servers_with_priority[key], report)
end
end
@out.puts "Done."
0
rescue BootstrapingStackServerError
@out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
rollback_stack!(stack)
@ -57,8 +64,8 @@ class StackBootstrapWorker < Worker
@out.puts "Rollback has been completed"
end
def create_stack(provider_name, stack_attrs)
stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out)
def create_stack(stack_attrs)
stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
mongo.stack_insert(stack)
operation_result = sync_stack_proc.call(@out, stack, mongo)
@ -76,15 +83,40 @@ class StackBootstrapWorker < Worker
def bootstrap_servers!(servers, report)
@out << "\nStart bootstraping stack servers\n"
bootstraping_results = {}
subreports = []
data = {}
servers.each do |server|
executor = Devops::Executor::ServerExecutor.new(server, @out)
executor.report = report
#TODO: can stack choose bootstrap template?
bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({bootstrap_template: 'omnibus'})
@out.flush
sjid = Worker.start_async(BootstrapWorker,
server_attrs: server.to_mongo_hash,
bootstrap_template: 'omnibus',
owner: server.created_by
)
subreports << sjid
@out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
data[server.id] = sjid
end
check_bootstrap_results!(bootstraping_results)
@out.puts
@out.flush
mongo.add_report_subreports(jid, subreports)
results = []
data.each do |server_id, subreport_id|
begin
sleep(5)
subreport = mongo.report(subreport_id)
status = subreport.status
if status == Worker::STATUS::COMPLETED
@out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
break
elsif status == Worker::STATUS::FAILED
results << subreport.job_result_code
@out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
break
end
end while(true)
end
@out.flush
results.empty? ? 0 : -5
end
def check_bootstrap_results!(results)
@ -124,42 +156,60 @@ class StackBootstrapWorker < Worker
"created_by" => stack_attrs['owner'],
"project" => stack_attrs["project"],
"deploy_env" => stack_attrs["deploy_env"],
"type" => ::Devops::Model::Report::STACK_TYPE
"type" => ::Devops::Model::Report::STACK_TYPE,
"subreports" => [],
"stack" => stack_attrs['name']
)
mongo.save_report(report)
report
end
def persist_stack_servers!(stack, provider)
# returns
# {
# "priority" => [Servers]
# }
def persist_stack_servers!(stack)
@out.puts "Start syncing stack servers with CID"
@out.flush
project = mongo.project(stack.project)
deploy_env = project.deploy_env(stack.deploy_env)
provider = stack.provider_instance
servers = provider.stack_servers(stack).map do |extended_info|
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info["key_name"] || provider.ssh_key,
'_id' => extended_info["id"],
'chef_node_name' => extended_info["name"],
'private_ip' => extended_info["private_ip"],
'public_ip' => extended_info["public_ip"],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.name
}
stack_servers = provider.stack_servers(stack)
stack_servers.each do |info|
info["tags"]["cid:priority"] = info["tags"]["cid:priority"].to_i
end
stack_servers_info = stack_servers.group_by{|info| info["tags"]["cid:priority"]}
stack_servers_with_priority = {}
stack_servers_info.each do |priority, info_array|
stack_servers_with_priority[priority] = info_array.map do |extended_info|
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info["key_name"] || provider.ssh_key,
'_id' => extended_info["id"],
'chef_node_name' => extended_info["name"],
'private_ip' => extended_info["private_ip"],
'public_ip' => extended_info["public_ip"],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.name
}
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# server.chef_node_name = provider.create_default_chef_node_name(server)
server
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
# server.chef_node_name = provider.create_default_chef_node_name(server)
server
end
end
@out.puts "Stack servers have been synced with CID"
stack_servers_with_priority.each do |priority, servers|
@out.puts "Servers with priority '#{priority}': #{servers.map(&:id).join(", ")}"
end
@out.flush
servers
stack_servers_with_priority
end
end

52
devops-service/workers/worker.rb
View File

@ -13,6 +13,8 @@ require "core/devops-db"
require "providers/provider_factory"
require "lib/knife/knife_factory"
# All options keys MUST be a symbol!!!
class Worker
include Sidekiq::Worker
@ -26,21 +28,25 @@ class Worker
IN_QUEUE = "queued"
end
def self.start_async(worker_class, request, job_options)
def self.start_async(worker_class, job_options)
jid = worker_class.perform_async(job_options.dup)
Worker.set_status jid, Worker::STATUS::IN_QUEUE
DevopsLogger.logger.info "Job '#{jid}' has been queued"
jid
=begin
uri = URI.parse(request.url)
uri.path = "#{DevopsConfig[:url_prefix]}/v2.0/report/#{jid}"
uri.to_s
=end
end
def self.start_sync(worker_class, request, job_options, out)
stringified_options = {}
stringified_options = job_options
=begin
job_options.each do |key, value|
stringified_options[key.to_s] = value
end
=end
w = worker_class.new
w.out = out
w.perform(stringified_options)
@ -50,25 +56,22 @@ class Worker
Sidekiq.redis {|con| con.hset "devops", id, status}
end
def call provider_name, &block
def call &block
begin
initialize_devops(provider_name)
provider = ::Provider::ProviderFactory.get(provider_name) if provider_name
initialize_devops()
if jid
call_async(provider, &block)
call_async(&block)
else
call_sync(provider, &block)
call_sync(&block)
end
rescue StandardError => e
puts e.message
puts e.backtrace.join("\n")
DevopsLogger.logger.error e.message
DevopsLogger.logger.error "#{e.message}:\n#{e.backtrace.join("\n")}"
end
end
private
def initialize_devops(provider_name)
def initialize_devops()
DevopsLogger.logger = logger
DevopsConfig.read
DevopsService.init
@ -76,33 +79,33 @@ class Worker
end
# outputs to file
def call_async(provider)
def call_async()
dir = DevopsConfig[:report_dir_v2]
# directory is created on server start in config.ru
file = File.join(dir, jid)
update_job_status(STATUS::INIT)
update_job_status(STATUS::INIT, nil)
File.open(file, "w") do |out|
begin
update_job_status(STATUS::RUNNING)
update_job_status(STATUS::RUNNING, nil)
self.out = out
job_result = yield(provider, out, file)
job_result = yield(out, file)
canonical_status = (job_result == 0 ? STATUS::COMPLETED : STATUS::FAILED)
update_job_status(canonical_status)
update_job_status(canonical_status, job_result)
rescue StandardError, RecordNotFound => e
out << "\n #{e.class}\n #{e.message}\n"
out << e.backtrace.join("\n")
update_job_status(STATUS::FAILED)
update_job_status(STATUS::FAILED, -100)
end
end
end
# outputs to STDOUT
def call_sync(provider)
def call_sync()
out = STDOUT
begin
yield(provider, out, '')
yield(out, '')
rescue StandardError, RecordNotFound => e
out << "\n"
out << e.message
@ -115,17 +118,12 @@ class Worker
::Devops::Db.connector
end
def update_job_status(status)
def update_job_status(status, job_result_code)
set_status(jid, status)
mongo.set_report_status(jid, status)
mongo.set_report_status(jid, status, job_result_code)
status
end
def init_provider(provider_name)
::Provider::ProviderFactory.init(DevopsConfig.config)
::Provider::ProviderFactory.get(provider_name) if provider_name
end
def convert_config conf
config = {}
conf.each {|k,v| config[k.is_a?(String) ? k.to_sym : k] = v}