Merge branch 'qa' of /home/git/repositories/cloudtechlab/devops-service into release

This commit is contained in:
Tim Lianov 2015-10-29 20:13:56 +04:00
commit 00f07fa953
35 changed files with 498 additions and 29 deletions

View File

@ -7,9 +7,12 @@ module Devops
class Flavor < RequestHandler
def flavors provider
::Provider::ProviderFactory.get(provider).flavors
flavors_with_account(provider, nil)
end
def flavors_with_account provider, account
::Provider::ProviderFactory.get(provider, account).flavors
end
end
end
end

View File

@ -8,7 +8,11 @@ module Devops
# TODO: vpc support for ec2
def groups provider
::Provider::ProviderFactory.get(provider).groups()#params
groups_with_account(provider, nil)
end
def groups_with_account provider, account
::Provider::ProviderFactory.get(provider, account).groups()#params
end
end
end

View File

@ -7,7 +7,11 @@ module Devops
class Network < RequestHandler
def networks provider
p = ::Provider::ProviderFactory.get provider
networks_with_account provider, nil
end
def networks_with_account provider, account
p = ::Provider::ProviderFactory.get(provider, account)
p.networks_detail
end
end

View File

@ -1,15 +1,39 @@
require "app/api2/parsers/provider"
require "providers/provider_factory"
require_relative "request_handler"
require "db/mongo/models/provider_accounts/static_provider_account"
require "db/mongo/models/provider_accounts/openstack_provider_account"
require "db/mongo/models/provider_accounts/ec2_provider_account"
module Devops
module API2_0
module Handler
class Provider < RequestHandler
set_parser Devops::API2_0::Parser::ProviderParser
def providers
::Provider::ProviderFactory.providers
end
def accounts provider
::Provider::ProviderFactory.get(provider).accounts
end
def add_account provider
account = ::Provider::ProviderFactory.get(provider).create_account(parser.account)
key = Devops::Db.connector.key account.ssh_key
Devops::Db.connector.provider_accounts_insert(account)
::Provider::ProviderFactory.add_account(provider, account)
account.to_hash
end
def delete_account name, provider
account = Devops::Db.connector.provider_accounts_show(name)
Devops::Db.connector.provider_accounts_delete(name)
account.to_hash
end
end
end
end

View File

@ -35,7 +35,11 @@ module Devops
end
def provider_servers provider
::Provider::ProviderFactory.get(provider).servers
provider_servers_with_account provider, nil
end
def provider_servers_with_account provider, account
::Provider::ProviderFactory.get(provider, account).servers
end
def server id

View File

@ -25,7 +25,10 @@ module Devops
raise InvalidRecord.new("Environment '#{env.identifier}' of project '#{project.id}' has no stack template") if env.stack_template.nil?
object["stack_template"] = env.stack_template
object["owner"] = parser.current_user
object["provider"] = env.provider
object["provider_account"] = env.provider_account
# TODO: without provider_name
uri = Worker.start_async(StackBootstrapWorker, @request,
provider_name: env.provider,
stack_attributes: object

View File

@ -0,0 +1,16 @@
require_relative "request_parser"
module Devops
module API2_0
module Parser
class ProviderParser < RequestParser
def account
create_object_from_json_body
end
end
end
end
end

View File

@ -37,6 +37,13 @@ module Devops
json Devops::API2_0::Handler::Flavor.new(request).flavors(provider)
end
# TODO: check account
app.get_with_headers "/flavors/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("flavor", "r")
check_provider(provider)
json Devops::API2_0::Handler::Flavor.new(request).flavors_with_account(provider, account)
end
puts "Flavor routes initialized"
end

View File

@ -50,6 +50,12 @@ module Devops
json Devops::API2_0::Handler::Group.new(request).groups(provider)
end
app.get_with_headers "/groups/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("group", "r")
check_provider(provider)
json Devops::API2_0::Handler::Group.new(request).groups_with_account(provider, account)
end
puts "Group routes initialized"
end

View File

@ -38,6 +38,12 @@ module Devops
json Devops::API2_0::Handler::Network.new(request).networks(provider)
end
app.get_with_headers "/networks/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("network", "r")
check_provider(provider)
json Devops::API2_0::Handler::Network.new(request).networks_with_account(provider, account)
end
puts "Network routes initialized"
end

View File

@ -26,6 +26,82 @@ module Devops
json Devops::API2_0::Handler::Provider.new(request).providers
end
# Get list of provider accounts
#
# * *Request*
# - method : GET
# - headers :
# - Accept: application/json
#
# * *Returns* : array of strings
# - ec2:
# {
# "account_name": "",
# "description": "",
# "access_key_id" : "",
# "ssh_key": "",
# "certificate" : "path to file",
# "availability_zone": ""
#
# }
# - openstack:
# {
# "account_name": "",
# "description": "",
# "username": "",
# "auth_url": "",
# "tenant": "",
# "ssh_key": "",
# "certificate" : "path to file"
# }
# - static:
# {
# "account_name": "",
# "description": "",
# "ssh_key": "",
# "certificate" : "path to file"
# }
app.get_with_headers "/provider/:provider/accounts", :headers => [:accept] do |provider|
check_privileges("provider", "r")
check_provider(provider)
json Devops::API2_0::Handler::Provider.new(request).accounts(provider)
end
# Create provider account for :provider
#
# * *Request*
# - method : POST
# - headers :
# - Accept: application/json
# - Content-Type: application/json
# - body :
# {
#
#
# }
#
# * *Returns* : 201
app.post_with_headers "/provider/:provider/account", :headers => [:accept, :content_type] do |provider|
check_privileges("provider", "w")
check_provider(provider)
create_response("Created", {:account => Devops::API2_0::Handler::Provider.new(request).add_account(provider)}, 201)
end
# Delete account with name :account_name for :provider
#
# * *Request*
# - method : DELETE
# - headers :
# - Accept: application/json
# - Content-Type: application/json
#
# * *Returns* : 200
app.delete_with_headers "/provider/:provider/account/:account_name", :headers => [:accept, :content_type] do |provider, account_name|
check_privileges("provider", "w")
check_provider(provider)
create_response("Deleted", {:account => Devops::API2_0::Handler::Provider.new(request).delete_account(provider)})
end
puts "Provider routes initialized"
end
end

View File

@ -85,6 +85,11 @@ module Devops
json Devops::API2_0::Handler::Server.new(request).provider_servers(provider)
end
app.get_with_headers "/servers/provider/:provider/:account", :headers => [:accept] do |provider, account|
check_privileges("server", "r")
json Devops::API2_0::Handler::Server.new(request).provider_servers_with_account(provider, account)
end
# Get server info by :name
#
# * *Request*

View File

@ -73,7 +73,9 @@ EOH
end
def knife cmd
o = `bundle exec knife #{cmd} -c #{self.config} 2>&1`
cmd = "bundle exec knife #{cmd} -c #{self.config} 2>&1"
DevopsLogger.logger.info("Going to invoke command: #{cmd}")
o = `#{cmd}`
return o, $?.success?
end

View File

@ -17,7 +17,9 @@ module Connectors
def insert(record)
begin
record.validate!
collection.insert(record.to_mongo_hash)
hash = record.to_mongo_hash
hash["created_at"] = Time.now.to_i
collection.insert(hash)
record
rescue Mongo::OperationFailure => e
# exception's message doesn't always start from error code

View File

@ -0,0 +1,30 @@
module Connectors
class ProviderAccounts < Base
include Helpers::InsertCommand,
Helpers::DeleteCommand
def initialize(db)
super(db)
end
def provider_accounts provider
c = Provider::ProviderFactory.get_account_class(provider)
collection.find({provider: provider}).to_a.map{|bson| c.build_from_bson(bson)}
end
def collection_name
'provider_accounts'
end
private
def model_from_bson(bson)
c = Provider::ProviderFactory.get_account_class(bson["provider"])
raise InvalidRecord.new "Invalid provider '#{bson["provider"]}' for account '#{bson["_id"]}'" if c.nil?
c.build_from_bson(bson)
end
end
end

View File

@ -26,6 +26,7 @@ module Devops
:stack_template => {:type => String, :empty => false, :nil => true}
=end
#TODO: account validator
set_validators ::Validators::DeployEnv::Flavor,
::Validators::DeployEnv::Image,
::Validators::DeployEnv::SubnetNotEmpty,

View File

@ -32,10 +32,10 @@ module Devops
def initialize d={}
self.identifier = d["identifier"]
set_provider(d)
b = d["run_list"] || []
self.run_list = b.uniq
self.expires = d["expires"]
self.provider = d["provider"]
b = d["users"] || []
self.users = b.uniq
end
@ -45,9 +45,8 @@ module Devops
"identifier" => self.identifier,
"run_list" => self.run_list,
"expires" => self.expires,
"provider" => self.provider,
"users" => self.users
}
}.merge(provider_hash)
end
def add_users users

View File

@ -4,7 +4,7 @@ module Devops
module Model
module ModelWithProvider
attr_accessor :provider
attr_accessor :provider, :provider_account
def ModelWithProvider.included(mod)
@ -15,8 +15,21 @@ module Devops
end
def provider_instance
@provider_instance ||= Provider::ProviderFactory.get(self.provider)
@provider_instance ||= Provider::ProviderFactory.get(self.provider, self.provider_account)
end
def set_provider hash
self.provider = hash["provider"]
self.provider_account = hash["provider_account"]
end
def provider_hash
{
"provider" => self.provider,
"provider_account" => self.provider_account
}
end
end
end
end

View File

@ -1,4 +1,4 @@
require "providers/provider_factory"
#require "providers/provider_factory"
require "exceptions/invalid_record"
require "json"
require 'db/validators/all'
@ -7,6 +7,8 @@ module Devops
module Model
class MongoModel
attr_accessor :created_at
# multi_json sends argument to 'to_json' method
def to_json arg=nil
JSON.pretty_generate self.to_hash
@ -76,6 +78,7 @@ module Devops
# :empty - can param be empty? (false)
# :nil - can param be nil? (false)
# :value_type - type of array element (String)
# TODO: @deprecated
def self.types types
define_method :validate_fields_types do
t = types.keys

View File

@ -148,7 +148,7 @@ module Devops
}
end
def deploy_info deploy_env, build_number
def deploy_info deploy_env, build_number=nil
{
"use_json_file" => true,
# "run_list" => Set.new.merge(self.run_list).merge(deploy_env.run_list).to_a,

View File

@ -0,0 +1,44 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class Ec2ProviderAccount < ProviderAccount
attr_accessor :access_key_id, :availability_zone, :secret_access_key
def initialize a={}
super(a)
self.provider = Provider::Ec2::PROVIDER
self.availability_zone = a["availability_zone"]
self.access_key_id = a["access_key_id"]
self.secret_access_key = a["secret_access_key"]
end
def to_list_hash
s = super
s["availability_zone"] = self.availability_zone
s["access_key_id"] = self.access_key_id
s
end
def to_mongo_hash
s = super
s["availability_zone"] = self.availability_zone
s["access_key_id"] = self.access_key_id
s["secret_access_key"] = self.secret_access_key
s
end
def self.build_from_bson a
a["account_name"] = a["_id"]
Ec2ProviderAccount.new a
end
# TODO: remove
def validate_fields_types
end
end
end
end

View File

@ -0,0 +1,40 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class OpenstackProviderAccount < ProviderAccount
attr_accessor :username, :auth_url, :tenant, :api_key
def initialize a={}
super(a)
self.username = a["username"]
self.auth_url = a["auth_url"]
self.tenant = a["tenant"]
self.api_key = a["api_key"]
end
def to_list_hash
s = super
s["username"] = self.username
s["auth_url"] = self.auth_url
s["tenant"] = self.tenant
end
def to_mongo_hash
s = super
s["username"] = self.username
s["auth_url"] = self.auth_url
s["tenant"] = self.tenant
s["api_key"] = self.api_key
end
def self.build_from_bson a
a["account_name"] = a["_id"]
OpenstackProviderAccount.new a
end
end
end
end

View File

@ -0,0 +1,48 @@
require "db/mongo/models/mongo_model"
require "db/mongo/models/model_with_provider"
module Devops
module Model
class ProviderAccount < MongoModel
include ModelWithProvider
attr_accessor :account_name, :description, :ssh_key, :certificate
def initialize a={}
self.account_name = a["account_name"]
self.description = a["description"]
self.ssh_key = a["ssh_key"]
self.provider = a["provider"]
self.certificate = a["certificate"]
self.created_at = a["created_at"]
end
def to_list_hash
to_hash
end
def to_hash
{
"account_name" => self.account_name,
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider,
"certificate" => self.certificate,
"created_at" => self.created_at
}
end
def to_mongo_hash
{
"_id" => self.account_name,
"description" => self.description,
"ssh_key" => self.ssh_key,
"provider" => self.provider,
"certificate" => self.certificate
}
end
end
end
end

View File

@ -0,0 +1,15 @@
require "db/mongo/models/provider_accounts/provider_account"
module Devops
module Model
class StaticProviderAccount < ProviderAccount
def self.build_from_bson a
a["account_name"] = a["_id"]
StaticProviderAccount.new a
end
end
end
end

View File

@ -33,7 +33,7 @@ module Devops
end
def initialize s={}
self.provider = s["provider"]
self.set_provider(s)
self.chef_node_name = s["chef_node_name"]
self.id = s["_id"]
self.remote_user = s["remote_user"]
@ -66,7 +66,6 @@ module Devops
def to_hash_without_id
{
"provider" => self.provider,
"chef_node_name" => self.chef_node_name,
"remote_user" => self.remote_user,
"project" => self.project,
@ -79,7 +78,7 @@ module Devops
"reserved_by" => self.reserved_by,
"stack" => stack,
"run_list" => self.run_list
}.delete_if { |k,v| v.nil? }
}.merge(provider_hash).delete_if { |k,v| v.nil? }
end
def self.build_from_bson s
@ -98,7 +97,7 @@ module Devops
end
def static?
(self.provider == Provider::Static::PROVIDER) || false
self.provider == Provider::Static::PROVIDER
end
end

View File

@ -21,7 +21,7 @@ module Devops
def initialize attrs={}
# self.provider = self.class.provider
self.set_provider(attrs)
self.id = attrs['id']
self.project = attrs['project']
self.deploy_env = attrs['deploy_env']
@ -36,7 +36,6 @@ module Devops
def to_hash_without_id
{
provider: provider,
project: project,
deploy_env: deploy_env,
stack_template: stack_template,
@ -46,7 +45,7 @@ module Devops
details: bson_safe_details,
stack_status: stack_status,
owner: owner
}
}.merge(provider_hash)
end
# overrided in ec2
@ -106,6 +105,8 @@ module Devops
# - id (String)
# - deploy_env (String)
# - stack_template (String)
# - provider (String)
# - provider_account (String)
def create(attrs, out)
model = new(attrs)
model.create_stack_in_cloud!(out)

View File

@ -32,7 +32,8 @@ class MongoConnector
:user_update, :create_root_user, :check_user_privileges] => :users_connector,
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports, :set_report_status, :set_report_server_data] => :reports_connector,
[:insert_statistic, :search_statistic] => :statistics_connector
[:insert_statistic, :search_statistic] => :statistics_connector,
[:provider_accounts, :provider_accounts_insert, :provider_accounts_delete, :provider_accounts_show] => :provider_accounts_connector
)
def initialize(db, host, port=27017, user=nil, password=nil)
@ -46,6 +47,10 @@ class MongoConnector
private
def provider_accounts_connector
@provider_accounts_connector ||= Connectors::ProviderAccounts.new(@db)
end
def images_connector
@image_connector ||= Connectors::Image.new(@db)
end

View File

@ -16,7 +16,8 @@ module Devops
deploy_failed: 8
}
MAX_SSH_RETRIES_AMOUNT = 20
# waiting for 5*60 seconds (5 min)
MAX_SSH_RETRIES_AMOUNT = 60
#params:
# out - container for output data
@ -74,11 +75,12 @@ module Devops
end
def create_server options
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider})
provider = @server.provider_instance
@server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider, "provider_account" => @deploy_env.provider_account})
provider = @deploy_env.provider_instance
mongo = ::Devops::Db.connector
begin
@out << "Create server...\n"
@out.puts "Using '#{@deploy_env.provider}' account '#{@deploy_env.provider_account}'\n" if @deploy_env.provider_account
@out.puts "Create server..."
@out.flush
@server.run_list = options["run_list"] || []
@ -137,6 +139,7 @@ module Devops
end
ja = {
:provider => @server.provider,
:provider_account => @server.provider_account,
:devops_host => `hostname`.strip
}
ip = @server.private_ip
@ -210,7 +213,7 @@ module Devops
bootstrap_options.push "--sudo" unless @server.remote_user == "root"
bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
rl = options[:run_list]
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil?# rl.empty?
bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil? or rl.empty?
bootstrap_options.push "-c #{options[:config]}" if options[:config]
bootstrap_options
end
@ -269,6 +272,7 @@ module Devops
end
rescue => e
@out << "\nError: #{e.message}\n"
DevopsLogger.logger.error(e.message + "\n" + e.backtrace.join("\n"))
result_code(:deploy_unknown_error)
end
end

View File

@ -0,0 +1,21 @@
module Provider
class AccountsFactory
def init config
end
# providers instances
def connection account_name
@connections[account_name]
end
def add_connection name, conn
@connections[name] = conn
end
def create_connection_from_account config, account
end
end
end

View File

@ -6,6 +6,14 @@ module Provider
attr_accessor :ssh_key, :certificate_path, :connection_options, :run_list
def accounts
Devops::Db.connector.provider_accounts(self.name)
end
def create_account hash
raise "override me"
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end

View File

@ -1,5 +1,7 @@
require "exceptions/conflict_exception"
require "providers/base_provider"
require "db/mongo/models/provider_accounts/ec2_provider_account"
require_relative "ec2_accounts_factory"
module Provider
# Provider for Amazon EC2
@ -33,6 +35,10 @@ module Provider
super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
end
def create_account hash
Devops::Model::Ec2ProviderAccount.new(hash)
end
def name
PROVIDER
end
@ -88,7 +94,7 @@ module Provider
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s == "running"}.map do |server|
convert_server server["instancesSet"][0]
end
end

View File

@ -0,0 +1,29 @@
require_relative "accounts_factory"
module Provider
class Ec2AccountsFactory < AccountsFactory
def init config
@connections = {}
Devops::Db.connector.provider_accounts(Ec2::PROVIDER).each do |account|
create_connection_from_account(config, account)
puts "\tFound ec2 account '#{account.account_name}'"
end
end
def create_connection_from_account config, account
options = {
aws_certificate: account.certificate,
aws_ssh_key: account.ssh_key,
aws_access_key_id: account.access_key_id,
aws_secret_access_key: account.secret_access_key,
aws_availability_zone: account.availability_zone,
aws_proxy: config[:aws_proxy],
aws_no_proxy: config[:aws_no_proxy],
aws_integration_run_list: config[:aws_integration_run_list]
}
add_connection(account.account_name, Ec2.new(options))
end
end
end

View File

@ -0,0 +1,6 @@
require_relative "accounts_factory"
module Provider
class OpenstackAccountsFactory < AccountsFactory
end
end

View File

@ -4,13 +4,19 @@ module Provider
module ProviderFactory
@@providers = {}
@@providers_with_accounts_factories = {}
def self.providers
@@providers.keys
end
def self.get provider
p = @@providers[provider]
def self.get provider, account=nil
p = if account.nil?
@@providers[provider]
else
@@providers_with_accounts_factories[provider].connection(account)
end
# TODO: new exception
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
@ -30,6 +36,9 @@ module Provider
@@providers[p] = o
puts "Provider '#{p}' has been loaded"
end
factory = Provider.const_get(p.capitalize + "AccountsFactory").new
factory.init(conf)
@@providers_with_accounts_factories[p] = factory
rescue => e
puts "Error while loading provider '#{p}': " + e.message
next
@ -37,15 +46,35 @@ module Provider
end
end
def self.add_account provider, account
factory = @@providers_with_accounts_factories[provider]
factory.create_connection_from_account(DevopsConfig.config, account)
DevopsLogger.logger.info("Added #{provider} account '#{account.account_name}'")
end
def self.require_all
["ec2", "openstack", "static"].each do |provider|
begin
require_relative provider
require_relative provider + "_accounts_factory"
rescue LoadError => e
puts "Can not load provider '#{provider}': " + e.message
end
end
end
def self.get_account_class provider
case(provider)
when ::Provider::Static::PROVIDER
::Devops::Model::StaticProviderAccount
when ::Provider::Ec2::PROVIDER
::Devops::Model::Ec2ProviderAccount
when ::Provider::Openstack::PROVIDER
::Devops::Model::OpenstackProviderAccount
else
nil
end
end
end
end

View File

@ -0,0 +1,6 @@
require_relative "accounts_factory"
module Provider
class StaticAccountsFactory < AccountsFactory
end
end