tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'devops_3' of http://git.stu.neva.ru/cloudtechlab/devops-service into devops_3

Browse Source
This commit is contained in:
amartynov 2015-08-07 10:04:12 +03:00
commit 37c84fe7b2
13 changed files with 93 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
devops-service/app/api2/routes/deploy.rb
View File

@ -46,6 +46,21 @@ module Devops
end
app.get "/deploy/data/:project/:env" do |project, env|
p = Devop::Db.connector.project project
data = p.to_chef_data(env)
content_type "application/json"
(JSON.pretty_generate data) << "\n"
end
app.get "/deploy/data/:file" do |file|
dir = DevopsConfig.config[:project_info_dir]
file_path = File.join(dir, file)
return [404, "Data for '#{file}' not found"] unless File.exists?(file_path)
content_type "application/json"
File.read(file_path) + "\n"
end
puts "Deploy routes initialized"
end

2
devops-service/app/api2/routes/report.rb
View File

@ -24,7 +24,7 @@ module Devops
app.get "/status/:id" do |id|
r = Devops::API2_0::Handler::Report.new(request).status(id)
return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
return [404, "Job with id '#{id}' not found"] if r.nil?
r
end

3
devops-service/app/api2/routes/v2.0.rb
View File

@ -4,7 +4,6 @@ require "json"
require "fog"
require "auth/devops_auth"
require "exceptions/invalid_record"
require "exceptions/record_not_found"
require "exceptions/dependency_error"
@ -106,7 +105,6 @@ module Devops
halt_response(e.message, 400)
end
=begin
error ::Excon::Errors::Unauthorized do
e = env["sinatra.error"]
resp = e.response
@ -124,7 +122,6 @@ module Devops
end
halt_response(msg, 500)
end
=end
error Fog::Compute::AWS::Error do
e = env["sinatra.error"]

17
devops-service/app/client/devops-client.rb
View File

@ -11,27 +11,32 @@ class Client < Sinatra::Base
configure :production do
config = DevopsConfig.config
log_file = File.join(config[:log_dir], "devops-client.log")
log_file = File.join(config[:log_dir], "devops-client.access.log")
logger = DevopsLogger.create(log_file, Logger::INFO)
use Rack::CommonLogger, logger
disable :dump_errors
disable :show_exceptions
# set :logging, Logger::INFO
logger.info "Production mode"
end
configure :development do
config = DevopsConfig.config
log_file = File.join(config[:log_dir], "devops-client.log")
logger = DevopsLogger.create(log_file, Logger::DEBUG)
# config = DevopsConfig.config
# log_file = File.join(config[:log_dir], "devops-client.log")
logger = DevopsLogger.create(STDOUT, Logger::DEBUG)
use Rack::CommonLogger, logger
# set :logging, Logger::DEBUG
disable :raise_errors
# disable :dump_errors
set :show_exceptions, :after_handler
logger.info "Development mode"
DevopsLogger.level = Logger::DEBUG
end
=begin
before do
env["rack.errors"] = error_logger
end
=end
# Route to download devops client
get "/devops-client.gem" do
begin

1
devops-service/app/devops-api2.rb
View File

@ -3,6 +3,7 @@ module Devops
class DevopsApi2Application < Application
def prepare
require "auth/devops_auth"
require_relative "api2/routes/v2.0"
require_relative "api2/handlers/provider"
require_relative "api2/handlers/flavor"

2
devops-service/auth/devops_auth.rb
View File

@ -30,6 +30,4 @@ module Sinatra
end
end
register Sinatra::DevopsAuth
end

2
devops-service/commands/deploy.rb
View File

@ -56,7 +56,7 @@ module DeployCommands
f.write json
end
out.flush if out.respond_to?(:flush)
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/chef/client/data/#{file}"
cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
end
ip = if server.public_ip.nil?
server.private_ip

52
devops-service/commands/deploy_env.rb
View File

@ -1,52 +0,0 @@
require "commands/image"
module DeployEnvCommands
include ImageCommands
# All these commands should be removed when all deploy envs are switched to new validation system
def check_expires! val
raise InvalidRecord.new "Parameter 'expires' is invalid" if val.match(/^[0-9]+[smhdw]$/).nil?
end
def check_flavor! p, val
f = p.flavors.detect{|f| f["id"] == val}
raise InvalidRecord.new "Invalid flavor '#{val}'" if f.nil?
end
def check_image! p, val
images = get_images(::Devops::Db.connector, p.name)
raise InvalidRecord.new "Invalid image '#{val}'" unless images.map{|i| i["id"]}.include?(val)
end
def check_subnets_and_groups! p, subnets, groups
networks = p.networks
n = subnets - networks.map{|n| n["name"]}
raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty?
filter = yield(networks)
=begin
if p.name == ::Provider::Ec2::PROVIDER
unless subnets.empty?
subnets = [ subnets[0] ] if subnets.size > 1
filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] }
end
elsif p.name == ::Provider::Openstack::PROVIDER
if subnets.empty?
raise InvalidRecord.new "Subnets array can not be empty"
end
end
=end
g = groups - p.groups(filter).keys
raise InvalidRecord.new "Invalid groups '#{g.join("', '")}'" unless g.empty?
end
def check_users! val
users = ::Devops::Db.connector.users_names(val)
buf = val - users
raise InvalidRecord.new("Invalid users: '#{buf.join("', '")}'") unless buf.empty?
end
end

86
devops-service/commands/server.rb
View File

@ -106,84 +106,6 @@ module ServerCommands
return status
end
def extract_servers provider, project, env, params, user
mongo = ::Devops::Db.connector
flavors = provider.flavors
projects = {}
env_name = env.identifier
project_name = project.id
servers_info = []
if project.multi?
#TODO: fix multi project
images = {}
env.servers.each do |name, server|
images[server["image"]] = mongo.image(server["image"]) unless images.has_key?(server["image"])
flavor = flavors.detect {|f| f["name"] == server["flavor"]}
raise RecordNotFound.new("Flavor with name '#{server["flavor"]}' not found") if flavor.nil?
run_list = []
project_ids = server["subprojects"].map{|sp| sp["project_id"]}
db_subprojects = mongo.projects project_ids
ids = project_ids - db_subprojects.map{|sp| sp.id}
unless ids.empty?
return [400, "Subproject(s) '#{ids.join("', '")}' is/are not exists"]
end
server["subprojects"].each do |sp|
p = db_subprojects.detect{|db_sp| db_sp.id == sp["project_id"]}
run_list += p.deploy_env(sp["project_env"]).run_list
end
o = {
:image => images[server["image"]],
:name => "#{name}_#{Time.now.to_i}",
:flavor => flavor["id"],
:groups => server["groups"],
:run_list => run_list
}
servers_info.push(o)
end
else
i = mongo.image env.image
flavor = flavors.detect {|f| f["id"] == env.flavor}
raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil?
rl = Set.new
rl.merge(project.run_list).merge(env.run_list)
o = {
:image => i,
:name => params["name"],
:flavor => flavor["id"],
:groups => params["groups"] || env.groups,
:run_list => rl,
:subnets => env.subnets,
:key => params["key"]
}
servers_info.push(o)
end
servers = []
servers_info.each do |info|
image = info[:image]
s = Devops::Model::Server.new
s.provider = provider.name
s.project = project_name
s.deploy_env = env_name
s.run_list = params["run_list"] || []
s.remote_user = image.remote_user
s.chef_node_name = info[:name]
s.key = info[:key] || provider.ssh_key
# s.options = {
# :image => image.id,
# :flavor => info[:flavor],
# :name => info[:name],
# :groups => info[:groups],
# :run_list => info[:run_list].merge(s.run_list),
# :bootstrap_template => image.bootstrap_template,
# :subnets => info[:subnets]
# }
s.created_by = user
servers.push s
end
return servers
end
def delete_from_chef_server node_name
{
:chef_node => KnifeCommands.chef_node_delete(node_name),
@ -191,6 +113,14 @@ module ServerCommands
}
end
def self.delete_etc_chef s, cert_path
cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
cmd += " sudo " unless s.remote_user == "root"
cmd += "rm -Rf /etc/chef"
r = `#{cmd}`
raise(r) unless $?.success?
end
def check_server s
KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
end

21
devops-service/commands/ssh.rb
View File

@ -11,4 +11,25 @@ module SshCommands
return res, $?
end
def self.ssh_with_status out, cert_path, user, ip, remote_cmd
cmd = "ssh -t -i #{cert_path} #{user}@#{ip} \"#{(user == "root" ? remote_cmd : "sudo #{remote_cmd}")}\""
out << "\nCommand: '#{cmd}'\n"
out.flush if out.respond_to?(:flush)
status = nil
IO.popen(cmd + " 2>&1") do |c|
buf = ""
while line = c.gets do
out << line
buf = line
end
c.close
status = $?.to_i
r = buf.scan(/exit\scode\s([0-9]{1,3})/)[0]
unless r.nil?
status = r[0].to_i
end
end
status
end
end

20
devops-service/core/devops-logger.rb
View File

@ -3,18 +3,26 @@ require "logger"
class DevopsLogger
def self.create out, level=Logger::INFO, format='%a %d-%m-%Y %H%M '
@_logger = Logger.new out
@_logger.level = level
@_logger.datetime_format = format
@_logger.debug("Logger has been created")
@_logger
logger = Logger.new out
logger.level = level
logger.datetime_format = format
logger.debug("Logger has been created")
logger
end
def self.logger
@_logger
@_logger ||= DevopsLogger.create(File.join(DevopsConfig.config[:log_dir], "devops.errors.log"))
end
def self.logger= logger
@_logger = logger
end
def self.level= level
@level = level
end
def self.level
@level || Logger::INFO
end
end

1
devops-service/db/mongo/models/server.rb
View File

@ -61,6 +61,7 @@ module Devops
self.reserved_by = s["reserved_by"]
self.stack = s["stack"]
self.run_list = s["run_list"] || []
self
end
def create provider, image, flavor, subnets, groups, out

44
devops-service/workers/stack_bootstrap_worker.rb
View File

@ -12,14 +12,12 @@ class StackBootstrapWorker < Worker
# :stack_id
def perform(options)
call(options['config'], options['provider'], options['dir']) do |provider, out, file|
mongo = Devops::Db.connector
mongo = ::Devops::Db.connector
stack = mongo.stack(options['stack_id'])
save_report(mongo, stack, file)
sync_bootstrap_proc.call(out, stack, mongo)
persist_stack_servers_in_mongo!(mongo, stack, provider)
# logger = DevopsLogger.logger
# bootstrap_stack_servers_proc.call(STDOUT, stack, mongo, provider, logger)
persist_stack_servers!(stack, provider)
end
end
@ -37,34 +35,32 @@ class StackBootstrapWorker < Worker
mongo.save_report(report)
end
def persist_stack_servers_in_mongo!(mongo, stack, provider)
def persist_stack_servers!(stack, provider)
mongo = ::Devops::Db.connector
project = mongo.project(stack.project)
deploy_env = project.deploy_env(stack.deploy_env)
stack_servers = []
stack.resources.each do |resource|
logical_name = resource.resource_name
extended_info = stack.resource(logical_name)
body = {
'name' => logical_name,
'key' => extended_info.key_name
extended_info = stack.resource(resource.resource_name)
server_attrs = {
'provider' => provider.name,
'project' => project.id,
'deploy_env' => deploy_env.identifier,
'remote_user' => mongo.image(deploy_env.image).remote_user,
'key' => extended_info.key_name || provider.ssh_key,
'id' => extended_info.id,
'chef_node_name' => extended_info.name,
'private_ip' => extended_info.addresses.values.first.first['addr'],
'created_by' => stack.owner,
'run_list' => stack.run_list || [],
'stack' => stack.id
}
servers = extract_servers(provider, project, deploy_env, body, stack.owner)
servers.each do |server|
# TODO: improve IP assigning logic
server.private_ip = extended_info.addresses.values.first.first['addr']
server.id = extended_info.id
server.chef_node_name = extended_info.name
end
stack_servers += servers
end
stack_servers.each_with_index do |server, i|
server.stack = stack.id
mongo.server_insert server
server = ::Devops::Model::Server.new(server_attrs)
mongo.server_insert(server)
end
end
end
WorkersStorage.add_worker(stack_bootstrap: StackBootstrapWorker)