tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

script, routes, deploy

Browse Source
This commit is contained in:
amartynov 2015-07-23 16:56:51 +03:00
parent 31f1fd5e2b
commit 644fd87ca4
10 changed files with 269 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
devops-service/app/api2/handlers/deploy.rb
View File

@ -3,65 +3,56 @@ require "commands/status"
require "workers/deploy_worker" require "workers/deploy_worker"
module Devops module Devops
module Version2_0 module API2_0
module Handler module Handler
class Deploy class Deploy
extend DeployCommands extend DeployCommands
extend StatusCommands extend StatusCommands
def self.deploy def initialize req, params
lambda { @request = req
check_privileges("server", "x") @params = params
# TODO: send message end
#broadcast(:devops_deploy, "deploy")
r = create_object_from_json_body
names = check_array(r["names"], "Parameter 'names' should be a not empty array of strings")
tags = check_array(r["tags"], "Parameter 'tags' should be an array of strings", String, true) || []
servers = settings.mongo.servers(nil, nil, names, true) def deploy names, tags
halt(404, "No reserved servers found for names '#{names.join("', '")}'") if servers.empty? dir = DevopsConfig.config[:report_dir_v2]
keys = {} files = []
servers.sort_by!{|s| names.index(s.chef_node_name)} uri = URI.parse(@request.url)
if r.key?("trace") servers(names).each do |s|
stream() do |out| project = begin
status = [] Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
begin rescue InvalidPrivileges, RecordNotFound => e
servers.each do |s| next
project = begin
settings.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
out << e.message + "\n"
status.push 2
next
end
res = deploy_server_proc.call(out, s, settings.mongo, tags)
status.push(res)
end
out << create_status(status)
rescue IOError => e
logger.error e.message
break
end
end # stream
else
dir = DevopsService.config[:report_dir_v2]
files = []
uri = URI.parse(request.url)
servers.each do |s|
project = begin
settings.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
next
end
jid = DeployWorker.perform_async(dir, s.to_hash, tags, request.env['REMOTE_USER'], DevopsService.config)
logger.info "Job '#{jid}' has been started"
uri.path = "#{DevopsService.config[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
end
sleep 1
json files
end end
} jid = DeployWorker.perform_async(dir, s.to_hash, tags, @request.env['REMOTE_USER'], DevopsConfig.config)
files.push jid
end
files
end
def deploy_stream out, names, tags
status = []
servers(names).each do |s|
project = begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
out << e.message + "\n"
status.push 2
next
end
res = deploy_server_proc.call(out, s, tags)
status.push(res)
end
out << create_status(status)
rescue RecordNotFound => e
out << e.message
end
def servers names
servers = Devops::Db.connector.servers(nil, nil, names, true)
raise RecordNotFound.new("No reserved servers found for names '#{names.join("', '")}'") if servers.empty?
servers.sort_by!{|s| names.index(s.chef_node_name)}
servers
end end
end end
end end

95
devops-service/app/api2/handlers/report.rb
View File

@ -1,61 +1,62 @@
require_relative "request_handler"
module Devops module Devops
module Version2_0 module Version2_0
module Handler module Handler
class Report class Report < RequestHandler
def self.reports_all def initialize params
lambda { @params = params
options = {}
["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "max_number", "chef_node_name"].each do |k|
options[k] = params[k] unless params[k].nil?
end
attributes_keys = params.keys.select{|k| k =~ /attributes\.*/}
attributes_keys.each do |ak|
options[ak] = params[ak]
end
json Devops::Db.connector.reports(options).map{|r| r.to_hash}
}
end end
def self.reports_latest def options
lambda { options = {}
options = {} ["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "chef_node_name", "max_number"].each do |k|
["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "chef_node_name"].each do |k| options[k] = @params[k] unless @params[k].nil?
options[k] = params[k] unless params[k].nil? end
end attributes_keys = @params.keys.select{|k| k =~ /attributes\.*/}
attributes_keys = params.keys.select{|k| k =~ /attributes\.*/} attributes_keys.each do |ak|
attributes_keys.each do |ak| options[ak] = @params[ak]
options[ak] = params[ak] end
end options
json Devops::Db.connector.latest_reports(options).map{|r| r.to_hash}
}
end end
def self.attributes_all def all
lambda{ Devops::Db.connector.reports(options)
json Devops::Db.connector.reports_attributes_values(params["name"])
}
end end
def self.report def all_latest
lambda{ Devops::Db.connector.latest_reports(options())
begin
r = Devops::Db.connector.report(params[:id])
file = r.file
return [404, "Report '#{params[:id]}' does not exist"] unless File.exists? file
@text = Rack::Utils.escape_html(File.read(file))
@done = completed?(params[:id])
rescue RecordNotFound => e
if task_status(params[:id]) == Worker::STATUS::IN_QUEUE
@text = "Task '#{params[:id]}' has been queued"
@done = false
else
raise e
end
end
erb :index
}
end end
def attributes name
Devops::Db.connector.reports_attributes_values(name)
end
def report id
r = Devops::Db.connector.report(id)
file = r.file
raise RecordNotFound.new("Report '#{id}' does not exist") unless File.exists? file
return Rack::Utils.escape_html(File.read(file)), completed?(id)
rescue RecordNotFound => e
if status(id) == Worker::STATUS::IN_QUEUE
return "Task '#{id}' has been queued", false
else
raise e
end
end
def status id
Sidekiq.redis do |connection|
connection.hget("devops", id)
end
end
def completed? id
r = self.status(id)
r == "completed" or r == "failed"
end
end end
end end
end end

12
devops-service/app/api2/handlers/request_handler.rb Normal file
View File

@ -0,0 +1,12 @@
module Devops
module API2_0
module Handler
class RequestHandler
def initialize request, params
@request = request
@params = params
end
end
end
end
end

159
devops-service/app/api2/handlers/script.rb
View File

@ -1,125 +1,96 @@
require "providers/provider_factory" require "providers/provider_factory"
require "fileutils" require "fileutils"
require "commands/status" require "commands/status"
require_relative "request_handler"
module Devops module Devops
module Version2_0 module API2_0
module Handler module Handler
class Script class Script < RequestHandler
def self.get_scripts def scripts
lambda { res = []
check_privileges("script", "r") Dir.foreach(DevopsConfig.config[:scripts_dir]) {|f| res.push(f) unless f.start_with?(".")}
res = []
Dir.foreach(DevopsService.config[:scripts_dir]) {|f| res.push(f) unless f.start_with?(".")}
json res
}
end end
def self.execute_command def execute_command out, cmd, node_name
lambda { user = @request.env['REMOTE_USER']
check_privileges("script", "x") s = ::Devops::Db.connector.server_by_chef_node_name node_name
user = request.env['REMOTE_USER'] ::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
s = ::Devops::Db.connector.server_by_chef_node_name params[:node_name] cert = ::Devops::Db.connector.key s.key
::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user addr = "#{s.remote_user}@#{s.public_ip || s.private_ip}"
cert = ::Devops::Db.connector.key s.key ssh_cmd = "ssh -i %s #{addr} '#{cmd}'"
cmd = request.body.read out << ssh_cmd % File.basename(cert.path)
addr = "#{s.remote_user}@#{s.public_ip || s.private_ip}" out << "\n"
ssh_cmd = "ssh -i %s #{addr} '#{cmd}'" IO.popen((ssh_cmd % cert.path) + " 2>&1") do |so|
stream() do |out| while line = so.gets do
begin out << line
out << ssh_cmd % File.basename(cert.path)
out << "\n"
IO.popen((ssh_cmd % cert.path) + " 2>&1") do |so|
while line = so.gets do
out << line
end
end
out << "\nDone"
rescue IOError => e
logger.error e.message
end
end end
} end
out << "\nDone"
end end
def self.run_script def run_script out, file_name, nodes, script_params
lambda { file = File.join(DevopsConfig.config[:scripts_dir], file_name)
check_privileges("script", "x") unless File.exists?(file)
file_name = params[:script_name] out << "File '#{file_name}' does not exist\n"
@file = File.join(DevopsService.config[:scripts_dir], check_filename(file_name, "Parameter 'script_name' must be a not empty string", false)) return
halt(404, "File '#{file_name}' does not exist") unless File.exists?(@file) end
body = create_object_from_json_body
nodes = check_array(body["nodes"], "Parameter 'nodes' must be a not empty array of strings")
p = check_array(body["params"], "Parameter 'params' should be a not empty array of strings", String, true)
servers = ::Devops::Db.connector.servers_by_names(nodes) servers = ::Devops::Db.connector.servers_by_names(nodes)
return [404, "No servers found for names '#{nodes.join("', '")}'"] if servers.empty? if servers.empty?
user = request.env['REMOTE_USER'] out << "No servers found for names '#{nodes.join("', '")}'\n"
return
end
user = @request.env['REMOTE_USER']
servers.each do |s| servers.each do |s|
::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user ::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
end end
stream() do |out| status = []
begin servers.each do |s|
status = [] cert = begin
servers.each do |s| ::Devops::Db.connector.key s.key
cert = begin rescue
::Devops::Db.connector.key s.key out << "No key found for '#{s.chef_node_name}'"
rescue status.push 2
out << "No key found for '#{s.chef_node_name}'" next
status.push 2 end
next ssh_cmd = "ssh -i #{cert.path} #{s.remote_user}@#{s.public_ip || s.private_ip} 'bash -s' < %s"
end out << "\nRun script on '#{s.chef_node_name}'\n"
ssh_cmd = "ssh -i #{cert.path} #{s.remote_user}@#{s.public_ip || s.private_ip} 'bash -s' < %s" unless script_params.nil?
out << "\nRun script on '#{s.chef_node_name}'\n" ssh_cmd += " " + script_params.join(" ")
unless p.nil? end
ssh_cmd += " " + p.join(" ") out << (ssh_cmd % [file_name])
end out << "\n"
out << (ssh_cmd % [params[:script_name]])
out << "\n"
begin begin
IO.popen( (ssh_cmd % [@file]) + " 2>&1") do |so| IO.popen( (ssh_cmd % [file]) + " 2>&1") do |so|
while line = so.gets do while line = so.gets do
out << line out << line
end
so.close
status.push $?.to_i
end
rescue IOError => e
logger.error e.message
out << e.message
status.push 3
end end
so.close
status.push $?.to_i
end end
out << create_status(status)
rescue IOError => e rescue IOError => e
logger.error e.message logger.error e.message
out << e.message
status.push 3
end end
end end
} out << create_status(status)
end end
def self.create_script def create_script file_name
lambda { file = File.join(settings.scripts_dir, file_name)
check_privileges("script", "w") raise RecordNotFound.new("File '#{file_name}' already exist") if File.exists?(file)
file_name = params[:script_name] File.open(file, "w") {|f| f.write(@request.body.read)}
file = File.join(settings.scripts_dir, check_filename(file_name, "Parameter 'script_name' must be a not empty string"))
halt_response("File '#{file_name}' already exist") if File.exists?(file)
File.open(file, "w") {|f| f.write(request.body.read)}
create_response("File '#{params[:script_name]}' created", nil, 201)
}
end end
def self.delete_script def delete_script file_name
lambda { file = File.join(settings.scripts_dir, file_nsme)
check_privileges("script", "w") raise RecordNotFound.new("File '#{file_name}' does not exist", 404) unless File.exists?(file)
file_name = params[:script_name] FileUtils.rm(file)
file = File.join(settings.scripts_dir, check_filename(file_name, "Parameter 'script_name' must be a not empty string"))
halt_response("File '#{file_name}' does not exist", 404) unless File.exists?(file)
FileUtils.rm(file)
create_response("File '#{params[:script_name]}' deleted")
}
end end
end end
end end
end end

22
devops-service/app/api2/handlers/status.rb
View File

@ -1,22 +0,0 @@
require "sidekiq"
module Devops
module Version2_0
module Handler
class Status
def self.get_status
lambda {
r = Sidekiq.redis do |connection|
connection.hget("devops", params[:id])
end
return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
r
}
end
end
end
end
end

33
devops-service/app/api2/routes/deploy.rb
View File

@ -1,6 +1,6 @@
module Devops module Devops
module Version2_0 module API2_0
module Routes module Routes
module DeployRoutes module DeployRoutes
@ -20,7 +20,36 @@ module Devops
# } # }
# #
# * *Returns* : text stream # * *Returns* : text stream
app.post_with_headers "/deploy", :headers => [:content_type], &Devops::Version2_0::Handler::Deploy.deploy app.post_with_headers "/deploy", :headers => [:content_type] do
check_privileges("server", "x")
# TODO: send message
#broadcast(:devops_deploy, "deploy")
r = create_object_from_json_body
names = check_array(r["names"], "Parameter 'names' should be a not empty array of strings")
tags = check_array(r["tags"], "Parameter 'tags' should be an array of strings", String, true) || []
if r.key?("trace")
stream() do |out|
status = []
begin
Devops::API2_0::Handler::Deploy.new(request, params).deploy_stream(out, names, tags)
rescue IOError => e
logger.error e.message
break
end
end # stream
else
ids = Devops::API2_0::Handler::Deploy.new(request, params).deploy(names, tags)
sleep 1
ids.each do |jid|
logger.info "Job '#{jid}' has been queued"
uri.path = "#{DevopsConfig.config[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
end
json files
end
end
puts "Deploy routes initialized" puts "Deploy routes initialized"
end end

40
devops-service/app/api2/routes/report.rb
View File

@ -1,26 +1,34 @@
module Devops module Devops
module Version2_0 module API2_0
module Routes module Routes
module ReportRoutes module ReportRoutes
def self.registered(app) def self.registered(app)
app.get_with_headers "/report/all", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_all app.get_with_headers "/report/all", headers: [:accept] do
app.get_with_headers "/report/all/latest", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_latest json Devops::API2_0::Handler::Report.new(request, params).all.map{|r| r.to_hash}
app.get_with_headers "/report/all/attributes/:name", headers: [:accept], &Devops::Version2_0::Handler::Report.attributes_all
app.get_with_headers "/report/:id", headers: [:accept], &Devops::Version2_0::Handler::Report.report
puts "Report routes initialized"
end
def completed? id
r = task_status(id)
r == "completed" or r == "failed"
end
def task_status id
r = Sidekiq.redis do |connection|
connection.hget("devops", id)
end end
app.get_with_headers "/report/all/latest", headers: [:accept] do
json Devops::API2_0::Handler::Report.new(request, params).all_latest.map{|r| r.to_hash}
end
app.get_with_headers "/report/all/attributes/:name", headers: [:accept] do |name|
json Devops::API2_0::Handler::Report.new(request, params).attributes(name)
end
app.get_with_headers "/report/:id", headers: [:accept] do |id|
@text, @done = Devops::API2_0::Handler::Report.new(request, params).report(id)
erb :index
end
app.get "/status/:id" do
r = Devops::API2_0::Handler::Report.new(request, params).status(params[:id])
return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
r
end
puts "Report routes initialized"
end end
end end

48
devops-service/app/api2/routes/script.rb
View File

@ -16,7 +16,10 @@ module Devops
# [ # [
# "script_1" # "script_1"
# ] # ]
app.get_with_headers "/scripts", :headers => [:accept], &Devops::Version2_0::Handler::Script.get_scripts app.get_with_headers "/scripts", :headers => [:accept] do
check_privileges("script", "r")
json Devops::API2_0::Handler::Script.new(request, params).scripts
end
# Run command on node :node_name # Run command on node :node_name
# #
@ -26,7 +29,16 @@ module Devops
# command to run # command to run
# #
# * *Returns* : text stream # * *Returns* : text stream
app.post_with_statistic "/script/command/:node_name", &Devops::Version2_0::Handler::Script.execute_command app.post_with_statistic "/script/command/:node_name" do |node_name|
check_privileges("script", "x")
stream() do |out|
begin
Devops::API2_0::Handler::Script.new(request, params).execute_command(out, request.body.read, node_name)
rescue IOError => e
logger.error e.message
end
end
end
# Run script :script_name on nodes # Run script :script_name on nodes
# #
@ -41,7 +53,21 @@ module Devops
# } # }
# #
# * *Returns* : text stream # * *Returns* : text stream
app.post_with_headers "/script/run/:script_name", :headers => [:content_type], &Devops::Version2_0::Handler::Script.run_script app.post_with_headers "/script/run/:script_name", :headers => [:content_type] do |script_name|
check_privileges("script", "x")
file_name = check_filename(script_name, "Parameter 'script_name' must be a not empty string", false)
body = create_object_from_json_body
nodes = check_array(body["nodes"], "Parameter 'nodes' must be a not empty array of strings")
p = check_array(body["params"], "Parameter 'params' should be a not empty array of strings", String, true)
stream() do |out|
begin
Devops::API2_0::Handler::Script.new(request, params).run_script out, file_name, nodes, p
rescue IOError => e
logger.error e.message
end
end
end
hash = {} hash = {}
# Create script :script_name # Create script :script_name
@ -54,7 +80,13 @@ module Devops
# #
# * *Returns* : # * *Returns* :
# 201 - Created # 201 - Created
hash["PUT"] = Devops::Version2_0::Handler::Script.create_script hash["PUT"] = lambda {
check_privileges("script", "w")
file_name = params[:script_name]
check_filename(file_name, "Parameter 'script_name' must be a not empty string")
Devops::API2_0::Handler::Script.new(request, params).create_script(file_name)
create_response("File '#{file_name}' created", nil, 201)
}
# Delete script :script_name # Delete script :script_name
# #
@ -65,7 +97,13 @@ module Devops
# #
# * *Returns* : # * *Returns* :
# 200 - Deleted # 200 - Deleted
hash["DELETE"] = Devops::Version2_0::Handler::Script.delete_script hash["DELETE"] = lambda {
check_privileges("script", "w")
file_name = params[:script_name]
check_filename(file_name, "Parameter 'script_name' must be a not empty string")
Devops::API2_0::Handler::Script.new(request, params).delete_script file_name
create_response("File '#{file_name}' deleted")
}
app.multi_routes "/script/:script_name", {:headers => [:accept]}, hash app.multi_routes "/script/:script_name", {:headers => [:accept]}, hash
puts "Script routes initialized" puts "Script routes initialized"

15
devops-service/app/api2/routes/status.rb
View File

@ -1,15 +0,0 @@
module Devops
module Version2_0
module Routes
module StatusRoutes
def self.registered(app)
app.get "/status/:id", &Devops::Version2_0::Handler::Status.get_status
puts "Status routes initialized"
end
end
end
end
end

8
devops-service/app/devops-api2.rb
View File

@ -8,20 +8,19 @@ module Devops
require_relative "api2/handlers/filter" require_relative "api2/handlers/filter"
require_relative "api2/handlers/group" require_relative "api2/handlers/group"
require_relative "api2/handlers/user" require_relative "api2/handlers/user"
require_relative "api2/handlers/report"
require_relative "api2/handlers/deploy"
require_relative "api2/handlers/script"
=begin =begin
require "routes/v2.0/handlers/bootstrap_templates" require "routes/v2.0/handlers/bootstrap_templates"
require "routes/v2.0/handlers/deploy"
require "routes/v2.0/handlers/image" require "routes/v2.0/handlers/image"
require "routes/v2.0/handlers/network" require "routes/v2.0/handlers/network"
require "routes/v2.0/handlers/key" require "routes/v2.0/handlers/key"
require "routes/v2.0/handlers/project" require "routes/v2.0/handlers/project"
require "routes/v2.0/handlers/script"
require "routes/v2.0/handlers/status"
require "routes/v2.0/handlers/tag" require "routes/v2.0/handlers/tag"
require "routes/v2.0/handlers/server" require "routes/v2.0/handlers/server"
require "routes/v2.0/handlers/stack" require "routes/v2.0/handlers/stack"
require "routes/v2.0/handlers/stack_template" require "routes/v2.0/handlers/stack_template"
require "routes/v2.0/handlers/report"
require_relative "api2/routes/handlers/stack_template_preset" require_relative "api2/routes/handlers/stack_template_preset"
=end =end
@ -57,7 +56,6 @@ module Devops
require_relative "api2/routes/tag" require_relative "api2/routes/tag"
require_relative "api2/routes/server" require_relative "api2/routes/server"
require_relative "api2/routes/script" require_relative "api2/routes/script"
require_relative "api2/routes/status"
require_relative "api2/routes/bootstrap_templates" require_relative "api2/routes/bootstrap_templates"
require_relative "api2/routes/stack" require_relative "api2/routes/stack"
require_relative "api2/routes/stack_template" require_relative "api2/routes/stack_template"