tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

script, routes, deploy

Browse Source
This commit is contained in:
amartynov 2015-07-23 16:56:51 +03:00
parent 31f1fd5e2b
commit 644fd87ca4
10 changed files with 269 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
devops-service/app/api2/handlers/deploy.rb
View File

@ -3,65 +3,56 @@ require "commands/status"
require "workers/deploy_worker"
module Devops
module Version2_0
module API2_0
module Handler
class Deploy
extend DeployCommands
extend StatusCommands
def self.deploy
lambda {
check_privileges("server", "x")
# TODO: send message
#broadcast(:devops_deploy, "deploy")
r = create_object_from_json_body
names = check_array(r["names"], "Parameter 'names' should be a not empty array of strings")
tags = check_array(r["tags"], "Parameter 'tags' should be an array of strings", String, true) || []
def initialize req, params
@request = req
@params = params
end
servers = settings.mongo.servers(nil, nil, names, true)
halt(404, "No reserved servers found for names '#{names.join("', '")}'") if servers.empty?
keys = {}
servers.sort_by!{|s| names.index(s.chef_node_name)}
if r.key?("trace")
stream() do |out|
status = []
begin
servers.each do |s|
project = begin
settings.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
out << e.message + "\n"
status.push 2
next
end
res = deploy_server_proc.call(out, s, settings.mongo, tags)
status.push(res)
end
out << create_status(status)
rescue IOError => e
logger.error e.message
break
end
end # stream
else
dir = DevopsService.config[:report_dir_v2]
files = []
uri = URI.parse(request.url)
servers.each do |s|
project = begin
settings.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
next
end
jid = DeployWorker.perform_async(dir, s.to_hash, tags, request.env['REMOTE_USER'], DevopsService.config)
logger.info "Job '#{jid}' has been started"
uri.path = "#{DevopsService.config[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
end
sleep 1
json files
def deploy names, tags
dir = DevopsConfig.config[:report_dir_v2]
files = []
uri = URI.parse(@request.url)
servers(names).each do |s|
project = begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
next
end
}
jid = DeployWorker.perform_async(dir, s.to_hash, tags, @request.env['REMOTE_USER'], DevopsConfig.config)
files.push jid
end
files
end
def deploy_stream out, names, tags
status = []
servers(names).each do |s|
project = begin
Devops::Db.connector.check_project_auth s.project, s.deploy_env, @request.env['REMOTE_USER']
rescue InvalidPrivileges, RecordNotFound => e
out << e.message + "\n"
status.push 2
next
end
res = deploy_server_proc.call(out, s, tags)
status.push(res)
end
out << create_status(status)
rescue RecordNotFound => e
out << e.message
end
def servers names
servers = Devops::Db.connector.servers(nil, nil, names, true)
raise RecordNotFound.new("No reserved servers found for names '#{names.join("', '")}'") if servers.empty?
servers.sort_by!{|s| names.index(s.chef_node_name)}
servers
end
end
end

95
devops-service/app/api2/handlers/report.rb
View File

@ -1,61 +1,62 @@
require_relative "request_handler"
module Devops
module Version2_0
module Handler
class Report
class Report < RequestHandler
def self.reports_all
lambda {
options = {}
["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "max_number", "chef_node_name"].each do |k|
options[k] = params[k] unless params[k].nil?
end
attributes_keys = params.keys.select{|k| k =~ /attributes\.*/}
attributes_keys.each do |ak|
options[ak] = params[ak]
end
json Devops::Db.connector.reports(options).map{|r| r.to_hash}
}
def initialize params
@params = params
end
def self.reports_latest
lambda {
options = {}
["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "chef_node_name"].each do |k|
options[k] = params[k] unless params[k].nil?
end
attributes_keys = params.keys.select{|k| k =~ /attributes\.*/}
attributes_keys.each do |ak|
options[ak] = params[ak]
end
json Devops::Db.connector.latest_reports(options).map{|r| r.to_hash}
}
def options
options = {}
["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "chef_node_name", "max_number"].each do |k|
options[k] = @params[k] unless @params[k].nil?
end
attributes_keys = @params.keys.select{|k| k =~ /attributes\.*/}
attributes_keys.each do |ak|
options[ak] = @params[ak]
end
options
end
def self.attributes_all
lambda{
json Devops::Db.connector.reports_attributes_values(params["name"])
}
def all
Devops::Db.connector.reports(options)
end
def self.report
lambda{
begin
r = Devops::Db.connector.report(params[:id])
file = r.file
return [404, "Report '#{params[:id]}' does not exist"] unless File.exists? file
@text = Rack::Utils.escape_html(File.read(file))
@done = completed?(params[:id])
rescue RecordNotFound => e
if task_status(params[:id]) == Worker::STATUS::IN_QUEUE
@text = "Task '#{params[:id]}' has been queued"
@done = false
else
raise e
end
end
erb :index
}
def all_latest
Devops::Db.connector.latest_reports(options())
end
def attributes name
Devops::Db.connector.reports_attributes_values(name)
end
def report id
r = Devops::Db.connector.report(id)
file = r.file
raise RecordNotFound.new("Report '#{id}' does not exist") unless File.exists? file
return Rack::Utils.escape_html(File.read(file)), completed?(id)
rescue RecordNotFound => e
if status(id) == Worker::STATUS::IN_QUEUE
return "Task '#{id}' has been queued", false
else
raise e
end
end
def status id
Sidekiq.redis do |connection|
connection.hget("devops", id)
end
end
def completed? id
r = self.status(id)
r == "completed" or r == "failed"
end
end
end
end

12
devops-service/app/api2/handlers/request_handler.rb Normal file
View File

@ -0,0 +1,12 @@
module Devops
module API2_0
module Handler
class RequestHandler
def initialize request, params
@request = request
@params = params
end
end
end
end
end

159
devops-service/app/api2/handlers/script.rb
View File

@ -1,125 +1,96 @@
require "providers/provider_factory"
require "fileutils"
require "commands/status"
require_relative "request_handler"
module Devops
module Version2_0
module API2_0
module Handler
class Script
class Script < RequestHandler
def self.get_scripts
lambda {
check_privileges("script", "r")
res = []
Dir.foreach(DevopsService.config[:scripts_dir]) {|f| res.push(f) unless f.start_with?(".")}
json res
}
def scripts
res = []
Dir.foreach(DevopsConfig.config[:scripts_dir]) {|f| res.push(f) unless f.start_with?(".")}
end
def self.execute_command
lambda {
check_privileges("script", "x")
user = request.env['REMOTE_USER']
s = ::Devops::Db.connector.server_by_chef_node_name params[:node_name]
::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
cert = ::Devops::Db.connector.key s.key
cmd = request.body.read
addr = "#{s.remote_user}@#{s.public_ip || s.private_ip}"
ssh_cmd = "ssh -i %s #{addr} '#{cmd}'"
stream() do |out|
begin
out << ssh_cmd % File.basename(cert.path)
out << "\n"
IO.popen((ssh_cmd % cert.path) + " 2>&1") do |so|
while line = so.gets do
out << line
end
end
out << "\nDone"
rescue IOError => e
logger.error e.message
end
def execute_command out, cmd, node_name
user = @request.env['REMOTE_USER']
s = ::Devops::Db.connector.server_by_chef_node_name node_name
::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
cert = ::Devops::Db.connector.key s.key
addr = "#{s.remote_user}@#{s.public_ip || s.private_ip}"
ssh_cmd = "ssh -i %s #{addr} '#{cmd}'"
out << ssh_cmd % File.basename(cert.path)
out << "\n"
IO.popen((ssh_cmd % cert.path) + " 2>&1") do |so|
while line = so.gets do
out << line
end
}
end
out << "\nDone"
end
def self.run_script
lambda {
check_privileges("script", "x")
file_name = params[:script_name]
@file = File.join(DevopsService.config[:scripts_dir], check_filename(file_name, "Parameter 'script_name' must be a not empty string", false))
halt(404, "File '#{file_name}' does not exist") unless File.exists?(@file)
body = create_object_from_json_body
nodes = check_array(body["nodes"], "Parameter 'nodes' must be a not empty array of strings")
p = check_array(body["params"], "Parameter 'params' should be a not empty array of strings", String, true)
def run_script out, file_name, nodes, script_params
file = File.join(DevopsConfig.config[:scripts_dir], file_name)
unless File.exists?(file)
out << "File '#{file_name}' does not exist\n"
return
end
servers = ::Devops::Db.connector.servers_by_names(nodes)
return [404, "No servers found for names '#{nodes.join("', '")}'"] if servers.empty?
user = request.env['REMOTE_USER']
if servers.empty?
out << "No servers found for names '#{nodes.join("', '")}'\n"
return
end
user = @request.env['REMOTE_USER']
servers.each do |s|
::Devops::Db.connector.check_project_auth s.project, s.deploy_env, user
end
stream() do |out|
begin
status = []
servers.each do |s|
cert = begin
::Devops::Db.connector.key s.key
rescue
out << "No key found for '#{s.chef_node_name}'"
status.push 2
next
end
ssh_cmd = "ssh -i #{cert.path} #{s.remote_user}@#{s.public_ip || s.private_ip} 'bash -s' < %s"
out << "\nRun script on '#{s.chef_node_name}'\n"
unless p.nil?
ssh_cmd += " " + p.join(" ")
end
out << (ssh_cmd % [params[:script_name]])
out << "\n"
status = []
servers.each do |s|
cert = begin
::Devops::Db.connector.key s.key
rescue
out << "No key found for '#{s.chef_node_name}'"
status.push 2
next
end
ssh_cmd = "ssh -i #{cert.path} #{s.remote_user}@#{s.public_ip || s.private_ip} 'bash -s' < %s"
out << "\nRun script on '#{s.chef_node_name}'\n"
unless script_params.nil?
ssh_cmd += " " + script_params.join(" ")
end
out << (ssh_cmd % [file_name])
out << "\n"
begin
IO.popen( (ssh_cmd % [@file]) + " 2>&1") do |so|
while line = so.gets do
out << line
end
so.close
status.push $?.to_i
end
rescue IOError => e
logger.error e.message
out << e.message
status.push 3
begin
IO.popen( (ssh_cmd % [file]) + " 2>&1") do |so|
while line = so.gets do
out << line
end
so.close
status.push $?.to_i
end
out << create_status(status)
rescue IOError => e
logger.error e.message
out << e.message
status.push 3
end
end
}
out << create_status(status)
end
def self.create_script
lambda {
check_privileges("script", "w")
file_name = params[:script_name]
file = File.join(settings.scripts_dir, check_filename(file_name, "Parameter 'script_name' must be a not empty string"))
halt_response("File '#{file_name}' already exist") if File.exists?(file)
File.open(file, "w") {|f| f.write(request.body.read)}
create_response("File '#{params[:script_name]}' created", nil, 201)
}
def create_script file_name
file = File.join(settings.scripts_dir, file_name)
raise RecordNotFound.new("File '#{file_name}' already exist") if File.exists?(file)
File.open(file, "w") {|f| f.write(@request.body.read)}
end
def self.delete_script
lambda {
check_privileges("script", "w")
file_name = params[:script_name]
file = File.join(settings.scripts_dir, check_filename(file_name, "Parameter 'script_name' must be a not empty string"))
halt_response("File '#{file_name}' does not exist", 404) unless File.exists?(file)
FileUtils.rm(file)
create_response("File '#{params[:script_name]}' deleted")
}
def delete_script file_name
file = File.join(settings.scripts_dir, file_nsme)
raise RecordNotFound.new("File '#{file_name}' does not exist", 404) unless File.exists?(file)
FileUtils.rm(file)
end
end
end
end

22
devops-service/app/api2/handlers/status.rb
View File

@ -1,22 +0,0 @@
require "sidekiq"
module Devops
module Version2_0
module Handler
class Status
def self.get_status
lambda {
r = Sidekiq.redis do |connection|
connection.hget("devops", params[:id])
end
return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
r
}
end
end
end
end
end

33
devops-service/app/api2/routes/deploy.rb
View File

@ -1,6 +1,6 @@
module Devops
module Version2_0
module API2_0
module Routes
module DeployRoutes
@ -20,7 +20,36 @@ module Devops
# }
#
# * *Returns* : text stream
app.post_with_headers "/deploy", :headers => [:content_type], &Devops::Version2_0::Handler::Deploy.deploy
app.post_with_headers "/deploy", :headers => [:content_type] do
check_privileges("server", "x")
# TODO: send message
#broadcast(:devops_deploy, "deploy")
r = create_object_from_json_body
names = check_array(r["names"], "Parameter 'names' should be a not empty array of strings")
tags = check_array(r["tags"], "Parameter 'tags' should be an array of strings", String, true) || []
if r.key?("trace")
stream() do |out|
status = []
begin
Devops::API2_0::Handler::Deploy.new(request, params).deploy_stream(out, names, tags)
rescue IOError => e
logger.error e.message
break
end
end # stream
else
ids = Devops::API2_0::Handler::Deploy.new(request, params).deploy(names, tags)
sleep 1
ids.each do |jid|
logger.info "Job '#{jid}' has been queued"
uri.path = "#{DevopsConfig.config[:url_prefix]}/v2.0/report/" + jid
files.push uri.to_s
end
json files
end
end
puts "Deploy routes initialized"
end

40
devops-service/app/api2/routes/report.rb
View File

@ -1,26 +1,34 @@
module Devops
module Version2_0
module API2_0
module Routes
module ReportRoutes
def self.registered(app)
app.get_with_headers "/report/all", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_all
app.get_with_headers "/report/all/latest", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_latest
app.get_with_headers "/report/all/attributes/:name", headers: [:accept], &Devops::Version2_0::Handler::Report.attributes_all
app.get_with_headers "/report/:id", headers: [:accept], &Devops::Version2_0::Handler::Report.report
puts "Report routes initialized"
end
def completed? id
r = task_status(id)
r == "completed" or r == "failed"
end
def task_status id
r = Sidekiq.redis do |connection|
connection.hget("devops", id)
app.get_with_headers "/report/all", headers: [:accept] do
json Devops::API2_0::Handler::Report.new(request, params).all.map{|r| r.to_hash}
end
app.get_with_headers "/report/all/latest", headers: [:accept] do
json Devops::API2_0::Handler::Report.new(request, params).all_latest.map{|r| r.to_hash}
end
app.get_with_headers "/report/all/attributes/:name", headers: [:accept] do |name|
json Devops::API2_0::Handler::Report.new(request, params).attributes(name)
end
app.get_with_headers "/report/:id", headers: [:accept] do |id|
@text, @done = Devops::API2_0::Handler::Report.new(request, params).report(id)
erb :index
end
app.get "/status/:id" do
r = Devops::API2_0::Handler::Report.new(request, params).status(params[:id])
return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
r
end
puts "Report routes initialized"
end
end

48
devops-service/app/api2/routes/script.rb
View File

@ -16,7 +16,10 @@ module Devops
# [
# "script_1"
# ]
app.get_with_headers "/scripts", :headers => [:accept], &Devops::Version2_0::Handler::Script.get_scripts
app.get_with_headers "/scripts", :headers => [:accept] do
check_privileges("script", "r")
json Devops::API2_0::Handler::Script.new(request, params).scripts
end
# Run command on node :node_name
#
@ -26,7 +29,16 @@ module Devops
# command to run
#
# * *Returns* : text stream
app.post_with_statistic "/script/command/:node_name", &Devops::Version2_0::Handler::Script.execute_command
app.post_with_statistic "/script/command/:node_name" do |node_name|
check_privileges("script", "x")
stream() do |out|
begin
Devops::API2_0::Handler::Script.new(request, params).execute_command(out, request.body.read, node_name)
rescue IOError => e
logger.error e.message
end
end
end
# Run script :script_name on nodes
#
@ -41,7 +53,21 @@ module Devops
# }
#
# * *Returns* : text stream
app.post_with_headers "/script/run/:script_name", :headers => [:content_type], &Devops::Version2_0::Handler::Script.run_script
app.post_with_headers "/script/run/:script_name", :headers => [:content_type] do |script_name|
check_privileges("script", "x")
file_name = check_filename(script_name, "Parameter 'script_name' must be a not empty string", false)
body = create_object_from_json_body
nodes = check_array(body["nodes"], "Parameter 'nodes' must be a not empty array of strings")
p = check_array(body["params"], "Parameter 'params' should be a not empty array of strings", String, true)
stream() do |out|
begin
Devops::API2_0::Handler::Script.new(request, params).run_script out, file_name, nodes, p
rescue IOError => e
logger.error e.message
end
end
end
hash = {}
# Create script :script_name
@ -54,7 +80,13 @@ module Devops
#
# * *Returns* :
# 201 - Created
hash["PUT"] = Devops::Version2_0::Handler::Script.create_script
hash["PUT"] = lambda {
check_privileges("script", "w")
file_name = params[:script_name]
check_filename(file_name, "Parameter 'script_name' must be a not empty string")
Devops::API2_0::Handler::Script.new(request, params).create_script(file_name)
create_response("File '#{file_name}' created", nil, 201)
}
# Delete script :script_name
#
@ -65,7 +97,13 @@ module Devops
#
# * *Returns* :
# 200 - Deleted
hash["DELETE"] = Devops::Version2_0::Handler::Script.delete_script
hash["DELETE"] = lambda {
check_privileges("script", "w")
file_name = params[:script_name]
check_filename(file_name, "Parameter 'script_name' must be a not empty string")
Devops::API2_0::Handler::Script.new(request, params).delete_script file_name
create_response("File '#{file_name}' deleted")
}
app.multi_routes "/script/:script_name", {:headers => [:accept]}, hash
puts "Script routes initialized"

15
devops-service/app/api2/routes/status.rb
View File

@ -1,15 +0,0 @@
module Devops
module Version2_0
module Routes
module StatusRoutes
def self.registered(app)
app.get "/status/:id", &Devops::Version2_0::Handler::Status.get_status
puts "Status routes initialized"
end
end
end
end
end

8
devops-service/app/devops-api2.rb
View File

@ -8,20 +8,19 @@ module Devops
require_relative "api2/handlers/filter"
require_relative "api2/handlers/group"
require_relative "api2/handlers/user"
require_relative "api2/handlers/report"
require_relative "api2/handlers/deploy"
require_relative "api2/handlers/script"
=begin
require "routes/v2.0/handlers/bootstrap_templates"
require "routes/v2.0/handlers/deploy"
require "routes/v2.0/handlers/image"
require "routes/v2.0/handlers/network"
require "routes/v2.0/handlers/key"
require "routes/v2.0/handlers/project"
require "routes/v2.0/handlers/script"
require "routes/v2.0/handlers/status"
require "routes/v2.0/handlers/tag"
require "routes/v2.0/handlers/server"
require "routes/v2.0/handlers/stack"
require "routes/v2.0/handlers/stack_template"
require "routes/v2.0/handlers/report"
require_relative "api2/routes/handlers/stack_template_preset"
=end
@ -57,7 +56,6 @@ module Devops
require_relative "api2/routes/tag"
require_relative "api2/routes/server"
require_relative "api2/routes/script"
require_relative "api2/routes/status"
require_relative "api2/routes/bootstrap_templates"
require_relative "api2/routes/stack"
require_relative "api2/routes/stack_template"