tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'devops_3_achuchkalov' of bitbucket.org:achuchkalov/devops_backup into devops_3_achuchkalov

Browse Source
This commit is contained in:
Anton Chuchkalov 2015-07-27 14:52:33 +02:00
commit 3ea39866b3
30 changed files with 302 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
devops-client/lib/devops-client/handler/helpers/outputtable.rb
View File

@ -4,17 +4,13 @@ module Outputtable
@list || @show
end
def additional_output_options
{}
end
def outputter
raise 'You should use "output_with" method to define outputter' unless defined?(outputter_class)
@outputter ||= outputter_class.new(data_to_output, options, additional_output_options)
@outputter ||= outputter_class.new(data_to_output, options.merge(current_command: current_command))
end
def output(preferred_format=nil)
outputter.output(preferred_format)
def output(options={})
outputter.output(options)
end

34
devops-client/lib/devops-client/handler/project.rb
View File

@ -48,7 +48,11 @@ class Project < Handler
when "servers"
self.options = @options_parser.servers_options
servers_handler @options_parser.args
output
output(output_type: :servers)
when "stacks"
self.options = @options_parser.stacks_options
stacks_handler @options_parser.args
output(output_type: :stacks)
when "set"
case ARGV[2]
when "run_list"
@ -61,7 +65,7 @@ class Project < Handler
when "show"
self.options = @options_parser.show_options
show_handler @options_parser.args
output
output(output_type: :show)
when "update"
self.options = @options_parser.update_options
update_handler @options_parser.args
@ -80,7 +84,7 @@ class Project < Handler
when "test"
self.options = @options_parser.test_options
test_handler @options_parser.args
output
output(output_type: :test)
else
@options_parser.invalid_command
end
@ -162,6 +166,20 @@ class Project < Handler
@servers = get "/project/#{args[2]}/servers", o
end
def stacks_handler args
project, deploy_env = args[2], args[3]
r = inspect_parameters @options_parser.stacks_params, project, deploy_env
unless r.nil?
@options_parser.invalid_stacks_command
abort(r)
end
options = {}
unless deploy_env.nil?
options[:deploy_env] = deploy_env
end
@list = get "/project/#{args[2]}/stacks", options
end
def user_add_handler args
r = inspect_parameters @options_parser.user_add_params, args[3], args[4]
unless r.nil?
@ -422,14 +440,4 @@ protected
@list || @show || @servers || @test
end
def additional_output_options
output_type = case ARGV[1]
when 'servers', 'test', 'show'
ARGV[1].to_sym
else
:list
end
{output_type: output_type}
end
end

6
devops-client/lib/devops-client/handler/stack.rb
View File

@ -30,7 +30,11 @@ class Stack < Handler
output
when :resources
resources_handler
output('json')
if @args[3]
output(format: 'json', resource: true)
else
output
end
end
end

3
devops-client/lib/devops-client/options/project_options.rb
View File

@ -3,7 +3,7 @@ require "set"
class ProjectOptions < CommonOptions
commands :create, :delete, :deploy, :list, {:multi => [:create]}, :servers, {:set => [:run_list]}, :show, :test, :update, {:user => [:add, :delete]}
commands :create, :delete, :deploy, :list, {:multi => [:create]}, :servers, :stacks, {:set => [:run_list]}, :show, :test, :update, {:user => [:add, :delete]}
def initialize args, def_options
super(args, def_options)
@ -17,6 +17,7 @@ class ProjectOptions < CommonOptions
self.deploy_params = [id, "[#{env}]"]
self.set_run_list_params = [id, env, "[(recipe[mycookbook::myrecipe])|(role[myrole]) ...]"]
self.servers_params = [id, "[#{env}]"]
self.stacks_params = [id, "[#{env}]"]
self.multi_create_params = [id]
self.update_params = [id, "FILE"]
self.user_add_params = [id, "USER_NAME", "[USER_NAME ...]"]

2
devops-client/lib/devops-client/options/stack_preset_options.rb
View File

@ -21,6 +21,8 @@ class StackPresetOptions < CommonOptions
parser.recognize_option_value(:project, 'stack_preset')
parser.recognize_option_value(:deploy_env, 'stack_preset')
parser.recognize_option_value(:stack, 'stack_preset')
parser.recognize_option_value(:project, 'stack_preset')
parser.recognize_option_value(:deploy_env, 'stack_preset')
parser.recognize_option_value(:parameters_file, 'stack_preset')
end
end

17
devops-client/lib/devops-client/output/base.rb
View File

@ -7,20 +7,13 @@ module Output
attr_reader :options
# QUESTION:
# Earlier I use additional parameter "output_type". Now I use
# detecting output_type from data_to_output.class:
# Array means we are outputting list command,
# Hash or something other - we are outputting show command.
# Is this OK?
def initialize(data_to_output, command_line_options={}, additional_options={})
@data, @options, @additional_options = data_to_output, command_line_options, additional_options
def initialize(data_to_output, options={})
@data, @options = data_to_output, options
end
def output(format = nil)
format ||= options[:format]
case format
def output(preferred_options)
@options = @options.merge(preferred_options)
case options[:format]
when CommonOptions::TABLE_FORMAT
table
when CommonOptions::JSON_FORMAT

25
devops-client/lib/devops-client/output/project.rb
View File

@ -9,22 +9,27 @@ module Output
def table
title = nil
with_separator = false
headers, rows = case @additional_options[:output_type]
when :list
title = I18n.t("output.title.project.list")
create_list(@data)
headers, rows = case options[:output_type]
when :show
title = I18n.t("output.title.project.show", :name => @data["name"])
with_separator = true
create_show(@data)
when :server
when :servers
title = ARGV[2]
title += " " + ARGV[3] unless ARGV[3].nil?
title = I18n.t("output.title.project.servers", :title => title)
create_servers(@data)
when :stacks
title = ARGV[2]
title += " " + ARGV[3] unless ARGV[3].nil?
title = I18n.t("output.title.project.stacks", :title => title)
create_stacks(@data)
when :test
title = I18n.t("output.title.project.test", :project => ARGV[2], :env => ARGV[3])
create_test(@data)
else
title = I18n.t("output.title.project.list")
create_list(@data)
end
create_table(headers, rows, title, with_num?, with_separator)
end
@ -101,7 +106,7 @@ module Output
end
def create_servers servers
abort(I18n.t("output.not_found.project.servers")) if servers.empty?
abort(I18n.t("output.not_found.project.servers", name: ARGV[2])) if servers.empty?
rows = []
servers.each do |s|
rows.push [ s["project"], s["deploy_env"], s["chef_node_name"], s["remote_user"], s["provider"], s["id"] ]
@ -136,5 +141,13 @@ module Output
return headers, rows
end
def create_stacks(stacks)
abort(I18n.t("output.not_found.project.stacks", name: ARGV[2])) if stacks.empty?
fields_to_output = %w(id deploy_env stack_template cloud_stack_id stack_status provider)
headers_and_rows(stacks, fields_to_output)
end
end
end

30
devops-client/lib/devops-client/output/stack.rb
View File

@ -6,13 +6,19 @@ module Output
include Concerns::HasProvider
def table
if outputting_list?
title = I18n.t("output.title.stack.list")
headers, rows = create_list
if options[:current_command] == :resources
if options[:resource].nil?
headers, rows = create_servers_list
end
else
puts 'Details are not displayed in table view'
title = I18n.t("output.title.stack.show", id: @data["id"])
headers, rows = create_show
if outputting_list?
title = I18n.t("output.title.stack.list")
headers, rows = create_list
else
puts 'Details are not displayed in table view'
title = I18n.t("output.title.stack.show", id: @data["id"])
headers, rows = create_show
end
end
create_table(headers, rows, title, with_num?)
end
@ -31,14 +37,22 @@ module Output
def create_list
abort(I18n.t("output.not_found.stack.list")) if @data.empty?
fields_to_output = %w(id deploy_env stack_template cloud_stack_id stack_status)
fields_to_output = %w(id project deploy_env stack_template cloud_stack_id stack_status)
fields_to_output << 'provider' unless provider_given?
headers_and_rows(@data, fields_to_output)
end
def create_show
headers_and_rows([@data], %w(id deploy_env stack_template cloud_stack_id stack_status))
headers_and_rows([@data], %w(id project deploy_env stack_template cloud_stack_id stack_status))
end
def create_servers_list
headers = ['Logical id', 'Physical id']
rows = @data.map do |resource|
[resource['resource_name'], resource['physical_resource_id']]
end
[headers, rows]
end
end

11
devops-client/lib/devops-client/output/stack_preset.rb
View File

@ -3,14 +3,15 @@ require "devops-client/output/base"
module Output
class StackPreset < Base
def table
def table
if outputting_list?
title = I18n.t("output.title.stack_preset.list")
headers, rows = create_list
create_table headers, rows, title, with_num?
else
@data["id"] + "\n" + @data["template_preset_body"]
title = I18n.t("output.title.stack.show", id: @data["id"])
headers, rows = create_apply
end
create_table headers, rows, title, with_num?
end
def csv
@ -36,5 +37,9 @@ module Output
headers_and_rows(@data, fields_to_output)
end
def create_apply
headers_and_rows([@data], %w(id deploy_env stack_template cloud_stack_id stack_status))
end
end
end

1
devops-client/locales/en.yml
View File

@ -259,6 +259,7 @@ en:
project:
list: "No project found"
servers: "No servers for project '%{name}' found"
stacks: "No stacks for project '%{name}' found"
provider:
list: "Empty providers list"
script:

9
devops-service/commands/commands_storage.rb Normal file
View File

@ -0,0 +1,9 @@
module CommandsStorage
def self.add_job_lambda(job_with_lambda)
job_lambdas.merge!(job_with_lambda)
end
def self.job_lambdas
@job_lambdas ||= {}
end
end

58
devops-service/commands/stack.rb Normal file
View File

@ -0,0 +1,58 @@
require "commands/server"
require 'commands/commands_storage'
module StackCommands
include ServerCommands
extend self
def sync_stack_till_not_in_progress_proc
lambda do |out, stack, mongo|
# two tries each 2 seconds, then 5 tries each 10 seconds, then 5 tries each minute.
sleep_times = [2]*2 + [10]*5 + [60]*5
begin
out << "Syncing stack '#{stack.id}'...\n"
sleep_times.each do |sleep_time|
sleep sleep_time
stack.sync_details!
if stack.stack_status != 'CREATE_IN_PROGRESS'
mongo.stack_update(stack)
out << "Stack '#{stack.id}' status is now #{stack.stack_status}"
break
end
out << "Next try...\n"
end
rescue StandardError => e
logger.error e.message
return 5
end
end
end
def create_devops_servers_from_stack_resources
lambda do |out, stack, mongo, owner|
project = mongo.project(stack.project)
deploy_env = project.deploy_envs.detect {|env| env.identifier == stack.deploy_env}
provider = ::Provider::ProviderFactory.get(stack.provider)
stack.resources.each do |resource|
logical_name = resource['resource_name']
attrs = stack.resource(logical_name)
body = {
'name' => logical_name,
'key' => attrs['key_name']
}
server = extract_servers(provider, project, deploy_env, body, owner, mongo)
server.private_ip = attrs['addresses']['devops-net-1'].first['addr']
end
end
end
CommandsStorage.add_job_lambda(
sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc,
create_devops_servers_from_stack_resources: create_devops_servers_from_stack_resources
)
end

4
devops-service/db/mongo/connectors/report.rb
View File

@ -38,6 +38,10 @@ module Connectors
list(options)
end
def set_report_status(jid, status)
# TODO: merge from novartis
end
private
def model_from_bson(bson)

5
devops-service/db/mongo/connectors/stack.rb
View File

@ -10,9 +10,8 @@ module Connectors
self.collection = db.collection('stacks')
end
def stacks(provider=nil)
query = (provider.nil? ? {} : {'provider' => provider})
list(query)
def stacks(options={})
list(options)
end
private

1
devops-service/db/mongo/models/mongo_model.rb
View File

@ -1,6 +1,7 @@
require "providers/provider_factory"
require "db/exceptions/invalid_record"
require "json"
require 'db/validators/all'
module Devops
module Model

1
devops-service/db/mongo/models/report.rb
View File

@ -9,6 +9,7 @@ module Devops
SERVER_TYPE = 2
BOOTSTRAP_TYPE = 3
PROJECT_TEST_TYPE = 4
STACK_TYPE = 5
attr_accessor :id, :file, :created_at, :created_by, :project, :deploy_env, :type

14
devops-service/db/mongo/models/stack/stack_base.rb
View File

@ -59,6 +59,7 @@ module Devops
provider_class.stack_resources(self)
end
# resource_id is logical
def resource(resource_id)
provider_class.stack_resource(self, resource_id)
end
@ -67,6 +68,18 @@ module Devops
Devops::Api2.settings.mongo.stack_template(stack_template).template_body
end
def stack_status
raise 'override me'
end
def stack_statuses
# maybe they differ in different providers, so use method instead of hardcoding
{
in_progress: 'CREATE_IN_PROGRESS',
complete: 'CREATE_COMPLETE'
}
end
class << self
attr_accessor :provider
@ -77,6 +90,7 @@ module Devops
def create(attrs)
model = new(attrs)
model.create_stack_in_cloud!
model.sync_details!
model
end

2
devops-service/db/mongo/models/stack/stack_openstack.rb
View File

@ -4,7 +4,7 @@ module Devops
self.provider = 'openstack'
def stack_status
details[:stack_status] if details
details[:stack_status] || details['stack_status'] if details
end
end
end

2
devops-service/db/mongo/mongo_connector.rb
View File

@ -26,7 +26,7 @@ class MongoConnector
[:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
:user_update, :create_root_user, :check_user_privileges] => :users_connector,
[:keys, :key, :key_insert, :key_delete] => :keys_connector,
[:save_report, :report, :reports] => :reports_connector,
[:save_report, :report, :reports, :set_report_status] => :reports_connector,
[:statistic] => :statistics_connector
)

5
devops-service/providers/openstack.rb
View File

@ -184,7 +184,7 @@ module Provider
end
def compute
connection_compute(self.connection_options)
@compute ||= connection_compute(connection_options)
end
def network
@ -218,7 +218,8 @@ module Provider
end
def stack_resource(stack, resource_id)
fog_stack(stack).resources.get(resource_id)
physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id
compute.servers.get(physical_id).attributes
end
private

10
devops-service/routes/v2.0/handlers/project.rb
View File

@ -42,6 +42,16 @@ module Devops
}
end
def self.get_project_stacks
lambda {
check_privileges("project", "r")
settings.mongo.project(params[:project])
options = {project: params[:project]}
options[:deploy_env] = params[:deploy_env] if params[:deploy_env]
json settings.mongo.stacks(options).map{|s| s.to_hash}
}
end
# TODO: multi project
def self.create_project
lambda {

2
devops-service/routes/v2.0/handlers/stack.rb
View File

@ -17,7 +17,7 @@ module Devops
lambda {
check_privileges("stack", "r")
check_provider(params[:provider])
stacks = settings.mongo.stacks(params[:provider])
stacks = settings.mongo.stacks(provider: params[:provider])
json stacks.map(&:to_hash)
}
end

10
devops-service/routes/v2.0/handlers/stack_preset.rb
View File

@ -1,5 +1,7 @@
require 'json'
require 'lib/stack_presets/factory'
require 'workers/stack_sync_worker'
require 'workers/job_starter'
module Devops
module Version2_0
@ -30,6 +32,14 @@ module Devops
stack = preset.create_stack_from_preset(attrs)
settings.mongo.stack_insert(stack)
file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress,
provider: stack.provider,
stack_id: stack.id,
request: request
)
puts "Syncing report is located here: #{file}"
create_response 'Created', stack.to_hash, 201
}
end

27
devops-service/routes/v2.0/project.rb
View File

@ -137,6 +137,33 @@ module Devops
# ]
app.get_with_headers "/project/:project/servers", :headers => [:accept], &Devops::Version2_0::Handler::Project.get_project_servers
# Get project stacks
#
# * *Request*
# - method : GET
# - headers :
# - Accept: application/json
# - parameters :
# - deploy_env=:env -> show stacks with environment :env
#
# * *Returns* :
# [
# "provider": "openstack",
# "project": "test_openstack",
# "deploy_env": "test",
# "stack_template": "openstack_template",
# "cloud_stack_id": "4c712026-dcd5-4664-90b8-0915494c1332",
# "parameters": {
# "admin_pass": "Pass12",
# "key_name": "devops",
# "image": "CirrOS_0.3.1"
# },
# "details": null,
# "stack_status": null,
# "id": "openstack_stack"
# ]
app.get_with_headers "/project/:project/stacks", :headers => [:accept], &Devops::Version2_0::Handler::Project.get_project_stacks
# Create project and chef roles
#
# * *Request*

2
devops-service/sinatra/methods_with_headers.rb
View File

@ -101,7 +101,7 @@ module Sinatra
# Can client works with JSON?
def accept_json
types = request.accept_media_types
unless types.include?('application/json')# or types.include?("*/*")
unless types.include?('application/json') or types.include?("*/*")
response.headers['Accept'] = 'application/json'
halt_response("Accept header should contains 'application/json' type", 406)
end

28
devops-service/workers/job_starter.rb Normal file
View File

@ -0,0 +1,28 @@
require 'commands/commands_storage'
require 'workers/workers_storage'
module JobStarter
def self.start_job(strategy, job_name, job_options)
case strategy
when :worker
start_job_as_worker(WorkersStorage.workers[job_name], job_options)
end
end
def self.start_job_as_worker(worker_class, options)
job_options = options.dup
job_options[:owner] ||= options[:request].env['REMOTE_USER']
job_options[:config] ||= DevopsConfig.config
job_options[:dir] ||= DevopsConfig[:report_dir_v2]
job_options[:url] ||= options[:request].url
jid = worker_class.perform_async(job_options)
Worker.set_status jid, Worker::STATUS::IN_QUEUE
DevopsLogger.logger.info "Job '#{jid}' has been started"
uri = URI.parse(job_options[:url])
uri.path = "#{job_options[:config][:url_prefix]}/v2.0/report/#{jid}"
uri.to_s
end
end

1
devops-service/workers/run_workers.rb
View File

@ -4,6 +4,7 @@ require File.join(root, "create_server_worker")
require File.join(root, "deploy_worker")
require File.join(root, "bootstrap_worker")
require File.join(root, "project_test_worker")
require File.join(root, "stack_sync_worker")
config = {}
#require File.join(root, "../proxy")

32
devops-service/workers/stack_sync_worker.rb Normal file
View File

@ -0,0 +1,32 @@
require "providers/provider_factory"
require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/report"
require 'workers/workers_storage'
class StackSyncWorker < Worker
include StackCommands
# besides options came from JobStarter we need:
# :provider
# :stack_id
def perform(options)
call(options['config'], options['provider'], options['dir']) do |mongo, provider, out, file|
stack = mongo.stack(options['stack_id'])
o = {
"file" => file,
"_id" => jid,
"created_by" => options['owner'],
"project" => stack.project,
"deploy_env" => stack.deploy_env,
"type" => ::Devops::Model::Report::STACK_TYPE
}
mongo.save_report(::Devops::Model::Report.new(o))
status = sync_stack_till_not_in_progress_proc.call(out, stack, mongo)
status
end
end
end
WorkersStorage.add_worker(sync_stack_till_not_in_progress: StackSyncWorker)

9
devops-service/workers/workers_storage.rb Normal file
View File

@ -0,0 +1,9 @@
module WorkersStorage
def self.add_worker(job_with_worker)
workers.merge!(job_with_worker)
end
def self.workers
@job_workers ||= {}
end
end