From 2e0229cfda8d553b297c15c2eb5aad5c67c10f72 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 29 Jul 2015 19:37:33 +0200 Subject: [PATCH 1/4] fix some bugs --- .../lib/devops-client/output/stack.rb | 1 + .../app/api2/handlers/stack_preset.rb | 36 ++++++++++++++++++- devops-service/app/api2/routes/stack.rb | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/devops-client/lib/devops-client/output/stack.rb b/devops-client/lib/devops-client/output/stack.rb index 2051fae..e3ad175 100644 --- a/devops-client/lib/devops-client/output/stack.rb +++ b/devops-client/lib/devops-client/output/stack.rb @@ -49,6 +49,7 @@ module Output def create_servers_list headers = ['Logical id', 'Physical id'] + byebug rows = @data.map do |resource| [resource['resource_name'], resource['physical_resource_id']] end diff --git a/devops-service/app/api2/handlers/stack_preset.rb b/devops-service/app/api2/handlers/stack_preset.rb index 160afc1..4729473 100644 --- a/devops-service/app/api2/handlers/stack_preset.rb +++ b/devops-service/app/api2/handlers/stack_preset.rb @@ -1,5 +1,7 @@ require 'json' require 'lib/stack_presets/factory' +require 'workers/stack_sync_worker' +require 'workers/job_starter' require_relative "request_handler" module Devops @@ -17,10 +19,42 @@ module Devops def apply id, body preset = Devops::StackPresetsFactory.get(id) - preset.create_stack_from_preset(body) + stack = preset.create_stack_from_preset(body) Devops::Db.connector.stack_insert(stack) + + file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, + provider: stack.provider, + stack_id: stack.id, + request: @request + ) + + puts "Syncing report is located here: #{file}" + + stack end + # def self.apply + # lambda { + # # check_privileges("stack_presets", "r") + # check_privileges('stack_template', 'w') + + # attrs = create_object_from_json_body + # preset = Devops::StackPresetsFactory.get(attrs.fetch('id')) + # stack = preset.create_stack_from_preset(attrs) + # settings.mongo.stack_insert(stack) + + # file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, + # provider: stack.provider, + # stack_id: stack.id, + # request: request + # ) + + # puts "Syncing report is located here: #{file}" + + # create_response 'Created', stack.to_hash, 201 + # } + # end + end end end diff --git a/devops-service/app/api2/routes/stack.rb b/devops-service/app/api2/routes/stack.rb index 0b0144c..396e14c 100644 --- a/devops-service/app/api2/routes/stack.rb +++ b/devops-service/app/api2/routes/stack.rb @@ -44,7 +44,7 @@ module Devops app.get_with_headers "/stack/:stack_id/resources", :headers => [:accept] do |stack_id| check_privileges("stack", "r") - json Devops::API2_0::Handler::Stack.new(request, params).resources(stack_id) + json Devops::API2_0::Handler::Stack.new(request, params).resources(stack_id, nil) end app.get_with_headers "/stack/:stack_id/resources/:resource_id", :headers => [:accept] do |stack_id, resource_id| From 52034554365b19143667cffed365aa24ce3fb450 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Thu, 30 Jul 2015 01:14:45 +0200 Subject: [PATCH 2/4] persist stack servers into mongo --- .../lib/devops-client/handler/image.rb | 2 +- devops-service/app/api2/handlers/stack.rb | 2 + .../app/api2/handlers/stack_preset.rb | 23 +------- devops-service/app/api2/routes/network.rb | 2 +- devops-service/commands/stack.rb | 28 ++-------- devops-service/db/mongo/models/server.rb | 9 ++-- .../db/mongo/models/stack/stack_base.rb | 19 ++++--- devops-service/providers/openstack.rb | 4 +- devops-service/workers/job_starter.rb | 1 - devops-service/workers/stack_sync_worker.rb | 54 +++++++++++++++---- 10 files changed, 72 insertions(+), 72 deletions(-) diff --git a/devops-client/lib/devops-client/handler/image.rb b/devops-client/lib/devops-client/handler/image.rb index fbd8165..3ee06d0 100644 --- a/devops-client/lib/devops-client/handler/image.rb +++ b/devops-client/lib/devops-client/handler/image.rb @@ -38,7 +38,7 @@ class Image < Handler bt.auth = self.auth list = bt.list_handler return list, nil if list.empty? - return list, bt.table + [list, bt.outputter.table] end def create_handler diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 58566d3..302c796 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -34,6 +34,8 @@ module Devops stack = self.stack(id) stack.sync_details! Devops::Db.connector.stack_update(stack) + + stack end def resources id diff --git a/devops-service/app/api2/handlers/stack_preset.rb b/devops-service/app/api2/handlers/stack_preset.rb index 4729473..8204710 100644 --- a/devops-service/app/api2/handlers/stack_preset.rb +++ b/devops-service/app/api2/handlers/stack_preset.rb @@ -20,6 +20,7 @@ module Devops def apply id, body preset = Devops::StackPresetsFactory.get(id) stack = preset.create_stack_from_preset(body) + stack.owner = @request.env['REMOTE_USER'] Devops::Db.connector.stack_insert(stack) file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, @@ -33,28 +34,6 @@ module Devops stack end - # def self.apply - # lambda { - # # check_privileges("stack_presets", "r") - # check_privileges('stack_template', 'w') - - # attrs = create_object_from_json_body - # preset = Devops::StackPresetsFactory.get(attrs.fetch('id')) - # stack = preset.create_stack_from_preset(attrs) - # settings.mongo.stack_insert(stack) - - # file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, - # provider: stack.provider, - # stack_id: stack.id, - # request: request - # ) - - # puts "Syncing report is located here: #{file}" - - # create_response 'Created', stack.to_hash, 201 - # } - # end - end end end diff --git a/devops-service/app/api2/routes/network.rb b/devops-service/app/api2/routes/network.rb index 13c13f4..2d6c738 100644 --- a/devops-service/app/api2/routes/network.rb +++ b/devops-service/app/api2/routes/network.rb @@ -35,7 +35,7 @@ module Devops app.get_with_headers "/networks/:provider", :headers => [:accept] do |provider| check_privileges("network", "r") check_provider(provider) - json Devops::API2_0::Handler::Network.new(request, params).networks + json Devops::API2_0::Handler::Network.new(request, params).networks(provider) end puts "Network routes initialized" diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb index fb60cfc..243281f 100644 --- a/devops-service/commands/stack.rb +++ b/devops-service/commands/stack.rb @@ -7,8 +7,8 @@ module StackCommands def sync_stack_till_not_in_progress_proc lambda do |out, stack, mongo| - # two tries each 2 seconds, then 5 tries each 10 seconds, then 5 tries each minute. - sleep_times = [2]*2 + [10]*5 + [60]*5 + # two tries each 4 seconds, then 5 tries each 10 seconds, then 5 tries each minute. + sleep_times = [4]*2 + [10]*5 + [60]*5 begin out << "Syncing stack '#{stack.id}'...\n" @@ -29,30 +29,8 @@ module StackCommands end end - def create_devops_servers_from_stack_resources - lambda do |out, stack, mongo, owner| - project = mongo.project(stack.project) - deploy_env = project.deploy_envs.detect {|env| env.identifier == stack.deploy_env} - provider = ::Provider::ProviderFactory.get(stack.provider) - - stack.resources.each do |resource| - logical_name = resource['resource_name'] - attrs = stack.resource(logical_name) - body = { - 'name' => logical_name, - 'key' => attrs['key_name'] - } - - server = extract_servers(provider, project, deploy_env, body, owner, mongo) - server.private_ip = attrs['addresses']['devops-net-1'].first['addr'] - - end - end - end - CommandsStorage.add_job_lambda( - sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc, - create_devops_servers_from_stack_resources: create_devops_servers_from_stack_resources + sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc ) end diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb index 96a7c6e..e4da64d 100644 --- a/devops-service/db/mongo/models/server.rb +++ b/devops-service/db/mongo/models/server.rb @@ -4,7 +4,7 @@ module Devops module Model class Server < MongoModel - attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by + attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack attr_accessor :options, :static, :key types :id => {:type => String, :empty => false}, @@ -17,7 +17,8 @@ module Devops :key => {:type => String, :empty => false}, :created_by => {:type => String, :empty => false}, :chef_node_name => {:type => String, :empty => true}, - :reserved_by => {:type => String, :empty => true} + :reserved_by => {:type => String, :empty => true}, + :stack => {:type => String, :nil => true} def self.fields ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "static", "key", "reserved_by"] @@ -37,6 +38,7 @@ module Devops self.static = s["static"] self.key = s["key"] self.reserved_by = s["reserved_by"] + self.stack = s["stack"] end def validate! @@ -57,7 +59,8 @@ module Devops "created_by" => self.created_by, "static" => self.static, "key" => self.key, - "reserved_by" => self.reserved_by + "reserved_by" => self.reserved_by, + "stack" => stack }.delete_if{|k,v| v.nil?} end diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index e148ac3..d95c435 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -3,14 +3,15 @@ module Devops class StackBase < MongoModel attr_accessor :id, :project, :deploy_env, :stack_template, - :cloud_stack_id, :provider, :parameters, :details + :cloud_stack_id, :provider, :parameters, :details, :owner types id: {type: String, empty: false}, provider: {type: String, empty: false}, project: {type: String}, deploy_env: {type: String}, stack_template: {type: String, empty: false}, - cloud_stack_id: {type: String, nil: true} + cloud_stack_id: {type: String, nil: true}, + owner: {type: String} # details: {type: Hash, nil: true} # Hash type isn't supported yet def initialize attrs={} @@ -23,6 +24,7 @@ module Devops self.cloud_stack_id = attrs['cloud_stack_id'] self.parameters = attrs['parameters'] self.details = attrs['details'] + self.owner = attrs['owner'] self end @@ -35,7 +37,8 @@ module Devops cloud_stack_id: cloud_stack_id, parameters: parameters, details: details, - stack_status: stack_status + stack_status: stack_status, + owner: owner } end @@ -52,7 +55,7 @@ module Devops end def sync_details! - self.details = provider_class.stack_details(self) + self.details = provider_class.stack_details(self).attributes end def resources @@ -64,10 +67,6 @@ module Devops provider_class.stack_resource(self, resource_id) end - def template_body - Devops::Api2.settings.mongo.stack_template(stack_template).template_body - end - def stack_status raise 'override me' end @@ -80,6 +79,10 @@ module Devops } end + def template_body + Devops::Db.connector.stack_template(stack_template).template_body + end + class << self attr_accessor :provider diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb index 011007d..95fd1f5 100644 --- a/devops-service/providers/openstack.rb +++ b/devops-service/providers/openstack.rb @@ -210,7 +210,7 @@ module Provider end def stack_details(stack) - fog_stack(stack).details.attributes + fog_stack(stack).details end def stack_resources(stack) @@ -219,7 +219,7 @@ module Provider def stack_resource(stack, resource_id) physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id - compute.servers.get(physical_id).attributes + compute.servers.get(physical_id) end private diff --git a/devops-service/workers/job_starter.rb b/devops-service/workers/job_starter.rb index 052874c..bfa6299 100644 --- a/devops-service/workers/job_starter.rb +++ b/devops-service/workers/job_starter.rb @@ -11,7 +11,6 @@ module JobStarter def self.start_job_as_worker(worker_class, options) job_options = options.dup - job_options[:owner] ||= options[:request].env['REMOTE_USER'] job_options[:config] ||= DevopsConfig.config job_options[:dir] ||= DevopsConfig[:report_dir_v2] job_options[:url] ||= options[:request].url diff --git a/devops-service/workers/stack_sync_worker.rb b/devops-service/workers/stack_sync_worker.rb index 1258beb..91a172c 100644 --- a/devops-service/workers/stack_sync_worker.rb +++ b/devops-service/workers/stack_sync_worker.rb @@ -1,6 +1,7 @@ require "providers/provider_factory" require "commands/stack" require "db/mongo/models/stack/stack_factory" +require "db/mongo/models/project" require "db/mongo/models/report" require 'workers/workers_storage' @@ -13,20 +14,55 @@ class StackSyncWorker < Worker def perform(options) call(options['config'], options['provider'], options['dir']) do |mongo, provider, out, file| stack = mongo.stack(options['stack_id']) - o = { - "file" => file, - "_id" => jid, - "created_by" => options['owner'], - "project" => stack.project, - "deploy_env" => stack.deploy_env, - "type" => ::Devops::Model::Report::STACK_TYPE - } - mongo.save_report(::Devops::Model::Report.new(o)) + save_report(mongo, stack, file) status = sync_stack_till_not_in_progress_proc.call(out, stack, mongo) + persist_stack_servers_in_mongo!(mongo, stack, provider) status end end + + private + + def save_report(mongo, stack, file) + report = ::Devops::Model::Report.new( + "file" => file, + "_id" => jid, + "created_by" => stack.owner, + "project" => stack.project, + "deploy_env" => stack.deploy_env, + "type" => ::Devops::Model::Report::STACK_TYPE + ) + mongo.save_report(report) + end + + def persist_stack_servers_in_mongo!(mongo, stack, provider) + project = mongo.project(stack.project) + deploy_env = project.deploy_env(stack.deploy_env) + + stack_servers = [] + + stack.resources.each do |resource| + logical_name = resource.resource_name + extended_info = stack.resource(logical_name) + body = { + 'name' => logical_name, + 'key' => extended_info.key_name + } + + servers = extract_servers(provider, project, deploy_env, body, stack.owner, mongo) + servers.each do |server| + # TODO: improve IP assigning logic + server.private_ip = extended_info.addresses.values.first.first['addr'] + end + stack_servers += servers + end + stack_servers.each_with_index do |server, i| + server.id = "stack-#{stack.id}-server-#{i+1}" + server.stack = stack.id + mongo.server_insert server + end + end end WorkersStorage.add_worker(sync_stack_till_not_in_progress: StackSyncWorker) From 9c8f685f44539e118556ae2ea14d1e0ec50fea88 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Thu, 30 Jul 2015 01:22:54 +0200 Subject: [PATCH 3/4] return model in mongo insert and update commands --- devops-client/lib/devops-client/output/stack.rb | 1 - devops-service/app/api2/handlers/stack.rb | 5 +---- devops-service/app/api2/routes/stack.rb | 2 +- .../db/mongo/connectors/helpers/insert_command.rb | 1 + .../db/mongo/connectors/helpers/update_command.rb | 15 +++++++++------ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/devops-client/lib/devops-client/output/stack.rb b/devops-client/lib/devops-client/output/stack.rb index e3ad175..2051fae 100644 --- a/devops-client/lib/devops-client/output/stack.rb +++ b/devops-client/lib/devops-client/output/stack.rb @@ -49,7 +49,6 @@ module Output def create_servers_list headers = ['Logical id', 'Physical id'] - byebug rows = @data.map do |resource| [resource['resource_name'], resource['physical_resource_id']] end diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 302c796..37512da 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -17,7 +17,6 @@ module Devops def create_stack object stack_model = Model::StackFactory.create(object['provider'], object) Devops::Db.connector.stack_insert(stack_model) - stack_model end def stack id @@ -34,8 +33,6 @@ module Devops stack = self.stack(id) stack.sync_details! Devops::Db.connector.stack_update(stack) - - stack end def resources id @@ -43,7 +40,7 @@ module Devops stack.resources end - def resources id, resource_id + def resource id, resource_id stack = Devops::Db.connector.stack(id) stack.resource(resource_id) end diff --git a/devops-service/app/api2/routes/stack.rb b/devops-service/app/api2/routes/stack.rb index 396e14c..0b0144c 100644 --- a/devops-service/app/api2/routes/stack.rb +++ b/devops-service/app/api2/routes/stack.rb @@ -44,7 +44,7 @@ module Devops app.get_with_headers "/stack/:stack_id/resources", :headers => [:accept] do |stack_id| check_privileges("stack", "r") - json Devops::API2_0::Handler::Stack.new(request, params).resources(stack_id, nil) + json Devops::API2_0::Handler::Stack.new(request, params).resources(stack_id) end app.get_with_headers "/stack/:stack_id/resources/:resource_id", :headers => [:accept] do |stack_id, resource_id| diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb index c5566b4..c00cc48 100644 --- a/devops-service/db/mongo/connectors/helpers/insert_command.rb +++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb @@ -18,6 +18,7 @@ module Connectors begin record.validate! collection.insert(record.to_mongo_hash) + record rescue Mongo::OperationFailure => e if e.message =~ /^11000/ resource_name = StringHelper.underscore_class(record.class) diff --git a/devops-service/db/mongo/connectors/helpers/update_command.rb b/devops-service/db/mongo/connectors/helpers/update_command.rb index b661196..e1ebec7 100644 --- a/devops-service/db/mongo/connectors/helpers/update_command.rb +++ b/devops-service/db/mongo/connectors/helpers/update_command.rb @@ -14,12 +14,15 @@ module Connectors end def update(record) - record.validate! - collection.update({"_id" => record.id}, record.to_mongo_hash) - rescue Mongo::OperationFailure => e - if e.message =~ /^11000/ - resource_name = StringHelper.underscore_class(record.class) - raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") + begin + record.validate! + collection.update({"_id" => record.id}, record.to_mongo_hash) + record + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + resource_name = StringHelper.underscore_class(record.class) + raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") + end end end From c1e2cee92020871efe8877988892837335a1cfd3 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Thu, 30 Jul 2015 01:29:53 +0200 Subject: [PATCH 4/4] delete stack servers on stack deleting --- devops-service/app/api2/handlers/stack.rb | 1 + devops-service/db/mongo/connectors/server.rb | 4 ++++ devops-service/db/mongo/mongo_connector.rb | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 37512da..8a476b6 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -26,6 +26,7 @@ module Devops def delete_stack id stack = self.stack(id) stack.delete_stack_in_cloud! + Devops::Db.connector.stack_servers_delete(id) Devops::Db.connector.stack_delete(id) end diff --git a/devops-service/db/mongo/connectors/server.rb b/devops-service/db/mongo/connectors/server.rb index 78e0798..7c40322 100644 --- a/devops-service/db/mongo/connectors/server.rb +++ b/devops-service/db/mongo/connectors/server.rb @@ -63,6 +63,10 @@ module Connectors collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}}) end + def stack_servers_delete(stack_id) + collection.remove('stack' => stack_id) + end + private def model_from_bson(bson) diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index 2522759..4dbfd7f 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -22,7 +22,7 @@ class MongoConnector :project_template_delete] => :projects_templates_connector, [:servers_find, :servers, :servers_by_names, :server_by_instance_id, :server_by_chef_node_name, :servers_by_key, :server_insert, - :server_delete, :server_update, :server_set_chef_node_name] => :servers_connector, + :server_delete, :server_update, :server_set_chef_node_name, :stack_servers_delete] => :servers_connector, [:user_auth, :user, :users, :users_names, :user_insert, :user_delete, :user_update, :create_root_user, :check_user_privileges] => :users_connector, [:keys, :key, :key_insert, :key_delete] => :keys_connector,