diff --git a/devops-client/lib/devops-client/handler/image.rb b/devops-client/lib/devops-client/handler/image.rb index fbd8165..3ee06d0 100644 --- a/devops-client/lib/devops-client/handler/image.rb +++ b/devops-client/lib/devops-client/handler/image.rb @@ -38,7 +38,7 @@ class Image < Handler bt.auth = self.auth list = bt.list_handler return list, nil if list.empty? - return list, bt.table + [list, bt.outputter.table] end def create_handler diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 281187c..46c2bc0 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -21,7 +21,6 @@ module Devops object = parser.create stack_model = Model::StackFactory.create(object['provider'], object) Devops::Db.connector.stack_insert(stack_model) - stack_model end def stack id @@ -31,6 +30,7 @@ module Devops def delete_stack id stack = self.stack(id) stack.delete_stack_in_cloud! + Devops::Db.connector.stack_servers_delete(id) Devops::Db.connector.stack_delete(id) end @@ -45,7 +45,7 @@ module Devops stack.resources end - def resources id, resource_id + def resource id, resource_id stack = Devops::Db.connector.stack(id) stack.resource(resource_id) end diff --git a/devops-service/app/api2/handlers/stack_preset.rb b/devops-service/app/api2/handlers/stack_preset.rb index 2056928..af66bc8 100644 --- a/devops-service/app/api2/handlers/stack_preset.rb +++ b/devops-service/app/api2/handlers/stack_preset.rb @@ -1,5 +1,7 @@ require 'json' require 'lib/stack_presets/factory' +require 'workers/stack_sync_worker' +require 'workers/job_starter' require_relative "request_handler" module Devops @@ -20,8 +22,19 @@ module Devops def apply id body = parser.apply preset = Devops::StackPresetsFactory.get(id) - preset.create_stack_from_preset(body) + stack = preset.create_stack_from_preset(body) + stack.owner = @request.env['REMOTE_USER'] Devops::Db.connector.stack_insert(stack) + + file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, + provider: stack.provider, + stack_id: stack.id, + request: @request + ) + + puts "Syncing report is located here: #{file}" + + stack end end diff --git a/devops-service/app/api2/routes/network.rb b/devops-service/app/api2/routes/network.rb index 306b457..3d31d64 100644 --- a/devops-service/app/api2/routes/network.rb +++ b/devops-service/app/api2/routes/network.rb @@ -35,7 +35,7 @@ module Devops app.get_with_headers "/networks/:provider", :headers => [:accept] do |provider| check_privileges("network", "r") check_provider(provider) - json Devops::API2_0::Handler::Network.new(request).networks + json Devops::API2_0::Handler::Network.new(request).networks(provider) end puts "Network routes initialized" diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb index fb60cfc..243281f 100644 --- a/devops-service/commands/stack.rb +++ b/devops-service/commands/stack.rb @@ -7,8 +7,8 @@ module StackCommands def sync_stack_till_not_in_progress_proc lambda do |out, stack, mongo| - # two tries each 2 seconds, then 5 tries each 10 seconds, then 5 tries each minute. - sleep_times = [2]*2 + [10]*5 + [60]*5 + # two tries each 4 seconds, then 5 tries each 10 seconds, then 5 tries each minute. + sleep_times = [4]*2 + [10]*5 + [60]*5 begin out << "Syncing stack '#{stack.id}'...\n" @@ -29,30 +29,8 @@ module StackCommands end end - def create_devops_servers_from_stack_resources - lambda do |out, stack, mongo, owner| - project = mongo.project(stack.project) - deploy_env = project.deploy_envs.detect {|env| env.identifier == stack.deploy_env} - provider = ::Provider::ProviderFactory.get(stack.provider) - - stack.resources.each do |resource| - logical_name = resource['resource_name'] - attrs = stack.resource(logical_name) - body = { - 'name' => logical_name, - 'key' => attrs['key_name'] - } - - server = extract_servers(provider, project, deploy_env, body, owner, mongo) - server.private_ip = attrs['addresses']['devops-net-1'].first['addr'] - - end - end - end - CommandsStorage.add_job_lambda( - sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc, - create_devops_servers_from_stack_resources: create_devops_servers_from_stack_resources + sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc ) end diff --git a/devops-service/db/mongo/connectors/helpers/insert_command.rb b/devops-service/db/mongo/connectors/helpers/insert_command.rb index c5566b4..c00cc48 100644 --- a/devops-service/db/mongo/connectors/helpers/insert_command.rb +++ b/devops-service/db/mongo/connectors/helpers/insert_command.rb @@ -18,6 +18,7 @@ module Connectors begin record.validate! collection.insert(record.to_mongo_hash) + record rescue Mongo::OperationFailure => e if e.message =~ /^11000/ resource_name = StringHelper.underscore_class(record.class) diff --git a/devops-service/db/mongo/connectors/helpers/update_command.rb b/devops-service/db/mongo/connectors/helpers/update_command.rb index b661196..e1ebec7 100644 --- a/devops-service/db/mongo/connectors/helpers/update_command.rb +++ b/devops-service/db/mongo/connectors/helpers/update_command.rb @@ -14,12 +14,15 @@ module Connectors end def update(record) - record.validate! - collection.update({"_id" => record.id}, record.to_mongo_hash) - rescue Mongo::OperationFailure => e - if e.message =~ /^11000/ - resource_name = StringHelper.underscore_class(record.class) - raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") + begin + record.validate! + collection.update({"_id" => record.id}, record.to_mongo_hash) + record + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + resource_name = StringHelper.underscore_class(record.class) + raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'") + end end end diff --git a/devops-service/db/mongo/connectors/server.rb b/devops-service/db/mongo/connectors/server.rb index 78e0798..7c40322 100644 --- a/devops-service/db/mongo/connectors/server.rb +++ b/devops-service/db/mongo/connectors/server.rb @@ -63,6 +63,10 @@ module Connectors collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}}) end + def stack_servers_delete(stack_id) + collection.remove('stack' => stack_id) + end + private def model_from_bson(bson) diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb index b64c52b..403ab8b 100644 --- a/devops-service/db/mongo/models/server.rb +++ b/devops-service/db/mongo/models/server.rb @@ -19,7 +19,7 @@ module Devops define_hook :before_bootstrap define_hook :after_bootstrap - attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by + attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack attr_accessor :options, :static, :key types :id => {:type => String, :empty => false}, @@ -32,7 +32,8 @@ module Devops :key => {:type => String, :empty => false}, :created_by => {:type => String, :empty => false}, :chef_node_name => {:type => String, :empty => true}, - :reserved_by => {:type => String, :empty => true} + :reserved_by => {:type => String, :empty => true}, + :stack => {:type => String, :nil => true} def self.fields ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "static", "key", "reserved_by"] @@ -52,6 +53,7 @@ module Devops self.static = s["static"] self.key = s["key"] self.reserved_by = s["reserved_by"] + self.stack = s["stack"] end def create @@ -80,7 +82,8 @@ module Devops "created_by" => self.created_by, "static" => self.static, "key" => self.key, - "reserved_by" => self.reserved_by + "reserved_by" => self.reserved_by, + "stack" => stack }.delete_if{|k,v| v.nil?} end diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index e148ac3..d95c435 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -3,14 +3,15 @@ module Devops class StackBase < MongoModel attr_accessor :id, :project, :deploy_env, :stack_template, - :cloud_stack_id, :provider, :parameters, :details + :cloud_stack_id, :provider, :parameters, :details, :owner types id: {type: String, empty: false}, provider: {type: String, empty: false}, project: {type: String}, deploy_env: {type: String}, stack_template: {type: String, empty: false}, - cloud_stack_id: {type: String, nil: true} + cloud_stack_id: {type: String, nil: true}, + owner: {type: String} # details: {type: Hash, nil: true} # Hash type isn't supported yet def initialize attrs={} @@ -23,6 +24,7 @@ module Devops self.cloud_stack_id = attrs['cloud_stack_id'] self.parameters = attrs['parameters'] self.details = attrs['details'] + self.owner = attrs['owner'] self end @@ -35,7 +37,8 @@ module Devops cloud_stack_id: cloud_stack_id, parameters: parameters, details: details, - stack_status: stack_status + stack_status: stack_status, + owner: owner } end @@ -52,7 +55,7 @@ module Devops end def sync_details! - self.details = provider_class.stack_details(self) + self.details = provider_class.stack_details(self).attributes end def resources @@ -64,10 +67,6 @@ module Devops provider_class.stack_resource(self, resource_id) end - def template_body - Devops::Api2.settings.mongo.stack_template(stack_template).template_body - end - def stack_status raise 'override me' end @@ -80,6 +79,10 @@ module Devops } end + def template_body + Devops::Db.connector.stack_template(stack_template).template_body + end + class << self attr_accessor :provider diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index 2522759..4dbfd7f 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -22,7 +22,7 @@ class MongoConnector :project_template_delete] => :projects_templates_connector, [:servers_find, :servers, :servers_by_names, :server_by_instance_id, :server_by_chef_node_name, :servers_by_key, :server_insert, - :server_delete, :server_update, :server_set_chef_node_name] => :servers_connector, + :server_delete, :server_update, :server_set_chef_node_name, :stack_servers_delete] => :servers_connector, [:user_auth, :user, :users, :users_names, :user_insert, :user_delete, :user_update, :create_root_user, :check_user_privileges] => :users_connector, [:keys, :key, :key_insert, :key_delete] => :keys_connector, diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb index 011007d..95fd1f5 100644 --- a/devops-service/providers/openstack.rb +++ b/devops-service/providers/openstack.rb @@ -210,7 +210,7 @@ module Provider end def stack_details(stack) - fog_stack(stack).details.attributes + fog_stack(stack).details end def stack_resources(stack) @@ -219,7 +219,7 @@ module Provider def stack_resource(stack, resource_id) physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id - compute.servers.get(physical_id).attributes + compute.servers.get(physical_id) end private diff --git a/devops-service/workers/job_starter.rb b/devops-service/workers/job_starter.rb index 052874c..bfa6299 100644 --- a/devops-service/workers/job_starter.rb +++ b/devops-service/workers/job_starter.rb @@ -11,7 +11,6 @@ module JobStarter def self.start_job_as_worker(worker_class, options) job_options = options.dup - job_options[:owner] ||= options[:request].env['REMOTE_USER'] job_options[:config] ||= DevopsConfig.config job_options[:dir] ||= DevopsConfig[:report_dir_v2] job_options[:url] ||= options[:request].url diff --git a/devops-service/workers/stack_sync_worker.rb b/devops-service/workers/stack_sync_worker.rb index d060ff1..eda9dab 100644 --- a/devops-service/workers/stack_sync_worker.rb +++ b/devops-service/workers/stack_sync_worker.rb @@ -1,6 +1,7 @@ require "providers/provider_factory" require "commands/stack" require "db/mongo/models/stack/stack_factory" +require "db/mongo/models/project" require "db/mongo/models/report" class StackSyncWorker < Worker @@ -13,20 +14,55 @@ class StackSyncWorker < Worker call(options['config'], options['provider'], options['dir']) do |provider, out, file| mongo = Devops::Db.connector stack = mongo.stack(options['stack_id']) - o = { - "file" => file, - "_id" => jid, - "created_by" => options['owner'], - "project" => stack.project, - "deploy_env" => stack.deploy_env, - "type" => ::Devops::Model::Report::STACK_TYPE - } - mongo.save_report(::Devops::Model::Report.new(o)) + save_report(mongo, stack, file) status = sync_stack_till_not_in_progress_proc.call(out, stack, mongo) + persist_stack_servers_in_mongo!(mongo, stack, provider) status end end + + private + + def save_report(mongo, stack, file) + report = ::Devops::Model::Report.new( + "file" => file, + "_id" => jid, + "created_by" => stack.owner, + "project" => stack.project, + "deploy_env" => stack.deploy_env, + "type" => ::Devops::Model::Report::STACK_TYPE + ) + mongo.save_report(report) + end + + def persist_stack_servers_in_mongo!(mongo, stack, provider) + project = mongo.project(stack.project) + deploy_env = project.deploy_env(stack.deploy_env) + + stack_servers = [] + + stack.resources.each do |resource| + logical_name = resource.resource_name + extended_info = stack.resource(logical_name) + body = { + 'name' => logical_name, + 'key' => extended_info.key_name + } + + servers = extract_servers(provider, project, deploy_env, body, stack.owner, mongo) + servers.each do |server| + # TODO: improve IP assigning logic + server.private_ip = extended_info.addresses.values.first.first['addr'] + end + stack_servers += servers + end + stack_servers.each_with_index do |server, i| + server.id = "stack-#{stack.id}-server-#{i+1}" + server.stack = stack.id + mongo.server_insert server + end + end end WorkersStorage.add_worker(sync_stack_till_not_in_progress: StackSyncWorker)