From 52034554365b19143667cffed365aa24ce3fb450 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Thu, 30 Jul 2015 01:14:45 +0200 Subject: [PATCH] persist stack servers into mongo --- .../lib/devops-client/handler/image.rb | 2 +- devops-service/app/api2/handlers/stack.rb | 2 + .../app/api2/handlers/stack_preset.rb | 23 +------- devops-service/app/api2/routes/network.rb | 2 +- devops-service/commands/stack.rb | 28 ++-------- devops-service/db/mongo/models/server.rb | 9 ++-- .../db/mongo/models/stack/stack_base.rb | 19 ++++--- devops-service/providers/openstack.rb | 4 +- devops-service/workers/job_starter.rb | 1 - devops-service/workers/stack_sync_worker.rb | 54 +++++++++++++++---- 10 files changed, 72 insertions(+), 72 deletions(-) diff --git a/devops-client/lib/devops-client/handler/image.rb b/devops-client/lib/devops-client/handler/image.rb index fbd8165..3ee06d0 100644 --- a/devops-client/lib/devops-client/handler/image.rb +++ b/devops-client/lib/devops-client/handler/image.rb @@ -38,7 +38,7 @@ class Image < Handler bt.auth = self.auth list = bt.list_handler return list, nil if list.empty? - return list, bt.table + [list, bt.outputter.table] end def create_handler diff --git a/devops-service/app/api2/handlers/stack.rb b/devops-service/app/api2/handlers/stack.rb index 58566d3..302c796 100644 --- a/devops-service/app/api2/handlers/stack.rb +++ b/devops-service/app/api2/handlers/stack.rb @@ -34,6 +34,8 @@ module Devops stack = self.stack(id) stack.sync_details! Devops::Db.connector.stack_update(stack) + + stack end def resources id diff --git a/devops-service/app/api2/handlers/stack_preset.rb b/devops-service/app/api2/handlers/stack_preset.rb index 4729473..8204710 100644 --- a/devops-service/app/api2/handlers/stack_preset.rb +++ b/devops-service/app/api2/handlers/stack_preset.rb @@ -20,6 +20,7 @@ module Devops def apply id, body preset = Devops::StackPresetsFactory.get(id) stack = preset.create_stack_from_preset(body) + stack.owner = @request.env['REMOTE_USER'] Devops::Db.connector.stack_insert(stack) file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, @@ -33,28 +34,6 @@ module Devops stack end - # def self.apply - # lambda { - # # check_privileges("stack_presets", "r") - # check_privileges('stack_template', 'w') - - # attrs = create_object_from_json_body - # preset = Devops::StackPresetsFactory.get(attrs.fetch('id')) - # stack = preset.create_stack_from_preset(attrs) - # settings.mongo.stack_insert(stack) - - # file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress, - # provider: stack.provider, - # stack_id: stack.id, - # request: request - # ) - - # puts "Syncing report is located here: #{file}" - - # create_response 'Created', stack.to_hash, 201 - # } - # end - end end end diff --git a/devops-service/app/api2/routes/network.rb b/devops-service/app/api2/routes/network.rb index 13c13f4..2d6c738 100644 --- a/devops-service/app/api2/routes/network.rb +++ b/devops-service/app/api2/routes/network.rb @@ -35,7 +35,7 @@ module Devops app.get_with_headers "/networks/:provider", :headers => [:accept] do |provider| check_privileges("network", "r") check_provider(provider) - json Devops::API2_0::Handler::Network.new(request, params).networks + json Devops::API2_0::Handler::Network.new(request, params).networks(provider) end puts "Network routes initialized" diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb index fb60cfc..243281f 100644 --- a/devops-service/commands/stack.rb +++ b/devops-service/commands/stack.rb @@ -7,8 +7,8 @@ module StackCommands def sync_stack_till_not_in_progress_proc lambda do |out, stack, mongo| - # two tries each 2 seconds, then 5 tries each 10 seconds, then 5 tries each minute. - sleep_times = [2]*2 + [10]*5 + [60]*5 + # two tries each 4 seconds, then 5 tries each 10 seconds, then 5 tries each minute. + sleep_times = [4]*2 + [10]*5 + [60]*5 begin out << "Syncing stack '#{stack.id}'...\n" @@ -29,30 +29,8 @@ module StackCommands end end - def create_devops_servers_from_stack_resources - lambda do |out, stack, mongo, owner| - project = mongo.project(stack.project) - deploy_env = project.deploy_envs.detect {|env| env.identifier == stack.deploy_env} - provider = ::Provider::ProviderFactory.get(stack.provider) - - stack.resources.each do |resource| - logical_name = resource['resource_name'] - attrs = stack.resource(logical_name) - body = { - 'name' => logical_name, - 'key' => attrs['key_name'] - } - - server = extract_servers(provider, project, deploy_env, body, owner, mongo) - server.private_ip = attrs['addresses']['devops-net-1'].first['addr'] - - end - end - end - CommandsStorage.add_job_lambda( - sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc, - create_devops_servers_from_stack_resources: create_devops_servers_from_stack_resources + sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc ) end diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb index 96a7c6e..e4da64d 100644 --- a/devops-service/db/mongo/models/server.rb +++ b/devops-service/db/mongo/models/server.rb @@ -4,7 +4,7 @@ module Devops module Model class Server < MongoModel - attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by + attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack attr_accessor :options, :static, :key types :id => {:type => String, :empty => false}, @@ -17,7 +17,8 @@ module Devops :key => {:type => String, :empty => false}, :created_by => {:type => String, :empty => false}, :chef_node_name => {:type => String, :empty => true}, - :reserved_by => {:type => String, :empty => true} + :reserved_by => {:type => String, :empty => true}, + :stack => {:type => String, :nil => true} def self.fields ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "static", "key", "reserved_by"] @@ -37,6 +38,7 @@ module Devops self.static = s["static"] self.key = s["key"] self.reserved_by = s["reserved_by"] + self.stack = s["stack"] end def validate! @@ -57,7 +59,8 @@ module Devops "created_by" => self.created_by, "static" => self.static, "key" => self.key, - "reserved_by" => self.reserved_by + "reserved_by" => self.reserved_by, + "stack" => stack }.delete_if{|k,v| v.nil?} end diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index e148ac3..d95c435 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -3,14 +3,15 @@ module Devops class StackBase < MongoModel attr_accessor :id, :project, :deploy_env, :stack_template, - :cloud_stack_id, :provider, :parameters, :details + :cloud_stack_id, :provider, :parameters, :details, :owner types id: {type: String, empty: false}, provider: {type: String, empty: false}, project: {type: String}, deploy_env: {type: String}, stack_template: {type: String, empty: false}, - cloud_stack_id: {type: String, nil: true} + cloud_stack_id: {type: String, nil: true}, + owner: {type: String} # details: {type: Hash, nil: true} # Hash type isn't supported yet def initialize attrs={} @@ -23,6 +24,7 @@ module Devops self.cloud_stack_id = attrs['cloud_stack_id'] self.parameters = attrs['parameters'] self.details = attrs['details'] + self.owner = attrs['owner'] self end @@ -35,7 +37,8 @@ module Devops cloud_stack_id: cloud_stack_id, parameters: parameters, details: details, - stack_status: stack_status + stack_status: stack_status, + owner: owner } end @@ -52,7 +55,7 @@ module Devops end def sync_details! - self.details = provider_class.stack_details(self) + self.details = provider_class.stack_details(self).attributes end def resources @@ -64,10 +67,6 @@ module Devops provider_class.stack_resource(self, resource_id) end - def template_body - Devops::Api2.settings.mongo.stack_template(stack_template).template_body - end - def stack_status raise 'override me' end @@ -80,6 +79,10 @@ module Devops } end + def template_body + Devops::Db.connector.stack_template(stack_template).template_body + end + class << self attr_accessor :provider diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb index 011007d..95fd1f5 100644 --- a/devops-service/providers/openstack.rb +++ b/devops-service/providers/openstack.rb @@ -210,7 +210,7 @@ module Provider end def stack_details(stack) - fog_stack(stack).details.attributes + fog_stack(stack).details end def stack_resources(stack) @@ -219,7 +219,7 @@ module Provider def stack_resource(stack, resource_id) physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id - compute.servers.get(physical_id).attributes + compute.servers.get(physical_id) end private diff --git a/devops-service/workers/job_starter.rb b/devops-service/workers/job_starter.rb index 052874c..bfa6299 100644 --- a/devops-service/workers/job_starter.rb +++ b/devops-service/workers/job_starter.rb @@ -11,7 +11,6 @@ module JobStarter def self.start_job_as_worker(worker_class, options) job_options = options.dup - job_options[:owner] ||= options[:request].env['REMOTE_USER'] job_options[:config] ||= DevopsConfig.config job_options[:dir] ||= DevopsConfig[:report_dir_v2] job_options[:url] ||= options[:request].url diff --git a/devops-service/workers/stack_sync_worker.rb b/devops-service/workers/stack_sync_worker.rb index 1258beb..91a172c 100644 --- a/devops-service/workers/stack_sync_worker.rb +++ b/devops-service/workers/stack_sync_worker.rb @@ -1,6 +1,7 @@ require "providers/provider_factory" require "commands/stack" require "db/mongo/models/stack/stack_factory" +require "db/mongo/models/project" require "db/mongo/models/report" require 'workers/workers_storage' @@ -13,20 +14,55 @@ class StackSyncWorker < Worker def perform(options) call(options['config'], options['provider'], options['dir']) do |mongo, provider, out, file| stack = mongo.stack(options['stack_id']) - o = { - "file" => file, - "_id" => jid, - "created_by" => options['owner'], - "project" => stack.project, - "deploy_env" => stack.deploy_env, - "type" => ::Devops::Model::Report::STACK_TYPE - } - mongo.save_report(::Devops::Model::Report.new(o)) + save_report(mongo, stack, file) status = sync_stack_till_not_in_progress_proc.call(out, stack, mongo) + persist_stack_servers_in_mongo!(mongo, stack, provider) status end end + + private + + def save_report(mongo, stack, file) + report = ::Devops::Model::Report.new( + "file" => file, + "_id" => jid, + "created_by" => stack.owner, + "project" => stack.project, + "deploy_env" => stack.deploy_env, + "type" => ::Devops::Model::Report::STACK_TYPE + ) + mongo.save_report(report) + end + + def persist_stack_servers_in_mongo!(mongo, stack, provider) + project = mongo.project(stack.project) + deploy_env = project.deploy_env(stack.deploy_env) + + stack_servers = [] + + stack.resources.each do |resource| + logical_name = resource.resource_name + extended_info = stack.resource(logical_name) + body = { + 'name' => logical_name, + 'key' => extended_info.key_name + } + + servers = extract_servers(provider, project, deploy_env, body, stack.owner, mongo) + servers.each do |server| + # TODO: improve IP assigning logic + server.private_ip = extended_info.addresses.values.first.first['addr'] + end + stack_servers += servers + end + stack_servers.each_with_index do |server, i| + server.id = "stack-#{stack.id}-server-#{i+1}" + server.stack = stack.id + mongo.server_insert server + end + end end WorkersStorage.add_worker(sync_stack_till_not_in_progress: StackSyncWorker)