Merge branch 'devops_3' of git.stu.neva.ru:cloudtechlab/devops-service into devops_3

devops-client/lib/devops-client/handler/image.rb

@@ -38,7 +38,7 @@ class Image < Handler
     bt.auth = self.auth
     list = bt.list_handler
     return list, nil if list.empty?
-    return list, bt.table
+    [list, bt.outputter.table]
   end
 
   def create_handler

devops-service/app/api2/handlers/stack.rb

@@ -21,7 +21,6 @@ module Devops
           object = parser.create
           stack_model = Model::StackFactory.create(object['provider'], object)
           Devops::Db.connector.stack_insert(stack_model)
-          stack_model
         end
 
         def stack id
@@ -31,6 +30,7 @@ module Devops
         def delete_stack id
           stack = self.stack(id)
           stack.delete_stack_in_cloud!
+          Devops::Db.connector.stack_servers_delete(id)
           Devops::Db.connector.stack_delete(id)
         end
 
@@ -45,7 +45,7 @@ module Devops
           stack.resources
         end
 
-        def resources id, resource_id
+        def resource id, resource_id
           stack = Devops::Db.connector.stack(id)
           stack.resource(resource_id)
         end

devops-service/app/api2/handlers/stack_preset.rb

@@ -1,5 +1,7 @@
 require 'json'
 require 'lib/stack_presets/factory'
+require 'workers/stack_sync_worker'
+require 'workers/job_starter'
 require_relative "request_handler"
 
 module Devops
@@ -20,8 +22,19 @@ module Devops
         def apply id
           body = parser.apply
           preset = Devops::StackPresetsFactory.get(id)
-          preset.create_stack_from_preset(body)
+          stack = preset.create_stack_from_preset(body)
+          stack.owner = @request.env['REMOTE_USER']
           Devops::Db.connector.stack_insert(stack)
+
+          file = JobStarter.start_job(:worker, :sync_stack_till_not_in_progress,
+            provider: stack.provider,
+            stack_id: stack.id,
+            request: @request
+          )
+
+          puts "Syncing report is located here: #{file}"
+
+          stack
         end
 
       end

devops-service/app/api2/routes/network.rb

@@ -35,7 +35,7 @@ module Devops
           app.get_with_headers "/networks/:provider", :headers => [:accept] do |provider|
             check_privileges("network", "r")
             check_provider(provider)
-            json Devops::API2_0::Handler::Network.new(request).networks
+            json Devops::API2_0::Handler::Network.new(request).networks(provider)
           end
 
           puts "Network routes initialized"

devops-service/commands/stack.rb

@@ -7,8 +7,8 @@ module StackCommands
 
   def sync_stack_till_not_in_progress_proc
     lambda do |out, stack, mongo|
-      # two tries each 2 seconds, then 5 tries each 10 seconds, then 5 tries each minute.
+      # two tries each 4 seconds, then 5 tries each 10 seconds, then 5 tries each minute.
-      sleep_times = [2]*2 + [10]*5 + [60]*5
+      sleep_times = [4]*2 + [10]*5 + [60]*5
 
       begin
         out << "Syncing stack '#{stack.id}'...\n"
@@ -29,30 +29,8 @@ module StackCommands
     end
   end
 
-  def create_devops_servers_from_stack_resources
-    lambda do |out, stack, mongo, owner|
-      project = mongo.project(stack.project)
-      deploy_env = project.deploy_envs.detect {|env| env.identifier == stack.deploy_env}
-      provider = ::Provider::ProviderFactory.get(stack.provider)
-
-      stack.resources.each do |resource|
-        logical_name = resource['resource_name']
-        attrs = stack.resource(logical_name)
-        body = {
-          'name' => logical_name,
-          'key' => attrs['key_name']
-        }
-
-        server = extract_servers(provider, project, deploy_env, body, owner, mongo)
-        server.private_ip = attrs['addresses']['devops-net-1'].first['addr']
-
-      end
-    end
-  end
-
   CommandsStorage.add_job_lambda(
-    sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc,
+    sync_stack_till_not_in_progress: sync_stack_till_not_in_progress_proc
-    create_devops_servers_from_stack_resources: create_devops_servers_from_stack_resources
   )
 
 end

devops-service/db/mongo/connectors/helpers/insert_command.rb

@@ -18,6 +18,7 @@ module Connectors
         begin
           record.validate!
           collection.insert(record.to_mongo_hash)
+          record
         rescue Mongo::OperationFailure => e
           if e.message =~ /^11000/
             resource_name = StringHelper.underscore_class(record.class)

devops-service/db/mongo/connectors/helpers/update_command.rb

@@ -14,12 +14,15 @@ module Connectors
       end
 
       def update(record)
-        record.validate!
+        begin
-        collection.update({"_id" => record.id}, record.to_mongo_hash)
+          record.validate!
-      rescue Mongo::OperationFailure => e
+          collection.update({"_id" => record.id}, record.to_mongo_hash)
-        if e.message =~ /^11000/
+          record
-          resource_name = StringHelper.underscore_class(record.class)
+        rescue Mongo::OperationFailure => e
-          raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
+          if e.message =~ /^11000/
+            resource_name = StringHelper.underscore_class(record.class)
+            raise InvalidRecord.new("Duplicate key error: #{resource_name} with id '#{record.id}'")
+          end
         end
       end
 

devops-service/db/mongo/connectors/server.rb

@@ -63,6 +63,10 @@ module Connectors
       collection.update({'_id' => server.id}, {'$set' => {'chef_node_name' => server.chef_node_name}})
     end
 
+    def stack_servers_delete(stack_id)
+      collection.remove('stack' => stack_id)
+    end
+
     private
 
     def model_from_bson(bson)

devops-service/db/mongo/models/server.rb

@@ -19,7 +19,7 @@ module Devops
       define_hook :before_bootstrap
       define_hook :after_bootstrap
 
-      attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by
+      attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by, :reserved_by, :stack
       attr_accessor :options, :static, :key
 
       types :id => {:type => String, :empty => false},
@@ -32,7 +32,8 @@ module Devops
             :key => {:type => String, :empty => false},
             :created_by => {:type => String, :empty => false},
             :chef_node_name => {:type => String, :empty => true},
-            :reserved_by => {:type => String, :empty => true}
+            :reserved_by => {:type => String, :empty => true},
+            :stack => {:type => String, :nil => true}
 
       def self.fields
         ["chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "public_ip", "created_at", "created_by", "static", "key", "reserved_by"]
@@ -52,6 +53,7 @@ module Devops
         self.static = s["static"]
         self.key = s["key"]
         self.reserved_by = s["reserved_by"]
+        self.stack = s["stack"]
       end
 
       def create
@@ -80,7 +82,8 @@ module Devops
           "created_by" => self.created_by,
           "static" => self.static,
           "key" => self.key,
-          "reserved_by" => self.reserved_by
+          "reserved_by" => self.reserved_by,
+          "stack" => stack
         }.delete_if{|k,v| v.nil?}
       end
 

devops-service/db/mongo/models/stack/stack_base.rb

@@ -3,14 +3,15 @@ module Devops
     class StackBase < MongoModel
 
       attr_accessor :id, :project, :deploy_env, :stack_template,
-        :cloud_stack_id, :provider, :parameters, :details
+        :cloud_stack_id, :provider, :parameters, :details, :owner
 
       types id: {type: String, empty: false},
             provider: {type: String, empty: false},
             project: {type: String},
             deploy_env: {type: String},
             stack_template: {type: String, empty: false},
-            cloud_stack_id: {type: String, nil: true}
+            cloud_stack_id: {type: String, nil: true},
+            owner: {type: String}
             # details: {type: Hash, nil: true} # Hash type isn't supported yet
 
       def initialize attrs={}
@@ -23,6 +24,7 @@ module Devops
         self.cloud_stack_id = attrs['cloud_stack_id']
         self.parameters = attrs['parameters']
         self.details = attrs['details']
+        self.owner = attrs['owner']
         self
       end
 
@@ -35,7 +37,8 @@ module Devops
           cloud_stack_id: cloud_stack_id,
           parameters: parameters,
           details: details,
-          stack_status: stack_status
+          stack_status: stack_status,
+          owner: owner
         }
       end
 
@@ -52,7 +55,7 @@ module Devops
       end
 
       def sync_details!
-        self.details = provider_class.stack_details(self)
+        self.details = provider_class.stack_details(self).attributes
       end
 
       def resources
@@ -64,10 +67,6 @@ module Devops
         provider_class.stack_resource(self, resource_id)
       end
 
-      def template_body
-        Devops::Api2.settings.mongo.stack_template(stack_template).template_body
-      end
-
       def stack_status
         raise 'override me'
       end
@@ -80,6 +79,10 @@ module Devops
         }
       end
 
+      def template_body
+        Devops::Db.connector.stack_template(stack_template).template_body
+      end
+
       class << self
         attr_accessor :provider
 

devops-service/db/mongo/mongo_connector.rb

@@ -22,7 +22,7 @@ class MongoConnector
     :project_template_delete] => :projects_templates_connector,
     [:servers_find, :servers, :servers_by_names, :server_by_instance_id,
     :server_by_chef_node_name, :servers_by_key, :server_insert,
-    :server_delete, :server_update, :server_set_chef_node_name] => :servers_connector,
+    :server_delete, :server_update, :server_set_chef_node_name, :stack_servers_delete] => :servers_connector,
     [:user_auth, :user, :users, :users_names, :user_insert, :user_delete,
     :user_update, :create_root_user, :check_user_privileges] => :users_connector,
     [:keys, :key, :key_insert, :key_delete] => :keys_connector,

devops-service/providers/openstack.rb

@@ -210,7 +210,7 @@ module Provider
     end
 
     def stack_details(stack)
-      fog_stack(stack).details.attributes
+      fog_stack(stack).details
     end
 
     def stack_resources(stack)
@@ -219,7 +219,7 @@ module Provider
 
     def stack_resource(stack, resource_id)
       physical_id = fog_stack(stack).resources.get(resource_id).physical_resource_id
-      compute.servers.get(physical_id).attributes
+      compute.servers.get(physical_id)
     end
 
   private

devops-service/workers/job_starter.rb

@@ -11,7 +11,6 @@ module JobStarter
 
   def self.start_job_as_worker(worker_class, options)
     job_options = options.dup
-    job_options[:owner] ||= options[:request].env['REMOTE_USER']
     job_options[:config] ||= DevopsConfig.config
     job_options[:dir] ||= DevopsConfig[:report_dir_v2]
     job_options[:url] ||= options[:request].url

devops-service/workers/stack_sync_worker.rb

@@ -1,6 +1,7 @@
 require "providers/provider_factory"
 require "commands/stack"
 require "db/mongo/models/stack/stack_factory"
+require "db/mongo/models/project"
 require "db/mongo/models/report"
 
 class StackSyncWorker < Worker
@@ -13,20 +14,55 @@ class StackSyncWorker < Worker
     call(options['config'], options['provider'], options['dir']) do |provider, out, file|
       mongo = Devops::Db.connector
       stack = mongo.stack(options['stack_id'])
-      o = {
+      save_report(mongo, stack, file)
-        "file" => file,
-        "_id" => jid,
-        "created_by" => options['owner'],
-        "project" => stack.project,
-        "deploy_env" => stack.deploy_env,
-        "type" => ::Devops::Model::Report::STACK_TYPE
-      }
-      mongo.save_report(::Devops::Model::Report.new(o))
 
       status = sync_stack_till_not_in_progress_proc.call(out, stack, mongo)
+      persist_stack_servers_in_mongo!(mongo, stack, provider)
       status
     end
   end
+
+  private
+
+  def save_report(mongo, stack, file)
+    report = ::Devops::Model::Report.new(
+      "file" => file,
+      "_id" => jid,
+      "created_by" => stack.owner,
+      "project" => stack.project,
+      "deploy_env" => stack.deploy_env,
+      "type" => ::Devops::Model::Report::STACK_TYPE
+    )
+    mongo.save_report(report)
+  end
+
+  def persist_stack_servers_in_mongo!(mongo, stack, provider)
+    project = mongo.project(stack.project)
+    deploy_env = project.deploy_env(stack.deploy_env)
+
+    stack_servers = []
+
+    stack.resources.each do |resource|
+      logical_name = resource.resource_name
+      extended_info = stack.resource(logical_name)
+      body = {
+        'name' => logical_name,
+        'key' => extended_info.key_name
+      }
+
+      servers = extract_servers(provider, project, deploy_env, body, stack.owner, mongo)
+      servers.each do |server|
+        # TODO: improve IP assigning logic
+        server.private_ip = extended_info.addresses.values.first.first['addr']
+      end
+      stack_servers += servers
+    end
+    stack_servers.each_with_index do |server, i|
+      server.id = "stack-#{stack.id}-server-#{i+1}"
+      server.stack = stack.id
+      mongo.server_insert server
+    end
+  end
 end
 
 WorkersStorage.add_worker(sync_stack_till_not_in_progress: StackSyncWorker)