From 32d6a1da1a27b71051ff8b7f494c713e7a8dded0 Mon Sep 17 00:00:00 2001
From: Anton Chuchkalov <a2new@yandex.ru>
Date: Wed, 5 Aug 2015 18:26:21 +0300
Subject: [PATCH 1/3] get rid of extract_servers method using

---
 devops-service/db/mongo/models/server.rb      |  1 +
 .../workers/stack_bootstrap_worker.rb         | 44 +++++++++----------
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb
index eeb4a9b..5433bae 100644
--- a/devops-service/db/mongo/models/server.rb
+++ b/devops-service/db/mongo/models/server.rb
@@ -61,6 +61,7 @@ module Devops
         self.reserved_by = s["reserved_by"]
         self.stack = s["stack"]
         self.run_list = s["run_list"] || []
+        self
       end
 
       def create provider, image, flavor, subnets, groups, out
diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb
index 55114e8..84f4628 100644
--- a/devops-service/workers/stack_bootstrap_worker.rb
+++ b/devops-service/workers/stack_bootstrap_worker.rb
@@ -12,14 +12,12 @@ class StackBootstrapWorker < Worker
   #   :stack_id
   def perform(options)
     call(options['config'], options['provider'], options['dir']) do |provider, out, file|
-      mongo = Devops::Db.connector
+      mongo = ::Devops::Db.connector
       stack = mongo.stack(options['stack_id'])
       save_report(mongo, stack, file)
 
       sync_bootstrap_proc.call(out, stack, mongo)
-      persist_stack_servers_in_mongo!(mongo, stack, provider)
-      # logger = DevopsLogger.logger
-      # bootstrap_stack_servers_proc.call(STDOUT, stack, mongo, provider, logger)
+      persist_stack_servers!(stack, provider)
     end
   end
 
@@ -37,34 +35,32 @@ class StackBootstrapWorker < Worker
     mongo.save_report(report)
   end
 
-  def persist_stack_servers_in_mongo!(mongo, stack, provider)
+  def persist_stack_servers!(stack, provider)
+    mongo = ::Devops::Db.connector
     project = mongo.project(stack.project)
     deploy_env = project.deploy_env(stack.deploy_env)
 
-    stack_servers = []
-
     stack.resources.each do |resource|
-      logical_name = resource.resource_name
-      extended_info = stack.resource(logical_name)
-      body = {
-        'name' => logical_name,
-        'key' => extended_info.key_name
+      extended_info = stack.resource(resource.resource_name)
+      server_attrs = {
+        'provider' => provider.name,
+        'project' => project.id,
+        'deploy_env' => deploy_env.identifier,
+        'remote_user' => mongo.image(deploy_env.image).remote_user,
+        'key' => extended_info.key_name || provider.ssh_key,
+        'id' => extended_info.id,
+        'chef_node_name' => extended_info.name,
+        'private_ip' => extended_info.addresses.values.first.first['addr'],
+        'created_by' => stack.owner,
+        'run_list' => stack.run_list || [],
+        'stack' => stack.id
       }
 
-      servers = extract_servers(provider, project, deploy_env, body, stack.owner)
-      servers.each do |server|
-        # TODO: improve IP assigning logic
-        server.private_ip = extended_info.addresses.values.first.first['addr']
-        server.id = extended_info.id
-        server.chef_node_name = extended_info.name
-      end
-      stack_servers += servers
-    end
-    stack_servers.each_with_index do |server, i|
-      server.stack = stack.id
-      mongo.server_insert server
+      server = ::Devops::Model::Server.new(server_attrs)
+      mongo.server_insert(server)
     end
   end
+
 end
 
 WorkersStorage.add_worker(stack_bootstrap: StackBootstrapWorker)

From 944c8df0974b114d4484960927fcfbb8103364f9 Mon Sep 17 00:00:00 2001
From: Anton Martynov <toshkamar@gmail.com>
Date: Thu, 6 Aug 2015 12:37:56 +0300
Subject: [PATCH 2/3] some changes

---
 devops-service/app/api2/routes/deploy.rb | 15 +++++
 devops-service/app/api2/routes/report.rb |  2 +-
 devops-service/app/api2/routes/v2.0.rb   |  3 -
 devops-service/app/devops-api2.rb        |  1 +
 devops-service/auth/devops_auth.rb       |  2 -
 devops-service/commands/deploy.rb        |  2 +-
 devops-service/commands/deploy_env.rb    | 52 --------------
 devops-service/commands/server.rb        | 86 +++---------------------
 devops-service/commands/ssh.rb           | 21 ++++++
 9 files changed, 47 insertions(+), 137 deletions(-)
 delete mode 100644 devops-service/commands/deploy_env.rb

diff --git a/devops-service/app/api2/routes/deploy.rb b/devops-service/app/api2/routes/deploy.rb
index 496bbf8..81955a7 100644
--- a/devops-service/app/api2/routes/deploy.rb
+++ b/devops-service/app/api2/routes/deploy.rb
@@ -46,6 +46,21 @@ module Devops
 
           end
 
+          app.get "/deploy/data/:project/:env" do |project, env|
+            p = Devop::Db.connector.project project
+            data = p.to_chef_data(env)
+            content_type "application/json"
+            (JSON.pretty_generate data) << "\n"
+          end
+
+          app.get "/deploy/data/:file" do |file|
+            dir = DevopsConfig.config[:project_info_dir]
+            file_path = File.join(dir, file)
+            return [404, "Data for '#{file}' not found"] unless File.exists?(file_path)
+            content_type "application/json"
+            File.read(file_path) + "\n"
+          end
+
           puts "Deploy routes initialized"
         end
 
diff --git a/devops-service/app/api2/routes/report.rb b/devops-service/app/api2/routes/report.rb
index 461c36f..75f5007 100644
--- a/devops-service/app/api2/routes/report.rb
+++ b/devops-service/app/api2/routes/report.rb
@@ -24,7 +24,7 @@ module Devops
 
           app.get "/status/:id" do |id|
             r = Devops::API2_0::Handler::Report.new(request).status(id)
-            return [404, "Job with id '#{params[:id]}' not found"] if r.nil?
+            return [404, "Job with id '#{id}' not found"] if r.nil?
             r
           end
 
diff --git a/devops-service/app/api2/routes/v2.0.rb b/devops-service/app/api2/routes/v2.0.rb
index a8cd0de..ca25757 100644
--- a/devops-service/app/api2/routes/v2.0.rb
+++ b/devops-service/app/api2/routes/v2.0.rb
@@ -4,7 +4,6 @@ require "json"
 
 require "fog"
 
-require "auth/devops_auth"
 require "exceptions/invalid_record"
 require "exceptions/record_not_found"
 require "exceptions/dependency_error"
@@ -106,7 +105,6 @@ module Devops
         halt_response(e.message, 400)
       end
 
-=begin
       error ::Excon::Errors::Unauthorized do
         e = env["sinatra.error"]
         resp = e.response
@@ -124,7 +122,6 @@ module Devops
         end
         halt_response(msg, 500)
       end
-=end
 
       error Fog::Compute::AWS::Error do
         e = env["sinatra.error"]
diff --git a/devops-service/app/devops-api2.rb b/devops-service/app/devops-api2.rb
index a0baa8c..32ef054 100644
--- a/devops-service/app/devops-api2.rb
+++ b/devops-service/app/devops-api2.rb
@@ -3,6 +3,7 @@ module Devops
   class DevopsApi2Application < Application
 
     def prepare
+      require "auth/devops_auth"
       require_relative "api2/routes/v2.0"
       require_relative "api2/handlers/provider"
       require_relative "api2/handlers/flavor"
diff --git a/devops-service/auth/devops_auth.rb b/devops-service/auth/devops_auth.rb
index 2e84e53..4287452 100644
--- a/devops-service/auth/devops_auth.rb
+++ b/devops-service/auth/devops_auth.rb
@@ -30,6 +30,4 @@ module Sinatra
     end
 
   end
-
-  register Sinatra::DevopsAuth
 end
diff --git a/devops-service/commands/deploy.rb b/devops-service/commands/deploy.rb
index 42c0274..8b725a2 100644
--- a/devops-service/commands/deploy.rb
+++ b/devops-service/commands/deploy.rb
@@ -56,7 +56,7 @@ module DeployCommands
         f.write json
       end
       out.flush if out.respond_to?(:flush)
-      cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/chef/client/data/#{file}"
+      cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
     end
     ip = if server.public_ip.nil?
       server.private_ip
diff --git a/devops-service/commands/deploy_env.rb b/devops-service/commands/deploy_env.rb
deleted file mode 100644
index 411db1c..0000000
--- a/devops-service/commands/deploy_env.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-require "commands/image"
-
-module DeployEnvCommands
-
-  include ImageCommands
-
-  # All these commands should be removed when all deploy envs are switched to new validation system
-
-  def check_expires! val
-    raise InvalidRecord.new "Parameter 'expires' is invalid" if val.match(/^[0-9]+[smhdw]$/).nil?
-  end
-
-  def check_flavor! p, val
-    f = p.flavors.detect{|f| f["id"] == val}
-    raise InvalidRecord.new "Invalid flavor '#{val}'" if f.nil?
-  end
-
-  def check_image! p, val
-    images = get_images(::Devops::Db.connector, p.name)
-    raise InvalidRecord.new "Invalid image '#{val}'" unless images.map{|i| i["id"]}.include?(val)
-  end
-
-  def check_subnets_and_groups! p, subnets, groups
-    networks = p.networks
-    n = subnets - networks.map{|n| n["name"]}
-    raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty?
-
-    filter = yield(networks)
-=begin
-    if p.name == ::Provider::Ec2::PROVIDER
-      unless subnets.empty?
-        subnets = [ subnets[0] ] if subnets.size > 1
-        filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] }
-      end
-    elsif p.name == ::Provider::Openstack::PROVIDER
-      if subnets.empty?
-        raise InvalidRecord.new "Subnets array can not be empty"
-      end
-    end
-=end
-
-    g = groups - p.groups(filter).keys
-    raise InvalidRecord.new "Invalid groups '#{g.join("', '")}'" unless g.empty?
-  end
-
-  def check_users! val
-    users = ::Devops::Db.connector.users_names(val)
-    buf = val - users
-    raise InvalidRecord.new("Invalid users: '#{buf.join("', '")}'") unless buf.empty?
-  end
-
-end
diff --git a/devops-service/commands/server.rb b/devops-service/commands/server.rb
index a72dcf2..7dd1aff 100644
--- a/devops-service/commands/server.rb
+++ b/devops-service/commands/server.rb
@@ -106,84 +106,6 @@ module ServerCommands
     return status
   end
 
-  def extract_servers provider, project, env, params, user
-    mongo = ::Devops::Db.connector
-    flavors = provider.flavors
-    projects = {}
-    env_name = env.identifier
-    project_name = project.id
-    servers_info = []
-    if project.multi?
-      #TODO: fix multi project
-      images = {}
-      env.servers.each do |name, server|
-        images[server["image"]] = mongo.image(server["image"]) unless images.has_key?(server["image"])
-        flavor = flavors.detect {|f| f["name"] == server["flavor"]}
-        raise RecordNotFound.new("Flavor with name '#{server["flavor"]}' not found") if flavor.nil?
-        run_list = []
-        project_ids = server["subprojects"].map{|sp| sp["project_id"]}
-        db_subprojects = mongo.projects project_ids
-        ids = project_ids - db_subprojects.map{|sp| sp.id}
-        unless ids.empty?
-          return [400, "Subproject(s) '#{ids.join("', '")}' is/are not exists"]
-        end
-        server["subprojects"].each do |sp|
-          p = db_subprojects.detect{|db_sp| db_sp.id == sp["project_id"]}
-          run_list += p.deploy_env(sp["project_env"]).run_list
-        end
-        o = {
-          :image => images[server["image"]],
-          :name => "#{name}_#{Time.now.to_i}",
-          :flavor => flavor["id"],
-          :groups => server["groups"],
-          :run_list => run_list
-        }
-        servers_info.push(o)
-      end
-    else
-      i = mongo.image env.image
-      flavor = flavors.detect {|f| f["id"] == env.flavor}
-      raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil?
-      rl = Set.new
-      rl.merge(project.run_list).merge(env.run_list)
-      o = {
-        :image => i,
-        :name => params["name"],
-        :flavor => flavor["id"],
-        :groups => params["groups"] || env.groups,
-        :run_list => rl,
-        :subnets => env.subnets,
-        :key => params["key"]
-      }
-      servers_info.push(o)
-    end
-
-    servers = []
-    servers_info.each do |info|
-      image = info[:image]
-      s = Devops::Model::Server.new
-      s.provider = provider.name
-      s.project = project_name
-      s.deploy_env = env_name
-      s.run_list = params["run_list"] || []
-      s.remote_user = image.remote_user
-      s.chef_node_name = info[:name]
-      s.key = info[:key] || provider.ssh_key
-      # s.options = {
-      #   :image => image.id,
-      #   :flavor => info[:flavor],
-      #   :name => info[:name],
-      #   :groups => info[:groups],
-      #   :run_list => info[:run_list].merge(s.run_list),
-      #   :bootstrap_template => image.bootstrap_template,
-      #   :subnets => info[:subnets]
-      # }
-      s.created_by = user
-      servers.push s
-    end
-    return servers
-  end
-
   def delete_from_chef_server node_name
     {
       :chef_node => KnifeCommands.chef_node_delete(node_name),
@@ -191,6 +113,14 @@ module ServerCommands
     }
   end
 
+  def self.delete_etc_chef s, cert_path
+    cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
+    cmd += " sudo " unless s.remote_user == "root"
+    cmd += "rm -Rf /etc/chef"
+    r = `#{cmd}`
+    raise(r) unless $?.success?
+  end
+
   def check_server s
     KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
   end
diff --git a/devops-service/commands/ssh.rb b/devops-service/commands/ssh.rb
index db52855..64515c6 100644
--- a/devops-service/commands/ssh.rb
+++ b/devops-service/commands/ssh.rb
@@ -11,4 +11,25 @@ module SshCommands
     return res, $?
   end
 
+ def self.ssh_with_status out, cert_path, user, ip, remote_cmd
+    cmd = "ssh -t -i #{cert_path} #{user}@#{ip} \"#{(user == "root" ? remote_cmd : "sudo #{remote_cmd}")}\""
+    out << "\nCommand: '#{cmd}'\n"
+    out.flush if out.respond_to?(:flush)
+    status = nil
+    IO.popen(cmd + " 2>&1") do |c|
+      buf = ""
+      while line = c.gets do
+        out << line
+        buf = line
+      end
+      c.close
+      status = $?.to_i
+      r = buf.scan(/exit\scode\s([0-9]{1,3})/)[0]
+      unless r.nil?
+        status = r[0].to_i
+      end
+    end
+    status
+  end
+
 end

From 54144d62f5c44fda577a8b11efc6eea4962726cc Mon Sep 17 00:00:00 2001
From: Anton Martynov <toshkamar@gmail.com>
Date: Thu, 6 Aug 2015 14:02:10 +0300
Subject: [PATCH 3/3] some play with logger

---
 devops-service/app/client/devops-client.rb | 17 +++++++++++------
 devops-service/core/devops-logger.rb       | 20 ++++++++++++++------
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/devops-service/app/client/devops-client.rb b/devops-service/app/client/devops-client.rb
index 36ac9f0..4aaaa52 100644
--- a/devops-service/app/client/devops-client.rb
+++ b/devops-service/app/client/devops-client.rb
@@ -11,27 +11,32 @@ class Client < Sinatra::Base
 
   configure :production do
     config = DevopsConfig.config
-    log_file = File.join(config[:log_dir], "devops-client.log")
+    log_file = File.join(config[:log_dir], "devops-client.access.log")
     logger = DevopsLogger.create(log_file, Logger::INFO)
     use Rack::CommonLogger, logger
     disable :dump_errors
     disable :show_exceptions
-#    set :logging, Logger::INFO
     logger.info "Production mode"
   end
 
   configure :development do
-    config = DevopsConfig.config
-    log_file = File.join(config[:log_dir], "devops-client.log")
-    logger = DevopsLogger.create(log_file, Logger::DEBUG)
+#    config = DevopsConfig.config
+#    log_file = File.join(config[:log_dir], "devops-client.log")
+    logger = DevopsLogger.create(STDOUT, Logger::DEBUG)
     use Rack::CommonLogger, logger
-#    set :logging, Logger::DEBUG
     disable :raise_errors
 #      disable :dump_errors
     set :show_exceptions, :after_handler
     logger.info "Development mode"
+    DevopsLogger.level = Logger::DEBUG
   end
 
+=begin
+  before do
+    env["rack.errors"] =  error_logger
+  end
+=end
+
   # Route to download devops client
   get "/devops-client.gem" do
     begin
diff --git a/devops-service/core/devops-logger.rb b/devops-service/core/devops-logger.rb
index 39a8dcd..0060aab 100644
--- a/devops-service/core/devops-logger.rb
+++ b/devops-service/core/devops-logger.rb
@@ -3,18 +3,26 @@ require "logger"
 class DevopsLogger
 
   def self.create out, level=Logger::INFO, format='%a %d-%m-%Y %H%M '
-    @_logger = Logger.new out
-    @_logger.level = level
-    @_logger.datetime_format = format
-    @_logger.debug("Logger has been created")
-    @_logger
+    logger = Logger.new out
+    logger.level = level
+    logger.datetime_format = format
+    logger.debug("Logger has been created")
+    logger
   end
 
   def self.logger
-    @_logger
+    @_logger ||= DevopsLogger.create(File.join(DevopsConfig.config[:log_dir], "devops.errors.log"))
   end
 
   def self.logger= logger
     @_logger = logger
   end
+
+  def self.level= level
+    @level = level
+  end
+
+  def self.level
+    @level || Logger::INFO
+  end
 end