From 947c7cdac4b35091b70c6794476afc6f3501df3c Mon Sep 17 00:00:00 2001 From: Anton Martynov Date: Fri, 18 Sep 2015 10:32:50 +0300 Subject: [PATCH 01/11] #731: fixed --- devops-service/app/api2/handlers/project.rb | 43 +++++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index 7a33657..f69e568 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -165,42 +165,56 @@ module Devops end def deploy_project_stream out, id + # check if project exist + project = Devops::Db.connector.project(id) deploy_env, servers = parser.deploy keys = {} - dbserver = servers(id, deploy_env, servers) + dbserver = Devops::Db.connector.servers(id, deploy_env, servers, true) out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") status = [] - servers.each do |s| - project = begin + deploy_info_buf = {} + dbservers.each do |s| + begin Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e out << e.message + "\n" status.push 2 next end - deploy_info = project.deploy_info(s.deploy_env, nil) + deploy_env_model = project.deploy_env(s.deploy_env) + deploy_info = if deploy_info_buf[s.deploy_env] + deploy_info_buf[s.deploy_env] + else + # мы не можем указать один build_number для всех окружений, поэтому nil + deploy_info_buf[s.deploy_env] = project.deploy_info(deploy_env_model, nil) + end status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info)) end status end def deploy_project id + # check if project exist + project_model = Devops::Db.connector.project(id) deploy_env, servers = parser.deploy files = [] - servers(id, deploy_env, servers).each do |s| + dbservers = Devops::Db.connector.servers(id, deploy_env, servers, true) + out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") + deploy_info_buf = {} + dbservers.each do |s| begin Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e next end - project_model = project(s.project) - deploy_env_model = project_model.deploy_env(deploy_env) - - puts '!!! WARNING !!!' - puts 'build_number is set to empty string' - build_number = '' - deploy_info = project_model.deploy_info(deploy_env_model, build_number) + deploy_env_model = project_model.deploy_env(s.deploy_env) + deploy_info = if deploy_info_buf[s.deploy_env] + deploy_info_buf[s.deploy_env] + else + # мы не можем указать один build_number для всех окружений, поэтому nil + deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil) + end uri = Worker.start_async(DeployWorker, @request, server_attrs: s.to_hash, @@ -213,11 +227,6 @@ module Devops files end - def servers project_id, deploy_env, servers - project = Devops::Db.connector.project(project_id) - dbservers = Devops::Db.connector.servers(project_id, deploy_env, servers, true) - end - def archive_project id project = Devops::Db.connector.project(id) Devops::Db.connector.archive_project(id) From 7b1a2c1dc56d966a8376d4360bc641393857ce7e Mon Sep 17 00:00:00 2001 From: Anton Martynov Date: Fri, 18 Sep 2015 12:28:16 +0300 Subject: [PATCH 02/11] fixed error with indexes if collection does not exist --- devops-service/db/mongo/connectors/base.rb | 14 ++++++++++++++ devops-service/db/mongo/connectors/filter.rb | 6 +++++- devops-service/db/mongo/connectors/image.rb | 6 +++++- devops-service/db/mongo/connectors/key.rb | 6 +++++- devops-service/db/mongo/connectors/project.rb | 6 +++++- .../db/mongo/connectors/project_template.rb | 6 +++++- devops-service/db/mongo/connectors/report.rb | 6 +++++- devops-service/db/mongo/connectors/server.rb | 6 +++++- devops-service/db/mongo/connectors/stack.rb | 6 +++++- .../db/mongo/connectors/stack_template.rb | 6 +++++- devops-service/db/mongo/connectors/statictic.rb | 9 +++++++-- devops-service/db/mongo/connectors/user.rb | 6 +++++- 12 files changed, 71 insertions(+), 12 deletions(-) diff --git a/devops-service/db/mongo/connectors/base.rb b/devops-service/db/mongo/connectors/base.rb index 0bc6499..9e5f762 100644 --- a/devops-service/db/mongo/connectors/base.rb +++ b/devops-service/db/mongo/connectors/base.rb @@ -6,10 +6,24 @@ require "exceptions/invalid_privileges" module Connectors class Base + # если хотим создавать индексы при старте приложения, нужно сначала создать коллекцию + def initialize db + names = db.collection_names + unless names.include?(self.collection_name) + db.create_collection(self.collection_name) + puts "Collection '#{self.collection_name}' has been created" + end + self.collection = db.collection(self.collection_name) + end + def create_indexes end + def collection_name + raise "owerride me" + end + # Yes, we can implement connectors without attr_accessor, storing collection directly # in instance variable like # @collection = db.collection('users') diff --git a/devops-service/db/mongo/connectors/filter.rb b/devops-service/db/mongo/connectors/filter.rb index 9cd4ba5..8fde9c1 100644 --- a/devops-service/db/mongo/connectors/filter.rb +++ b/devops-service/db/mongo/connectors/filter.rb @@ -2,7 +2,11 @@ module Connectors class Filter < Base def initialize(db) - self.collection = db.collection('filters') + super(db) + end + + def collection_name + 'filters' end def available_images provider diff --git a/devops-service/db/mongo/connectors/image.rb b/devops-service/db/mongo/connectors/image.rb index f681064..c3ecac8 100644 --- a/devops-service/db/mongo/connectors/image.rb +++ b/devops-service/db/mongo/connectors/image.rb @@ -9,7 +9,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('images') + super(db) + end + + def collection_name + 'images' end def images(provider=nil) diff --git a/devops-service/db/mongo/connectors/key.rb b/devops-service/db/mongo/connectors/key.rb index c2bfd9e..cdbd2c4 100644 --- a/devops-service/db/mongo/connectors/key.rb +++ b/devops-service/db/mongo/connectors/key.rb @@ -7,7 +7,11 @@ module Connectors Helpers::DeleteCommand def initialize(db) - self.collection = db.collection('keys') + super(db) + end + + def collection_name + 'keys' end def key(id, scope=nil) diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index 14e89b9..058ebff 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -8,7 +8,11 @@ module Connectors def initialize(db) - @collection = db.collection('projects') + super(db) + end + + def collection_name + 'projects' end def is_project_exists?(project) diff --git a/devops-service/db/mongo/connectors/project_template.rb b/devops-service/db/mongo/connectors/project_template.rb index e461edd..11bcfbd 100644 --- a/devops-service/db/mongo/connectors/project_template.rb +++ b/devops-service/db/mongo/connectors/project_template.rb @@ -8,7 +8,11 @@ module Connectors def initialize(db) - @collection = db.collection('project_templates') + super(db) + end + + def collection_name + 'project_templates' end end end diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb index 4c7075e..8c5b199 100644 --- a/devops-service/db/mongo/connectors/report.rb +++ b/devops-service/db/mongo/connectors/report.rb @@ -6,7 +6,11 @@ module Connectors Helpers::ListCommand def initialize(db) - self.collection = db.collection('reports') + super(db) + end + + def collection_name + 'reports' end def save_report r diff --git a/devops-service/db/mongo/connectors/server.rb b/devops-service/db/mongo/connectors/server.rb index bfebcc2..4ab4fea 100644 --- a/devops-service/db/mongo/connectors/server.rb +++ b/devops-service/db/mongo/connectors/server.rb @@ -5,7 +5,11 @@ module Connectors def initialize(db) - self.collection = db.collection('servers') + super(db) + end + + def collection_name + 'servers' end def servers_find(query, fields=nil) diff --git a/devops-service/db/mongo/connectors/stack.rb b/devops-service/db/mongo/connectors/stack.rb index 958ba16..feee768 100644 --- a/devops-service/db/mongo/connectors/stack.rb +++ b/devops-service/db/mongo/connectors/stack.rb @@ -7,7 +7,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('stacks') + super(db) + end + + def collection_name + 'stacks' end def stacks(options={}) diff --git a/devops-service/db/mongo/connectors/stack_template.rb b/devops-service/db/mongo/connectors/stack_template.rb index 172f1a1..51f8b4b 100644 --- a/devops-service/db/mongo/connectors/stack_template.rb +++ b/devops-service/db/mongo/connectors/stack_template.rb @@ -8,7 +8,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('stack_templates') + super(db) + end + + def collection_name + "stack_templates" end def stack_templates(provider=nil) diff --git a/devops-service/db/mongo/connectors/statictic.rb b/devops-service/db/mongo/connectors/statictic.rb index b797c23..83927f2 100644 --- a/devops-service/db/mongo/connectors/statictic.rb +++ b/devops-service/db/mongo/connectors/statictic.rb @@ -2,7 +2,11 @@ module Connectors class Statistic < Base def initialize(db) - self.collection = db.collection('statistic') + super(db) + end + + def collection_name + 'statistic' end def insert_statistic(user, path, method, body, response_code) @@ -45,7 +49,8 @@ module Connectors private def mongo_sort_order(order) - raise "Wrong sort order" unless %w(asc desc).include?(order) + # asc by default and if order id invalid value, it is not a reason for response with code 500 + #raise "Wrong sort order" unless %w(asc desc).include?(order) order == 'asc' ? 1 : -1 end diff --git a/devops-service/db/mongo/connectors/user.rb b/devops-service/db/mongo/connectors/user.rb index 526839d..5699811 100644 --- a/devops-service/db/mongo/connectors/user.rb +++ b/devops-service/db/mongo/connectors/user.rb @@ -8,7 +8,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('users') + super(db) + end + + def collection_name + 'users' end def user_auth user, password From ddae928c30d5f043b110b78ca148ece7fe65a1c8 Mon Sep 17 00:00:00 2001 From: Anton Martynov Date: Fri, 18 Sep 2015 12:31:33 +0300 Subject: [PATCH 03/11] Build information -> Deploy Input Parameters --- devops-service/lib/executors/server_executor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 5488c85..7cd0490 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -284,7 +284,7 @@ module Devops cmd = "chef-client --no-color" if deploy_info["use_json_file"] deploy_info.delete("use_json_file") - @out << "Build information:\n" + @out << "Deploy Input Parameters:\n" json = JSON.pretty_generate(deploy_info) @out << json @out << "\n" From 17b64b24227a1a821ef73c5f749ef9b350603447 Mon Sep 17 00:00:00 2001 From: Anton Martynov Date: Fri, 18 Sep 2015 16:50:08 +0300 Subject: [PATCH 04/11] #731: fixed --- devops-service/app/api2/handlers/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index f69e568..591df96 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -199,7 +199,7 @@ module Devops deploy_env, servers = parser.deploy files = [] dbservers = Devops::Db.connector.servers(id, deploy_env, servers, true) - out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") + #out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") deploy_info_buf = {} dbservers.each do |s| begin From 08f826586a1f3041b34311bc41d70db0c7963688 Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 11:53:40 +0300 Subject: [PATCH 05/11] #739: fixed --- devops-service/app/api2/parsers/project.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/devops-service/app/api2/parsers/project.rb b/devops-service/app/api2/parsers/project.rb index 1a799f4..c82d934 100644 --- a/devops-service/app/api2/parsers/project.rb +++ b/devops-service/app/api2/parsers/project.rb @@ -52,7 +52,6 @@ module Devops def update body = create_object_from_json_body - check_string(body["name"], "Parameter 'name' must be a not empty string", true, false) check_string(body["description"], "Parameter 'description' must be a string", true, true) #check_array(body["deploy_envs"], "Parameter 'deploy_envs' must be a not empty array of objects", Hash) rl = check_array(body["run_list"], "Parameter 'run_list' must be an array of string", String, true, true) From f922c689e20a928b786af75c5b4c76e21c407c37 Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 11:56:01 +0300 Subject: [PATCH 06/11] #739: without name parameter --- devops-service/db/mongo/connectors/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index 058ebff..fa453c6 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -123,7 +123,7 @@ module Connectors end def project_update id, params - raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"] + #raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"] keys = %w(run_list description) params.delete_if{|k,v| !keys.include?(k)} @collection.update({"_id" => id}, {'$set' => params }) From 1cccd2b5e773823769cdb9d58220b267abeb07a9 Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 12:44:29 +0300 Subject: [PATCH 07/11] #742: fixed --- devops-service/app/api2/handlers/project.rb | 2 ++ devops-service/db/mongo/connectors/project.rb | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index 591df96..5b52990 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -79,6 +79,7 @@ module Devops def add_or_update_deploy_env id, deploy_env project = Devops::Db.connector.project(id) env = parser.add_or_update_deploy_env + env.identifier = deploy_env if env.identifier.nil? begin db_env = project.deploy_env(deploy_env) unless env.identifier == deploy_env @@ -90,6 +91,7 @@ module Devops raise InvalidRecord.new("Can not change environment '#{deploy_env}' to '#{env.identifier}', environment '#{env.identifier}' already exist") unless deploy_env == env.identifier rescue RecordNotFound => e end + env.validate! project.delete_deploy_env(deploy_env) project.add_deploy_env(env) "Deploy environment '#{deploy_env}' has been updated in project '#{project.id}'" diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index fa453c6..d601584 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -108,7 +108,6 @@ module Connectors end def add_deploy_env_to_project id, env - env.validate! @collection.update({"_id" => id}, {'$push' => {deploy_envs: env.to_hash} }) end From 2ac1ce76c23e75d04629a358164bafdf09e2cdcd Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 13:34:08 +0300 Subject: [PATCH 08/11] test for tags --- devops-service/providers/ec2.rb | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 8060dcf..5da703d 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -103,7 +103,14 @@ module Provider options = { "InstanceType" => flavor, # "Placement.AvailabilityZone" => s.options[:availability_zone], - "KeyName" => self.ssh_key + "KeyName" => self.ssh_key, + "Tags" => { + "Name" => s.chef_node_name, + "cid:project" => s.project, + "cid:deployEnv" => s.deploy_env, + "cid:user" => s.created_by, + "cid:remoteUser" => s.remote_user + } } vpcId = nil unless subnets.empty? @@ -150,7 +157,7 @@ module Provider end s.public_ip = details["ipAddress"] s.private_ip = details["privateIpAddress"] - set_server_tags(s) +# set_server_tags(s) out << "\nDone\n\n" out << s.info @@ -159,11 +166,6 @@ module Provider def set_server_tags s tags = { - "Name" => s.chef_node_name, - "cid:project" => s.project, - "cid:deployEnv" => s.deploy_env, - "cid:user" => s.created_by, - "cid:remoteUser" => s.remote_user } compute.create_tags(s.id, tags) end @@ -226,12 +228,18 @@ module Provider { 'TemplateBody' => stack.template_body, 'Parameters' => stack.parameters || {}, - 'Capabilities' => ['CAPABILITY_IAM'] + 'Capabilities' => ['CAPABILITY_IAM'], + 'Tags' => { + "StackName" => stack.name, + "cid:project" => stack.project, + "cid:deployEnv" => stack.deploy_env, + "cid:user" => stack.owner + } } ) stack.id = response.body['StackId'] out << "Stack id: #{stack.id}\n" - set_stack_tags(stack, out) + #set_stack_tags(stack, out) out.flush rescue Excon::Errors::Conflict => e raise ProviderErrors::NameConflict @@ -250,11 +258,7 @@ module Provider end def set_stack_tags stack, out="" - tags = { - "StackName" => stack.name, - "cid:project" => stack.project, - "cid:deployEnv" => stack.deploy_env, - "cid:user" => stack.owner + tags = { # "cid:remoteUser" => s.remote_user } #ids = stack_resources(stack).map {|resource| resource['PhysicalResourceId']} From 324992a13e7d3e49f10874933ab4ed70b167a392 Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 17:34:20 +0300 Subject: [PATCH 09/11] server without Tags, some tests for stack --- .../db/mongo/models/stack/stack_ec2.rb | 3 ++- devops-service/providers/ec2.rb | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/devops-service/db/mongo/models/stack/stack_ec2.rb b/devops-service/db/mongo/models/stack/stack_ec2.rb index fcfec56..03ae847 100644 --- a/devops-service/db/mongo/models/stack/stack_ec2.rb +++ b/devops-service/db/mongo/models/stack/stack_ec2.rb @@ -8,7 +8,8 @@ module Devops end def stack_status - details['StackStatus'] if details + puts self.details.inspect + self.details['StackStatus'] if self.details end def bson_safe_details diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 5da703d..74bdaa1 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -103,14 +103,7 @@ module Provider options = { "InstanceType" => flavor, # "Placement.AvailabilityZone" => s.options[:availability_zone], - "KeyName" => self.ssh_key, - "Tags" => { - "Name" => s.chef_node_name, - "cid:project" => s.project, - "cid:deployEnv" => s.deploy_env, - "cid:user" => s.created_by, - "cid:remoteUser" => s.remote_user - } + "KeyName" => self.ssh_key } vpcId = nil unless subnets.empty? @@ -157,7 +150,8 @@ module Provider end s.public_ip = details["ipAddress"] s.private_ip = details["privateIpAddress"] -# set_server_tags(s) + tags = set_server_tags(s) + out << "Server tags: #{tags.inspect}\n" out << "\nDone\n\n" out << s.info @@ -166,8 +160,14 @@ module Provider def set_server_tags s tags = { + "Name" => s.chef_node_name, + "cid:project" => s.project, + "cid:deployEnv" => s.deploy_env, + "cid:user" => s.created_by, + "cid:remoteUser" => s.remote_user } compute.create_tags(s.id, tags) + tags end def delete_server s @@ -283,7 +283,10 @@ module Provider end def stack_details(stack) - cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'][0] + #cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'][0] + b = cloud_formation.describe_stacks({'StackName' => stack.name}).body + pp b + b['Stacks'][0] end def stack_resources(stack) From 1129079627e5b461d7636d7967ccfd749c35b7de Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 17:41:40 +0300 Subject: [PATCH 10/11] some new lines --- devops-service/lib/executors/server_executor.rb | 2 +- devops-service/providers/ec2.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 7cd0490..ab84f27 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -132,7 +132,7 @@ module Devops cmd << " 2>&1" @out << "\nWaiting for SSH..." - @out << "Test command: '#{cmd}'\n" + @out << "\nTest command: '#{cmd}'\n" @out.flush retries_amount = 0 diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 74bdaa1..0d09b38 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -151,7 +151,7 @@ module Provider s.public_ip = details["ipAddress"] s.private_ip = details["privateIpAddress"] tags = set_server_tags(s) - out << "Server tags: #{tags.inspect}\n" + out << "\nServer tags: #{tags.inspect}\n" out << "\nDone\n\n" out << s.info From b2c666ea95581341f5322f23b0d216430c4f901e Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 21 Sep 2015 18:00:21 +0300 Subject: [PATCH 11/11] fixed stack status --- devops-service/providers/ec2.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 0d09b38..d69d996 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -285,8 +285,7 @@ module Provider def stack_details(stack) #cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'][0] b = cloud_formation.describe_stacks({'StackName' => stack.name}).body - pp b - b['Stacks'][0] + b['Stacks'].detect{|s| s.key?("StackStatus")} || {} end def stack_resources(stack)