diff --git a/devops-service/app/api2/handlers/project.rb b/devops-service/app/api2/handlers/project.rb index 7a33657..5b52990 100644 --- a/devops-service/app/api2/handlers/project.rb +++ b/devops-service/app/api2/handlers/project.rb @@ -79,6 +79,7 @@ module Devops def add_or_update_deploy_env id, deploy_env project = Devops::Db.connector.project(id) env = parser.add_or_update_deploy_env + env.identifier = deploy_env if env.identifier.nil? begin db_env = project.deploy_env(deploy_env) unless env.identifier == deploy_env @@ -90,6 +91,7 @@ module Devops raise InvalidRecord.new("Can not change environment '#{deploy_env}' to '#{env.identifier}', environment '#{env.identifier}' already exist") unless deploy_env == env.identifier rescue RecordNotFound => e end + env.validate! project.delete_deploy_env(deploy_env) project.add_deploy_env(env) "Deploy environment '#{deploy_env}' has been updated in project '#{project.id}'" @@ -165,42 +167,56 @@ module Devops end def deploy_project_stream out, id + # check if project exist + project = Devops::Db.connector.project(id) deploy_env, servers = parser.deploy keys = {} - dbserver = servers(id, deploy_env, servers) + dbserver = Devops::Db.connector.servers(id, deploy_env, servers, true) out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") status = [] - servers.each do |s| - project = begin + deploy_info_buf = {} + dbservers.each do |s| + begin Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e out << e.message + "\n" status.push 2 next end - deploy_info = project.deploy_info(s.deploy_env, nil) + deploy_env_model = project.deploy_env(s.deploy_env) + deploy_info = if deploy_info_buf[s.deploy_env] + deploy_info_buf[s.deploy_env] + else + # мы не можем указать один build_number для всех окружений, поэтому nil + deploy_info_buf[s.deploy_env] = project.deploy_info(deploy_env_model, nil) + end status.push(Devops::Executor::ServerExecutor.new(s, out).deploy_server(deploy_info)) end status end def deploy_project id + # check if project exist + project_model = Devops::Db.connector.project(id) deploy_env, servers = parser.deploy files = [] - servers(id, deploy_env, servers).each do |s| + dbservers = Devops::Db.connector.servers(id, deploy_env, servers, true) + #out << (dbservers.empty? ? "No reserved servers to deploy\n" : "Deploy servers: '#{dbservers.map{|s| s.chef_node_name}.join("', '")}'\n") + deploy_info_buf = {} + dbservers.each do |s| begin Devops::Db.connector.check_project_auth s.project, s.deploy_env, parser.current_user rescue InvalidPrivileges, RecordNotFound => e next end - project_model = project(s.project) - deploy_env_model = project_model.deploy_env(deploy_env) - - puts '!!! WARNING !!!' - puts 'build_number is set to empty string' - build_number = '' - deploy_info = project_model.deploy_info(deploy_env_model, build_number) + deploy_env_model = project_model.deploy_env(s.deploy_env) + deploy_info = if deploy_info_buf[s.deploy_env] + deploy_info_buf[s.deploy_env] + else + # мы не можем указать один build_number для всех окружений, поэтому nil + deploy_info_buf[s.deploy_env] = project_model.deploy_info(deploy_env_model, nil) + end uri = Worker.start_async(DeployWorker, @request, server_attrs: s.to_hash, @@ -213,11 +229,6 @@ module Devops files end - def servers project_id, deploy_env, servers - project = Devops::Db.connector.project(project_id) - dbservers = Devops::Db.connector.servers(project_id, deploy_env, servers, true) - end - def archive_project id project = Devops::Db.connector.project(id) Devops::Db.connector.archive_project(id) diff --git a/devops-service/app/api2/parsers/project.rb b/devops-service/app/api2/parsers/project.rb index 1a799f4..c82d934 100644 --- a/devops-service/app/api2/parsers/project.rb +++ b/devops-service/app/api2/parsers/project.rb @@ -52,7 +52,6 @@ module Devops def update body = create_object_from_json_body - check_string(body["name"], "Parameter 'name' must be a not empty string", true, false) check_string(body["description"], "Parameter 'description' must be a string", true, true) #check_array(body["deploy_envs"], "Parameter 'deploy_envs' must be a not empty array of objects", Hash) rl = check_array(body["run_list"], "Parameter 'run_list' must be an array of string", String, true, true) diff --git a/devops-service/commands/stack.rb b/devops-service/commands/stack.rb index 862794d..3c45863 100644 --- a/devops-service/commands/stack.rb +++ b/devops-service/commands/stack.rb @@ -1,48 +1,54 @@ module StackCommands extend self + def self.result_codes + { + stack_rolled_back: 1, + unkown_status: 2, + timeout: 3, + error: 5 + } + end + + def self.result_code(code) + result_codes.fetch(code) + end + def sync_stack_proc lambda do |out, stack, mongo| - # two tries each 4 seconds, then 5 tries each 10 seconds, then 10 tries each 30 seconds. - sleep_times = [4]*2 + [10]*5 + [30]*10 + # 5 tries each 5 seconds, then 200 tries each 10 seconds + sleep_times = [5]*5 + [10]*200 begin out << "Syncing stack '#{stack.id}'...\n" sleep_times.each do |sleep_time| sleep sleep_time stack.sync_details! - case stack.stack_status - when 'CREATE_IN_PROGRESS' + case stack.stack_status + when 'CREATE_IN_PROGRESS' out << "." - out.flush - when 'CREATE_COMPLETE' + out.flush + when 'CREATE_COMPLETE' mongo.stack_update(stack) out << "\nStack '#{stack.id}' status is now #{stack.stack_status}\n" out.flush return 0 - when 'ROLLBACK_COMPLETE' + when 'ROLLBACK_COMPLETE' out << "\nStack '#{stack.id}' status is rolled back\n" - return 1 - else - out << "\nUnknown status: '#{stack.stack_status}'" - return 2 - end + return StackCommands.result_code(:stack_rolled_back) + else + out.puts "\nUnknown stack status: '#{stack.stack_status}'" + return StackCommands.result_code(:unkown_status) + end end + out.puts "Stack hasn't synced in #{sleep_times.inject(&:+)} seconds." + return StackCommands.result_code(:timeout) rescue StandardError => e logger.error e.message out << "Error: #{e.message}\n" - return 5 + return StackCommands.result_code(:error) end end end -=begin - def bootstrap_stack_servers_proc(out, stack, mongo, provider, logger) - mongo.stack_servers(stack.id).each do |server| - key = mongo.key(server.key) - two_phase_bootstrap(server, out, provider, mongo, key.path, logger) - end - end -=end - end diff --git a/devops-service/db/mongo/connectors/base.rb b/devops-service/db/mongo/connectors/base.rb index 0bc6499..9e5f762 100644 --- a/devops-service/db/mongo/connectors/base.rb +++ b/devops-service/db/mongo/connectors/base.rb @@ -6,10 +6,24 @@ require "exceptions/invalid_privileges" module Connectors class Base + # если хотим создавать индексы при старте приложения, нужно сначала создать коллекцию + def initialize db + names = db.collection_names + unless names.include?(self.collection_name) + db.create_collection(self.collection_name) + puts "Collection '#{self.collection_name}' has been created" + end + self.collection = db.collection(self.collection_name) + end + def create_indexes end + def collection_name + raise "owerride me" + end + # Yes, we can implement connectors without attr_accessor, storing collection directly # in instance variable like # @collection = db.collection('users') diff --git a/devops-service/db/mongo/connectors/filter.rb b/devops-service/db/mongo/connectors/filter.rb index 9cd4ba5..8fde9c1 100644 --- a/devops-service/db/mongo/connectors/filter.rb +++ b/devops-service/db/mongo/connectors/filter.rb @@ -2,7 +2,11 @@ module Connectors class Filter < Base def initialize(db) - self.collection = db.collection('filters') + super(db) + end + + def collection_name + 'filters' end def available_images provider diff --git a/devops-service/db/mongo/connectors/image.rb b/devops-service/db/mongo/connectors/image.rb index f681064..c3ecac8 100644 --- a/devops-service/db/mongo/connectors/image.rb +++ b/devops-service/db/mongo/connectors/image.rb @@ -9,7 +9,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('images') + super(db) + end + + def collection_name + 'images' end def images(provider=nil) diff --git a/devops-service/db/mongo/connectors/key.rb b/devops-service/db/mongo/connectors/key.rb index c2bfd9e..cdbd2c4 100644 --- a/devops-service/db/mongo/connectors/key.rb +++ b/devops-service/db/mongo/connectors/key.rb @@ -7,7 +7,11 @@ module Connectors Helpers::DeleteCommand def initialize(db) - self.collection = db.collection('keys') + super(db) + end + + def collection_name + 'keys' end def key(id, scope=nil) diff --git a/devops-service/db/mongo/connectors/project.rb b/devops-service/db/mongo/connectors/project.rb index 14e89b9..d601584 100644 --- a/devops-service/db/mongo/connectors/project.rb +++ b/devops-service/db/mongo/connectors/project.rb @@ -8,7 +8,11 @@ module Connectors def initialize(db) - @collection = db.collection('projects') + super(db) + end + + def collection_name + 'projects' end def is_project_exists?(project) @@ -104,7 +108,6 @@ module Connectors end def add_deploy_env_to_project id, env - env.validate! @collection.update({"_id" => id}, {'$push' => {deploy_envs: env.to_hash} }) end @@ -119,7 +122,7 @@ module Connectors end def project_update id, params - raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"] + #raise InvalidRecord.new("You can not change project name for '#{id}'.") if params["name"] keys = %w(run_list description) params.delete_if{|k,v| !keys.include?(k)} @collection.update({"_id" => id}, {'$set' => params }) diff --git a/devops-service/db/mongo/connectors/project_template.rb b/devops-service/db/mongo/connectors/project_template.rb index e461edd..11bcfbd 100644 --- a/devops-service/db/mongo/connectors/project_template.rb +++ b/devops-service/db/mongo/connectors/project_template.rb @@ -8,7 +8,11 @@ module Connectors def initialize(db) - @collection = db.collection('project_templates') + super(db) + end + + def collection_name + 'project_templates' end end end diff --git a/devops-service/db/mongo/connectors/report.rb b/devops-service/db/mongo/connectors/report.rb index 4c7075e..8c5b199 100644 --- a/devops-service/db/mongo/connectors/report.rb +++ b/devops-service/db/mongo/connectors/report.rb @@ -6,7 +6,11 @@ module Connectors Helpers::ListCommand def initialize(db) - self.collection = db.collection('reports') + super(db) + end + + def collection_name + 'reports' end def save_report r diff --git a/devops-service/db/mongo/connectors/server.rb b/devops-service/db/mongo/connectors/server.rb index bfebcc2..4ab4fea 100644 --- a/devops-service/db/mongo/connectors/server.rb +++ b/devops-service/db/mongo/connectors/server.rb @@ -5,7 +5,11 @@ module Connectors def initialize(db) - self.collection = db.collection('servers') + super(db) + end + + def collection_name + 'servers' end def servers_find(query, fields=nil) diff --git a/devops-service/db/mongo/connectors/stack.rb b/devops-service/db/mongo/connectors/stack.rb index 958ba16..feee768 100644 --- a/devops-service/db/mongo/connectors/stack.rb +++ b/devops-service/db/mongo/connectors/stack.rb @@ -7,7 +7,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('stacks') + super(db) + end + + def collection_name + 'stacks' end def stacks(options={}) diff --git a/devops-service/db/mongo/connectors/stack_template.rb b/devops-service/db/mongo/connectors/stack_template.rb index 172f1a1..51f8b4b 100644 --- a/devops-service/db/mongo/connectors/stack_template.rb +++ b/devops-service/db/mongo/connectors/stack_template.rb @@ -8,7 +8,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('stack_templates') + super(db) + end + + def collection_name + "stack_templates" end def stack_templates(provider=nil) diff --git a/devops-service/db/mongo/connectors/statictic.rb b/devops-service/db/mongo/connectors/statictic.rb index b797c23..83927f2 100644 --- a/devops-service/db/mongo/connectors/statictic.rb +++ b/devops-service/db/mongo/connectors/statictic.rb @@ -2,7 +2,11 @@ module Connectors class Statistic < Base def initialize(db) - self.collection = db.collection('statistic') + super(db) + end + + def collection_name + 'statistic' end def insert_statistic(user, path, method, body, response_code) @@ -45,7 +49,8 @@ module Connectors private def mongo_sort_order(order) - raise "Wrong sort order" unless %w(asc desc).include?(order) + # asc by default and if order id invalid value, it is not a reason for response with code 500 + #raise "Wrong sort order" unless %w(asc desc).include?(order) order == 'asc' ? 1 : -1 end diff --git a/devops-service/db/mongo/connectors/user.rb b/devops-service/db/mongo/connectors/user.rb index 526839d..5699811 100644 --- a/devops-service/db/mongo/connectors/user.rb +++ b/devops-service/db/mongo/connectors/user.rb @@ -8,7 +8,11 @@ module Connectors Helpers::UpdateCommand def initialize(db) - self.collection = db.collection('users') + super(db) + end + + def collection_name + 'users' end def user_auth user, password diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 5488c85..9db4c6f 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -35,6 +35,25 @@ module Devops @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush) end + def result_codes + self.class.result_codes + end + + def self.result_codes + { + server_bootstrap_fail: 2, + server_not_in_chef_nodes: 5 + } + end + + def self.result_code(code) + result_codes.fetch(code) + end + + def result_code(code) + self.class.result_code(code) + end + def report= r @report = r end @@ -95,7 +114,8 @@ module Devops DevopsLogger.logger.error e.message roll_back mongo.server_delete @server.id - return 5 + # return 5 + return result_code(:server_not_in_chef_nodes) end end @@ -127,12 +147,13 @@ module Devops cmd = 'ssh ' cmd << "-i #{cert_path} " cmd << '-q ' + cmd << '-o StrictHostKeyChecking=no ' cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 ' cmd << "#{address} 'exit'" cmd << " 2>&1" - @out << "\nWaiting for SSH..." - @out << "Test command: '#{cmd}'\n" + @out.puts "\nWaiting for SSH..." + @out.puts "Test command: '#{cmd}'" @out.flush retries_amount = 0 @@ -140,14 +161,17 @@ module Devops sleep(5) res = `#{cmd}` retries_amount += 1 - if retries_amount == max_retries_amount - @out << "\nCan not connect to #{address}" - @out << "\n" + res + if retries_amount > max_retries_amount + @out.puts "Can not connect to #{address}" + @out.puts res + @out.flush DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}" - return false + return result_code(:server_bootstrap_fail) end raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success? rescue ArgumentError => e + @out.puts "SSH command failed, retry (#{retries_amount}/#{max_retries_amount})" + @out.flush retry end @@ -195,7 +219,8 @@ module Devops else @out << roll_back mongo.server_delete @server.id - return 5 + # return 5 + return result_code(:server_not_in_chef_nodes) end @out << "\n" @out.flush @@ -284,7 +309,7 @@ module Devops cmd = "chef-client --no-color" if deploy_info["use_json_file"] deploy_info.delete("use_json_file") - @out << "Build information:\n" + @out << "Deploy Input Parameters:\n" json = JSON.pretty_generate(deploy_info) @out << json @out << "\n" @@ -394,7 +419,7 @@ module Devops private def max_retries_amount - 120 + 20 end def schedule_expiration(server) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 8060dcf..1c99df0 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -103,7 +103,14 @@ module Provider options = { "InstanceType" => flavor, # "Placement.AvailabilityZone" => s.options[:availability_zone], - "KeyName" => self.ssh_key + "KeyName" => self.ssh_key, + "Tags" => { + "Name" => s.chef_node_name, + "cid:project" => s.project, + "cid:deployEnv" => s.deploy_env, + "cid:user" => s.created_by, + "cid:remoteUser" => s.remote_user + } } vpcId = nil unless subnets.empty? @@ -150,7 +157,7 @@ module Provider end s.public_ip = details["ipAddress"] s.private_ip = details["privateIpAddress"] - set_server_tags(s) +# set_server_tags(s) out << "\nDone\n\n" out << s.info @@ -159,11 +166,6 @@ module Provider def set_server_tags s tags = { - "Name" => s.chef_node_name, - "cid:project" => s.project, - "cid:deployEnv" => s.deploy_env, - "cid:user" => s.created_by, - "cid:remoteUser" => s.remote_user } compute.create_tags(s.id, tags) end @@ -226,12 +228,18 @@ module Provider { 'TemplateBody' => stack.template_body, 'Parameters' => stack.parameters || {}, - 'Capabilities' => ['CAPABILITY_IAM'] + 'Capabilities' => ['CAPABILITY_IAM'], + 'Tags' => { + "StackName" => stack.name, + "cid:project" => stack.project, + "cid:deployEnv" => stack.deploy_env, + "cid:user" => stack.owner + } } ) stack.id = response.body['StackId'] out << "Stack id: #{stack.id}\n" - set_stack_tags(stack, out) + #set_stack_tags(stack, out) out.flush rescue Excon::Errors::Conflict => e raise ProviderErrors::NameConflict @@ -250,11 +258,7 @@ module Provider end def set_stack_tags stack, out="" - tags = { - "StackName" => stack.name, - "cid:project" => stack.project, - "cid:deployEnv" => stack.deploy_env, - "cid:user" => stack.owner + tags = { # "cid:remoteUser" => s.remote_user } #ids = stack_resources(stack).map {|resource| resource['PhysicalResourceId']} @@ -279,7 +283,11 @@ module Provider end def stack_details(stack) - cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'][0] + response = cloud_formation.describe_stacks({'StackName' => stack.name}).body['Stacks'] + # somewhy it sometimes returns blank results + response.detect do |t| + t.has_key?('StackName') + end end def stack_resources(stack) diff --git a/devops-service/workers/stack_bootstrap_worker.rb b/devops-service/workers/stack_bootstrap_worker.rb index f4f36bf..c509112 100644 --- a/devops-service/workers/stack_bootstrap_worker.rb +++ b/devops-service/workers/stack_bootstrap_worker.rb @@ -4,6 +4,10 @@ require "db/mongo/models/stack/stack_factory" require "db/mongo/models/project" require "db/mongo/models/report" +class StackCreatingError < StandardError; end +class BootstrapingStackServerError < StandardError; end +class DeployingStackServerError < StandardError; end + class StackBootstrapWorker < Worker include StackCommands @@ -12,31 +16,98 @@ class StackBootstrapWorker < Worker stack_attrs = options.fetch('stack_attributes') call(provider_name) do |provider, out, file| + @out = out without_bootstrap = stack_attrs.delete('without_bootstrap') report = save_report(file, stack_attrs) - stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, out) - mongo.stack_insert(stack) - - operation_result = sync_stack_proc.call(out, stack, mongo) - if operation_result == 0 - out << "\nStack '#{stack.name}' has been created\n" - out.flush - servers = persist_stack_servers!(stack, provider) - bootstrap_servers(servers, out, report) unless without_bootstrap + begin + stack = create_stack(provider_name, stack_attrs) + rescue StackCreatingError + return 1 + end + + begin + servers = persist_stack_servers!(stack, provider) + bootstrap_servers!(servers, report) unless without_bootstrap + rescue BootstrapingStackServerError + @out.puts "An error occured during bootstraping stack servers. Initiating stack rollback.\n\n" + rollback_stack!(stack) + 1 + rescue DeployingStackServerError => e + @out.puts "Stack was launched, but an error occured during deploying stack servers." + @out.puts "You can redeploy stack after fixing the error.\n\n" + 1 + rescue StandardError => e + @out.puts "An error occured. Initiating stack rollback.\n\n" + rollback_stack!(stack) + raise e end - operation_result end end private - def bootstrap_servers(servers, out, report) - out << "\n" - servers.map do |server| - executor = Devops::Executor::ServerExecutor.new(server, out) + def rollback_stack!(stack) + @out.puts "\nStart rollback of a stack" + stack.delete_stack_in_cloud! + Devops::Db.connector.stack_servers_delete(stack.name) + Devops::Db.connector.stack_delete(stack.id) + @out.puts "\nRollback has been completed" + end + + def create_stack(provider_name, stack_attrs) + stack = Devops::Model::StackFactory.create(provider_name, stack_attrs, @out) + mongo.stack_insert(stack) + operation_result = sync_stack_proc.call(@out, stack, mongo) + + if operation_result == 0 + @out.puts "\nStack '#{stack.name}' has been created" + @out.flush + stack + else + human_readable_code = StackCommands.result_codes.key(operation_result) + @out.puts "An error ocurred during stack creating" + @out.puts "Stack creating operation result was #{human_readable_code}" + raise StackCreatingError + end + end + + def bootstrap_servers!(servers, report) + @out << "\nStart bootstraping stack servers\n" + bootstraping_results = {} + servers.each do |server| + executor = Devops::Executor::ServerExecutor.new(server, @out) executor.report = report - executor.two_phase_bootstrap({}) + bootstraping_results[server.chef_node_name] = executor.two_phase_bootstrap({}) + end + check_bootstrap_results!(bootstraping_results) + end + + def check_bootstrap_results!(results) + if results.values.all?(&:zero?) + # everything is OK + @out.puts "Stack servers have been bootstraped" + @out.flush + return 0 + end + + results.each do |chef_node_name, code| + human_readable_code = Devops::Executor::ServerExecutor.result_codes.key(code) + @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}" + end + + if errors_in_bootstrapping_present?(results) + # An error occured during servers bootsraping, so rollback stack + raise BootstrapingStackServerError + else + # An error occured during servers deploying, rollback isn't needed + raise DeployingStackServerError + end + end + + def errors_in_bootstrapping_present?(results) + results.values.any? do |result| + result == Devops::Executor::ServerExecutor.result_code(:server_bootstrap_fail) end end @@ -54,10 +125,12 @@ class StackBootstrapWorker < Worker end def persist_stack_servers!(stack, provider) + @out.puts "Start syncing stack servers with CID" + @out.flush project = mongo.project(stack.project) deploy_env = project.deploy_env(stack.deploy_env) - provider.stack_servers(stack).map do |extended_info| + servers = provider.stack_servers(stack).map do |extended_info| server_attrs = { 'provider' => provider.name, 'project' => project.id, @@ -78,6 +151,9 @@ class StackBootstrapWorker < Worker # server.chef_node_name = provider.create_default_chef_node_name(server) server end + @out.puts "Stack servers have been synced with CID" + @out.flush + servers end end