From 231c79ab4878dc6d3270c0cf8ffa8d87b7e1420c Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Fri, 4 Mar 2016 17:50:01 +0300 Subject: [PATCH 1/6] CID-443: rollbacks servers with failed bootstrap --- devops-service/lib/executors/server_executor.rb | 6 +++--- devops-service/spec/executors/server_executor_spec.rb | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 7cf84e8..520d4d6 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -262,8 +262,8 @@ module Devops return error_code(:server_not_in_chef_nodes) end else - # @out << roll_back - # mongo.server_delete @server.id + roll_back + mongo.server_delete @server.id msg = "Failed while bootstraping server with id '#{@server.id}'\n" msg << "Bootstraping operation result was #{bootstrap_status}" DevopsLogger.logger.error msg @@ -510,7 +510,7 @@ module Devops def schedule_expiration if @deploy_env.expires - @out << "Planning expiration in #{@deploy_env.expires}" + puts_and_flush "Planning expiration in #{@deploy_env.expires}" ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration! end end diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb index db48152..ca2be93 100644 --- a/devops-service/spec/executors/server_executor_spec.rb +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -366,10 +366,19 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec end context "when bootstrap wasn't successful" do - it 'returns :server_bootstrap_fail error code' do + before do allow(executor).to receive(:bootstrap) { 1 } + allow(executor).to receive(:roll_back) + end + + it 'returns :server_bootstrap_fail error code' do expect(two_phase_bootstrap).to eq 2 end + + it 'rollbacks server' do + allow(executor).to receive(:roll_back) + two_phase_bootstrap + end end context 'when an error occured during bootstrap' do From 02e92be1cd31744d93fa0c5143424e15b5b3e519 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 9 Mar 2016 15:44:57 +0300 Subject: [PATCH 2/6] CID-449: fix expiration worker --- .../lib/executors/expiration_scheduler.rb | 2 +- .../lib/executors/server_executor.rb | 4 ++-- .../executors/expiration_scheduler_spec.rb | 2 +- .../spec/executors/server_executor_spec.rb | 2 +- .../workers/delete_expired_server_worker.rb | 19 +++++++------------ 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/devops-service/lib/executors/expiration_scheduler.rb b/devops-service/lib/executors/expiration_scheduler.rb index d84702d..33cc2e7 100644 --- a/devops-service/lib/executors/expiration_scheduler.rb +++ b/devops-service/lib/executors/expiration_scheduler.rb @@ -9,7 +9,7 @@ module Devops def schedule_expiration! return unless @expires - DeleteExpiredServerWorker.perform_in(interval_in_seconds, server_chef_node_name: @server.chef_node_name) + DeleteExpiredServerWorker.perform_in(interval_in_seconds, server_id: @server.id) end def interval_in_seconds diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index 54090d4..a31d471 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -511,8 +511,8 @@ module Devops def schedule_expiration if @deploy_env.expires - puts_and_flush "Planning expiration in #{@deploy_env.expires}" - ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration! + job_id = ExpirationScheduler.new(@deploy_env.expires, @server).schedule_expiration! + puts_and_flush "Planning expiration in #{@deploy_env.expires}, job_id: #{job_id}" end end diff --git a/devops-service/spec/executors/expiration_scheduler_spec.rb b/devops-service/spec/executors/expiration_scheduler_spec.rb index f8009f8..c3882aa 100644 --- a/devops-service/spec/executors/expiration_scheduler_spec.rb +++ b/devops-service/spec/executors/expiration_scheduler_spec.rb @@ -5,7 +5,7 @@ RSpec.describe Devops::Executor::ExpirationScheduler do describe '#schedule_expiration!' do it 'schedules server deleting at given time' do - expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, server_chef_node_name: 'chef_node_name') + expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, server_id: server.id) described_class.new('2m', server).schedule_expiration! end diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb index 7c57f75..d3ff57a 100644 --- a/devops-service/spec/executors/server_executor_spec.rb +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -140,7 +140,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec it 'schedules expiration for server' do deploy_env.expires = '2m' allow(DeleteExpiredServerWorker).to receive(:perform_in) - expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, {server_chef_node_name: 'node_name'}) + expect(DeleteExpiredServerWorker).to receive(:perform_in).with(120, hash_including(:server_id)) create_server end diff --git a/devops-service/workers/delete_expired_server_worker.rb b/devops-service/workers/delete_expired_server_worker.rb index a749e73..0d2ee7f 100644 --- a/devops-service/workers/delete_expired_server_worker.rb +++ b/devops-service/workers/delete_expired_server_worker.rb @@ -6,12 +6,11 @@ require "workers/worker" class DeleteExpiredServerWorker < Worker def perform(options) - chef_node_name = options.fetch('server_chef_node_name') - - call() do |out, file| - out.puts "Expire server '#{chef_node_name}'." - server = mongo.server_by_chef_node_name(chef_node_name) - report = save_report(file, server) + call do + server_id = options.fetch('server_id') + puts_and_flush "Expire server '#{server_id}'." + server = mongo.server_by_instance_id(server_id) + report = save_report(server) e = Devops::Executor::ServerExecutor.new(server, out) e.report = report @@ -21,17 +20,13 @@ class DeleteExpiredServerWorker < Worker private - def save_report(file, server) - report = Devops::Model::Report.new( - "file" => file, - "_id" => jid, + def save_report(server) + update_report( "created_by" => 'SYSTEM', "project" => server.project, "deploy_env" => server.deploy_env, "type" => Devops::Model::Report::EXPIRE_SERVER_TYPE ) - mongo.save_report(report) - report end end From e5561ab5f7f1e01b0d12b79a22d29862f57be931 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 9 Mar 2016 17:21:46 +0300 Subject: [PATCH 3/6] CID-457: regard provider account on stack servers persisting --- .../spec/workers/stack_servers_persister_spec.rb | 11 ++++++++++- .../stack_bootstrap/stack_servers_persister.rb | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/devops-service/spec/workers/stack_servers_persister_spec.rb b/devops-service/spec/workers/stack_servers_persister_spec.rb index 93a7cb0..e7e9a76 100644 --- a/devops-service/spec/workers/stack_servers_persister_spec.rb +++ b/devops-service/spec/workers/stack_servers_persister_spec.rb @@ -3,7 +3,7 @@ require 'workers/stack_bootstrap/stack_servers_persister' RSpec.describe StackServersPersister, stubbed_connector: true do let(:out) { double(:out, puts: nil, flush: nil) } let(:run_list) { ['role[asd]'] } - let(:stack) { build(:stack, deploy_env: 'foo', run_list: run_list) } + let(:stack) { build(:stack_ec2, deploy_env: 'foo', run_list: run_list) } let(:project) { build(:project, id: 'name') } let(:persister) { described_class.new(stack, out) } let(:provider) { instance_double(Provider::Ec2, name: 'ec2') } @@ -113,6 +113,15 @@ RSpec.describe StackServersPersister, stubbed_connector: true do persister.persist end + it "sets provider and provider account from stack" do + stack.provider_account = 'foo' + expect(stubbed_connector).to receive(:server_insert) do |server| + expect(server.provider).to eq 'ec2' + expect(server.provider_account).to eq 'foo' + end + persister.persist + end + describe 'incremented variables' do it 'substitutes :increment-groupid: with incrementing numbers' do allow(provider).to receive(:stack_servers) {[ diff --git a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb index cb022f3..c186e2a 100644 --- a/devops-service/workers/stack_bootstrap/stack_servers_persister.rb +++ b/devops-service/workers/stack_bootstrap/stack_servers_persister.rb @@ -50,7 +50,8 @@ class StackServersPersister 'deploy_env' => @deploy_env.identifier, 'key' => server_info['key_name'] || @provider.ssh_key, 'project' => @project.id, - 'provider' => @provider.name, + 'provider' => @stack.provider, + 'provider_account' => @stack.provider_account, 'remote_user' => mongo.image(@deploy_env.image).remote_user, 'private_ip' => server_info['private_ip'], 'public_ip' => server_info['public_ip'], From ac540e6f9c005183ac72d3c272628780a0f0b237 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Tue, 15 Mar 2016 12:22:16 +0200 Subject: [PATCH 4/6] CID-471: set EbsOptimized for instances with certain flavors --- devops-service/providers/ec2.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index d07b699..3a83a98 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -100,7 +100,8 @@ module Provider "InstanceType" => flavor, # "Placement.AvailabilityZone" => s.options[:availability_zone], "KeyName" => self.ssh_key, - "PrivateIpAddress" => s.private_ip + "PrivateIpAddress" => s.private_ip, + "EbsOptimized" => ebs_optimized?(flavor) } vpcId = nil unless subnets.empty? @@ -412,5 +413,10 @@ module Provider end end + def ebs_optimized?(instance_type) + always_ebs_optimized = ['c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', 'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', 'm4.large', 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge'] + always_ebs_optimized.include?(instance_type) + end + end end From 7fb5862f8590c71522d5001405f9d4744456b4c0 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Tue, 15 Mar 2016 16:56:58 +0200 Subject: [PATCH 5/6] CID-471: add mappings for ephemeral storages --- devops-service/providers/ec2.rb | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 3a83a98..e122c1b 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -101,7 +101,8 @@ module Provider # "Placement.AvailabilityZone" => s.options[:availability_zone], "KeyName" => self.ssh_key, "PrivateIpAddress" => s.private_ip, - "EbsOptimized" => ebs_optimized?(flavor) + "EbsOptimized" => ebs_optimized?(flavor), + "BlockDeviceMapping" => ephemeral_storages_mappings(flavor) } vpcId = nil unless subnets.empty? @@ -413,9 +414,28 @@ module Provider end end - def ebs_optimized?(instance_type) + # When you create an instance with one of these flavors within AWS console, + # EbsOptimized is automatically set to true. But in case of creating within API we should set it manually. + def ebs_optimized?(flavor) always_ebs_optimized = ['c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', 'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', 'm4.large', 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge'] - always_ebs_optimized.include?(instance_type) + always_ebs_optimized.include?(flavor) + end + + # When you create an instance with with ephemeral storages available within AWS console, + # they are mapped automatically. But in case of creating within API we should map them manually. + def ephemeral_storages_mappings(flavor) + require 'fog/aws/models/compute/flavors' + details = Fog::Compute::AWS::FLAVORS.detect {|f| f[:id] == flavor} + return [] unless details + + mappings = [] + details[:instance_store_volumes].times do |i| + mappings << { + 'VirtualName' => "ephemeral#{i}", + 'DeviceName' => "/dev/xvd#{('b'.ord + i).chr}" + } + end + mappings end end From cfd25be46a6b017a071674df189b2efb87bf3425 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 16 Mar 2016 21:39:19 +0200 Subject: [PATCH 6/6] CID-471: add docs --- devops-service/providers/ec2.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index e122c1b..5aeaeb9 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -423,6 +423,7 @@ module Provider # When you create an instance with with ephemeral storages available within AWS console, # they are mapped automatically. But in case of creating within API we should map them manually. + # Result example: [{'VirtualName' => 'ephemeral0', 'DeviceName' => '/dev/xvdb'}] def ephemeral_storages_mappings(flavor) require 'fog/aws/models/compute/flavors' details = Fog::Compute::AWS::FLAVORS.detect {|f| f[:id] == flavor}