From cf19294772f87d190ce1656fc36e9531311d860c Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Mon, 28 Dec 2015 13:07:13 +0300 Subject: [PATCH] add specs for ServerExecutor#bootstrap --- devops-service/Guardfile | 1 + .../lib/executors/server_executor.rb | 82 ++++++++---- .../spec/executors/server_executor_spec.rb | 126 +++++++++++++++--- .../spec/shared_contexts/stubbed_knife.rb | 6 + 4 files changed, 172 insertions(+), 43 deletions(-) create mode 100644 devops-service/spec/shared_contexts/stubbed_knife.rb diff --git a/devops-service/Guardfile b/devops-service/Guardfile index 13d5834..a66e388 100644 --- a/devops-service/Guardfile +++ b/devops-service/Guardfile @@ -42,4 +42,5 @@ guard :rspec, cmd: "rspec" do # Devops files watch(%r{db/.+\.rb}) { rspec.spec_dir } + watch(%r{lib/executors/.+\.rb}) { "#{rspec.spec_dir}/executors" } end diff --git a/devops-service/lib/executors/server_executor.rb b/devops-service/lib/executors/server_executor.rb index fda02fb..23c6420 100644 --- a/devops-service/lib/executors/server_executor.rb +++ b/devops-service/lib/executors/server_executor.rb @@ -10,11 +10,13 @@ module Devops RESULT_CODES = { server_bootstrap_fail: 2, + server_bootstrap_private_ip_unset: 3, server_not_in_chef_nodes: 5, server_bootstrap_unknown_error: 7, deploy_unknown_error: 6, deploy_failed: 8, - creating_server_unknown_error: 9 + creating_server_unknown_error: 9, + creating_server_in_cloud_failed: 10 } # waiting for 5*60 seconds (5 min) @@ -44,7 +46,6 @@ module Devops @project = Devops::Db.connector.project(server.project) @deploy_env = @project.deploy_env(server.deploy_env) end - @knife_instance = KnifeFactory.instance @server = server @out = out @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush) @@ -103,7 +104,9 @@ module Devops res[:before] = self.run_hook :before_create @out << "Done\n" - return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out) + unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out) + return result_code(:creating_server_in_cloud_failed) + end mongo.server_insert @server @out << "\nAfter create hooks...\n" @@ -133,6 +136,11 @@ module Devops end end + # options: + # :run_list (optional) + # :bootstrap_template (optional) + # :chef_environment (optional) + # :config (optional) def bootstrap options @out << "\n\nBootstrap...\n" @out.flush @@ -144,7 +152,7 @@ module Devops @out << "Done\n" if @server.private_ip.nil? @out << "Error: Private IP is null" - return false + return result_code(:server_bootstrap_private_ip_unset) end ja = { :provider => @server.provider, @@ -159,13 +167,7 @@ module Devops address = "#{@server.remote_user}@#{ip}" - cmd = 'ssh ' - cmd << "-i #{cert_path} " - cmd << '-q ' - cmd << '-o StrictHostKeyChecking=no ' - cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 ' - cmd << "#{address} 'exit'" - cmd << " 2>&1" + cmd = check_ssh_command(cert_path, address) @out << "\nWaiting for SSH..." @out << "\nTest command: '#{cmd}'\n" @@ -174,16 +176,16 @@ module Devops retries_amount = 0 begin sleep(5) - res = `#{cmd}` + res = execute_system_command(cmd) retries_amount += 1 - if retries_amount > MAX_SSH_RETRIES_AMOUNT + if retries_amount >= MAX_SSH_RETRIES_AMOUNT @out.puts "Can not connect to #{address}" @out.puts res @out.flush DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}" return result_code(:server_bootstrap_fail) end - raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success? + raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless connected_successfully? rescue ArgumentError => e @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})" @out.flush @@ -193,7 +195,7 @@ module Devops provider = @server.provider_instance @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil? - r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options)) + r = knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options)) if r == 0 @out << "Chef node name: #{@server.chef_node_name}\n" @@ -214,6 +216,12 @@ module Devops end end + # options: + # :cert_path (required) + # :run_list (optional) + # :bootstrap_template (optional) + # :chef_environment (optional) + # :config (optional) def bootstrap_options attributes, options bootstrap_options = [ "-x #{@server.remote_user}", @@ -276,7 +284,7 @@ module Devops run_list = compute_run_list @out << "\nComputed run list: #{run_list.join(", ")}" @out.flush - @knife_instance.set_run_list(@server.chef_node_name, run_list) + knife_instance.set_run_list(@server.chef_node_name, run_list) deploy_info = options[:deploy_info] || @project.deploy_info(@deploy_env) deploy_status = deploy_server(deploy_info) if deploy_status == 0 @@ -296,7 +304,7 @@ module Devops end def check_server - @knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name) + knife_instance.chef_node_list.include?(@server.chef_node_name) and knife_instance.chef_client_list.include?(@server.chef_node_name) end def unbootstrap @@ -330,13 +338,13 @@ module Devops old_tags_str = nil new_tags_str = nil unless tags.empty? - old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ") + old_tags_str = knife_instance.tags_list(@server.chef_node_name).join(" ") @out << "Server tags: #{old_tags_str}\n" - @knife_instance.tags_delete(@server.chef_node_name, old_tags_str) + knife_instance.tags_delete(@server.chef_node_name, old_tags_str) new_tags_str = tags.join(" ") @out << "Server new tags: #{new_tags_str}\n" - cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str) + cmd = knife_instance.tags_create(@server.chef_node_name, new_tags_str) unless cmd[1] m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'" DevopsLogger.logger.error(m) @@ -350,9 +358,9 @@ module Devops unless tags.empty? @out << "Restore tags\n" - cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str) + cmd = knife_instance.tags_delete(@server.chef_node_name, new_tags_str) DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}") - cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str) + cmd = knife_instance.tags_create(@server.chef_node_name, old_tags_str) DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}") end return r @@ -393,7 +401,7 @@ module Devops end @out.flush k = Devops::Db.connector.key(@server.key) - lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path) + lline = knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path) r = /Chef\sClient\sfinished/i if lline && lline[r] @@ -413,8 +421,8 @@ module Devops def delete_from_chef_server node_name { - :chef_node => @knife_instance.chef_node_delete(node_name), - :chef_client => @knife_instance.chef_client_delete(node_name) + :chef_node => knife_instance.chef_node_delete(node_name), + :chef_client => knife_instance.chef_client_delete(node_name) } end @@ -533,6 +541,30 @@ module Devops end end + def check_ssh_command(cert_path, address) + cmd = 'ssh ' + cmd << "-i #{cert_path} " + cmd << '-q ' + cmd << '-o StrictHostKeyChecking=no ' + cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 ' + cmd << "#{address} 'exit'" + cmd << " 2>&1" + cmd + end + + # to simplify testing + def execute_system_command(cmd) + `#{cmd}` + end + + def connected_successfully? + $?.success? + end + + def knife_instance + @knife_instance ||= KnifeFactory.instance + end + end end end diff --git a/devops-service/spec/executors/server_executor_spec.rb b/devops-service/spec/executors/server_executor_spec.rb index e2fc1b2..de2c118 100644 --- a/devops-service/spec/executors/server_executor_spec.rb +++ b/devops-service/spec/executors/server_executor_spec.rb @@ -5,10 +5,14 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec let(:deploy_env) { project.deploy_env('foo') } let(:server) { build(:server, project: project.id, deploy_env: 'foo') } let(:output) { File.open(File::NULL, "w") } + let(:provider) { double('Provider instance') } let(:executor) { described_class.new(server, output) } + before do allow(stubbed_connector).to receive(:project) { project } + allow(executor.deploy_env).to receive(:provider_instance) { provider } + allow(server).to receive(:provider_instance) { provider } end describe '#initialize' do @@ -19,11 +23,6 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec expect(executor).to have_instance_variable_value(:out, output) end - it 'set knife_instance instance variable' do - allow(KnifeFactory).to receive(:instance) - expect(executor).to be_instance_variable_defined(:@knife_instance) - end - it 'defines :flush method on @out if it is absent' do out = Class.new.new expect(out).not_to respond_to(:flush) @@ -52,6 +51,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec let!(:without_bootstrap) { @without_bootstrap = true } let!(:run_list) { @run_list = %w(role[asd]) } let!(:key) { @key = 'key' } + let!(:image) { double('Image instance', remote_user: 'remote_user') } subject { executor.create_server( @@ -64,14 +64,8 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec } before do - @provider = double('Provider instance') - allow(executor.deploy_env).to receive(:provider_instance) { @provider } - allow(@provider).to receive(:create_server) { true } - - @image = double('Image instance') - allow(@image).to receive(:remote_user) { 'remote_user' } - - allow(stubbed_connector).to receive(:image) { @image} + allow(provider).to receive(:create_server) { true } + allow(stubbed_connector).to receive(:image) { image } allow(stubbed_connector).to receive(:server_insert) end @@ -91,7 +85,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec it 'sets key to default provider ssh key by default' do @key = nil - allow(@provider).to receive(:ssh_key) { 'default_key' } + allow(provider).to receive(:ssh_key) { 'default_key' } subject expect(executor.server.key).to eq 'default_key' end @@ -103,7 +97,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec end it 'creates server in cloud' do - expect(@provider).to receive(:create_server).with( + expect(provider).to receive(:create_server).with( an_instance_of(Devops::Model::Server), deploy_env.image, deploy_env.flavor, deploy_env.subnets, deploy_env.groups, output ) subject @@ -122,7 +116,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec context 'without_bootstrap option is false' do it 'launches bootstrap' do @without_bootstrap = false - allow(@image).to receive(:bootstrap_template) { 'template' } + allow(image).to receive(:bootstrap_template) { 'template' } allow(executor).to receive(:two_phase_bootstrap) expect(executor).to receive(:two_phase_bootstrap) subject @@ -132,7 +126,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec context 'without_bootstrap option is nil' do it 'launches bootstrap' do @without_bootstrap = nil - allow(@image).to receive(:bootstrap_template) { 'template' } + allow(image).to receive(:bootstrap_template) { 'template' } allow(executor).to receive(:two_phase_bootstrap) expect(executor).to receive(:two_phase_bootstrap) subject @@ -150,7 +144,7 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec context 'if error has been raised during execution' do before do allow(stubbed_connector).to receive(:server_delete) - allow(@provider).to receive(:create_server) { raise } + allow(provider).to receive(:create_server) { raise } end it 'rollbacks server creating' do @@ -162,7 +156,103 @@ RSpec.describe Devops::Executor::ServerExecutor, type: :executor, stubbed_connec expect(stubbed_connector).to receive(:server_delete) subject end + end + context "if creating server in cloud wasn't successful" do + it 'returns proper error code' do + allow(provider).to receive(:create_server) { false } + expect(subject).to eq 10 + end + end + end + + + + describe '#bootstrap', stubbed_logger: true, stubbed_knife: true do + subject { executor.bootstrap({}) } + let(:image) { double('Key instance', path: 'path') } + + before do + allow(executor).to receive(:sleep) + allow(executor).to receive(:connected_successfully?).and_return(true) + allow(executor).to receive(:execute_system_command) + allow(provider).to receive(:create_default_chef_node_name).and_return('chef_node') + allow(stubbed_connector).to receive(:key).and_return(image) + allow(stubbed_connector).to receive(:server_set_chef_node_name) + allow(stubbed_knife).to receive(:knife_bootstrap).and_return(0) + end + + it 'run before hook' do + expect(executor).to receive(:run_hook).with(:before_bootstrap, output).ordered + expect(executor).to receive(:run_hook).with(:after_bootstrap, output).ordered + subject + end + + context "when server's private ip is unset" do + it 'returns proper error code' do + server.private_ip = nil + expect(subject).to eq 3 + end + end + + it 'tries to ssh to server' do + expect(executor).to receive(:execute_system_command).with(/ssh/) + subject + end + + context "couldn't ssh to server" do + before { allow(executor).to receive(:connected_successfully?) { false } } + + it 'tries to ssh to server maximum MAX_SSH_RETRIES_AMOUNT times' do + max_retries = Devops::Executor::ServerExecutor::MAX_SSH_RETRIES_AMOUNT + expect(executor).to receive(:execute_system_command).exactly(max_retries).times + subject + end + + it 'returns proper error code' do + expect(subject).to eq 2 + end + end + + + context 'after successful ssh check' do + before { allow(executor).to receive(:connected_successfully?).and_return(false, true) } + + it "sets default chef node name if it's nil" do + executor.server.chef_node_name = nil + expect {subject}.to change {executor.server.chef_node_name}.to 'chef_node' + end + + it 'executes knife bootstrap' do + expect(stubbed_knife).to receive(:knife_bootstrap).with(output, server.private_ip, instance_of(Array)) + subject + end + + it "bootstraps to public ip if it's set" do + server.public_ip = '8.8.8.8' + expect(stubbed_knife).to receive(:knife_bootstrap).with(output, '8.8.8.8', instance_of(Array)) + subject + end + + context 'after successful bootstrap' do + it "updates server's chef node name in db" do + expect(stubbed_connector).to receive(:server_set_chef_node_name).with(instance_of(Devops::Model::Server)) + subject + end + end + + context "if bootstraping wasn't successful" do + before { allow(stubbed_knife).to receive(:knife_bootstrap).and_return(123) } + + it 'returns proper code' do + expect(subject).to eq 2 + end + + it "doesn't run after hook" do + expect(executor).to receive(:run_hook).with(:before_bootstrap, output) + subject + end + end end end end \ No newline at end of file diff --git a/devops-service/spec/shared_contexts/stubbed_knife.rb b/devops-service/spec/shared_contexts/stubbed_knife.rb new file mode 100644 index 0000000..cd49d05 --- /dev/null +++ b/devops-service/spec/shared_contexts/stubbed_knife.rb @@ -0,0 +1,6 @@ +RSpec.shared_context 'stubbed calls to KnifeFactory.instance', stubbed_knife: true do + let(:stubbed_knife) { double('KnifeCommands') } + before do + allow(KnifeFactory).to receive(:instance) { stubbed_knife } + end +end \ No newline at end of file