From b46dffe3fc42d5b9608624f70f969db5de10ecc6 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Tue, 15 Dec 2015 18:56:14 +0300 Subject: [PATCH 1/6] storing templates in s3 --- .../devops-client/handler/stack_template.rb | 13 +++++++++- .../options/stack_template_options.rb | 3 ++- .../app/api2/handlers/stack_template.rb | 8 ++++++ .../app/api2/routes/stack_template.rb | 5 ++++ devops-service/config.rb | 3 ++- .../db/mongo/models/stack/stack_base.rb | 6 ++++- .../stack_template/stack_template_ec2.rb | 25 ++++++------------- devops-service/providers/ec2.rb | 21 +++++++++++++++- 8 files changed, 62 insertions(+), 22 deletions(-) diff --git a/devops-client/lib/devops-client/handler/stack_template.rb b/devops-client/lib/devops-client/handler/stack_template.rb index c305e18..d1754c4 100644 --- a/devops-client/lib/devops-client/handler/stack_template.rb +++ b/devops-client/lib/devops-client/handler/stack_template.rb @@ -14,7 +14,6 @@ class StackTemplate < Handler end def handle - current_command = ARGV[1].to_sym @options, @args = @options_parser.parse_options_for!(current_command) case current_command when :list @@ -27,6 +26,8 @@ class StackTemplate < Handler create_handler when :delete delete_handler + when :update_url + update_url_handler end end @@ -76,6 +77,16 @@ class StackTemplate < Handler end end + def update_url_handler + r = inspect_parameters @options_parser.update_url_params, @args[2] + unless r.nil? + @options_parser.invalid_update_url_command + abort(r) + end + stack_template = post "/stack_template/#{@args[2]}/update_template_url" + puts stack_template['template_url'] + end + def provider_stack_templates(provider) if Providers.has_functionality?(provider, :stack_templates) @provider = true diff --git a/devops-client/lib/devops-client/options/stack_template_options.rb b/devops-client/lib/devops-client/options/stack_template_options.rb index 67f8498..74b9d9e 100644 --- a/devops-client/lib/devops-client/options/stack_template_options.rb +++ b/devops-client/lib/devops-client/options/stack_template_options.rb @@ -2,7 +2,7 @@ require "devops-client/options/common_options" class StackTemplateOptions < CommonOptions - commands :create, :delete, :list, :show + commands :create, :delete, :list, :show, :update_url def initialize args, def_options super(args, def_options) @@ -11,6 +11,7 @@ class StackTemplateOptions < CommonOptions self.list_params = ["[provider]", "[ec2|openstack]"] self.show_params = ["STACK_TEMPLATE"] self.delete_params = ["STACK_TEMPLATE"] + self.update_url_params = ["STACK_TEMPLATE"] end def create_options diff --git a/devops-service/app/api2/handlers/stack_template.rb b/devops-service/app/api2/handlers/stack_template.rb index 49718a9..ecb28a2 100644 --- a/devops-service/app/api2/handlers/stack_template.rb +++ b/devops-service/app/api2/handlers/stack_template.rb @@ -39,6 +39,14 @@ module Devops end end + # temp solution to update url on existing stacks + def update_template_url(id) + template = Devops::Db.connector.stack_template(id) + template.update_template_url + Devops::Db.connector.stack_template_update(template) + template + end + private # returns: diff --git a/devops-service/app/api2/routes/stack_template.rb b/devops-service/app/api2/routes/stack_template.rb index d825191..1b18fd9 100644 --- a/devops-service/app/api2/routes/stack_template.rb +++ b/devops-service/app/api2/routes/stack_template.rb @@ -22,6 +22,11 @@ module Devops create_response 'Created', model.to_hash, 201 end + app.post_with_headers "/stack_template/:id/update_template_url", :headers => [:accept] do |template_id| + check_privileges('stack_template', 'w') + json Devops::API2_0::Handler::StackTemplate.new(request).update_template_url(template_id).to_hash + end + hash = {} hash['GET'] = lambda {|stack_template_id| diff --git a/devops-service/config.rb b/devops-service/config.rb index 5bc27b3..1751eb7 100644 --- a/devops-service/config.rb +++ b/devops-service/config.rb @@ -37,6 +37,7 @@ config[:aws_certificate] = "/path/to/.ssh/ec2.pem" config[:aws_availability_zone] = "aws_zone" config[:aws_proxy] = "" config[:aws_no_proxy] = "" +config[:aws_stack_templates_bucket] = 'stacktemplates' # static settings config[:static_ssh_key] = "ssh_key" # or nil @@ -46,4 +47,4 @@ config[:debug] = true # set it to :all or [:ec2] to stub calls to selected providers # or to false to disable stubbing -config[:stub_providers] = false +config[:stub_providers] = false \ No newline at end of file diff --git a/devops-service/db/mongo/models/stack/stack_base.rb b/devops-service/db/mongo/models/stack/stack_base.rb index cf118e5..e631bf6 100644 --- a/devops-service/db/mongo/models/stack/stack_base.rb +++ b/devops-service/db/mongo/models/stack/stack_base.rb @@ -131,7 +131,11 @@ module Devops end def template_body - Devops::Db.connector.stack_template(stack_template).template_body + stack_template_model.template_body + end + + def stack_template_model + Devops::Db.connector.stack_template(stack_template) end class << self diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb index 89f7a0d..3c159bf 100644 --- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb +++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb @@ -20,34 +20,25 @@ module Devops raise 'Implement me' end - class << self + def update_template_url + self.template_url = generate_template_file_and_upload_to_storage(id, template_body) + end + class << self def create(attrs) template = attrs['template_body'] attrs['template_url'] = generate_template_file_and_upload_to_storage(attrs['id'], template) super(attrs) end + end - private + private def generate_template_file_and_upload_to_storage(id, json) - begin - tempfile = Tempfile.new('foo') - tempfile.write(json) - tempfile.close - secure_filename = "#{id}-#{SecureRandom.hex}.template" - upload_file_to_storage(secure_filename, tempfile.path) - ensure - tempfile.unlink - end + uniq_filename = "#{id}-#{SecureRandom.hex}.template" + provider_instance.store_file(uniq_filename, json)['url'] end - def upload_file_to_storage(filename, file_path) - "https://s3.amazonaws.com/#{filename}" - end - - end - end end end diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index a51b423..6725974 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -222,7 +222,7 @@ module Provider out.flush response = cloud_formation.create_stack(stack.name, { - 'TemplateBody' => stack.template_body, + 'TemplateURL' => stack.stack_template_model.template_url, 'Parameters' => stack.parameters || {}, 'Capabilities' => ['CAPABILITY_IAM'], 'Tags' => stack_tags(stack) @@ -319,6 +319,15 @@ module Provider def describe_vpcs self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } } end + + # use public_url for a while because there is no possibility to create + # non expiring private links via fog + def store_file(filename, json) + { + 'url' => stack_templates_bucket.files.create(key: filename, body: json, public: true).public_url + } + end + private def convert_groups list @@ -375,6 +384,16 @@ module Provider @orchestration ||= Fog::AWS::CloudFormation.new(connection_options) end + def storage + @storage ||= Fog::Storage.new(connection_options) + end + + def stack_templates_bucket + bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplates' + bucket = storage.directories.get(bucket_name) + bucket ||= storage.directories.create(key: bucket_name) + end + def instance_name(instance) return instance["tagSet"]["Name"] if instance["tagSet"]["Name"] if instance['tagSet']['aws:autoscaling:groupName'] From 22ad5e2b9853dc4638fe9d8cbc8d9693d4a1f02a Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Tue, 15 Dec 2015 19:36:33 +0300 Subject: [PATCH 2/6] fix --- .../db/mongo/models/stack_template/stack_template_ec2.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb index 3c159bf..371a6d4 100644 --- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb +++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb @@ -26,9 +26,9 @@ module Devops class << self def create(attrs) - template = attrs['template_body'] - attrs['template_url'] = generate_template_file_and_upload_to_storage(attrs['id'], template) - super(attrs) + model = super(attrs) + model.update_template_url + model end end From 1cc6fb3e78440dd40ac0f431aab2278669c41326 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Tue, 15 Dec 2015 19:53:09 +0300 Subject: [PATCH 3/6] change default bucket name --- devops-service/providers/ec2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index 6725974..b62a32f 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -389,7 +389,7 @@ module Provider end def stack_templates_bucket - bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplates' + bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplatesnibrdev' bucket = storage.directories.get(bucket_name) bucket ||= storage.directories.create(key: bucket_name) end From fae0c54b4b62ae520a7350888c3258705a9d53a9 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Mon, 21 Dec 2015 13:59:28 +0400 Subject: [PATCH 4/6] fix specs --- .../spec/models/stack_template/stack_template_ec2_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb index 1cb48af..eece64f 100644 --- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb +++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true } + allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_file') { {'url' => nil} } end it_behaves_like 'stack template' From 3c5362c7fd773867b8b1e2338c3cbb910d4d4073 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 13 Jan 2016 14:57:01 +0300 Subject: [PATCH 5/6] set more semantic name for uploading stack template method --- .../mongo/models/stack_template/stack_template_ec2.rb | 2 +- devops-service/providers/ec2.rb | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb index 371a6d4..20b3d03 100644 --- a/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb +++ b/devops-service/db/mongo/models/stack_template/stack_template_ec2.rb @@ -36,7 +36,7 @@ module Devops def generate_template_file_and_upload_to_storage(id, json) uniq_filename = "#{id}-#{SecureRandom.hex}.template" - provider_instance.store_file(uniq_filename, json)['url'] + provider_instance.store_stack_template(uniq_filename, json)['url'] end end diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index b62a32f..b0e8e2a 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -320,11 +320,13 @@ module Provider self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } } end - # use public_url for a while because there is no possibility to create - # non expiring private links via fog - def store_file(filename, json) + def store_stack_template(filename, json) + store_file(stack_templates_bucket, filename, json) + end + + def store_file(bucket, filename, body) { - 'url' => stack_templates_bucket.files.create(key: filename, body: json, public: true).public_url + 'url' => bucket.files.create(key: filename, body: body, public: true).public_url } end From 065537b4dc8b4f171bf7e649bdf05571ce0663d5 Mon Sep 17 00:00:00 2001 From: Anton Chuchkalov Date: Wed, 27 Jan 2016 11:37:45 +0300 Subject: [PATCH 6/6] fix failing stack_template spec --- .../stack_template/stack_template_ec2_spec.rb | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb index eece64f..d4a1d2c 100644 --- a/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb +++ b/devops-service/spec/models/stack_template/stack_template_ec2_spec.rb @@ -6,9 +6,20 @@ RSpec.describe Devops::Model::StackTemplateEc2, type: :model do before do allow(Provider::ProviderFactory).to receive(:providers).and_return(%w(ec2)) allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.validate_stack_template') { true } - allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_file') { {'url' => nil} } + allow_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') { {'url' => nil} } end it_behaves_like 'stack template' + it 'uploads file to S3' do + expect_any_instance_of(Devops::Model::StackTemplateEc2).to receive_message_chain('provider_instance.store_stack_template') + params = { + 'id' => 'foo', + 'template_body' => '{}', + 'owner' => 'root', + 'provider' => 'ec2' + } + expect(described_class.create(params)).to be_an_instance_of(described_class) + end + end \ No newline at end of file