storing templates in s3

devops-client/lib/devops-client/handler/stack_template.rb

@@ -14,7 +14,6 @@ class StackTemplate < Handler
   end
 
   def handle
-    current_command = ARGV[1].to_sym
     @options, @args = @options_parser.parse_options_for!(current_command)
     case current_command
     when :list
@@ -27,6 +26,8 @@ class StackTemplate < Handler
       create_handler
     when :delete
       delete_handler
+    when :update_url
+      update_url_handler
     end
   end
 
@@ -76,6 +77,16 @@ class StackTemplate < Handler
     end
   end
 
+  def update_url_handler
+    r = inspect_parameters @options_parser.update_url_params, @args[2]
+    unless r.nil?
+      @options_parser.invalid_update_url_command
+      abort(r)
+    end
+    stack_template = post "/stack_template/#{@args[2]}/update_template_url"
+    puts stack_template['template_url']
+  end
+
   def provider_stack_templates(provider)
     if Providers.has_functionality?(provider, :stack_templates)
       @provider = true

devops-client/lib/devops-client/options/stack_template_options.rb

@@ -2,7 +2,7 @@ require "devops-client/options/common_options"
 
 class StackTemplateOptions < CommonOptions
 
-  commands :create, :delete, :list, :show
+  commands :create, :delete, :list, :show, :update_url
 
   def initialize args, def_options
     super(args, def_options)
@@ -11,6 +11,7 @@ class StackTemplateOptions < CommonOptions
     self.list_params = ["[provider]", "[ec2|openstack]"]
     self.show_params = ["STACK_TEMPLATE"]
     self.delete_params = ["STACK_TEMPLATE"]
+    self.update_url_params = ["STACK_TEMPLATE"]
   end
 
   def create_options

devops-service/app/api2/handlers/stack_template.rb

@@ -39,6 +39,14 @@ module Devops
           end
         end
 
+        # temp solution to update url on existing stacks
+        def update_template_url(id)
+          template = Devops::Db.connector.stack_template(id)
+          template.update_template_url
+          Devops::Db.connector.stack_template_update(template)
+          template
+        end
+
         private
 
         # returns:

devops-service/app/api2/routes/stack_template.rb

@@ -22,6 +22,11 @@ module Devops
             create_response 'Created', model.to_hash, 201
           end
 
+          app.post_with_headers "/stack_template/:id/update_template_url", :headers => [:accept] do |template_id|
+            check_privileges('stack_template', 'w')
+            json Devops::API2_0::Handler::StackTemplate.new(request).update_template_url(template_id).to_hash
+          end
+
           hash = {}
 
           hash['GET'] = lambda {|stack_template_id|

devops-service/config.rb

@@ -37,6 +37,7 @@ config[:aws_certificate]   = "/path/to/.ssh/ec2.pem"
 config[:aws_availability_zone]   = "aws_zone"
 config[:aws_proxy] = ""
 config[:aws_no_proxy] = ""
+config[:aws_stack_templates_bucket] = 'stacktemplates'
 
 # static settings
 config[:static_ssh_key]   = "ssh_key" # or nil

devops-service/db/mongo/models/stack/stack_base.rb

@@ -131,7 +131,11 @@ module Devops
       end
 
       def template_body
-        Devops::Db.connector.stack_template(stack_template).template_body
+        stack_template_model.template_body
+      end
+
+      def stack_template_model
+        Devops::Db.connector.stack_template(stack_template)
       end
 
       class << self

devops-service/db/mongo/models/stack_template/stack_template_ec2.rb

@@ -20,34 +20,25 @@ module Devops
         raise 'Implement me'
       end
 
-      class << self
+      def update_template_url
+        self.template_url = generate_template_file_and_upload_to_storage(id, template_body)
+      end
 
+      class << self
         def create(attrs)
           template = attrs['template_body']
           attrs['template_url'] = generate_template_file_and_upload_to_storage(attrs['id'], template)
           super(attrs)
         end
+      end
 
-        private
+      private
 
         def generate_template_file_and_upload_to_storage(id, json)
-          begin
-            tempfile = Tempfile.new('foo')
-            tempfile.write(json)
-            tempfile.close
-            secure_filename = "#{id}-#{SecureRandom.hex}.template"
-            upload_file_to_storage(secure_filename, tempfile.path)
-          ensure
-            tempfile.unlink
-          end
+          uniq_filename = "#{id}-#{SecureRandom.hex}.template"
+          provider_instance.store_file(uniq_filename, json)['url']
         end
 
-        def upload_file_to_storage(filename, file_path)
-          "https://s3.amazonaws.com/#{filename}"
-        end
-
-      end
-
     end
   end
 end

devops-service/providers/ec2.rb

@@ -222,7 +222,7 @@ module Provider
         out.flush
         response = cloud_formation.create_stack(stack.name,
           {
-            'TemplateBody' => stack.template_body,
+            'TemplateURL' => stack.stack_template_model.template_url,
             'Parameters' => stack.parameters || {},
             'Capabilities' => ['CAPABILITY_IAM'],
             'Tags' => stack_tags(stack)
@@ -319,6 +319,15 @@ module Provider
     def describe_vpcs
       self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } }
     end
+
+    # use public_url for a while because there is no possibility to create
+    # non expiring private links via fog
+    def store_file(filename, json)
+      {
+        'url' => stack_templates_bucket.files.create(key: filename, body: json, public: true).public_url
+      }
+    end
+
   private
 
     def convert_groups list
@@ -375,6 +384,16 @@ module Provider
       @orchestration ||= Fog::AWS::CloudFormation.new(connection_options)
     end
 
+    def storage
+      @storage ||= Fog::Storage.new(connection_options)
+    end
+
+    def stack_templates_bucket
+      bucket_name = DevopsConfig.config[:aws_stack_templates_bucket] || 'stacktemplates'
+      bucket = storage.directories.get(bucket_name)
+      bucket ||= storage.directories.create(key: bucket_name)
+    end
+
     def instance_name(instance)
       return instance["tagSet"]["Name"] if instance["tagSet"]["Name"]
       if instance['tagSet']['aws:autoscaling:groupName']