fluke/devops-service/providers/aws/aws_connector.rb

require "exceptions/conflict_exception"
require "providers/abstract_provider_connector"
require "providers/aws/aws_provider_account"

module Provider
  # Provider for Aws
  class AwsConnector < AbstractProviderConnector

    PROVIDER = "aws"

    attr_accessor :availability_zone, :storage_bucket_name

    def initialize config
      options = {
        :provider => PROVIDER
      }
      if config[:aws_use_iam_profile]
        options[:use_iam_profile] = true
      else
        options[:aws_access_key_id] = config[:aws_access_key_id]
        options[:aws_secret_access_key] = config[:aws_secret_access_key]
      end
      if config[:aws_proxy] and config[:aws_no_proxy]
        options[:connection_options] = {
          :proxy => config[:aws_proxy],
          :no_proxy => config[:no_proxy]
        }
      end
      self.connection_options = options
      self.availability_zone = config[:aws_availability_zone] || "us-east-1a"
      self.run_list = config[:aws_integration_run_list] || []
      self.storage_bucket_name = config[:storage_bucket_name]
    end

    def configured?
      o = self.connection_options
      super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
    end

    def name
      PROVIDER
    end

    def check_node_name name
      set = self.compute.describe_instances({"tag-key" => "Name", "tag-value" => name, 'instance-state-name' => "running"}).body["reservationSet"]
      return if set.empty?
      if set[0]["instancesSet"].size != 0
        raise Devops::Exception::ValidationError.new("Aws node with name '#{name}' already exist")
      end
    end

    def flavors
      self.compute.flavors.all.map do |f|
        {
          "id" => f.id,
          "cores" => f.cores,
          "disk" => f.disk,
          "name" => f.name,
          "ram" => f.ram
        }
      end
    end

    def security_groups filters={}
      g = self.compute.describe_security_groups(filters)
      convert_groups(g.body["securityGroupInfo"])
    end

    def images filters
      self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i|
        {
          "id" => i["imageId"],
          "name" => i["name"],
          "status" => i["imageState"]
        }
      end
    end

    def networks filters={}
      self.compute.describe_subnets(filters).body["subnetSet"].select{|n| n["state"] == "available"}.map do |n|
        {
          "cidr" => n["cidrBlock"],
          "vpcId" => n["vpcId"],
          "subnetId" => n["subnetId"],
          "name" => n["subnetId"],
          "zone" => n["availabilityZone"]
        }
      end
    end

    def servers
      list = self.compute.describe_instances('instance-state-name' => "running").body["reservationSet"]
      list.map do |server|
        convert_server server["instancesSet"][0]
      end
    end

    def server id
      list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"]
      convert_server list[0]["instancesSet"][0]
    end

    def create_server s, image, flavor, subnet, vpc_id, groups, out, options={}
      out << "Creating server for project '#{s.project}', environment '#{s.environment}', category '#{s.category}'\n"
      options = {
        "InstanceType" => flavor,
#        "Placement.AvailabilityZone" => s.options[:availability_zone],
        "KeyName" => s.ssh_key,
        #TODO: private ip is nil ???
        "PrivateIpAddress" => s.private_ip,
        "EbsOptimized" => ebs_optimized?(flavor),
        "BlockDeviceMapping" => ephemeral_storages_mappings(flavor)
      }
      if vpc_id.nil?
        unless subnet.nil?
          options["SubnetId"] = subnet
          network = self.networks.detect{|n| n["name"] == options["SubnetId"]}
          vpc_id = network["vpcId"] if network
          if vpcId.nil?
            out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
            return false
          end
        end
      end
      groupIds = extract_group_ids(groups, vpc_id).join(",")
      options["SecurityGroupId"] = groupIds unless groupIds.empty?

      aws_server = nil
      begin
        out.puts "Run instance with image '#{image}' and options: #{options.inspect}"
        aws_server = self.compute.run_instances(image, 1, 1, options)
      rescue Excon::Errors::Unauthorized => ue
        #root = XML::Parser.string(ue.response.body).parse.root
        #msg = root.children.find { |node| node.name == "Message" }
        #code = root.children.find { |node| node.name == "Code" }
        code = "TODO"
        msg = ue.response.body
        out << "\nERROR: Unauthorized (#{code}: #{msg})"
        return false
      rescue Fog::Compute::AWS::Error => e
        out << e.message
        return false
      end

      abody = aws_server.body
      instance = abody["instancesSet"][0]
      s.id = instance["instanceId"]
      return instance["instanceState"]["name"] == "pending"
    end

    def waiting_server server, out
      details, state = nil, nil
      until state == "running"
        sleep(5)
        details = self.compute.describe_instances("instance-id" => [server.id]).body["reservationSet"][0]["instancesSet"][0]
        state = details["instanceState"]["name"].to_s
        if state == "pending" or state == "running"
          out << "."
          out.flush
          next
        end
        out << "Server returns state '#{state}'"
        return false
      end
      server.public_ip = details["ipAddress"]
      server.private_ip = details["privateIpAddress"]
      tags = server_tags(server)
      self.compute.create_tags(server.id, tags)
      out << "\nServer tags: #{tags.inspect}\n"
      out << "\nDone\n\n"
      out.flush
      true
    end

    def server_tags server
      {
        "Name" => server.name,
        "cid:project" => server.project,
        "cid:environment" => server.environment,
        "cid:category" => server.category,
        "cid:user" => server.created_by,
        "cid:remoteUser" => server.remote_user
      }
    end

    def delete_server s
      r = self.compute.terminate_instances(s.id)
      i = r.body["instancesSet"][0]
      old_state = i["previousState"]["name"]
      state = i["currentState"]["name"]
      return r.status == 200 ? "Server with id '#{s.id}' changed state '#{old_state}' to '#{state}'" : r.body
    end

    def pause_server s
      es = self.server s.id
      if es["state"] == "running"
        self.compute.stop_instances [ s.id ]
        return nil
      else
        return es["state"]
      end
    end

    def unpause_server s
      es = self.server s.id
      if es["state"] == "stopped"
        self.compute.start_instances [ s.id ]
        return nil
      else
        return es["state"]
      end
    end

    def set_tags instance_id, tags
      raise ConflictException.new("You can not change 'Name' tag") if tags.key?("Name")
      compute.create_tags(instance_id, tags)
    end

    def unset_tags instance_id, tags
      raise ConflictException.new("You can not change 'Name' tag") if tags.key?("Name")
      compute.delete_tags(instance_id, tags)
    end

    def compute
      connection_compute(connection_options)
    end

    def create_stack(stack, out)
      begin
        out << "Creating stack for project '#{stack.project}' and environment '#{stack.environment}'...\n"
        stack.name = create_default_stack_name(stack) unless stack.name
        out << "Stack name: #{stack.name}\n"
        out << "Stack template: #{stack.stack_template}\n"
        out << "Stack parameters: #{stack.parameters}\n"
        out << "Stack tags: #{JSON.pretty_generate(stack_tags(stack))}\n"
        out.flush
        response = cloud_formation.create_stack(stack.name,
          {
            'TemplateURL' => stack.stack_template_model.template_url,
            'Parameters' => stack.parameters || {},
            'Capabilities' => ['CAPABILITY_IAM'],
            'Tags' => stack_tags(stack)
          }
        )
        stack.id = response.body['StackId']
        out << "Stack id: #{stack.id}\n"
        out.flush
      rescue Excon::Errors::Conflict => e
        raise ProviderErrors::NameConflict
      rescue Excon::Errors::BadRequest => br
        response = ::Chef::JSONCompat.from_json(br.response.body)
        if response['code'] == 400
          out << "\nERROR: Bad request (400): #{response['explanation']}"
          out << "\n"
          raise InvalidRecord.new(response['explanation'])
        else
          out << "\nERROR: Unknown server error (#{response['code']}): #{response['explanation']}"
          out << "\n"
          raise InvalidRecord.new(response['explanation'])
        end
      end
    end

    def stack_tags stack
      {
        "StackTemplate" => stack.stack_template,
        "cid:project" => stack.project,
        "cid:deployEnv" => stack.environment,
        "cid:user" => stack.owner
      }.merge(stack.tags)
    end

    def update_stack(stack, params)
      cloud_formation.update_stack(stack.name, params)
    end

    def validate_stack_template template
      #r = cloud_formation.validate_template({'TemplateBody' => template})
      #pp r.body
      true
    end

    def delete_stack(stack)
      cloud_formation.delete_stack(stack.name)
    end

    def stack_details(stack)
      b = cloud_formation.describe_stacks({'StackName' => stack.name}).body
      details = b['Stacks'].detect{|s| s.key?("StackStatus")} || {}
      {
        stack_status: details['StackStatus'],
        raw: details
      }
    end

    def stack_resources(stack)
      cloud_formation.describe_stack_resources({'StackName' => stack.name}).body['StackResources']
    end

    def stack_events(stack)
      cloud_formation.describe_stack_events(stack.name).body['StackEvents'].map{|se| {"timestamp" => se["Timestamp"], "stack_name" => se["StackName"], "stack_id" => se["StackId"], "event_id" => se["EventId"], "reason" => se["ResourceStatusReason"], "status" => se["ResourceStatus"]}}.sort{|se1, se2| se1["timestamp"] <=> se2["timestamp"]}
    end

    def stack_servers(stack)
      resources = compute.describe_instances(
        'tag-key' => 'aws:cloudformation:stack-id',
        'tag-value' => stack.id
      ).body["reservationSet"]


      # There may be not only instances but autoscaling groups with several instances.
      # Handle such situations.
      instances = resources.map { |resource| resource["instancesSet"] }.flatten
      instances.delete_if {|i| i['instanceState']['name'] == 'terminated'}

      instances.map do |instance|
        {
          'name' => [stack.name, instance_name(instance)].join('-'),
          'id' => instance["instanceId"],
          'key_name' => instance["keyName"],
          'private_ip' => instance["privateIpAddress"],
          'public_ip' => instance["ipAddress"],
          'tags' => instance["tagSet"]
        }
      end
    end

    def create_default_stack_name s
      "stack-#{self.ssh_key}-#{s.project}-#{s.environment}-#{Time.now.to_i}".gsub('_', '-')
    end

    def stack_id_of_autoscaling_group(id)
      response = auto_scaling.describe_auto_scaling_groups('AutoScalingGroupNames' => [id])
      tags = response.body['DescribeAutoScalingGroupsResult']['AutoScalingGroups'].first['Tags']
      stack_id_tag = tags.find {|t| t['Key'] == 'aws:cloudformation:stack-id'}
      if stack_id_tag
        stack_id_tag['Value']
      end
    rescue StandardError => e
      puts "An error occured while analyzing group '#{id}': #{[e.name, e.message, e.backtrace].join("\n")}"
      nil
    end

    def describe_vpcs
      self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } }
    end

    def store_stack_template(filename, json)
      store_file(stack_templates_bucket, filename, json)
    end

    def store_file(bucket, filename, body)
      {
        'url' => bucket.files.create(key: filename, body: body, public: true).public_url
      }
    end

  private

    def convert_groups list
      res = {}
      list.each do |g|
        next if g["groupName"].nil?
        res[g["groupName"]] = {
          "description" => g["groupDescription"],
          "id" => g["groupId"]
        }
        rules = []
        g["ipPermissions"].each do |r|
          cidr = r["ipRanges"][0] || {}
          rules.push({
            "protocol" => r["ipProtocol"],
            "from" => r["fromPort"],
            "to" => r["toPort"],
            "cidr" => cidr["cidrIp"]
          })
        end
        res[g["groupName"]]["rules"] = rules
      end
      res
    end

    def convert_server s
      {
        "state" => s["instanceState"]["name"],
        "name" => s["tagSet"]["Name"],
        "image" => s["imageId"],
        "flavor" => s["instanceType"],
        "keypair" => s["keyName"],
        "instance_id" => s["instanceId"],
        "dns_name" => s["dnsName"],
        "zone" => s["placement"]["availabilityZone"],
        "private_ip" => s["privateIpAddress"],
        "public_ip" => s["ipAddress"],
        "launched_at" => s["launchTime"]
      }
    end

    def extract_group_ids names, vpcId
      return [] if names.nil?
      p = nil
      p = {"vpc-id" => vpcId} unless vpcId.nil?
      groups = self.security_groups(p)
      r = []
      names.each do |name|
        g = groups[name]
        r.push g["id"] unless g.nil?
      end
      r
    end

    def storage
      @storage ||= Fog::Storage.new(connection_options)
    end

    def cloud_formation
      @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options)
    end

    def auto_scaling
      @auto_scaling ||= Fog::AWS::AutoScaling.new(connection_options)
    end

    def stack_templates_bucket
      bucket = storage.directories.get(self.storage_bucket_name)
      bucket ||= storage.directories.create(key: self.storage_bucket_name)
    end

    def instance_name(instance)
      return instance["tagSet"]["Name"] if instance["tagSet"]["Name"]
      if instance['tagSet']['aws:autoscaling:groupName']
        instance["instanceId"]
      else
        instance['tagSet']['aws:cloudformation:logical-id']
      end
    end

    # When you create an instance with one of these flavors within AWS console,
    # EbsOptimized is automatically set to true. But in case of creating within API we should set it manually.
    ALWAYS_EBS_OPTIMIZED = ['c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', 'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', 'm4.large', 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge']
    def ebs_optimized?(flavor)
      ALWAYS_EBS_OPTIMIZED.include?(flavor)
    end

    # When you create an instance with ephemeral storages available within AWS console,
    # they are mapped automatically. But in case of creating within API we should map them manually.
    # Result example: [{'VirtualName' => 'ephemeral0', 'DeviceName' => '/dev/xvdb'}]
    def ephemeral_storages_mappings(flavor)
      require 'fog/aws/models/compute/flavors'
      details = Fog::Compute::AWS::FLAVORS.detect {|f| f[:id] == flavor}
      return [] unless details

      mappings = []
      details[:instance_store_volumes].times do |i|
        mappings << {
          'VirtualName' => "ephemeral#{i}",
          'DeviceName' => "/dev/xvd#{('b'.ord + i).chr}"
        }
      end
      mappings
    end

  end
end