require "exceptions/conflict_exception" require "providers/abstract_provider_connector" require "providers/aws/aws_provider_account" module Provider # Provider for Aws class AwsConnector < AbstractProviderConnector PROVIDER = "aws" attr_accessor :availability_zone, :storage_bucket_name def initialize config options = { :provider => PROVIDER } if config[:aws_use_iam_profile] options[:use_iam_profile] = true else options[:aws_access_key_id] = config[:aws_access_key_id] options[:aws_secret_access_key] = config[:aws_secret_access_key] end if config[:aws_proxy] and config[:aws_no_proxy] options[:connection_options] = { :proxy => config[:aws_proxy], :no_proxy => config[:no_proxy] } end self.connection_options = options self.availability_zone = config[:aws_availability_zone] || "us-east-1a" self.run_list = config[:aws_integration_run_list] || [] self.storage_bucket_name = config[:storage_bucket_name] end def configured? o = self.connection_options super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key])) end def name PROVIDER end def check_node_name name set = self.compute.describe_instances({"tag-key" => "Name", "tag-value" => name, 'instance-state-name' => "running"}).body["reservationSet"] return if set.empty? if set[0]["instancesSet"].size != 0 raise Devops::Exception::ValidationError.new("Aws node with name '#{name}' already exist") end end def flavors self.compute.flavors.all.map do |f| { "id" => f.id, "cores" => f.cores, "disk" => f.disk, "name" => f.name, "ram" => f.ram } end end def security_groups filters={} g = self.compute.describe_security_groups(filters) convert_groups(g.body["securityGroupInfo"]) end def images filters self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i| { "id" => i["imageId"], "name" => i["name"], "status" => i["imageState"] } end end def networks filters={} self.compute.describe_subnets(filters).body["subnetSet"].select{|n| n["state"] == "available"}.map do |n| { "cidr" => n["cidrBlock"], "vpcId" => n["vpcId"], "subnetId" => n["subnetId"], "name" => n["subnetId"], "zone" => n["availabilityZone"] } end end def servers list = self.compute.describe_instances('instance-state-name' => "running").body["reservationSet"] list.map do |server| convert_server server["instancesSet"][0] end end def server id list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"] convert_server list[0]["instancesSet"][0] end def create_server s, image, flavor, subnet, vpc_id, groups, out, options={} out << "Creating server for project '#{s.project}', environment '#{s.environment}', category '#{s.category}'\n" options = { "InstanceType" => flavor, # "Placement.AvailabilityZone" => s.options[:availability_zone], "KeyName" => s.ssh_key, #TODO: private ip is nil ??? "PrivateIpAddress" => s.private_ip, "EbsOptimized" => ebs_optimized?(flavor), "BlockDeviceMapping" => ephemeral_storages_mappings(flavor) } if vpc_id.nil? unless subnet.nil? options["SubnetId"] = subnet network = self.networks.detect{|n| n["name"] == options["SubnetId"]} vpc_id = network["vpcId"] if network if vpcId.nil? out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n" return false end end end groupIds = extract_group_ids(groups, vpc_id).join(",") options["SecurityGroupId"] = groupIds unless groupIds.empty? aws_server = nil begin out.puts "Run instance with image '#{image}' and options: #{options.inspect}" aws_server = self.compute.run_instances(image, 1, 1, options) rescue Excon::Errors::Unauthorized => ue #root = XML::Parser.string(ue.response.body).parse.root #msg = root.children.find { |node| node.name == "Message" } #code = root.children.find { |node| node.name == "Code" } code = "TODO" msg = ue.response.body out << "\nERROR: Unauthorized (#{code}: #{msg})" return false rescue Fog::Compute::AWS::Error => e out << e.message return false end abody = aws_server.body instance = abody["instancesSet"][0] s.id = instance["instanceId"] return instance["instanceState"]["name"] == "pending" end def waiting_server server, out details, state = nil, nil until state == "running" sleep(5) details = self.compute.describe_instances("instance-id" => [server.id]).body["reservationSet"][0]["instancesSet"][0] state = details["instanceState"]["name"].to_s if state == "pending" or state == "running" out << "." out.flush next end out << "Server returns state '#{state}'" return false end server.public_ip = details["ipAddress"] server.private_ip = details["privateIpAddress"] tags = server_tags(server) self.compute.create_tags(server.id, tags) out << "\nServer tags: #{tags.inspect}\n" out << "\nDone\n\n" out.flush true end def server_tags server { "Name" => server.name, "cid:project" => server.project, "cid:environment" => server.environment, "cid:category" => server.category, "cid:user" => server.created_by, "cid:remoteUser" => server.remote_user } end def delete_server s r = self.compute.terminate_instances(s.id) i = r.body["instancesSet"][0] old_state = i["previousState"]["name"] state = i["currentState"]["name"] return r.status == 200 ? "Server with id '#{s.id}' changed state '#{old_state}' to '#{state}'" : r.body end def pause_server s es = self.server s.id if es["state"] == "running" self.compute.stop_instances [ s.id ] return nil else return es["state"] end end def unpause_server s es = self.server s.id if es["state"] == "stopped" self.compute.start_instances [ s.id ] return nil else return es["state"] end end def set_tags instance_id, tags raise ConflictException.new("You can not change 'Name' tag") if tags.key?("Name") compute.create_tags(instance_id, tags) end def unset_tags instance_id, tags raise ConflictException.new("You can not change 'Name' tag") if tags.key?("Name") compute.delete_tags(instance_id, tags) end def compute connection_compute(connection_options) end def create_stack(stack, out) begin out << "Creating stack for project '#{stack.project}' and environment '#{stack.environment}'...\n" stack.name = create_default_stack_name(stack) unless stack.name out << "Stack name: #{stack.name}\n" out << "Stack template: #{stack.stack_template}\n" out << "Stack parameters: #{stack.parameters}\n" out << "Stack tags: #{JSON.pretty_generate(stack_tags(stack))}\n" out.flush response = cloud_formation.create_stack(stack.name, { 'TemplateURL' => stack.stack_template_model.template_url, 'Parameters' => stack.parameters || {}, 'Capabilities' => ['CAPABILITY_IAM'], 'Tags' => stack_tags(stack) } ) stack.id = response.body['StackId'] out << "Stack id: #{stack.id}\n" out.flush rescue Excon::Errors::Conflict => e raise ProviderErrors::NameConflict rescue Excon::Errors::BadRequest => br response = ::Chef::JSONCompat.from_json(br.response.body) if response['code'] == 400 out << "\nERROR: Bad request (400): #{response['explanation']}" out << "\n" raise InvalidRecord.new(response['explanation']) else out << "\nERROR: Unknown server error (#{response['code']}): #{response['explanation']}" out << "\n" raise InvalidRecord.new(response['explanation']) end end end def stack_tags stack { "StackTemplate" => stack.stack_template, "cid:project" => stack.project, "cid:deployEnv" => stack.environment, "cid:user" => stack.owner }.merge(stack.tags) end def update_stack(stack, params) cloud_formation.update_stack(stack.name, params) end def validate_stack_template template #r = cloud_formation.validate_template({'TemplateBody' => template}) #pp r.body true end def delete_stack(stack) cloud_formation.delete_stack(stack.name) end def stack_details(stack) b = cloud_formation.describe_stacks({'StackName' => stack.name}).body details = b['Stacks'].detect{|s| s.key?("StackStatus")} || {} { stack_status: details['StackStatus'], raw: details } end def stack_resources(stack) cloud_formation.describe_stack_resources({'StackName' => stack.name}).body['StackResources'] end def stack_events(stack) cloud_formation.describe_stack_events(stack.name).body['StackEvents'].map{|se| {"timestamp" => se["Timestamp"], "stack_name" => se["StackName"], "stack_id" => se["StackId"], "event_id" => se["EventId"], "reason" => se["ResourceStatusReason"], "status" => se["ResourceStatus"]}}.sort{|se1, se2| se1["timestamp"] <=> se2["timestamp"]} end def stack_servers(stack) resources = compute.describe_instances( 'tag-key' => 'aws:cloudformation:stack-id', 'tag-value' => stack.id ).body["reservationSet"] # There may be not only instances but autoscaling groups with several instances. # Handle such situations. instances = resources.map { |resource| resource["instancesSet"] }.flatten instances.delete_if {|i| i['instanceState']['name'] == 'terminated'} instances.map do |instance| { 'name' => [stack.name, instance_name(instance)].join('-'), 'id' => instance["instanceId"], 'key_name' => instance["keyName"], 'private_ip' => instance["privateIpAddress"], 'public_ip' => instance["ipAddress"], 'tags' => instance["tagSet"] } end end def create_default_stack_name s "stack-#{self.ssh_key}-#{s.project}-#{s.environment}-#{Time.now.to_i}".gsub('_', '-') end def stack_id_of_autoscaling_group(id) response = auto_scaling.describe_auto_scaling_groups('AutoScalingGroupNames' => [id]) tags = response.body['DescribeAutoScalingGroupsResult']['AutoScalingGroups'].first['Tags'] stack_id_tag = tags.find {|t| t['Key'] == 'aws:cloudformation:stack-id'} if stack_id_tag stack_id_tag['Value'] end rescue StandardError => e puts "An error occured while analyzing group '#{id}': #{[e.name, e.message, e.backtrace].join("\n")}" nil end def describe_vpcs self.compute.describe_vpcs.body["vpcSet"].select{|v| v["state"] == "available"}.map{|v| {"vpc_id" => v["vpcId"], "cidr" => v["cidrBlock"] } } end def store_stack_template(filename, json) store_file(stack_templates_bucket, filename, json) end def store_file(bucket, filename, body) { 'url' => bucket.files.create(key: filename, body: body, public: true).public_url } end private def convert_groups list res = {} list.each do |g| next if g["groupName"].nil? res[g["groupName"]] = { "description" => g["groupDescription"], "id" => g["groupId"] } rules = [] g["ipPermissions"].each do |r| cidr = r["ipRanges"][0] || {} rules.push({ "protocol" => r["ipProtocol"], "from" => r["fromPort"], "to" => r["toPort"], "cidr" => cidr["cidrIp"] }) end res[g["groupName"]]["rules"] = rules end res end def convert_server s { "state" => s["instanceState"]["name"], "name" => s["tagSet"]["Name"], "image" => s["imageId"], "flavor" => s["instanceType"], "keypair" => s["keyName"], "instance_id" => s["instanceId"], "dns_name" => s["dnsName"], "zone" => s["placement"]["availabilityZone"], "private_ip" => s["privateIpAddress"], "public_ip" => s["ipAddress"], "launched_at" => s["launchTime"] } end def extract_group_ids names, vpcId return [] if names.nil? p = nil p = {"vpc-id" => vpcId} unless vpcId.nil? groups = self.security_groups(p) r = [] names.each do |name| g = groups[name] r.push g["id"] unless g.nil? end r end def storage @storage ||= Fog::Storage.new(connection_options) end def cloud_formation @cloud_formation ||= Fog::AWS::CloudFormation.new(connection_options) end def auto_scaling @auto_scaling ||= Fog::AWS::AutoScaling.new(connection_options) end def stack_templates_bucket bucket = storage.directories.get(self.storage_bucket_name) bucket ||= storage.directories.create(key: self.storage_bucket_name) end def instance_name(instance) return instance["tagSet"]["Name"] if instance["tagSet"]["Name"] if instance['tagSet']['aws:autoscaling:groupName'] instance["instanceId"] else instance['tagSet']['aws:cloudformation:logical-id'] end end # When you create an instance with one of these flavors within AWS console, # EbsOptimized is automatically set to true. But in case of creating within API we should set it manually. ALWAYS_EBS_OPTIMIZED = ['c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', 'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', 'm4.large', 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge'] def ebs_optimized?(flavor) ALWAYS_EBS_OPTIMIZED.include?(flavor) end # When you create an instance with ephemeral storages available within AWS console, # they are mapped automatically. But in case of creating within API we should map them manually. # Result example: [{'VirtualName' => 'ephemeral0', 'DeviceName' => '/dev/xvdb'}] def ephemeral_storages_mappings(flavor) require 'fog/aws/models/compute/flavors' details = Fog::Compute::AWS::FLAVORS.detect {|f| f[:id] == flavor} return [] unless details mappings = [] details[:instance_store_volumes].times do |i| mappings << { 'VirtualName' => "ephemeral#{i}", 'DeviceName' => "/dev/xvd#{('b'.ord + i).chr}" } end mappings end end end