diff --git a/devops-client/lib/devops-client/handler/server.rb b/devops-client/lib/devops-client/handler/server.rb index 37cf7e0..58574d6 100644 --- a/devops-client/lib/devops-client/handler/server.rb +++ b/devops-client/lib/devops-client/handler/server.rb @@ -63,7 +63,7 @@ class Server < Handler @list = case args[2] when "chef" get("/servers/chef").map {|l| {"chef_node_name" => l}} - when "ec2", "openstack" + when "ec2", "openstack", "static" get("/servers/#{args[2]}") else @options_parser.invalid_list_command @@ -149,7 +149,7 @@ class Server < Handler :key => args[6] } q[:public_ip] = self.options[:public_ip] unless self.options[:public_ip].nil? - post_chunk "/server/add", q + post "/server/add", q end def pause_handler args diff --git a/devops-client/lib/devops-client/options/server_options.rb b/devops-client/lib/devops-client/options/server_options.rb index b18aee8..d208645 100644 --- a/devops-client/lib/devops-client/options/server_options.rb +++ b/devops-client/lib/devops-client/options/server_options.rb @@ -8,7 +8,7 @@ class ServerOptions < CommonOptions super(args, def_options) self.header = I18n.t("headers.server") self.banner_header = "server" - self.list_params = ["[chef|ec2|openstack]"] + self.list_params = ["[chef|ec2|openstack|static]"] self.create_params = ["PROJECT_ID", "DEPLOY_ENV"] node_params = ["NODE_NAME"] self.delete_params = node_params diff --git a/devops-service/commands/deploy_env.rb b/devops-service/commands/deploy_env.rb index c70d8c5..0a6efea 100644 --- a/devops-service/commands/deploy_env.rb +++ b/devops-service/commands/deploy_env.rb @@ -25,12 +25,12 @@ module DeployEnvCommands raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty? filter = nil - if p.name == ::Version2_0::Provider::Ec2::PROVIDER + if p.name == ::Provider::Ec2::PROVIDER unless subnets.empty? subnets = [ subnets[0] ] if subnets.size > 1 filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] } end - elsif p.name == ::Version2_0::Provider::Openstack::PROVIDER + elsif p.name == ::Provider::Openstack::PROVIDER if subnets.empty? raise InvalidRecord.new "Subnets array can not be empty" end diff --git a/devops-service/commands/image.rb b/devops-service/commands/image.rb index da21645..b405b4f 100644 --- a/devops-service/commands/image.rb +++ b/devops-service/commands/image.rb @@ -7,7 +7,7 @@ module ImageCommands if filters.empty? [] else - ::Version2_0::Provider::ProviderFactory.get(provider).images(filters) + ::Provider::ProviderFactory.get(provider).images(filters) end end end diff --git a/devops-service/commands/server.rb b/devops-service/commands/server.rb index 615e8e1..8cff5ae 100644 --- a/devops-service/commands/server.rb +++ b/devops-service/commands/server.rb @@ -142,17 +142,19 @@ module ServerCommands return status end - def unbootstrap s, cert_path + def self.unbootstrap s, cert_path i = 0 begin - `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef` - raise unless $?.success? + r = `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef` + raise(r) unless $?.success? rescue => e - logger.error "Unbootstrap eeror: " + e.message + logger.error "Unbootstrap error: " + e.message i += 1 sleep(1) retry unless i == 5 + return e.message end + nil end def delete_server s, mongo, logger @@ -163,26 +165,16 @@ module ServerCommands return msg, nil end r = delete_from_chef_server(s.chef_node_name) - info = if s.static - cert = mongo.key(s.key).path - unbootstrap(s, cert) - mongo.server_delete s.id - msg = "Static server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" - logger.info msg - msg - else - provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) - begin - r[:server] = provider.delete_server s.id - rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound - r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" - logger.warn r[:server] - end - mongo.server_delete s.id - msg = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" - logger.info msg - msg + provider = ::Provider::ProviderFactory.get(s.provider) + begin + r[:server] = provider.delete_server s + rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound + r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" + logger.warn r[:server] end + mongo.server_delete s.id + info = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" + logger.info info r.each{|key, log| logger.info("#{key} - #{log}")} return info, r end @@ -193,7 +185,7 @@ module ServerCommands str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n" str << delete_from_chef_server(s.chef_node_name).values.join("\n") begin - str << provider.delete_server(s.id) unless s.static + str << provider.delete_server(s) rescue => e str << e.message end diff --git a/devops-service/config.rb b/devops-service/config.rb index 3097f9b..9450f15 100644 --- a/devops-service/config.rb +++ b/devops-service/config.rb @@ -34,3 +34,7 @@ config[:aws_secret_access_key] = "secret_access_key" config[:aws_ssh_key] = "ssh_key" config[:aws_certificate] = "/path/to/.ssh/ec2.pem" config[:aws_availability_zone] = "aws_zone" + +# static settings +config[:static_ssh_key] = "ssh_key" # or nil +config[:static_certificate] = "/path/to/.ssh/static.pem" diff --git a/devops-service/db/mongo/models/deploy_env.rb b/devops-service/db/mongo/models/deploy_env.rb index 7103a34..717cd05 100644 --- a/devops-service/db/mongo/models/deploy_env.rb +++ b/devops-service/db/mongo/models/deploy_env.rb @@ -44,7 +44,7 @@ class DeployEnv < MongoModel check_expires!(self.expires) end - p = ::Version2_0::Provider::ProviderFactory.get(self.provider) + p = ::Provider::ProviderFactory.get(self.provider) check_flavor!(p, self.flavor) check_image!(p, self.image) check_subnets_and_groups!(p, self.subnets, self.groups) diff --git a/devops-service/db/mongo/models/deploy_env_multi.rb b/devops-service/db/mongo/models/deploy_env_multi.rb index edafdbb..4e17396 100644 --- a/devops-service/db/mongo/models/deploy_env_multi.rb +++ b/devops-service/db/mongo/models/deploy_env_multi.rb @@ -65,7 +65,7 @@ class DeployEnvMulti < MongoModel end end - p = ::Version2_0::Provider::ProviderFactory.get(server["provider"]) + p = ::Provider::ProviderFactory.get(server["provider"]) check_flavor!(p, server["flavor"]) check_image!(p, server["image"]) check_subnets_and_groups!(p, server["subnets"], server["groups"]) diff --git a/devops-service/db/mongo/models/mongo_model.rb b/devops-service/db/mongo/models/mongo_model.rb index 5fd374d..d8137f1 100644 --- a/devops-service/db/mongo/models/mongo_model.rb +++ b/devops-service/db/mongo/models/mongo_model.rb @@ -43,7 +43,7 @@ class MongoModel end def check_provider provider=self.provider - unless ::Version2_0::Provider::ProviderFactory.providers.include?(provider) or provider == "static" + unless ::Provider::ProviderFactory.providers.include?(provider) raise InvalidRecord.new "Invalid provider '#{provider}'" end end diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index 850543c..ffd56c1 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -191,12 +191,16 @@ class MongoConnector end end + def servers_find q + @servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs} + end + def servers p=nil, env=nil, names=nil q = {} q["project"] = p unless p.nil? or p.empty? q["deploy_env"] = env unless env.nil? or env.empty? q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array - @servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs} + servers_find(q) end def servers_by_names names diff --git a/devops-service/devops-service.rb b/devops-service/devops-service.rb index 38402d0..2432053 100644 --- a/devops-service/devops-service.rb +++ b/devops-service/devops-service.rb @@ -29,7 +29,8 @@ class DevopsService < Sinatra::Base [:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } mongo = DevopsService.mongo mongo.create_root_user - ::Version2_0::Provider::ProviderFactory.all.each do |p| + ::Provider::ProviderFactory.all.each do |p| + next if p.certificate_path.nil? begin mongo.key p.ssh_key, Key::SYSTEM rescue RecordNotFound => e diff --git a/devops-service/providers/base_provider.rb b/devops-service/providers/base_provider.rb index 8388c6f..f60e5f2 100644 --- a/devops-service/providers/base_provider.rb +++ b/devops-service/providers/base_provider.rb @@ -1,28 +1,26 @@ require "fog" -module Version2_0 - module Provider - class BaseProvider +module Provider + class BaseProvider - attr_accessor :ssh_key, :certificate_path, :connection_options - - protected - def connection_compute options - Fog::Compute.new( options ) - end - - def connection_network options - Fog::Network.new( options ) - end - - def configured? - !(empty_param?(self.ssh_key) or empty_param?(self.certificate_path)) - end - - def empty_param? param - param.nil? or param.empty? - end + attr_accessor :ssh_key, :certificate_path, :connection_options + protected + def connection_compute options + Fog::Compute.new( options ) end + + def connection_network options + Fog::Network.new( options ) + end + + def configured? + !(empty_param?(self.ssh_key) or empty_param?(self.certificate_path)) + end + + def empty_param? param + param.nil? or param.empty? + end + end end diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb index db07409..8ed3b23 100644 --- a/devops-service/providers/ec2.rb +++ b/devops-service/providers/ec2.rb @@ -1,247 +1,240 @@ require "providers/base_provider" -#require 'xml' -module Version2_0 - module Provider - # Provider for Amazon EC2 - class Ec2 < BaseProvider +module Provider + # Provider for Amazon EC2 + class Ec2 < BaseProvider - PROVIDER = "ec2" + PROVIDER = "ec2" - attr_accessor :availability_zone + attr_accessor :availability_zone - def initialize config - self.certificate_path = config[:aws_certificate] - self.ssh_key = config[:aws_ssh_key] - self.connection_options = { - :provider => "aws", - :aws_access_key_id => config[:aws_access_key_id], - :aws_secret_access_key => config[:aws_secret_access_key] - } - self.availability_zone = config[:aws_availability_zone] || "us-east-1a" - end + def initialize config + self.certificate_path = config[:aws_certificate] + self.ssh_key = config[:aws_ssh_key] + self.connection_options = { + :provider => "aws", + :aws_access_key_id => config[:aws_access_key_id], + :aws_secret_access_key => config[:aws_secret_access_key] + } + self.availability_zone = config[:aws_availability_zone] || "us-east-1a" + end - def configured? - o = self.connection_options - super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key])) - end + def configured? + o = self.connection_options + super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key])) + end - def name - PROVIDER - end + def name + PROVIDER + end - def compute - connection_compute(connection_options) - end - - def network - nil - end - - def flavors - self.compute.flavors.all.map do |f| - { - "id" => f.id, - "cores" => f.cores, - "disk" => f.disk, - "name" => f.name, - "ram" => f.ram - } - end - end - - def groups filters=nil - buf = {} - buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil? - g = if buf.empty? - self.compute.describe_security_groups - else - self.compute.describe_security_groups(buf) - end - convert_groups(g.body["securityGroupInfo"]) - end - - def images filters - self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i| - { - "id" => i["imageId"], - "name" => i["name"], - "status" => i["imageState"] - } - end - end - - def networks_detail - self.networks - end - - def networks - self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n| - { - "cidr" => n["cidrBlock"], - "vpcId" => n["vpcId"], - "subnetId" => n["subnetId"], - "name" => n["subnetId"], - "zone" => n["availabilityZone"] - } - end - end - - def servers - list = self.compute.describe_instances.body["reservationSet"] - list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server| - convert_server server["instancesSet"][0] - end - end - - def server id - list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"] - convert_server list[0]["instancesSet"][0] - end - - def create_server s, out - out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" - options = { - "InstanceType" => s.options[:flavor], - "Placement.AvailabilityZone" => s.options[:availability_zone], - "KeyName" => self.ssh_key - } - vpcId = nil - unless s.options[:subnets].empty? - options["SubnetId"] = s.options[:subnets][0] - vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"] - if vpcId.nil? - out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n" - return false - end - end - options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",") - - aws_server = nil - compute = self.compute - begin - aws_server = compute.run_instances(s.options[:image], 1, 1, options) - rescue Excon::Errors::Unauthorized => ue - #root = XML::Parser.string(ue.response.body).parse.root - #msg = root.children.find { |node| node.name == "Message" } - #code = root.children.find { |node| node.name == "Code" } - code = "TODO" - msg = ue.response.body - out << "\nERROR: Unauthorized (#{code}: #{msg})" - return false - rescue Fog::Compute::AWS::Error => e - out << e.message - return false - end - - abody = aws_server.body - instance = abody["instancesSet"][0] - s.id = instance["instanceId"] - - out << "\nInstance Name: #{s.chef_node_name}" - out << "\nInstance ID: #{s.id}\n" - out << "\nWaiting for server..." - - details, state = nil, instance["instanceState"]["name"] - until state == "running" - sleep(2) - details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0] - state = details["instanceState"]["name"].to_s - next if state == "pending" or state == "running" - out << "Server returns state '#{state}'" - return false - end - s.public_ip = details["ipAddress"] - s.private_ip = details["privateIpAddress"] - compute.create_tags(s.id, {"Name" => s.chef_node_name}) - out << "\nDone\n\n" - out << s.info - - true - end - - def create_default_chef_node_name s - "#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" - end - - def delete_server id - r = self.compute.terminate_instances(id) - i = r.body["instancesSet"][0] - old_state = i["previousState"]["name"] - state = i["currentState"]["name"] - return r.status == 200 ? "Server with id '#{id}' changed state '#{old_state}' to '#{state}'" : r.body - end - - def pause_server id - s = self.server id - if s["state"] == "running" - self.compute.stop_instances [ id ] - return nil - else - return s["state"] - end - end - - def unpause_server id - s = self.server id - if s["state"] == "stopped" - self.compute.start_instances [ id ] - return nil - else - return s["state"] - end - end - - private - def convert_groups list - res = {} - list.each do |g| - res[g["groupName"]] = { - "description" => g["groupDescription"], - "id" => g["groupId"] - } - rules = [] - g["ipPermissions"].each do |r| - cidr = r["ipRanges"][0] || {} - rules.push({ - "protocol" => r["ipProtocol"], - "from" => r["fromPort"], - "to" => r["toPort"], - "cidr" => cidr["cidrIp"] - }) - end - res[g["groupName"]]["rules"] = rules - end - res - end - - def convert_server s + def flavors + self.compute.flavors.all.map do |f| { - "state" => s["instanceState"]["name"], - "name" => s["tagSet"]["Name"], - "image" => s["imageId"], - "flavor" => s["instanceType"], - "keypair" => s["keyName"], - "instance_id" => s["instanceId"], - "dns_name" => s["dnsName"], - "zone" => s["placement"]["availabilityZone"], - "private_ip" => s["privateIpAddress"], - "public_ip" => s["ipAddress"], - "launched_at" => s["launchTime"] + "id" => f.id, + "cores" => f.cores, + "disk" => f.disk, + "name" => f.name, + "ram" => f.ram } end + end - def extract_group_ids names, vpcId - return [] if names.nil? - p = nil - p = {"vpc-id" => vpcId} unless vpcId.nil? - groups = self.groups(p) - r = names.map do |name| - groups[name]["id"] + def groups filters=nil + buf = {} + buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil? + g = if buf.empty? + self.compute.describe_security_groups + else + self.compute.describe_security_groups(buf) + end + convert_groups(g.body["securityGroupInfo"]) + end + + def images filters + self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i| + { + "id" => i["imageId"], + "name" => i["name"], + "status" => i["imageState"] + } + end + end + + def networks_detail + self.networks + end + + def networks + self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n| + { + "cidr" => n["cidrBlock"], + "vpcId" => n["vpcId"], + "subnetId" => n["subnetId"], + "name" => n["subnetId"], + "zone" => n["availabilityZone"] + } + end + end + + def servers + list = self.compute.describe_instances.body["reservationSet"] + list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server| + convert_server server["instancesSet"][0] + end + end + + def server id + list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"] + convert_server list[0]["instancesSet"][0] + end + + def create_server s, out + out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" + options = { + "InstanceType" => s.options[:flavor], + "Placement.AvailabilityZone" => s.options[:availability_zone], + "KeyName" => self.ssh_key + } + vpcId = nil + unless s.options[:subnets].empty? + options["SubnetId"] = s.options[:subnets][0] + vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"] + if vpcId.nil? + out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n" + return false end - r + end + options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",") + + aws_server = nil + compute = self.compute + begin + aws_server = compute.run_instances(s.options[:image], 1, 1, options) + rescue Excon::Errors::Unauthorized => ue + #root = XML::Parser.string(ue.response.body).parse.root + #msg = root.children.find { |node| node.name == "Message" } + #code = root.children.find { |node| node.name == "Code" } + code = "TODO" + msg = ue.response.body + out << "\nERROR: Unauthorized (#{code}: #{msg})" + return false + rescue Fog::Compute::AWS::Error => e + out << e.message + return false end + abody = aws_server.body + instance = abody["instancesSet"][0] + s.id = instance["instanceId"] + + out << "\nInstance Name: #{s.chef_node_name}" + out << "\nInstance ID: #{s.id}\n" + out << "\nWaiting for server..." + + details, state = nil, instance["instanceState"]["name"] + until state == "running" + sleep(2) + details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0] + state = details["instanceState"]["name"].to_s + next if state == "pending" or state == "running" + out << "Server returns state '#{state}'" + return false + end + s.public_ip = details["ipAddress"] + s.private_ip = details["privateIpAddress"] + compute.create_tags(s.id, {"Name" => s.chef_node_name}) + out << "\nDone\n\n" + out << s.info + + true + end + + def create_default_chef_node_name s + "#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" + end + + def delete_server s + r = self.compute.terminate_instances(s.id) + i = r.body["instancesSet"][0] + old_state = i["previousState"]["name"] + state = i["currentState"]["name"] + return r.status == 200 ? "Server with id '#{s.id}' changed state '#{old_state}' to '#{state}'" : r.body + end + + def pause_server s + es = self.server s.id + if es["state"] == "running" + self.compute.stop_instances [ s.id ] + return nil + else + return es["state"] + end + end + + def unpause_server s + es = self.server s.id + if es["state"] == "stopped" + self.compute.start_instances [ s.id ] + return nil + else + return es["state"] + end + end + + private + def convert_groups list + res = {} + list.each do |g| + res[g["groupName"]] = { + "description" => g["groupDescription"], + "id" => g["groupId"] + } + rules = [] + g["ipPermissions"].each do |r| + cidr = r["ipRanges"][0] || {} + rules.push({ + "protocol" => r["ipProtocol"], + "from" => r["fromPort"], + "to" => r["toPort"], + "cidr" => cidr["cidrIp"] + }) + end + res[g["groupName"]]["rules"] = rules + end + res + end + + def convert_server s + { + "state" => s["instanceState"]["name"], + "name" => s["tagSet"]["Name"], + "image" => s["imageId"], + "flavor" => s["instanceType"], + "keypair" => s["keyName"], + "instance_id" => s["instanceId"], + "dns_name" => s["dnsName"], + "zone" => s["placement"]["availabilityZone"], + "private_ip" => s["privateIpAddress"], + "public_ip" => s["ipAddress"], + "launched_at" => s["launchTime"] + } + end + + def extract_group_ids names, vpcId + return [] if names.nil? + p = nil + p = {"vpc-id" => vpcId} unless vpcId.nil? + groups = self.groups(p) + r = names.map do |name| + groups[name]["id"] + end + r + end + + private + def compute + connection_compute(connection_options) end end end diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb index 124cd9e..97fa2eb 100644 --- a/devops-service/providers/openstack.rb +++ b/devops-service/providers/openstack.rb @@ -1,217 +1,215 @@ require "providers/base_provider" -module Version2_0 - module Provider - # Provider for 'openstack' - class Openstack < BaseProvider +module Provider + # Provider for 'openstack' + class Openstack < BaseProvider - PROVIDER = "openstack" - - def initialize config - self.certificate_path = config[:openstack_certificate] - self.ssh_key = config[:openstack_ssh_key] - self.connection_options = { - :provider => PROVIDER, - :openstack_username => config[:openstack_username], - :openstack_api_key => config[:openstack_api_key], - :openstack_auth_url => config[:openstack_auth_url], - :openstack_tenant => config[:openstack_tenant] - } - end - - # Returns 'true' if all parameters defined - def configured? - o = self.connection_options - super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant])) - end - - def name - PROVIDER - end - - def compute - connection_compute(self.connection_options) - end - - def network - connection_network(self.connection_options) - end - - def groups filter=nil - convert_groups(compute.list_security_groups.body["security_groups"]) - end - - def flavors - self.compute.list_flavors_detail.body["flavors"].map do |f| - { - "id" => f["name"], - "v_cpus" => f["vcpus"], - "ram" => f["ram"], - "disk" => f["disk"] - } - end - end - - def images filters - self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i| - { - "id" => i["id"], - "name" => i["name"], - "status" => i["status"] - } - end - end - - def networks_detail - net = self.network - subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]} - net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| - sn = subnets.detect{|s| n["subnets"][0] == s["id"]} - { - "cidr" => sn["cidr"], - "name" => n["name"], - "id" => n["id"] - } - } - end - - def networks - net = self.network - net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| - { - "name" => n["name"], - "id" => n["id"] - } - } - end - - def servers - list = self.compute.list_servers_detail.body["servers"] - list.map do |s| - o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]} - s["addresses"].each_value do |a| - a.each do |addr| - o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed" - end - end - o - end - end - - def create_server s, out - out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" - networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])} - buf = s.options[:subnets] - networks.map{|n| n["name"]} - unless buf.empty? - out << "No networks with names '#{buf.join("', '")}' found" - return false - end - s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"] - out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n" - compute = self.compute - begin - o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor], - "nics" => networks.map{|n| {"net_id" => n["id"]}}, - "security_groups" => s.options[:groups], - "key_name" => s.key) - rescue Excon::Errors::BadRequest => e - response = ::Chef::JSONCompat.from_json(e.response.body) - if response['badRequest']['code'] == 400 - if response['badRequest']['message'] =~ /Invalid flavorRef/ - out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}" - elsif response['badRequest']['message'] =~ /Invalid imageRef/ - out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}" - else - out << "\nERROR: Bad request (400): #{response['badRequest']['message']}" - end - out << "\n" - return false - else - out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}" - out << "\n" - return false - end - rescue Excon::Errors::InternalServerError => ise - out << "\nError: openstack internal server error " + ise.message - out << "\n" - return false - rescue => e2 - out << "\nError: Unknown error: " + e2.message - out << "\n" - return false - end - sbody = o_server.body - s.id = sbody["server"]["id"] - - out << "\nInstance Name: #{s.chef_node_name}" - out << "\nInstance ID: #{s.id}\n" - out << "\nWaiting for server..." - - details, status = nil, nil - until status == "ACTIVE" - sleep(1) - details = compute.get_server_details(s.id).body - status = details["server"]["status"].upcase - if status == "ERROR" - out << "Server returns status 'ERROR'" - return false - end - end - network = networks[0]["name"] - s.private_ip = details["server"]["addresses"][network][0]["addr"] - out << "\nDone\n\n" - out << s.info - true - end - - def create_default_chef_node_name s - "#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" - end - - def delete_server id - r = self.compute.delete_server(id) - return r.status == 204 ? "Server with id '#{id}' terminated" : r.body - end - - def pause_server id - begin - self.compute.pause_server id - rescue Excon::Errors::Conflict => e - return "pause" - end - return nil - end - - def unpause_server id - begin - self.compute.unpause_server id - rescue Excon::Errors::Conflict => e - return "unpause" - end - return nil - end - - private - def convert_groups list - res = {} - list.map do |g| - res[g["name"]] = { - "description" => g["description"] - } - rules = [] - g["rules"].each do |r| - rules.push({ - "protocol" => r["ip_protocol"], - "from" => r["from_port"], - "to" => r["to_port"], - "cidr" => r["ip_range"]["cidr"] - }) - end - res[g["name"]]["rules"] = rules - end - res - end + PROVIDER = "openstack" + def initialize config + self.certificate_path = config[:openstack_certificate] + self.ssh_key = config[:openstack_ssh_key] + self.connection_options = { + :provider => PROVIDER, + :openstack_username => config[:openstack_username], + :openstack_api_key => config[:openstack_api_key], + :openstack_auth_url => config[:openstack_auth_url], + :openstack_tenant => config[:openstack_tenant] + } end + + # Returns 'true' if all parameters defined + def configured? + o = self.connection_options + super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant])) + end + + def name + PROVIDER + end + + def groups filter=nil + convert_groups(compute.list_security_groups.body["security_groups"]) + end + + def flavors + self.compute.list_flavors_detail.body["flavors"].map do |f| + { + "id" => f["name"], + "v_cpus" => f["vcpus"], + "ram" => f["ram"], + "disk" => f["disk"] + } + end + end + + def images filters + self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i| + { + "id" => i["id"], + "name" => i["name"], + "status" => i["status"] + } + end + end + + def networks_detail + net = self.network + subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]} + net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| + sn = subnets.detect{|s| n["subnets"][0] == s["id"]} + { + "cidr" => sn["cidr"], + "name" => n["name"], + "id" => n["id"] + } + } + end + + def networks + net = self.network + net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| + { + "name" => n["name"], + "id" => n["id"] + } + } + end + + def servers + list = self.compute.list_servers_detail.body["servers"] + list.map do |s| + o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]} + s["addresses"].each_value do |a| + a.each do |addr| + o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed" + end + end + o + end + end + + def create_server s, out + out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" + networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])} + buf = s.options[:subnets] - networks.map{|n| n["name"]} + unless buf.empty? + out << "No networks with names '#{buf.join("', '")}' found" + return false + end + s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"] + out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n" + compute = self.compute + begin + o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor], + "nics" => networks.map{|n| {"net_id" => n["id"]}}, + "security_groups" => s.options[:groups], + "key_name" => s.key) + rescue Excon::Errors::BadRequest => e + response = ::Chef::JSONCompat.from_json(e.response.body) + if response['badRequest']['code'] == 400 + if response['badRequest']['message'] =~ /Invalid flavorRef/ + out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}" + elsif response['badRequest']['message'] =~ /Invalid imageRef/ + out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}" + else + out << "\nERROR: Bad request (400): #{response['badRequest']['message']}" + end + out << "\n" + return false + else + out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}" + out << "\n" + return false + end + rescue Excon::Errors::InternalServerError => ise + out << "\nError: openstack internal server error " + ise.message + out << "\n" + return false + rescue => e2 + out << "\nError: Unknown error: " + e2.message + out << "\n" + return false + end + sbody = o_server.body + s.id = sbody["server"]["id"] + + out << "\nInstance Name: #{s.chef_node_name}" + out << "\nInstance ID: #{s.id}\n" + out << "\nWaiting for server..." + + details, status = nil, nil + until status == "ACTIVE" + sleep(1) + details = compute.get_server_details(s.id).body + status = details["server"]["status"].upcase + if status == "ERROR" + out << "Server returns status 'ERROR'" + return false + end + end + network = networks[0]["name"] + s.private_ip = details["server"]["addresses"][network][0]["addr"] + out << "\nDone\n\n" + out << s.info + true + end + + def create_default_chef_node_name s + "#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" + end + + def delete_server s + r = self.compute.delete_server(s.id) + return r.status == 204 ? "Server with id '#{s.id}' terminated" : r.body + end + + def pause_server s + begin + self.compute.pause_server s.id + rescue Excon::Errors::Conflict => e + return "pause" + end + return nil + end + + def unpause_server s + begin + self.compute.unpause_server s.id + rescue Excon::Errors::Conflict => e + return "unpause" + end + return nil + end + + private + def convert_groups list + res = {} + list.map do |g| + res[g["name"]] = { + "description" => g["description"] + } + rules = [] + g["rules"].each do |r| + rules.push({ + "protocol" => r["ip_protocol"], + "from" => r["from_port"], + "to" => r["to_port"], + "cidr" => r["ip_range"]["cidr"] + }) + end + res[g["name"]]["rules"] = rules + end + res + end + + def compute + connection_compute(self.connection_options) + end + + def network + connection_network(self.connection_options) + end + end end diff --git a/devops-service/providers/provider_factory.rb b/devops-service/providers/provider_factory.rb index ca37f77..17fe878 100644 --- a/devops-service/providers/provider_factory.rb +++ b/devops-service/providers/provider_factory.rb @@ -1,42 +1,40 @@ -module Version2_0 - module Provider - class ProviderFactory +module Provider + class ProviderFactory - @@providers = nil - - def self.providers - @@providers.keys - end - - def self.get provider - p = @@providers[provider] - raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil? - p - end - - def self.all - if @@providers.nil? - ProviderFactory.init - end - @@providers.values - end - - def self.init - conf = DevopsService.config - @@providers = {} - ["ec2", "openstack"].each do |p| - begin - require "providers/#{p}" - o = Version2_0::Provider.const_get(p.capitalize).new(conf) - @@providers[p] = o if o.configured? - rescue => e - next - rescue LoadError - next - end - end - end + @@providers = nil + def self.providers + @@providers.keys end + + def self.get provider + p = @@providers[provider] + raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil? + p + end + + def self.all + if @@providers.nil? + ProviderFactory.init + end + @@providers.values + end + + def self.init + conf = DevopsService.config + @@providers = {} + ["ec2", "openstack", "static"].each do |p| + begin + require "providers/#{p}" + o = Provider.const_get(p.capitalize).new(conf) + @@providers[p] = o if o.configured? + rescue => e + next + rescue LoadError + next + end + end + end + end end diff --git a/devops-service/providers/static.rb b/devops-service/providers/static.rb new file mode 100644 index 0000000..1166b6b --- /dev/null +++ b/devops-service/providers/static.rb @@ -0,0 +1,63 @@ +require "providers/base_provider" +require "commands/server" + +module Provider + class Static < BaseProvider + PROVIDER = "static" + + def initialize config + self.certificate_path = config[:static_certificate] + self.ssh_key = "static" + @@mongo ||= MongoConnector.new(config[:mongo_db], config[:mongo_host], config[:mongo_port], config[:mongo_user], config[:mongo_password]) + end + + def configured? + true + end + + def name + PROVIDER + end + + def flavors + [] + end + + def images filters + [] + end + + def networks + [] + end + + def servers + @@mongo.servers_find({:provider => PROVIDER}).map{|s| s.to_hash} + end + + def create_default_chef_node_name s + "static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}" + end + + def create_server s, out + out << "Unsupported operation: ca not create server for provider 'static'" + false + end + + def delete_server s + cert = @@mongo.key(s.key).path + res = ::ServerCommands.unbootstrap(s, cert) + m = "Static server with id '#{s.id}' and name '#{s.chef_node_name}' " + return m + (res.nil? ? "has been unbootstraped" : "can not be unbootstraped: #{res}") + end + + def pause_server s + nil + end + + def unpause_server s + nil + end + + end +end diff --git a/devops-service/routes/v2.0/base_routes.rb b/devops-service/routes/v2.0/base_routes.rb index 29f9c5e..ec63d3a 100644 --- a/devops-service/routes/v2.0/base_routes.rb +++ b/devops-service/routes/v2.0/base_routes.rb @@ -68,7 +68,7 @@ module Version2_0 end def check_provider provider - list = ::Version2_0::Provider::ProviderFactory.providers + list = ::Provider::ProviderFactory.providers halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider) end diff --git a/devops-service/routes/v2.0/flavor.rb b/devops-service/routes/v2.0/flavor.rb index efb72aa..951dac4 100644 --- a/devops-service/routes/v2.0/flavor.rb +++ b/devops-service/routes/v2.0/flavor.rb @@ -41,7 +41,7 @@ module Version2_0 check_headers :accept check_privileges("flavor", "r") check_provider(params[:provider]) - p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + p = ::Provider::ProviderFactory.get params[:provider] json p.flavors end diff --git a/devops-service/routes/v2.0/group.rb b/devops-service/routes/v2.0/group.rb index 6c27d0a..7d0b0d3 100644 --- a/devops-service/routes/v2.0/group.rb +++ b/devops-service/routes/v2.0/group.rb @@ -53,7 +53,7 @@ module Version2_0 check_headers :accept check_privileges("group", "r") check_provider(params[:provider]) - p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + p = ::Provider::ProviderFactory.get params[:provider] json p.groups(params) end diff --git a/devops-service/routes/v2.0/network.rb b/devops-service/routes/v2.0/network.rb index 2f8201f..d209b25 100644 --- a/devops-service/routes/v2.0/network.rb +++ b/devops-service/routes/v2.0/network.rb @@ -41,7 +41,7 @@ module Version2_0 check_headers :accept check_privileges("network", "r") check_provider(params[:provider]) - p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + p = ::Provider::ProviderFactory.get params[:provider] json p.networks_detail end diff --git a/devops-service/routes/v2.0/project.rb b/devops-service/routes/v2.0/project.rb index bfda09b..6c4f1f9 100644 --- a/devops-service/routes/v2.0/project.rb +++ b/devops-service/routes/v2.0/project.rb @@ -476,7 +476,7 @@ module Version2_0 project = BaseRoutes.mongo.project(params[:id]) env = project.deploy_env params[:env] user = request.env['REMOTE_USER'] - provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) + provider = ::Provider::ProviderFactory.get(env.provider) header = "Test project '#{project.id}' and environment '#{env.identifier}'" logger.info header servers = extract_servers(provider, project, env, {}, user, BaseRoutes.mongo) diff --git a/devops-service/routes/v2.0/provider.rb b/devops-service/routes/v2.0/provider.rb index 61415d1..de340f0 100644 --- a/devops-service/routes/v2.0/provider.rb +++ b/devops-service/routes/v2.0/provider.rb @@ -26,7 +26,7 @@ module Version2_0 get "/providers" do check_headers :accept check_privileges("provider", "r") - json ::Version2_0::Provider::ProviderFactory.providers + json ::Provider::ProviderFactory.providers end end diff --git a/devops-service/routes/v2.0/server.rb b/devops-service/routes/v2.0/server.rb index 66c2078..95c693b 100644 --- a/devops-service/routes/v2.0/server.rb +++ b/devops-service/routes/v2.0/server.rb @@ -139,7 +139,7 @@ module Version2_0 # } # ] get "/servers/:provider" do - json ::Version2_0::Provider::ProviderFactory.get(params[:provider]).servers + json ::Provider::ProviderFactory.get(params[:provider]).servers end # Get server info by :name @@ -223,7 +223,7 @@ module Version2_0 p = BaseRoutes.mongo.check_project_auth(project_name, env_name, user) env = p.deploy_env(env_name) - provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) + provider = ::Provider::ProviderFactory.get(env.provider) check_chef_node_name(server_name, provider) unless server_name.nil? unless groups.nil? buf = groups - provider.groups.keys @@ -286,8 +286,8 @@ module Version2_0 s = get_server(params[:node_name], @key) ## Authorization BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] - provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) - r = provider.pause_server s.id + provider = ::Provider::ProviderFactory.get(s.provider) + r = provider.pause_server s if r.nil? create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is paused") else @@ -313,8 +313,8 @@ module Version2_0 s = get_server(params[:node_name], @key) ## Authorization BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] - provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) - r = provider.unpause_server s.id + provider = ::Provider::ProviderFactory.get(s.provider) + r = provider.unpause_server s if r.nil? create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is unpaused") else @@ -401,27 +401,34 @@ module Version2_0 p = BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] d = p.deploy_env s.deploy_env + provider = ::Provider::ProviderFactory.get(s.provider) + s.options = { :run_list => rl || d.run_list, } s.options[:bootstrap_template] = t unless t.nil? + status = [] stream() do |out| begin - s.chef_node_name = name || "static_#{s.key}-#{Time.now.to_i}" + s.chef_node_name = name || provider.create_default_chef_node_name(s) cert = BaseRoutes.mongo.key s.key logger.debug "Bootstrap certificate path: #{cert.path}" bootstrap s, out, cert.path, logger str = nil - if check_server(s) + r = if check_server(s) BaseRoutes.mongo.server_update s str = "Server with id '#{s.id}' is bootstraped" logger.info str + 0 else str = "Server with id '#{s.id}' is not bootstraped" logger.warn str + 1 end + status.push r out << str out << "\n" + out << create_status(status) rescue IOError => e logger.error e.message end @@ -461,8 +468,9 @@ module Version2_0 d = p.deploy_env(deploy_env) cert = BaseRoutes.mongo.key(key) + provider = ::Provider::ProviderFactory.get("static") s = Server.new - s.provider = "static" + s.provider = provider.name s.project = project s.deploy_env = deploy_env s.remote_user = remote_user