tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

provider static, todo: project env with provider static

Browse Source
This commit is contained in:
amartynov 2014-06-18 15:11:47 +04:00
parent 0ffdd8261c
commit 10eaf25454
23 changed files with 606 additions and 547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
devops-client/lib/devops-client/handler/server.rb
View File

@ -63,7 +63,7 @@ class Server < Handler
@list = case args[2] @list = case args[2]
when "chef" when "chef"
get("/servers/chef").map {|l| {"chef_node_name" => l}} get("/servers/chef").map {|l| {"chef_node_name" => l}}
when "ec2", "openstack" when "ec2", "openstack", "static"
get("/servers/#{args[2]}") get("/servers/#{args[2]}")
else else
@options_parser.invalid_list_command @options_parser.invalid_list_command
@ -149,7 +149,7 @@ class Server < Handler
:key => args[6] :key => args[6]
} }
q[:public_ip] = self.options[:public_ip] unless self.options[:public_ip].nil? q[:public_ip] = self.options[:public_ip] unless self.options[:public_ip].nil?
post_chunk "/server/add", q post "/server/add", q
end end
def pause_handler args def pause_handler args

2
devops-client/lib/devops-client/options/server_options.rb
View File

@ -8,7 +8,7 @@ class ServerOptions < CommonOptions
super(args, def_options) super(args, def_options)
self.header = I18n.t("headers.server") self.header = I18n.t("headers.server")
self.banner_header = "server" self.banner_header = "server"
self.list_params = ["[chef|ec2|openstack]"] self.list_params = ["[chef|ec2|openstack|static]"]
self.create_params = ["PROJECT_ID", "DEPLOY_ENV"] self.create_params = ["PROJECT_ID", "DEPLOY_ENV"]
node_params = ["NODE_NAME"] node_params = ["NODE_NAME"]
self.delete_params = node_params self.delete_params = node_params

4
devops-service/commands/deploy_env.rb
View File

@ -25,12 +25,12 @@ module DeployEnvCommands
raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty? raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty?
filter = nil filter = nil
if p.name == ::Version2_0::Provider::Ec2::PROVIDER if p.name == ::Provider::Ec2::PROVIDER
unless subnets.empty? unless subnets.empty?
subnets = [ subnets[0] ] if subnets.size > 1 subnets = [ subnets[0] ] if subnets.size > 1
filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] } filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] }
end end
elsif p.name == ::Version2_0::Provider::Openstack::PROVIDER elsif p.name == ::Provider::Openstack::PROVIDER
if subnets.empty? if subnets.empty?
raise InvalidRecord.new "Subnets array can not be empty" raise InvalidRecord.new "Subnets array can not be empty"
end end

2
devops-service/commands/image.rb
View File

@ -7,7 +7,7 @@ module ImageCommands
if filters.empty? if filters.empty?
[] []
else else
::Version2_0::Provider::ProviderFactory.get(provider).images(filters) ::Provider::ProviderFactory.get(provider).images(filters)
end end
end end
end end

40
devops-service/commands/server.rb
View File

@ -142,17 +142,19 @@ module ServerCommands
return status return status
end end
def unbootstrap s, cert_path def self.unbootstrap s, cert_path
i = 0 i = 0
begin begin
`ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef` r = `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef`
raise unless $?.success? raise(r) unless $?.success?
rescue => e rescue => e
logger.error "Unbootstrap eeror: " + e.message logger.error "Unbootstrap error: " + e.message
i += 1 i += 1
sleep(1) sleep(1)
retry unless i == 5 retry unless i == 5
return e.message
end end
nil
end end
def delete_server s, mongo, logger def delete_server s, mongo, logger
@ -163,26 +165,16 @@ module ServerCommands
return msg, nil return msg, nil
end end
r = delete_from_chef_server(s.chef_node_name) r = delete_from_chef_server(s.chef_node_name)
info = if s.static provider = ::Provider::ProviderFactory.get(s.provider)
cert = mongo.key(s.key).path begin
unbootstrap(s, cert) r[:server] = provider.delete_server s
mongo.server_delete s.id rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
msg = "Static server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
logger.info msg logger.warn r[:server]
msg
else
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider)
begin
r[:server] = provider.delete_server s.id
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
logger.warn r[:server]
end
mongo.server_delete s.id
msg = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
logger.info msg
msg
end end
mongo.server_delete s.id
info = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
logger.info info
r.each{|key, log| logger.info("#{key} - #{log}")} r.each{|key, log| logger.info("#{key} - #{log}")}
return info, r return info, r
end end
@ -193,7 +185,7 @@ module ServerCommands
str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n" str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n"
str << delete_from_chef_server(s.chef_node_name).values.join("\n") str << delete_from_chef_server(s.chef_node_name).values.join("\n")
begin begin
str << provider.delete_server(s.id) unless s.static str << provider.delete_server(s)
rescue => e rescue => e
str << e.message str << e.message
end end

4
devops-service/config.rb
View File

@ -34,3 +34,7 @@ config[:aws_secret_access_key] = "secret_access_key"
config[:aws_ssh_key] = "ssh_key" config[:aws_ssh_key] = "ssh_key"
config[:aws_certificate] = "/path/to/.ssh/ec2.pem" config[:aws_certificate] = "/path/to/.ssh/ec2.pem"
config[:aws_availability_zone] = "aws_zone" config[:aws_availability_zone] = "aws_zone"
# static settings
config[:static_ssh_key] = "ssh_key" # or nil
config[:static_certificate] = "/path/to/.ssh/static.pem"

2
devops-service/db/mongo/models/deploy_env.rb
View File

@ -44,7 +44,7 @@ class DeployEnv < MongoModel
check_expires!(self.expires) check_expires!(self.expires)
end end
p = ::Version2_0::Provider::ProviderFactory.get(self.provider) p = ::Provider::ProviderFactory.get(self.provider)
check_flavor!(p, self.flavor) check_flavor!(p, self.flavor)
check_image!(p, self.image) check_image!(p, self.image)
check_subnets_and_groups!(p, self.subnets, self.groups) check_subnets_and_groups!(p, self.subnets, self.groups)

2
devops-service/db/mongo/models/deploy_env_multi.rb
View File

@ -65,7 +65,7 @@ class DeployEnvMulti < MongoModel
end end
end end
p = ::Version2_0::Provider::ProviderFactory.get(server["provider"]) p = ::Provider::ProviderFactory.get(server["provider"])
check_flavor!(p, server["flavor"]) check_flavor!(p, server["flavor"])
check_image!(p, server["image"]) check_image!(p, server["image"])
check_subnets_and_groups!(p, server["subnets"], server["groups"]) check_subnets_and_groups!(p, server["subnets"], server["groups"])

2
devops-service/db/mongo/models/mongo_model.rb
View File

@ -43,7 +43,7 @@ class MongoModel
end end
def check_provider provider=self.provider def check_provider provider=self.provider
unless ::Version2_0::Provider::ProviderFactory.providers.include?(provider) or provider == "static" unless ::Provider::ProviderFactory.providers.include?(provider)
raise InvalidRecord.new "Invalid provider '#{provider}'" raise InvalidRecord.new "Invalid provider '#{provider}'"
end end
end end

6
devops-service/db/mongo/mongo_connector.rb
View File

@ -191,12 +191,16 @@ class MongoConnector
end end
end end
def servers_find q
@servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs}
end
def servers p=nil, env=nil, names=nil def servers p=nil, env=nil, names=nil
q = {} q = {}
q["project"] = p unless p.nil? or p.empty? q["project"] = p unless p.nil? or p.empty?
q["deploy_env"] = env unless env.nil? or env.empty? q["deploy_env"] = env unless env.nil? or env.empty?
q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array
@servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs} servers_find(q)
end end
def servers_by_names names def servers_by_names names

3
devops-service/devops-service.rb
View File

@ -29,7 +29,8 @@ class DevopsService < Sinatra::Base
[:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } [:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) }
mongo = DevopsService.mongo mongo = DevopsService.mongo
mongo.create_root_user mongo.create_root_user
::Version2_0::Provider::ProviderFactory.all.each do |p| ::Provider::ProviderFactory.all.each do |p|
next if p.certificate_path.nil?
begin begin
mongo.key p.ssh_key, Key::SYSTEM mongo.key p.ssh_key, Key::SYSTEM
rescue RecordNotFound => e rescue RecordNotFound => e

40
devops-service/providers/base_provider.rb
View File

@ -1,28 +1,26 @@
require "fog" require "fog"
module Version2_0 module Provider
module Provider class BaseProvider
class BaseProvider
attr_accessor :ssh_key, :certificate_path, :connection_options attr_accessor :ssh_key, :certificate_path, :connection_options
protected
def connection_compute options
Fog::Compute.new( options )
end
def connection_network options
Fog::Network.new( options )
end
def configured?
!(empty_param?(self.ssh_key) or empty_param?(self.certificate_path))
end
def empty_param? param
param.nil? or param.empty?
end
protected
def connection_compute options
Fog::Compute.new( options )
end end
def connection_network options
Fog::Network.new( options )
end
def configured?
!(empty_param?(self.ssh_key) or empty_param?(self.certificate_path))
end
def empty_param? param
param.nil? or param.empty?
end
end end
end end

451
devops-service/providers/ec2.rb
View File

@ -1,247 +1,240 @@
require "providers/base_provider" require "providers/base_provider"
#require 'xml'
module Version2_0 module Provider
module Provider # Provider for Amazon EC2
# Provider for Amazon EC2 class Ec2 < BaseProvider
class Ec2 < BaseProvider
PROVIDER = "ec2" PROVIDER = "ec2"
attr_accessor :availability_zone attr_accessor :availability_zone
def initialize config def initialize config
self.certificate_path = config[:aws_certificate] self.certificate_path = config[:aws_certificate]
self.ssh_key = config[:aws_ssh_key] self.ssh_key = config[:aws_ssh_key]
self.connection_options = { self.connection_options = {
:provider => "aws", :provider => "aws",
:aws_access_key_id => config[:aws_access_key_id], :aws_access_key_id => config[:aws_access_key_id],
:aws_secret_access_key => config[:aws_secret_access_key] :aws_secret_access_key => config[:aws_secret_access_key]
} }
self.availability_zone = config[:aws_availability_zone] || "us-east-1a" self.availability_zone = config[:aws_availability_zone] || "us-east-1a"
end end
def configured? def configured?
o = self.connection_options o = self.connection_options
super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key])) super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
end end
def name def name
PROVIDER PROVIDER
end end
def compute def flavors
connection_compute(connection_options) self.compute.flavors.all.map do |f|
end
def network
nil
end
def flavors
self.compute.flavors.all.map do |f|
{
"id" => f.id,
"cores" => f.cores,
"disk" => f.disk,
"name" => f.name,
"ram" => f.ram
}
end
end
def groups filters=nil
buf = {}
buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil?
g = if buf.empty?
self.compute.describe_security_groups
else
self.compute.describe_security_groups(buf)
end
convert_groups(g.body["securityGroupInfo"])
end
def images filters
self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i|
{
"id" => i["imageId"],
"name" => i["name"],
"status" => i["imageState"]
}
end
end
def networks_detail
self.networks
end
def networks
self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n|
{
"cidr" => n["cidrBlock"],
"vpcId" => n["vpcId"],
"subnetId" => n["subnetId"],
"name" => n["subnetId"],
"zone" => n["availabilityZone"]
}
end
end
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
convert_server server["instancesSet"][0]
end
end
def server id
list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"]
convert_server list[0]["instancesSet"][0]
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
options = {
"InstanceType" => s.options[:flavor],
"Placement.AvailabilityZone" => s.options[:availability_zone],
"KeyName" => self.ssh_key
}
vpcId = nil
unless s.options[:subnets].empty?
options["SubnetId"] = s.options[:subnets][0]
vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
if vpcId.nil?
out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
return false
end
end
options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
aws_server = nil
compute = self.compute
begin
aws_server = compute.run_instances(s.options[:image], 1, 1, options)
rescue Excon::Errors::Unauthorized => ue
#root = XML::Parser.string(ue.response.body).parse.root
#msg = root.children.find { |node| node.name == "Message" }
#code = root.children.find { |node| node.name == "Code" }
code = "TODO"
msg = ue.response.body
out << "\nERROR: Unauthorized (#{code}: #{msg})"
return false
rescue Fog::Compute::AWS::Error => e
out << e.message
return false
end
abody = aws_server.body
instance = abody["instancesSet"][0]
s.id = instance["instanceId"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, state = nil, instance["instanceState"]["name"]
until state == "running"
sleep(2)
details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0]
state = details["instanceState"]["name"].to_s
next if state == "pending" or state == "running"
out << "Server returns state '#{state}'"
return false
end
s.public_ip = details["ipAddress"]
s.private_ip = details["privateIpAddress"]
compute.create_tags(s.id, {"Name" => s.chef_node_name})
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server id
r = self.compute.terminate_instances(id)
i = r.body["instancesSet"][0]
old_state = i["previousState"]["name"]
state = i["currentState"]["name"]
return r.status == 200 ? "Server with id '#{id}' changed state '#{old_state}' to '#{state}'" : r.body
end
def pause_server id
s = self.server id
if s["state"] == "running"
self.compute.stop_instances [ id ]
return nil
else
return s["state"]
end
end
def unpause_server id
s = self.server id
if s["state"] == "stopped"
self.compute.start_instances [ id ]
return nil
else
return s["state"]
end
end
private
def convert_groups list
res = {}
list.each do |g|
res[g["groupName"]] = {
"description" => g["groupDescription"],
"id" => g["groupId"]
}
rules = []
g["ipPermissions"].each do |r|
cidr = r["ipRanges"][0] || {}
rules.push({
"protocol" => r["ipProtocol"],
"from" => r["fromPort"],
"to" => r["toPort"],
"cidr" => cidr["cidrIp"]
})
end
res[g["groupName"]]["rules"] = rules
end
res
end
def convert_server s
{ {
"state" => s["instanceState"]["name"], "id" => f.id,
"name" => s["tagSet"]["Name"], "cores" => f.cores,
"image" => s["imageId"], "disk" => f.disk,
"flavor" => s["instanceType"], "name" => f.name,
"keypair" => s["keyName"], "ram" => f.ram
"instance_id" => s["instanceId"],
"dns_name" => s["dnsName"],
"zone" => s["placement"]["availabilityZone"],
"private_ip" => s["privateIpAddress"],
"public_ip" => s["ipAddress"],
"launched_at" => s["launchTime"]
} }
end end
end
def extract_group_ids names, vpcId def groups filters=nil
return [] if names.nil? buf = {}
p = nil buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil?
p = {"vpc-id" => vpcId} unless vpcId.nil? g = if buf.empty?
groups = self.groups(p) self.compute.describe_security_groups
r = names.map do |name| else
groups[name]["id"] self.compute.describe_security_groups(buf)
end
convert_groups(g.body["securityGroupInfo"])
end
def images filters
self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i|
{
"id" => i["imageId"],
"name" => i["name"],
"status" => i["imageState"]
}
end
end
def networks_detail
self.networks
end
def networks
self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n|
{
"cidr" => n["cidrBlock"],
"vpcId" => n["vpcId"],
"subnetId" => n["subnetId"],
"name" => n["subnetId"],
"zone" => n["availabilityZone"]
}
end
end
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
convert_server server["instancesSet"][0]
end
end
def server id
list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"]
convert_server list[0]["instancesSet"][0]
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
options = {
"InstanceType" => s.options[:flavor],
"Placement.AvailabilityZone" => s.options[:availability_zone],
"KeyName" => self.ssh_key
}
vpcId = nil
unless s.options[:subnets].empty?
options["SubnetId"] = s.options[:subnets][0]
vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
if vpcId.nil?
out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
return false
end end
r end
options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
aws_server = nil
compute = self.compute
begin
aws_server = compute.run_instances(s.options[:image], 1, 1, options)
rescue Excon::Errors::Unauthorized => ue
#root = XML::Parser.string(ue.response.body).parse.root
#msg = root.children.find { |node| node.name == "Message" }
#code = root.children.find { |node| node.name == "Code" }
code = "TODO"
msg = ue.response.body
out << "\nERROR: Unauthorized (#{code}: #{msg})"
return false
rescue Fog::Compute::AWS::Error => e
out << e.message
return false
end end
abody = aws_server.body
instance = abody["instancesSet"][0]
s.id = instance["instanceId"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, state = nil, instance["instanceState"]["name"]
until state == "running"
sleep(2)
details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0]
state = details["instanceState"]["name"].to_s
next if state == "pending" or state == "running"
out << "Server returns state '#{state}'"
return false
end
s.public_ip = details["ipAddress"]
s.private_ip = details["privateIpAddress"]
compute.create_tags(s.id, {"Name" => s.chef_node_name})
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server s
r = self.compute.terminate_instances(s.id)
i = r.body["instancesSet"][0]
old_state = i["previousState"]["name"]
state = i["currentState"]["name"]
return r.status == 200 ? "Server with id '#{s.id}' changed state '#{old_state}' to '#{state}'" : r.body
end
def pause_server s
es = self.server s.id
if es["state"] == "running"
self.compute.stop_instances [ s.id ]
return nil
else
return es["state"]
end
end
def unpause_server s
es = self.server s.id
if es["state"] == "stopped"
self.compute.start_instances [ s.id ]
return nil
else
return es["state"]
end
end
private
def convert_groups list
res = {}
list.each do |g|
res[g["groupName"]] = {
"description" => g["groupDescription"],
"id" => g["groupId"]
}
rules = []
g["ipPermissions"].each do |r|
cidr = r["ipRanges"][0] || {}
rules.push({
"protocol" => r["ipProtocol"],
"from" => r["fromPort"],
"to" => r["toPort"],
"cidr" => cidr["cidrIp"]
})
end
res[g["groupName"]]["rules"] = rules
end
res
end
def convert_server s
{
"state" => s["instanceState"]["name"],
"name" => s["tagSet"]["Name"],
"image" => s["imageId"],
"flavor" => s["instanceType"],
"keypair" => s["keyName"],
"instance_id" => s["instanceId"],
"dns_name" => s["dnsName"],
"zone" => s["placement"]["availabilityZone"],
"private_ip" => s["privateIpAddress"],
"public_ip" => s["ipAddress"],
"launched_at" => s["launchTime"]
}
end
def extract_group_ids names, vpcId
return [] if names.nil?
p = nil
p = {"vpc-id" => vpcId} unless vpcId.nil?
groups = self.groups(p)
r = names.map do |name|
groups[name]["id"]
end
r
end
private
def compute
connection_compute(connection_options)
end end
end end
end end

418
devops-service/providers/openstack.rb
View File

@ -1,217 +1,215 @@
require "providers/base_provider" require "providers/base_provider"
module Version2_0 module Provider
module Provider # Provider for 'openstack'
# Provider for 'openstack' class Openstack < BaseProvider
class Openstack < BaseProvider
PROVIDER = "openstack" PROVIDER = "openstack"
def initialize config
self.certificate_path = config[:openstack_certificate]
self.ssh_key = config[:openstack_ssh_key]
self.connection_options = {
:provider => PROVIDER,
:openstack_username => config[:openstack_username],
:openstack_api_key => config[:openstack_api_key],
:openstack_auth_url => config[:openstack_auth_url],
:openstack_tenant => config[:openstack_tenant]
}
end
# Returns 'true' if all parameters defined
def configured?
o = self.connection_options
super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant]))
end
def name
PROVIDER
end
def compute
connection_compute(self.connection_options)
end
def network
connection_network(self.connection_options)
end
def groups filter=nil
convert_groups(compute.list_security_groups.body["security_groups"])
end
def flavors
self.compute.list_flavors_detail.body["flavors"].map do |f|
{
"id" => f["name"],
"v_cpus" => f["vcpus"],
"ram" => f["ram"],
"disk" => f["disk"]
}
end
end
def images filters
self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i|
{
"id" => i["id"],
"name" => i["name"],
"status" => i["status"]
}
end
end
def networks_detail
net = self.network
subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]}
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
sn = subnets.detect{|s| n["subnets"][0] == s["id"]}
{
"cidr" => sn["cidr"],
"name" => n["name"],
"id" => n["id"]
}
}
end
def networks
net = self.network
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
{
"name" => n["name"],
"id" => n["id"]
}
}
end
def servers
list = self.compute.list_servers_detail.body["servers"]
list.map do |s|
o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]}
s["addresses"].each_value do |a|
a.each do |addr|
o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed"
end
end
o
end
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
buf = s.options[:subnets] - networks.map{|n| n["name"]}
unless buf.empty?
out << "No networks with names '#{buf.join("', '")}' found"
return false
end
s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
compute = self.compute
begin
o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
"nics" => networks.map{|n| {"net_id" => n["id"]}},
"security_groups" => s.options[:groups],
"key_name" => s.key)
rescue Excon::Errors::BadRequest => e
response = ::Chef::JSONCompat.from_json(e.response.body)
if response['badRequest']['code'] == 400
if response['badRequest']['message'] =~ /Invalid flavorRef/
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
elsif response['badRequest']['message'] =~ /Invalid imageRef/
out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
else
out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
end
out << "\n"
return false
else
out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}"
out << "\n"
return false
end
rescue Excon::Errors::InternalServerError => ise
out << "\nError: openstack internal server error " + ise.message
out << "\n"
return false
rescue => e2
out << "\nError: Unknown error: " + e2.message
out << "\n"
return false
end
sbody = o_server.body
s.id = sbody["server"]["id"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, status = nil, nil
until status == "ACTIVE"
sleep(1)
details = compute.get_server_details(s.id).body
status = details["server"]["status"].upcase
if status == "ERROR"
out << "Server returns status 'ERROR'"
return false
end
end
network = networks[0]["name"]
s.private_ip = details["server"]["addresses"][network][0]["addr"]
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server id
r = self.compute.delete_server(id)
return r.status == 204 ? "Server with id '#{id}' terminated" : r.body
end
def pause_server id
begin
self.compute.pause_server id
rescue Excon::Errors::Conflict => e
return "pause"
end
return nil
end
def unpause_server id
begin
self.compute.unpause_server id
rescue Excon::Errors::Conflict => e
return "unpause"
end
return nil
end
private
def convert_groups list
res = {}
list.map do |g|
res[g["name"]] = {
"description" => g["description"]
}
rules = []
g["rules"].each do |r|
rules.push({
"protocol" => r["ip_protocol"],
"from" => r["from_port"],
"to" => r["to_port"],
"cidr" => r["ip_range"]["cidr"]
})
end
res[g["name"]]["rules"] = rules
end
res
end
def initialize config
self.certificate_path = config[:openstack_certificate]
self.ssh_key = config[:openstack_ssh_key]
self.connection_options = {
:provider => PROVIDER,
:openstack_username => config[:openstack_username],
:openstack_api_key => config[:openstack_api_key],
:openstack_auth_url => config[:openstack_auth_url],
:openstack_tenant => config[:openstack_tenant]
}
end end
# Returns 'true' if all parameters defined
def configured?
o = self.connection_options
super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant]))
end
def name
PROVIDER
end
def groups filter=nil
convert_groups(compute.list_security_groups.body["security_groups"])
end
def flavors
self.compute.list_flavors_detail.body["flavors"].map do |f|
{
"id" => f["name"],
"v_cpus" => f["vcpus"],
"ram" => f["ram"],
"disk" => f["disk"]
}
end
end
def images filters
self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i|
{
"id" => i["id"],
"name" => i["name"],
"status" => i["status"]
}
end
end
def networks_detail
net = self.network
subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]}
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
sn = subnets.detect{|s| n["subnets"][0] == s["id"]}
{
"cidr" => sn["cidr"],
"name" => n["name"],
"id" => n["id"]
}
}
end
def networks
net = self.network
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
{
"name" => n["name"],
"id" => n["id"]
}
}
end
def servers
list = self.compute.list_servers_detail.body["servers"]
list.map do |s|
o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]}
s["addresses"].each_value do |a|
a.each do |addr|
o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed"
end
end
o
end
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
buf = s.options[:subnets] - networks.map{|n| n["name"]}
unless buf.empty?
out << "No networks with names '#{buf.join("', '")}' found"
return false
end
s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
compute = self.compute
begin
o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
"nics" => networks.map{|n| {"net_id" => n["id"]}},
"security_groups" => s.options[:groups],
"key_name" => s.key)
rescue Excon::Errors::BadRequest => e
response = ::Chef::JSONCompat.from_json(e.response.body)
if response['badRequest']['code'] == 400
if response['badRequest']['message'] =~ /Invalid flavorRef/
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
elsif response['badRequest']['message'] =~ /Invalid imageRef/
out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
else
out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
end
out << "\n"
return false
else
out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}"
out << "\n"
return false
end
rescue Excon::Errors::InternalServerError => ise
out << "\nError: openstack internal server error " + ise.message
out << "\n"
return false
rescue => e2
out << "\nError: Unknown error: " + e2.message
out << "\n"
return false
end
sbody = o_server.body
s.id = sbody["server"]["id"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, status = nil, nil
until status == "ACTIVE"
sleep(1)
details = compute.get_server_details(s.id).body
status = details["server"]["status"].upcase
if status == "ERROR"
out << "Server returns status 'ERROR'"
return false
end
end
network = networks[0]["name"]
s.private_ip = details["server"]["addresses"][network][0]["addr"]
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server s
r = self.compute.delete_server(s.id)
return r.status == 204 ? "Server with id '#{s.id}' terminated" : r.body
end
def pause_server s
begin
self.compute.pause_server s.id
rescue Excon::Errors::Conflict => e
return "pause"
end
return nil
end
def unpause_server s
begin
self.compute.unpause_server s.id
rescue Excon::Errors::Conflict => e
return "unpause"
end
return nil
end
private
def convert_groups list
res = {}
list.map do |g|
res[g["name"]] = {
"description" => g["description"]
}
rules = []
g["rules"].each do |r|
rules.push({
"protocol" => r["ip_protocol"],
"from" => r["from_port"],
"to" => r["to_port"],
"cidr" => r["ip_range"]["cidr"]
})
end
res[g["name"]]["rules"] = rules
end
res
end
def compute
connection_compute(self.connection_options)
end
def network
connection_network(self.connection_options)
end
end end
end end

72
devops-service/providers/provider_factory.rb
View File

@ -1,42 +1,40 @@
module Version2_0 module Provider
module Provider class ProviderFactory
class ProviderFactory
@@providers = nil @@providers = nil
def self.providers
@@providers.keys
end
def self.get provider
p = @@providers[provider]
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
def self.all
if @@providers.nil?
ProviderFactory.init
end
@@providers.values
end
def self.init
conf = DevopsService.config
@@providers = {}
["ec2", "openstack"].each do |p|
begin
require "providers/#{p}"
o = Version2_0::Provider.const_get(p.capitalize).new(conf)
@@providers[p] = o if o.configured?
rescue => e
next
rescue LoadError
next
end
end
end
def self.providers
@@providers.keys
end end
def self.get provider
p = @@providers[provider]
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
def self.all
if @@providers.nil?
ProviderFactory.init
end
@@providers.values
end
def self.init
conf = DevopsService.config
@@providers = {}
["ec2", "openstack", "static"].each do |p|
begin
require "providers/#{p}"
o = Provider.const_get(p.capitalize).new(conf)
@@providers[p] = o if o.configured?
rescue => e
next
rescue LoadError
next
end
end
end
end end
end end

63
devops-service/providers/static.rb Normal file
View File

@ -0,0 +1,63 @@
require "providers/base_provider"
require "commands/server"
module Provider
class Static < BaseProvider
PROVIDER = "static"
def initialize config
self.certificate_path = config[:static_certificate]
self.ssh_key = "static"
@@mongo ||= MongoConnector.new(config[:mongo_db], config[:mongo_host], config[:mongo_port], config[:mongo_user], config[:mongo_password])
end
def configured?
true
end
def name
PROVIDER
end
def flavors
[]
end
def images filters
[]
end
def networks
[]
end
def servers
@@mongo.servers_find({:provider => PROVIDER}).map{|s| s.to_hash}
end
def create_default_chef_node_name s
"static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def create_server s, out
out << "Unsupported operation: ca not create server for provider 'static'"
false
end
def delete_server s
cert = @@mongo.key(s.key).path
res = ::ServerCommands.unbootstrap(s, cert)
m = "Static server with id '#{s.id}' and name '#{s.chef_node_name}' "
return m + (res.nil? ? "has been unbootstraped" : "can not be unbootstraped: #{res}")
end
def pause_server s
nil
end
def unpause_server s
nil
end
end
end

2
devops-service/routes/v2.0/base_routes.rb
View File

@ -68,7 +68,7 @@ module Version2_0
end end
def check_provider provider def check_provider provider
list = ::Version2_0::Provider::ProviderFactory.providers list = ::Provider::ProviderFactory.providers
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider) halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider)
end end

2
devops-service/routes/v2.0/flavor.rb
View File

@ -41,7 +41,7 @@ module Version2_0
check_headers :accept check_headers :accept
check_privileges("flavor", "r") check_privileges("flavor", "r")
check_provider(params[:provider]) check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider] p = ::Provider::ProviderFactory.get params[:provider]
json p.flavors json p.flavors
end end

2
devops-service/routes/v2.0/group.rb
View File

@ -53,7 +53,7 @@ module Version2_0
check_headers :accept check_headers :accept
check_privileges("group", "r") check_privileges("group", "r")
check_provider(params[:provider]) check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider] p = ::Provider::ProviderFactory.get params[:provider]
json p.groups(params) json p.groups(params)
end end

2
devops-service/routes/v2.0/network.rb
View File

@ -41,7 +41,7 @@ module Version2_0
check_headers :accept check_headers :accept
check_privileges("network", "r") check_privileges("network", "r")
check_provider(params[:provider]) check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider] p = ::Provider::ProviderFactory.get params[:provider]
json p.networks_detail json p.networks_detail
end end

2
devops-service/routes/v2.0/project.rb
View File

@ -476,7 +476,7 @@ module Version2_0
project = BaseRoutes.mongo.project(params[:id]) project = BaseRoutes.mongo.project(params[:id])
env = project.deploy_env params[:env] env = project.deploy_env params[:env]
user = request.env['REMOTE_USER'] user = request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) provider = ::Provider::ProviderFactory.get(env.provider)
header = "Test project '#{project.id}' and environment '#{env.identifier}'" header = "Test project '#{project.id}' and environment '#{env.identifier}'"
logger.info header logger.info header
servers = extract_servers(provider, project, env, {}, user, BaseRoutes.mongo) servers = extract_servers(provider, project, env, {}, user, BaseRoutes.mongo)

2
devops-service/routes/v2.0/provider.rb
View File

@ -26,7 +26,7 @@ module Version2_0
get "/providers" do get "/providers" do
check_headers :accept check_headers :accept
check_privileges("provider", "r") check_privileges("provider", "r")
json ::Version2_0::Provider::ProviderFactory.providers json ::Provider::ProviderFactory.providers
end end
end end

26
devops-service/routes/v2.0/server.rb
View File

@ -139,7 +139,7 @@ module Version2_0
# } # }
# ] # ]
get "/servers/:provider" do get "/servers/:provider" do
json ::Version2_0::Provider::ProviderFactory.get(params[:provider]).servers json ::Provider::ProviderFactory.get(params[:provider]).servers
end end
# Get server info by :name # Get server info by :name
@ -223,7 +223,7 @@ module Version2_0
p = BaseRoutes.mongo.check_project_auth(project_name, env_name, user) p = BaseRoutes.mongo.check_project_auth(project_name, env_name, user)
env = p.deploy_env(env_name) env = p.deploy_env(env_name)
provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) provider = ::Provider::ProviderFactory.get(env.provider)
check_chef_node_name(server_name, provider) unless server_name.nil? check_chef_node_name(server_name, provider) unless server_name.nil?
unless groups.nil? unless groups.nil?
buf = groups - provider.groups.keys buf = groups - provider.groups.keys
@ -286,8 +286,8 @@ module Version2_0
s = get_server(params[:node_name], @key) s = get_server(params[:node_name], @key)
## Authorization ## Authorization
BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) provider = ::Provider::ProviderFactory.get(s.provider)
r = provider.pause_server s.id r = provider.pause_server s
if r.nil? if r.nil?
create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is paused") create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is paused")
else else
@ -313,8 +313,8 @@ module Version2_0
s = get_server(params[:node_name], @key) s = get_server(params[:node_name], @key)
## Authorization ## Authorization
BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) provider = ::Provider::ProviderFactory.get(s.provider)
r = provider.unpause_server s.id r = provider.unpause_server s
if r.nil? if r.nil?
create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is unpaused") create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is unpaused")
else else
@ -401,27 +401,34 @@ module Version2_0
p = BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] p = BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
d = p.deploy_env s.deploy_env d = p.deploy_env s.deploy_env
provider = ::Provider::ProviderFactory.get(s.provider)
s.options = { s.options = {
:run_list => rl || d.run_list, :run_list => rl || d.run_list,
} }
s.options[:bootstrap_template] = t unless t.nil? s.options[:bootstrap_template] = t unless t.nil?
status = []
stream() do |out| stream() do |out|
begin begin
s.chef_node_name = name || "static_#{s.key}-#{Time.now.to_i}" s.chef_node_name = name || provider.create_default_chef_node_name(s)
cert = BaseRoutes.mongo.key s.key cert = BaseRoutes.mongo.key s.key
logger.debug "Bootstrap certificate path: #{cert.path}" logger.debug "Bootstrap certificate path: #{cert.path}"
bootstrap s, out, cert.path, logger bootstrap s, out, cert.path, logger
str = nil str = nil
if check_server(s) r = if check_server(s)
BaseRoutes.mongo.server_update s BaseRoutes.mongo.server_update s
str = "Server with id '#{s.id}' is bootstraped" str = "Server with id '#{s.id}' is bootstraped"
logger.info str logger.info str
0
else else
str = "Server with id '#{s.id}' is not bootstraped" str = "Server with id '#{s.id}' is not bootstraped"
logger.warn str logger.warn str
1
end end
status.push r
out << str out << str
out << "\n" out << "\n"
out << create_status(status)
rescue IOError => e rescue IOError => e
logger.error e.message logger.error e.message
end end
@ -461,8 +468,9 @@ module Version2_0
d = p.deploy_env(deploy_env) d = p.deploy_env(deploy_env)
cert = BaseRoutes.mongo.key(key) cert = BaseRoutes.mongo.key(key)
provider = ::Provider::ProviderFactory.get("static")
s = Server.new s = Server.new
s.provider = "static" s.provider = provider.name
s.project = project s.project = project
s.deploy_env = deploy_env s.deploy_env = deploy_env
s.remote_user = remote_user s.remote_user = remote_user