tim/fluke
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

provider static, todo: project env with provider static

Browse Source
This commit is contained in:
amartynov 2014-06-18 15:11:47 +04:00
parent 0ffdd8261c
commit 10eaf25454
23 changed files with 606 additions and 547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
devops-client/lib/devops-client/handler/server.rb
View File

@ -63,7 +63,7 @@ class Server < Handler
@list = case args[2]
when "chef"
get("/servers/chef").map {|l| {"chef_node_name" => l}}
when "ec2", "openstack"
when "ec2", "openstack", "static"
get("/servers/#{args[2]}")
else
@options_parser.invalid_list_command
@ -149,7 +149,7 @@ class Server < Handler
:key => args[6]
}
q[:public_ip] = self.options[:public_ip] unless self.options[:public_ip].nil?
post_chunk "/server/add", q
post "/server/add", q
end
def pause_handler args

2
devops-client/lib/devops-client/options/server_options.rb
View File

@ -8,7 +8,7 @@ class ServerOptions < CommonOptions
super(args, def_options)
self.header = I18n.t("headers.server")
self.banner_header = "server"
self.list_params = ["[chef|ec2|openstack]"]
self.list_params = ["[chef|ec2|openstack|static]"]
self.create_params = ["PROJECT_ID", "DEPLOY_ENV"]
node_params = ["NODE_NAME"]
self.delete_params = node_params

4
devops-service/commands/deploy_env.rb
View File

@ -25,12 +25,12 @@ module DeployEnvCommands
raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty?
filter = nil
if p.name == ::Version2_0::Provider::Ec2::PROVIDER
if p.name == ::Provider::Ec2::PROVIDER
unless subnets.empty?
subnets = [ subnets[0] ] if subnets.size > 1
filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] }
end
elsif p.name == ::Version2_0::Provider::Openstack::PROVIDER
elsif p.name == ::Provider::Openstack::PROVIDER
if subnets.empty?
raise InvalidRecord.new "Subnets array can not be empty"
end

2
devops-service/commands/image.rb
View File

@ -7,7 +7,7 @@ module ImageCommands
if filters.empty?
[]
else
::Version2_0::Provider::ProviderFactory.get(provider).images(filters)
::Provider::ProviderFactory.get(provider).images(filters)
end
end
end

40
devops-service/commands/server.rb
View File

@ -142,17 +142,19 @@ module ServerCommands
return status
end
def unbootstrap s, cert_path
def self.unbootstrap s, cert_path
i = 0
begin
`ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef`
raise unless $?.success?
r = `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef`
raise(r) unless $?.success?
rescue => e
logger.error "Unbootstrap eeror: " + e.message
logger.error "Unbootstrap error: " + e.message
i += 1
sleep(1)
retry unless i == 5
return e.message
end
nil
end
def delete_server s, mongo, logger
@ -163,26 +165,16 @@ module ServerCommands
return msg, nil
end
r = delete_from_chef_server(s.chef_node_name)
info = if s.static
cert = mongo.key(s.key).path
unbootstrap(s, cert)
mongo.server_delete s.id
msg = "Static server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
logger.info msg
msg
else
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider)
begin
r[:server] = provider.delete_server s.id
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
logger.warn r[:server]
end
mongo.server_delete s.id
msg = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
logger.info msg
msg
provider = ::Provider::ProviderFactory.get(s.provider)
begin
r[:server] = provider.delete_server s
rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
logger.warn r[:server]
end
mongo.server_delete s.id
info = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
logger.info info
r.each{|key, log| logger.info("#{key} - #{log}")}
return info, r
end
@ -193,7 +185,7 @@ module ServerCommands
str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n"
str << delete_from_chef_server(s.chef_node_name).values.join("\n")
begin
str << provider.delete_server(s.id) unless s.static
str << provider.delete_server(s)
rescue => e
str << e.message
end

4
devops-service/config.rb
View File

@ -34,3 +34,7 @@ config[:aws_secret_access_key] = "secret_access_key"
config[:aws_ssh_key] = "ssh_key"
config[:aws_certificate] = "/path/to/.ssh/ec2.pem"
config[:aws_availability_zone] = "aws_zone"
# static settings
config[:static_ssh_key] = "ssh_key" # or nil
config[:static_certificate] = "/path/to/.ssh/static.pem"

2
devops-service/db/mongo/models/deploy_env.rb
View File

@ -44,7 +44,7 @@ class DeployEnv < MongoModel
check_expires!(self.expires)
end
p = ::Version2_0::Provider::ProviderFactory.get(self.provider)
p = ::Provider::ProviderFactory.get(self.provider)
check_flavor!(p, self.flavor)
check_image!(p, self.image)
check_subnets_and_groups!(p, self.subnets, self.groups)

2
devops-service/db/mongo/models/deploy_env_multi.rb
View File

@ -65,7 +65,7 @@ class DeployEnvMulti < MongoModel
end
end
p = ::Version2_0::Provider::ProviderFactory.get(server["provider"])
p = ::Provider::ProviderFactory.get(server["provider"])
check_flavor!(p, server["flavor"])
check_image!(p, server["image"])
check_subnets_and_groups!(p, server["subnets"], server["groups"])

2
devops-service/db/mongo/models/mongo_model.rb
View File

@ -43,7 +43,7 @@ class MongoModel
end
def check_provider provider=self.provider
unless ::Version2_0::Provider::ProviderFactory.providers.include?(provider) or provider == "static"
unless ::Provider::ProviderFactory.providers.include?(provider)
raise InvalidRecord.new "Invalid provider '#{provider}'"
end
end

6
devops-service/db/mongo/mongo_connector.rb
View File

@ -191,12 +191,16 @@ class MongoConnector
end
end
def servers_find q
@servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs}
end
def servers p=nil, env=nil, names=nil
q = {}
q["project"] = p unless p.nil? or p.empty?
q["deploy_env"] = env unless env.nil? or env.empty?
q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array
@servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs}
servers_find(q)
end
def servers_by_names names

3
devops-service/devops-service.rb
View File

@ -29,7 +29,8 @@ class DevopsService < Sinatra::Base
[:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) }
mongo = DevopsService.mongo
mongo.create_root_user
::Version2_0::Provider::ProviderFactory.all.each do |p|
::Provider::ProviderFactory.all.each do |p|
next if p.certificate_path.nil?
begin
mongo.key p.ssh_key, Key::SYSTEM
rescue RecordNotFound => e

40
devops-service/providers/base_provider.rb
View File

@ -1,28 +1,26 @@
require "fog"
module Version2_0
module Provider
class BaseProvider
module Provider
class BaseProvider
attr_accessor :ssh_key, :certificate_path, :connection_options
protected
def connection_compute options
Fog::Compute.new( options )
end
def connection_network options
Fog::Network.new( options )
end
def configured?
!(empty_param?(self.ssh_key) or empty_param?(self.certificate_path))
end
def empty_param? param
param.nil? or param.empty?
end
attr_accessor :ssh_key, :certificate_path, :connection_options
protected
def connection_compute options
Fog::Compute.new( options )
end
def connection_network options
Fog::Network.new( options )
end
def configured?
!(empty_param?(self.ssh_key) or empty_param?(self.certificate_path))
end
def empty_param? param
param.nil? or param.empty?
end
end
end

451
devops-service/providers/ec2.rb
View File

@ -1,247 +1,240 @@
require "providers/base_provider"
#require 'xml'
module Version2_0
module Provider
# Provider for Amazon EC2
class Ec2 < BaseProvider
module Provider
# Provider for Amazon EC2
class Ec2 < BaseProvider
PROVIDER = "ec2"
PROVIDER = "ec2"
attr_accessor :availability_zone
attr_accessor :availability_zone
def initialize config
self.certificate_path = config[:aws_certificate]
self.ssh_key = config[:aws_ssh_key]
self.connection_options = {
:provider => "aws",
:aws_access_key_id => config[:aws_access_key_id],
:aws_secret_access_key => config[:aws_secret_access_key]
}
self.availability_zone = config[:aws_availability_zone] || "us-east-1a"
end
def initialize config
self.certificate_path = config[:aws_certificate]
self.ssh_key = config[:aws_ssh_key]
self.connection_options = {
:provider => "aws",
:aws_access_key_id => config[:aws_access_key_id],
:aws_secret_access_key => config[:aws_secret_access_key]
}
self.availability_zone = config[:aws_availability_zone] || "us-east-1a"
end
def configured?
o = self.connection_options
super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
end
def configured?
o = self.connection_options
super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key]))
end
def name
PROVIDER
end
def name
PROVIDER
end
def compute
connection_compute(connection_options)
end
def network
nil
end
def flavors
self.compute.flavors.all.map do |f|
{
"id" => f.id,
"cores" => f.cores,
"disk" => f.disk,
"name" => f.name,
"ram" => f.ram
}
end
end
def groups filters=nil
buf = {}
buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil?
g = if buf.empty?
self.compute.describe_security_groups
else
self.compute.describe_security_groups(buf)
end
convert_groups(g.body["securityGroupInfo"])
end
def images filters
self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i|
{
"id" => i["imageId"],
"name" => i["name"],
"status" => i["imageState"]
}
end
end
def networks_detail
self.networks
end
def networks
self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n|
{
"cidr" => n["cidrBlock"],
"vpcId" => n["vpcId"],
"subnetId" => n["subnetId"],
"name" => n["subnetId"],
"zone" => n["availabilityZone"]
}
end
end
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
convert_server server["instancesSet"][0]
end
end
def server id
list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"]
convert_server list[0]["instancesSet"][0]
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
options = {
"InstanceType" => s.options[:flavor],
"Placement.AvailabilityZone" => s.options[:availability_zone],
"KeyName" => self.ssh_key
}
vpcId = nil
unless s.options[:subnets].empty?
options["SubnetId"] = s.options[:subnets][0]
vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
if vpcId.nil?
out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
return false
end
end
options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
aws_server = nil
compute = self.compute
begin
aws_server = compute.run_instances(s.options[:image], 1, 1, options)
rescue Excon::Errors::Unauthorized => ue
#root = XML::Parser.string(ue.response.body).parse.root
#msg = root.children.find { |node| node.name == "Message" }
#code = root.children.find { |node| node.name == "Code" }
code = "TODO"
msg = ue.response.body
out << "\nERROR: Unauthorized (#{code}: #{msg})"
return false
rescue Fog::Compute::AWS::Error => e
out << e.message
return false
end
abody = aws_server.body
instance = abody["instancesSet"][0]
s.id = instance["instanceId"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, state = nil, instance["instanceState"]["name"]
until state == "running"
sleep(2)
details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0]
state = details["instanceState"]["name"].to_s
next if state == "pending" or state == "running"
out << "Server returns state '#{state}'"
return false
end
s.public_ip = details["ipAddress"]
s.private_ip = details["privateIpAddress"]
compute.create_tags(s.id, {"Name" => s.chef_node_name})
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server id
r = self.compute.terminate_instances(id)
i = r.body["instancesSet"][0]
old_state = i["previousState"]["name"]
state = i["currentState"]["name"]
return r.status == 200 ? "Server with id '#{id}' changed state '#{old_state}' to '#{state}'" : r.body
end
def pause_server id
s = self.server id
if s["state"] == "running"
self.compute.stop_instances [ id ]
return nil
else
return s["state"]
end
end
def unpause_server id
s = self.server id
if s["state"] == "stopped"
self.compute.start_instances [ id ]
return nil
else
return s["state"]
end
end
private
def convert_groups list
res = {}
list.each do |g|
res[g["groupName"]] = {
"description" => g["groupDescription"],
"id" => g["groupId"]
}
rules = []
g["ipPermissions"].each do |r|
cidr = r["ipRanges"][0] || {}
rules.push({
"protocol" => r["ipProtocol"],
"from" => r["fromPort"],
"to" => r["toPort"],
"cidr" => cidr["cidrIp"]
})
end
res[g["groupName"]]["rules"] = rules
end
res
end
def convert_server s
def flavors
self.compute.flavors.all.map do |f|
{
"state" => s["instanceState"]["name"],
"name" => s["tagSet"]["Name"],
"image" => s["imageId"],
"flavor" => s["instanceType"],
"keypair" => s["keyName"],
"instance_id" => s["instanceId"],
"dns_name" => s["dnsName"],
"zone" => s["placement"]["availabilityZone"],
"private_ip" => s["privateIpAddress"],
"public_ip" => s["ipAddress"],
"launched_at" => s["launchTime"]
"id" => f.id,
"cores" => f.cores,
"disk" => f.disk,
"name" => f.name,
"ram" => f.ram
}
end
end
def extract_group_ids names, vpcId
return [] if names.nil?
p = nil
p = {"vpc-id" => vpcId} unless vpcId.nil?
groups = self.groups(p)
r = names.map do |name|
groups[name]["id"]
def groups filters=nil
buf = {}
buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil?
g = if buf.empty?
self.compute.describe_security_groups
else
self.compute.describe_security_groups(buf)
end
convert_groups(g.body["securityGroupInfo"])
end
def images filters
self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i|
{
"id" => i["imageId"],
"name" => i["name"],
"status" => i["imageState"]
}
end
end
def networks_detail
self.networks
end
def networks
self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n|
{
"cidr" => n["cidrBlock"],
"vpcId" => n["vpcId"],
"subnetId" => n["subnetId"],
"name" => n["subnetId"],
"zone" => n["availabilityZone"]
}
end
end
def servers
list = self.compute.describe_instances.body["reservationSet"]
list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server|
convert_server server["instancesSet"][0]
end
end
def server id
list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"]
convert_server list[0]["instancesSet"][0]
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
options = {
"InstanceType" => s.options[:flavor],
"Placement.AvailabilityZone" => s.options[:availability_zone],
"KeyName" => self.ssh_key
}
vpcId = nil
unless s.options[:subnets].empty?
options["SubnetId"] = s.options[:subnets][0]
vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"]
if vpcId.nil?
out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n"
return false
end
r
end
options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",")
aws_server = nil
compute = self.compute
begin
aws_server = compute.run_instances(s.options[:image], 1, 1, options)
rescue Excon::Errors::Unauthorized => ue
#root = XML::Parser.string(ue.response.body).parse.root
#msg = root.children.find { |node| node.name == "Message" }
#code = root.children.find { |node| node.name == "Code" }
code = "TODO"
msg = ue.response.body
out << "\nERROR: Unauthorized (#{code}: #{msg})"
return false
rescue Fog::Compute::AWS::Error => e
out << e.message
return false
end
abody = aws_server.body
instance = abody["instancesSet"][0]
s.id = instance["instanceId"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, state = nil, instance["instanceState"]["name"]
until state == "running"
sleep(2)
details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0]
state = details["instanceState"]["name"].to_s
next if state == "pending" or state == "running"
out << "Server returns state '#{state}'"
return false
end
s.public_ip = details["ipAddress"]
s.private_ip = details["privateIpAddress"]
compute.create_tags(s.id, {"Name" => s.chef_node_name})
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server s
r = self.compute.terminate_instances(s.id)
i = r.body["instancesSet"][0]
old_state = i["previousState"]["name"]
state = i["currentState"]["name"]
return r.status == 200 ? "Server with id '#{s.id}' changed state '#{old_state}' to '#{state}'" : r.body
end
def pause_server s
es = self.server s.id
if es["state"] == "running"
self.compute.stop_instances [ s.id ]
return nil
else
return es["state"]
end
end
def unpause_server s
es = self.server s.id
if es["state"] == "stopped"
self.compute.start_instances [ s.id ]
return nil
else
return es["state"]
end
end
private
def convert_groups list
res = {}
list.each do |g|
res[g["groupName"]] = {
"description" => g["groupDescription"],
"id" => g["groupId"]
}
rules = []
g["ipPermissions"].each do |r|
cidr = r["ipRanges"][0] || {}
rules.push({
"protocol" => r["ipProtocol"],
"from" => r["fromPort"],
"to" => r["toPort"],
"cidr" => cidr["cidrIp"]
})
end
res[g["groupName"]]["rules"] = rules
end
res
end
def convert_server s
{
"state" => s["instanceState"]["name"],
"name" => s["tagSet"]["Name"],
"image" => s["imageId"],
"flavor" => s["instanceType"],
"keypair" => s["keyName"],
"instance_id" => s["instanceId"],
"dns_name" => s["dnsName"],
"zone" => s["placement"]["availabilityZone"],
"private_ip" => s["privateIpAddress"],
"public_ip" => s["ipAddress"],
"launched_at" => s["launchTime"]
}
end
def extract_group_ids names, vpcId
return [] if names.nil?
p = nil
p = {"vpc-id" => vpcId} unless vpcId.nil?
groups = self.groups(p)
r = names.map do |name|
groups[name]["id"]
end
r
end
private
def compute
connection_compute(connection_options)
end
end
end

418
devops-service/providers/openstack.rb
View File

@ -1,217 +1,215 @@
require "providers/base_provider"
module Version2_0
module Provider
# Provider for 'openstack'
class Openstack < BaseProvider
module Provider
# Provider for 'openstack'
class Openstack < BaseProvider
PROVIDER = "openstack"
def initialize config
self.certificate_path = config[:openstack_certificate]
self.ssh_key = config[:openstack_ssh_key]
self.connection_options = {
:provider => PROVIDER,
:openstack_username => config[:openstack_username],
:openstack_api_key => config[:openstack_api_key],
:openstack_auth_url => config[:openstack_auth_url],
:openstack_tenant => config[:openstack_tenant]
}
end
# Returns 'true' if all parameters defined
def configured?
o = self.connection_options
super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant]))
end
def name
PROVIDER
end
def compute
connection_compute(self.connection_options)
end
def network
connection_network(self.connection_options)
end
def groups filter=nil
convert_groups(compute.list_security_groups.body["security_groups"])
end
def flavors
self.compute.list_flavors_detail.body["flavors"].map do |f|
{
"id" => f["name"],
"v_cpus" => f["vcpus"],
"ram" => f["ram"],
"disk" => f["disk"]
}
end
end
def images filters
self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i|
{
"id" => i["id"],
"name" => i["name"],
"status" => i["status"]
}
end
end
def networks_detail
net = self.network
subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]}
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
sn = subnets.detect{|s| n["subnets"][0] == s["id"]}
{
"cidr" => sn["cidr"],
"name" => n["name"],
"id" => n["id"]
}
}
end
def networks
net = self.network
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
{
"name" => n["name"],
"id" => n["id"]
}
}
end
def servers
list = self.compute.list_servers_detail.body["servers"]
list.map do |s|
o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]}
s["addresses"].each_value do |a|
a.each do |addr|
o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed"
end
end
o
end
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
buf = s.options[:subnets] - networks.map{|n| n["name"]}
unless buf.empty?
out << "No networks with names '#{buf.join("', '")}' found"
return false
end
s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
compute = self.compute
begin
o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
"nics" => networks.map{|n| {"net_id" => n["id"]}},
"security_groups" => s.options[:groups],
"key_name" => s.key)
rescue Excon::Errors::BadRequest => e
response = ::Chef::JSONCompat.from_json(e.response.body)
if response['badRequest']['code'] == 400
if response['badRequest']['message'] =~ /Invalid flavorRef/
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
elsif response['badRequest']['message'] =~ /Invalid imageRef/
out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
else
out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
end
out << "\n"
return false
else
out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}"
out << "\n"
return false
end
rescue Excon::Errors::InternalServerError => ise
out << "\nError: openstack internal server error " + ise.message
out << "\n"
return false
rescue => e2
out << "\nError: Unknown error: " + e2.message
out << "\n"
return false
end
sbody = o_server.body
s.id = sbody["server"]["id"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, status = nil, nil
until status == "ACTIVE"
sleep(1)
details = compute.get_server_details(s.id).body
status = details["server"]["status"].upcase
if status == "ERROR"
out << "Server returns status 'ERROR'"
return false
end
end
network = networks[0]["name"]
s.private_ip = details["server"]["addresses"][network][0]["addr"]
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server id
r = self.compute.delete_server(id)
return r.status == 204 ? "Server with id '#{id}' terminated" : r.body
end
def pause_server id
begin
self.compute.pause_server id
rescue Excon::Errors::Conflict => e
return "pause"
end
return nil
end
def unpause_server id
begin
self.compute.unpause_server id
rescue Excon::Errors::Conflict => e
return "unpause"
end
return nil
end
private
def convert_groups list
res = {}
list.map do |g|
res[g["name"]] = {
"description" => g["description"]
}
rules = []
g["rules"].each do |r|
rules.push({
"protocol" => r["ip_protocol"],
"from" => r["from_port"],
"to" => r["to_port"],
"cidr" => r["ip_range"]["cidr"]
})
end
res[g["name"]]["rules"] = rules
end
res
end
PROVIDER = "openstack"
def initialize config
self.certificate_path = config[:openstack_certificate]
self.ssh_key = config[:openstack_ssh_key]
self.connection_options = {
:provider => PROVIDER,
:openstack_username => config[:openstack_username],
:openstack_api_key => config[:openstack_api_key],
:openstack_auth_url => config[:openstack_auth_url],
:openstack_tenant => config[:openstack_tenant]
}
end
# Returns 'true' if all parameters defined
def configured?
o = self.connection_options
super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant]))
end
def name
PROVIDER
end
def groups filter=nil
convert_groups(compute.list_security_groups.body["security_groups"])
end
def flavors
self.compute.list_flavors_detail.body["flavors"].map do |f|
{
"id" => f["name"],
"v_cpus" => f["vcpus"],
"ram" => f["ram"],
"disk" => f["disk"]
}
end
end
def images filters
self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i|
{
"id" => i["id"],
"name" => i["name"],
"status" => i["status"]
}
end
end
def networks_detail
net = self.network
subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]}
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
sn = subnets.detect{|s| n["subnets"][0] == s["id"]}
{
"cidr" => sn["cidr"],
"name" => n["name"],
"id" => n["id"]
}
}
end
def networks
net = self.network
net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n|
{
"name" => n["name"],
"id" => n["id"]
}
}
end
def servers
list = self.compute.list_servers_detail.body["servers"]
list.map do |s|
o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]}
s["addresses"].each_value do |a|
a.each do |addr|
o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed"
end
end
o
end
end
def create_server s, out
out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n"
networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])}
buf = s.options[:subnets] - networks.map{|n| n["name"]}
unless buf.empty?
out << "No networks with names '#{buf.join("', '")}' found"
return false
end
s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"]
out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n"
compute = self.compute
begin
o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor],
"nics" => networks.map{|n| {"net_id" => n["id"]}},
"security_groups" => s.options[:groups],
"key_name" => s.key)
rescue Excon::Errors::BadRequest => e
response = ::Chef::JSONCompat.from_json(e.response.body)
if response['badRequest']['code'] == 400
if response['badRequest']['message'] =~ /Invalid flavorRef/
out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}"
elsif response['badRequest']['message'] =~ /Invalid imageRef/
out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}"
else
out << "\nERROR: Bad request (400): #{response['badRequest']['message']}"
end
out << "\n"
return false
else
out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}"
out << "\n"
return false
end
rescue Excon::Errors::InternalServerError => ise
out << "\nError: openstack internal server error " + ise.message
out << "\n"
return false
rescue => e2
out << "\nError: Unknown error: " + e2.message
out << "\n"
return false
end
sbody = o_server.body
s.id = sbody["server"]["id"]
out << "\nInstance Name: #{s.chef_node_name}"
out << "\nInstance ID: #{s.id}\n"
out << "\nWaiting for server..."
details, status = nil, nil
until status == "ACTIVE"
sleep(1)
details = compute.get_server_details(s.id).body
status = details["server"]["status"].upcase
if status == "ERROR"
out << "Server returns status 'ERROR'"
return false
end
end
network = networks[0]["name"]
s.private_ip = details["server"]["addresses"][network][0]["addr"]
out << "\nDone\n\n"
out << s.info
true
end
def create_default_chef_node_name s
"#{self.ssh_key}-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def delete_server s
r = self.compute.delete_server(s.id)
return r.status == 204 ? "Server with id '#{s.id}' terminated" : r.body
end
def pause_server s
begin
self.compute.pause_server s.id
rescue Excon::Errors::Conflict => e
return "pause"
end
return nil
end
def unpause_server s
begin
self.compute.unpause_server s.id
rescue Excon::Errors::Conflict => e
return "unpause"
end
return nil
end
private
def convert_groups list
res = {}
list.map do |g|
res[g["name"]] = {
"description" => g["description"]
}
rules = []
g["rules"].each do |r|
rules.push({
"protocol" => r["ip_protocol"],
"from" => r["from_port"],
"to" => r["to_port"],
"cidr" => r["ip_range"]["cidr"]
})
end
res[g["name"]]["rules"] = rules
end
res
end
def compute
connection_compute(self.connection_options)
end
def network
connection_network(self.connection_options)
end
end
end

72
devops-service/providers/provider_factory.rb
View File

@ -1,42 +1,40 @@
module Version2_0
module Provider
class ProviderFactory
module Provider
class ProviderFactory
@@providers = nil
def self.providers
@@providers.keys
end
def self.get provider
p = @@providers[provider]
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
def self.all
if @@providers.nil?
ProviderFactory.init
end
@@providers.values
end
def self.init
conf = DevopsService.config
@@providers = {}
["ec2", "openstack"].each do |p|
begin
require "providers/#{p}"
o = Version2_0::Provider.const_get(p.capitalize).new(conf)
@@providers[p] = o if o.configured?
rescue => e
next
rescue LoadError
next
end
end
end
@@providers = nil
def self.providers
@@providers.keys
end
def self.get provider
p = @@providers[provider]
raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil?
p
end
def self.all
if @@providers.nil?
ProviderFactory.init
end
@@providers.values
end
def self.init
conf = DevopsService.config
@@providers = {}
["ec2", "openstack", "static"].each do |p|
begin
require "providers/#{p}"
o = Provider.const_get(p.capitalize).new(conf)
@@providers[p] = o if o.configured?
rescue => e
next
rescue LoadError
next
end
end
end
end
end

63
devops-service/providers/static.rb Normal file
View File

@ -0,0 +1,63 @@
require "providers/base_provider"
require "commands/server"
module Provider
class Static < BaseProvider
PROVIDER = "static"
def initialize config
self.certificate_path = config[:static_certificate]
self.ssh_key = "static"
@@mongo ||= MongoConnector.new(config[:mongo_db], config[:mongo_host], config[:mongo_port], config[:mongo_user], config[:mongo_password])
end
def configured?
true
end
def name
PROVIDER
end
def flavors
[]
end
def images filters
[]
end
def networks
[]
end
def servers
@@mongo.servers_find({:provider => PROVIDER}).map{|s| s.to_hash}
end
def create_default_chef_node_name s
"static-#{s.project}-#{s.deploy_env}-#{Time.now.to_i}"
end
def create_server s, out
out << "Unsupported operation: ca not create server for provider 'static'"
false
end
def delete_server s
cert = @@mongo.key(s.key).path
res = ::ServerCommands.unbootstrap(s, cert)
m = "Static server with id '#{s.id}' and name '#{s.chef_node_name}' "
return m + (res.nil? ? "has been unbootstraped" : "can not be unbootstraped: #{res}")
end
def pause_server s
nil
end
def unpause_server s
nil
end
end
end

2
devops-service/routes/v2.0/base_routes.rb
View File

@ -68,7 +68,7 @@ module Version2_0
end
def check_provider provider
list = ::Version2_0::Provider::ProviderFactory.providers
list = ::Provider::ProviderFactory.providers
halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider)
end

2
devops-service/routes/v2.0/flavor.rb
View File

@ -41,7 +41,7 @@ module Version2_0
check_headers :accept
check_privileges("flavor", "r")
check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider]
p = ::Provider::ProviderFactory.get params[:provider]
json p.flavors
end

2
devops-service/routes/v2.0/group.rb
View File

@ -53,7 +53,7 @@ module Version2_0
check_headers :accept
check_privileges("group", "r")
check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider]
p = ::Provider::ProviderFactory.get params[:provider]
json p.groups(params)
end

2
devops-service/routes/v2.0/network.rb
View File

@ -41,7 +41,7 @@ module Version2_0
check_headers :accept
check_privileges("network", "r")
check_provider(params[:provider])
p = ::Version2_0::Provider::ProviderFactory.get params[:provider]
p = ::Provider::ProviderFactory.get params[:provider]
json p.networks_detail
end

2
devops-service/routes/v2.0/project.rb
View File

@ -476,7 +476,7 @@ module Version2_0
project = BaseRoutes.mongo.project(params[:id])
env = project.deploy_env params[:env]
user = request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(env.provider)
provider = ::Provider::ProviderFactory.get(env.provider)
header = "Test project '#{project.id}' and environment '#{env.identifier}'"
logger.info header
servers = extract_servers(provider, project, env, {}, user, BaseRoutes.mongo)

2
devops-service/routes/v2.0/provider.rb
View File

@ -26,7 +26,7 @@ module Version2_0
get "/providers" do
check_headers :accept
check_privileges("provider", "r")
json ::Version2_0::Provider::ProviderFactory.providers
json ::Provider::ProviderFactory.providers
end
end

26
devops-service/routes/v2.0/server.rb
View File

@ -139,7 +139,7 @@ module Version2_0
# }
# ]
get "/servers/:provider" do
json ::Version2_0::Provider::ProviderFactory.get(params[:provider]).servers
json ::Provider::ProviderFactory.get(params[:provider]).servers
end
# Get server info by :name
@ -223,7 +223,7 @@ module Version2_0
p = BaseRoutes.mongo.check_project_auth(project_name, env_name, user)
env = p.deploy_env(env_name)
provider = ::Version2_0::Provider::ProviderFactory.get(env.provider)
provider = ::Provider::ProviderFactory.get(env.provider)
check_chef_node_name(server_name, provider) unless server_name.nil?
unless groups.nil?
buf = groups - provider.groups.keys
@ -286,8 +286,8 @@ module Version2_0
s = get_server(params[:node_name], @key)
## Authorization
BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider)
r = provider.pause_server s.id
provider = ::Provider::ProviderFactory.get(s.provider)
r = provider.pause_server s
if r.nil?
create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is paused")
else
@ -313,8 +313,8 @@ module Version2_0
s = get_server(params[:node_name], @key)
## Authorization
BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
provider = ::Version2_0::Provider::ProviderFactory.get(s.provider)
r = provider.unpause_server s.id
provider = ::Provider::ProviderFactory.get(s.provider)
r = provider.unpause_server s
if r.nil?
create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is unpaused")
else
@ -401,27 +401,34 @@ module Version2_0
p = BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER']
d = p.deploy_env s.deploy_env
provider = ::Provider::ProviderFactory.get(s.provider)
s.options = {
:run_list => rl || d.run_list,
}
s.options[:bootstrap_template] = t unless t.nil?
status = []
stream() do |out|
begin
s.chef_node_name = name || "static_#{s.key}-#{Time.now.to_i}"
s.chef_node_name = name || provider.create_default_chef_node_name(s)
cert = BaseRoutes.mongo.key s.key
logger.debug "Bootstrap certificate path: #{cert.path}"
bootstrap s, out, cert.path, logger
str = nil
if check_server(s)
r = if check_server(s)
BaseRoutes.mongo.server_update s
str = "Server with id '#{s.id}' is bootstraped"
logger.info str
0
else
str = "Server with id '#{s.id}' is not bootstraped"
logger.warn str
1
end
status.push r
out << str
out << "\n"
out << create_status(status)
rescue IOError => e
logger.error e.message
end
@ -461,8 +468,9 @@ module Version2_0
d = p.deploy_env(deploy_env)
cert = BaseRoutes.mongo.key(key)
provider = ::Provider::ProviderFactory.get("static")
s = Server.new
s.provider = "static"
s.provider = provider.name
s.project = project
s.deploy_env = deploy_env
s.remote_user = remote_user