diff --git a/devops-service/Gemfile b/devops-service/Gemfile index 5897ed1..2b14f50 100644 --- a/devops-service/Gemfile +++ b/devops-service/Gemfile @@ -1,23 +1,25 @@ source 'https://rubygems.org' -ruby "2.0.0" +#ruby "2.0.0" gem "thin", "~>1.5.1" gem "mime-types", "~>1.25.1" -gem "sinatra", "1.4.3" +gem "sinatra", "1.4.5" gem "sinatra-contrib"#, "1.4.1" gem "sinatra-websocket"#, "~>0.3.0" gem "fog", "~>1.20" gem "mixlib-shellout" -gem "chef", ">=11" +gem "chef", ">=12" gem "mongo" gem "bson_ext" gem "multi_json", "1.7.8" gem "rufus-scheduler", "2.0.24" gem "sidekiq", "3.2.6" gem 'wisper' -gem 'rake', '10.0.0' +gem 'rake', '10.2.0' +gem 'rack-accept-media-types' gem 'rack', '1.5.2' +gem 'hooks' # gem "devops-nibr", :path => "plugins/devops-nibr" diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock index 7bb6d1b..eb3077a 100644 --- a/devops-service/Gemfile.lock +++ b/devops-service/Gemfile.lock @@ -1,61 +1,84 @@ GEM remote: https://rubygems.org/ specs: - CFPropertyList (2.3.0) - addressable (2.3.6) + CFPropertyList (2.3.1) + addressable (2.3.8) backports (3.6.4) - bson (1.11.1) - bson_ext (1.11.1) - bson (~> 1.11.1) + bson (1.12.3) + bson_ext (1.12.3) + bson (~> 1.12.3) builder (3.2.2) - byebug (3.5.1) - columnize (~> 0.8) - debugger-linecache (~> 1.2) - slop (~> 3.6) + byebug (5.0.0) + columnize (= 0.9.0) celluloid (0.15.2) timers (~> 1.1.0) - chef (11.6.0) - erubis - highline (>= 1.6.9) - json (>= 1.4.4, <= 1.7.7) - mixlib-authentication (>= 1.3.0) - mixlib-cli (~> 1.3.0) - mixlib-config (>= 1.1.2) - mixlib-log (>= 1.3.0) - mixlib-shellout + chef (12.1.2) + chef-zero (~> 4.0) + diff-lcs (~> 1.2, >= 1.2.4) + erubis (~> 2.7) + ffi-yajl (~> 1.2) + highline (~> 1.6, >= 1.6.9) + mixlib-authentication (~> 1.3) + mixlib-cli (~> 1.4) + mixlib-config (~> 2.0) + mixlib-log (~> 1.3) + mixlib-shellout (>= 2.0.0.rc.0, < 3.0) net-ssh (~> 2.6) - net-ssh-multi (~> 1.1.0) - ohai (>= 0.6.0) - rest-client (>= 1.0.4, < 1.7.0) - yajl-ruby (~> 1.1) + net-ssh-multi (~> 1.1) + ohai (~> 8.0) + plist (~> 3.1.0) + pry (~> 0.9) + rspec-core (~> 3.2) + rspec-expectations (~> 3.2) + rspec-mocks (~> 3.2) + rspec_junit_formatter (~> 0.2.0) + serverspec (~> 2.7) + specinfra (~> 2.10) + chef-zero (4.2.3) + ffi-yajl (>= 1.1, < 3.0) + hashie (~> 2.0) + mixlib-log (~> 1.3) + rack + uuidtools (~> 2.1) + coderay (1.1.0) columnize (0.9.0) - connection_pool (2.1.1) - cucumber (1.3.18) + connection_pool (2.2.0) + cucumber (2.0.0) builder (>= 2.1.2) + cucumber-core (~> 1.1.3) diff-lcs (>= 1.1.3) gherkin (~> 2.12) multi_json (>= 1.7.5, < 2.0) - multi_test (>= 0.1.1) - daemons (1.1.9) - debugger-linecache (1.2.0) + multi_test (>= 0.1.2) + cucumber-core (1.1.3) + gherkin (~> 2.12.0) + daemons (1.2.3) diff-lcs (1.2.5) em-websocket (0.3.8) addressable (>= 2.1.1) eventmachine (>= 0.12.9) erubis (2.7.0) - eventmachine (1.0.4) - excon (0.43.0) + eventmachine (1.0.7) + excon (0.45.3) + ffi (1.9.9) + ffi-yajl (1.4.0) + ffi (~> 1.5) + libyajl2 (~> 1.2) fission (0.5.0) CFPropertyList (~> 2.2) - fog (1.27.0) + fog (1.31.0) fog-atmos fog-aws (~> 0.0) fog-brightbox (~> 0.4) - fog-core (~> 1.27, >= 1.27.3) + fog-core (~> 1.30) fog-ecloud + fog-google (>= 0.0.2) fog-json + fog-local + fog-powerdns (>= 0.1.1) fog-profitbricks fog-radosgw (>= 0.0.2) + fog-riakcs fog-sakuracloud (>= 0.0.4) fog-serverlove fog-softlayer @@ -69,150 +92,206 @@ GEM fog-atmos (0.1.0) fog-core fog-xml - fog-aws (0.0.7) + fog-aws (0.6.0) fog-core (~> 1.27) fog-json (~> 1.0) fog-xml (~> 0.1) ipaddress (~> 0.8) - fog-brightbox (0.7.1) + fog-brightbox (0.7.2) fog-core (~> 1.22) fog-json inflecto (~> 0.0.2) - fog-core (1.27.3) + fog-core (1.31.1) builder - excon (~> 0.38) + excon (~> 0.45) formatador (~> 0.2) mime-types net-scp (~> 1.1) net-ssh (>= 2.1.3) - fog-ecloud (0.0.2) + fog-ecloud (0.3.0) fog-core fog-xml - fog-json (1.0.0) + fog-google (0.0.6) + fog-core + fog-json + fog-xml + fog-json (1.0.1) + fog-core (~> 1.0) multi_json (~> 1.0) - fog-profitbricks (0.0.1) + fog-local (0.2.1) + fog-core (~> 1.27) + fog-powerdns (0.1.1) + fog-core (~> 1.27) + fog-json (~> 1.0) + fog-xml (~> 0.1) + fog-profitbricks (0.0.3) fog-core fog-xml nokogiri - fog-radosgw (0.0.3) + fog-radosgw (0.0.4) fog-core (>= 1.21.0) fog-json fog-xml (>= 0.0.1) - fog-sakuracloud (1.0.0) + fog-riakcs (0.1.0) fog-core fog-json - fog-serverlove (0.1.1) + fog-xml + fog-sakuracloud (1.0.1) fog-core fog-json - fog-softlayer (0.3.30) + fog-serverlove (0.1.2) fog-core fog-json - fog-storm_on_demand (0.1.0) + fog-softlayer (0.4.7) fog-core fog-json - fog-terremark (0.0.3) + fog-storm_on_demand (0.1.1) + fog-core + fog-json + fog-terremark (0.1.0) fog-core fog-xml - fog-vmfusion (0.0.1) + fog-vmfusion (0.1.0) fission fog-core - fog-voxel (0.0.2) + fog-voxel (0.1.0) fog-core fog-xml - fog-xml (0.1.1) + fog-xml (0.1.2) fog-core nokogiri (~> 1.5, >= 1.5.11) formatador (0.2.5) gherkin (2.12.2) multi_json (~> 1.3) - highline (1.6.21) + hashie (2.1.2) + highline (1.7.2) + hooks (0.4.0) + uber (~> 0.0.4) httpclient (2.6.0.1) inflecto (0.0.2) ipaddress (0.8.0) - json (1.7.7) + json (1.8.3) + libyajl2 (1.2.0) + method_source (0.8.2) mime-types (1.25.1) mini_portile (0.6.2) mixlib-authentication (1.3.0) mixlib-log - mixlib-cli (1.3.0) - mixlib-config (2.1.0) + mixlib-cli (1.5.0) + mixlib-config (2.2.1) mixlib-log (1.6.0) - mixlib-shellout (2.0.1) - mongo (1.11.1) - bson (= 1.11.1) + mixlib-shellout (2.1.0) + mongo (1.12.3) + bson (= 1.12.3) multi_json (1.7.8) - multi_test (0.1.1) + multi_test (0.1.2) + net-dhcp (1.3.2) net-scp (1.2.1) net-ssh (>= 2.6.5) net-ssh (2.9.2) net-ssh-gateway (1.2.0) net-ssh (>= 2.6.5) - net-ssh-multi (1.1) - net-ssh (>= 2.1.4) - net-ssh-gateway (>= 0.99.0) + net-ssh-multi (1.2.1) + net-ssh (>= 2.6.5) + net-ssh-gateway (>= 1.2.0) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) - ohai (6.24.2) + ohai (8.0.1) + ffi (~> 1.9) + ffi-yajl (~> 1.1) ipaddress + mime-types (~> 1.16) mixlib-cli - mixlib-config + mixlib-config (~> 2.0) mixlib-log - mixlib-shellout - systemu (~> 2.5.2) - yajl-ruby - power_assert (0.2.2) + mixlib-shellout (~> 2.0) + net-dhcp + rake (~> 10.1) + systemu (~> 2.6.4) + wmi-lite (~> 1.0) + plist (3.1.0) + power_assert (0.2.3) + pry (0.10.1) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) rack (1.5.2) + rack-accept-media-types (0.9) rack-protection (1.5.3) rack rack-test (0.6.3) rack (>= 1.0) - rake (10.0.0) - rdoc (4.2.0) - json (~> 1.4) - redis (3.2.0) - redis-namespace (1.5.1) + rake (10.2.0) + redis (3.2.1) + redis-namespace (1.5.2) redis (~> 3.0, >= 3.0.4) - rest-client (1.6.8) - mime-types (~> 1.16) - rdoc (>= 2.4.2) + rspec (3.3.0) + rspec-core (~> 3.3.0) + rspec-expectations (~> 3.3.0) + rspec-mocks (~> 3.3.0) + rspec-core (3.3.1) + rspec-support (~> 3.3.0) + rspec-expectations (3.3.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.3.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-support (3.3.0) + rspec_junit_formatter (0.2.3) + builder (< 4) + rspec-core (>= 2, < 4, != 2.12.0) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) + serverspec (2.19.0) + multi_json + rspec (~> 3.0) + rspec-its + specinfra (~> 2.35) sidekiq (3.2.6) celluloid (= 0.15.2) connection_pool (>= 2.0.0) json redis (>= 3.0.6) redis-namespace (>= 1.3.1) - sinatra (1.4.3) + sinatra (1.4.5) rack (~> 1.4) rack-protection (~> 1.4) tilt (~> 1.3, >= 1.3.4) - sinatra-contrib (1.4.2) + sinatra-contrib (1.4.4) backports (>= 2.0) multi_json rack-protection rack-test sinatra (~> 1.4.0) - tilt (~> 1.3) + tilt (>= 1.3, < 3) sinatra-websocket (0.3.1) em-websocket (~> 0.3.6) eventmachine thin (>= 1.3.1, < 2.0.0) slop (3.6.0) - systemu (2.5.2) - test-unit (3.0.9) + specinfra (2.36.6) + net-scp + net-ssh + systemu (2.6.5) + test-unit (3.1.2) power_assert thin (1.5.1) daemons (>= 1.0.9) eventmachine (>= 0.12.6) rack (>= 1.0.0) - thread_safe (0.3.4) + thread_safe (0.3.5) tilt (1.4.1) timers (1.1.0) tzinfo (1.2.2) thread_safe (~> 0.1) + uber (0.0.13) + uuidtools (2.1.5) wisper (1.6.0) - yajl-ruby (1.2.1) + wmi-lite (1.0.0) PLATFORMS ruby @@ -220,19 +299,21 @@ PLATFORMS DEPENDENCIES bson_ext byebug - chef (>= 11) + chef (>= 12) cucumber fog (~> 1.20) + hooks httpclient mime-types (~> 1.25.1) mixlib-shellout mongo multi_json (= 1.7.8) rack (= 1.5.2) - rake (= 10.0.0) + rack-accept-media-types + rake (= 10.2.0) rufus-scheduler (= 2.0.24) sidekiq (= 3.2.6) - sinatra (= 1.4.3) + sinatra (= 1.4.5) sinatra-contrib sinatra-websocket test-unit diff --git a/devops-service/client.rb b/devops-service/app/client/devops-client.rb similarity index 92% rename from devops-service/client.rb rename to devops-service/app/client/devops-client.rb index c676b69..80a3448 100644 --- a/devops-service/client.rb +++ b/devops-service/app/client/devops-client.rb @@ -2,9 +2,9 @@ require 'sinatra/base' class Client < Sinatra::Base - def initialize config + def initialize super() - @@config = config + @@config = DevopsConfig.config end # Route to download devops client @@ -30,3 +30,4 @@ class Client < Sinatra::Base end end + diff --git a/devops-service/app/devops-api2.rb b/devops-service/app/devops-api2.rb new file mode 100644 index 0000000..c039dba --- /dev/null +++ b/devops-service/app/devops-api2.rb @@ -0,0 +1,97 @@ +module Devops + module Application + class DevopsApi2Application < Application + + def prepare + require "routes/v2.0" + require "routes/v2.0/handlers/provider" + require "routes/v2.0/handlers/bootstrap_templates" + require "routes/v2.0/handlers/deploy" + require "routes/v2.0/handlers/filter" + require "routes/v2.0/handlers/flavor" + require "routes/v2.0/handlers/group" + require "routes/v2.0/handlers/image" + require "routes/v2.0/handlers/network" + require "routes/v2.0/handlers/key" + require "routes/v2.0/handlers/project" + require "routes/v2.0/handlers/script" + require "routes/v2.0/handlers/status" + require "routes/v2.0/handlers/tag" + require "routes/v2.0/handlers/user" + require "routes/v2.0/handlers/server" + require "routes/v2.0/handlers/stack" + require "routes/v2.0/handlers/stack_template" + require "routes/v2.0/stack_template_presets" + require "routes/v2.0/handlers/report" + + require 'lib/stubber' + end + + def init + config = DevopsConfig.config + Devops::Api2.set :devops_home, config[:devops_dir] + #set :config, config + + Devops::Api2.set :keys_dir, (config[:keys_dir] || File.join(config[:devops_dir], "files/keys")) + Devops::Api2.set :scripts_dir, (config[:scripts_dir] || File.join(config[:devops_dir], "files/scripts")) + [:keys_dir, :scripts_dir].each {|key| d = Devops::Api2.settings.send(key); FileUtils.mkdir_p(d) unless File.exists?(d) } + init_mongo + Devops::Api2.settings.mongo.create_root_user + ::Provider::ProviderFactory.init(config) + #set_up_providers_keys!(::Provider::ProviderFactory.all, mongo) + + if Devops::Api2.settings.test? || config[:stub_classes] + Stubber.stub_providers! + Stubber.stub_models! + end + end + + def routes + require "routes/v2.0/flavor" + require "routes/v2.0/image" + require "routes/v2.0/filter" + require "routes/v2.0/network" + require "routes/v2.0/group" + require "routes/v2.0/deploy" + require "routes/v2.0/project" + require "routes/v2.0/key" + require "routes/v2.0/user" + require "routes/v2.0/provider" + require "routes/v2.0/tag" + require "routes/v2.0/server" + require "routes/v2.0/script" + require "routes/v2.0/status" + require "routes/v2.0/bootstrap_templates" + require "routes/v2.0/stack" + require "routes/v2.0/stack_template" + require "routes/v2.0/handlers/stack_template_preset" + require "routes/v2.0/report" + + routes = Devops::Version2_0::Routes.constants.collect{|s| Devops::Version2_0::Routes.const_get(s)}.select {|const| const.class == Module} + routes.each do |r| + Devops::Api2.register r + end + Routes.route "/v2.0", Devops::Api2 + end + + private + def init_mongo + Devops::Api2.set :mongo, Devops::Db.connector + end + + def set_up_providers_keys!(providers, mongo) + providers.each do |provider| + next if provider.certificate_path.nil? + begin + mongo.key provider.ssh_key, Key::SYSTEM + rescue RecordNotFound => e + k = Key.new({"id" => provider.ssh_key, "path" => provider.certificate_path, "scope" => Key::SYSTEM}) + mongo.key_insert k + end + end + end + + end + end +end + diff --git a/devops-service/app/devops-application.rb b/devops-service/app/devops-application.rb new file mode 100644 index 0000000..d11676f --- /dev/null +++ b/devops-service/app/devops-application.rb @@ -0,0 +1,28 @@ +module Devops + module Application + class Application + + @@applications = [] + + def self.inherited(base) + @@applications << base.new + puts "Devops application '#{base}' has been added" + end + + def self.applications + @@applications + end + + def prepare + + end + + def init + + end + + def routes + end + end + end +end diff --git a/devops-service/app/devops-client.rb b/devops-service/app/devops-client.rb new file mode 100644 index 0000000..a303a53 --- /dev/null +++ b/devops-service/app/devops-client.rb @@ -0,0 +1,14 @@ +module Devops + module Application + class DevopsClientApplication < Application + + def prepare + require "app/client/devops-client" + end + + def routes + Routes.route "/client", ::Client.new + end + end + end +end diff --git a/devops-service/app/devops-version.rb b/devops-service/app/devops-version.rb new file mode 100644 index 0000000..0cfadc0 --- /dev/null +++ b/devops-service/app/devops-version.rb @@ -0,0 +1,15 @@ +module Devops + module Application + class DevopsVersionApplication < Application + + def prepare + require "app/version/version" + end + + def routes + Routes.route "/version", ::DevopsVersion.new + end + end + end +end + diff --git a/devops-service/app/sidekiq_web.rb b/devops-service/app/sidekiq_web.rb new file mode 100644 index 0000000..c53d3db --- /dev/null +++ b/devops-service/app/sidekiq_web.rb @@ -0,0 +1,17 @@ +module Devops + module Application + + class SidekiqApplication < Application + + def prepare + require "sidekiq/web" + end + + def routes + Devops::Routes.route "/sidekiq", Sidekiq::Web + end + + end + end +end + diff --git a/devops-service/version.rb b/devops-service/app/version/version.rb similarity index 100% rename from devops-service/version.rb rename to devops-service/app/version/version.rb diff --git a/devops-service/config.ru b/devops-service/config.ru index d2f4065..4354bf3 100644 --- a/devops-service/config.ru +++ b/devops-service/config.ru @@ -6,7 +6,7 @@ require 'byebug' require_relative "devops-service" require_relative "devops_config" -require_relative "routes" +require_relative "devops-routes" root = File.dirname(__FILE__) @@ -26,4 +26,9 @@ config[:report_dir_v2] = File.expand_path(File.join(config[:devops_dir], "report ].each {|key| d = config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } DevopsService.init -run Rack::URLMap.new(Devops::Routes.routes) +puts Devops::Routes.routes +Devops::Routes.routes.each do |p, c| + map(p) do + run c + end +end diff --git a/devops-service/db/mongo/connectors/project_template.rb b/devops-service/db/mongo/connectors/project_template.rb new file mode 100644 index 0000000..e461edd --- /dev/null +++ b/devops-service/db/mongo/connectors/project_template.rb @@ -0,0 +1,15 @@ +module Connectors + class ProjectTemplates < Base + include Helpers::InsertCommand, + Helpers::ShowCommand, + Helpers::ListCommand, + Helpers::DeleteCommand, + Helpers::UpdateCommand + + + def initialize(db) + @collection = db.collection('project_templates') + end + end +end + diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb index d38e557..98a010a 100644 --- a/devops-service/db/mongo/models/project.rb +++ b/devops-service/db/mongo/models/project.rb @@ -5,11 +5,20 @@ require "db/mongo/models/user" require "db/mongo/models/deploy_env/deploy_env_multi" require "db/mongo/models/mongo_model" require "json" +require "lib/project/handler/types_factory" module Devops module Model class Project < MongoModel + include Hooks + include Hooks::InstanceHooks + define_hook :before_create + define_hook :after_create + + define_hook :before_delete + define_hook :after_delete + attr_accessor :id, :deploy_envs, :type, :archived, :description types :id => {:type => String, :empty => false}, @@ -25,9 +34,13 @@ module Devops def initialize p={} self.id = p["name"] #raise InvalidRecord.new "No deploy envirenments for project #{self.id}" if p["deploy_envs"].nil? or p["deploy_envs"].empty? - self.type = p["type"] + self.type = p["type"] || Devops::GenericType.name self.description = p["description"] self.archived = p["archived"] || false + + handler = Devops::TypesFactory.type self.type + handler.prepare(self) unless handler.nil? + env_class = ( self.multi? ? DeployEnvMulti : DeployEnvFactory ) unless p["deploy_envs"].nil? self.deploy_envs = [] @@ -90,12 +103,14 @@ module Devops raise InvalidRecord.new "Deploy environment(s) '#{non_uniq.join("', '")}' is/are not unique" unless non_uniq.empty? self.deploy_envs.each do |d| d.validate! +=begin unless self.multi? rn = "#{self.id}#{DevopsConfig.config[:role_separator] || "_"}#{d.identifier}" role = "role[#{rn}]" d.run_list = d.run_list - [rn, role] d.run_list.unshift(role) end +=end end true @@ -133,6 +148,22 @@ module Devops self.type == MULTI_TYPE end + def create + res = {} + res[:before] = self.run_hook :before_create + Devops::Db.connector.project_insert self + res[:after] = self.run_hook :after_create + res + end + + def delete + res = {} + res[:before] = self.run_hook :before_delete + Devops::Db.connector.project_delete self.id + res[:after] = self.run_hook :after_delete + res + end + def self.build_from_bson p p["name"] = p["_id"] Project.new p diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb index 781223b..c095c23 100644 --- a/devops-service/db/mongo/mongo_connector.rb +++ b/devops-service/db/mongo/mongo_connector.rb @@ -18,6 +18,9 @@ class MongoConnector [:project, :projects_all, :projects, :project_names_with_envs, :projects_by_image, :projects_by_user, :project_insert, :project_update, :project_delete, :is_project_exists?, :check_project_auth] => :projects_connector, + [:project_templates, :projects_all, :projects, :project_names_with_envs, + :projects_by_image, :projects_by_user, :project_insert, :project_update, + :project_delete, :is_project_exists?, :check_project_auth] => :projects_templates_connector, [:servers_find, :servers, :servers_by_names, :server_by_instance_id, :server_by_chef_node_name, :servers_by_key, :server_insert, :server_delete, :server_update, :server_set_chef_node_name] => :servers_connector, @@ -55,6 +58,10 @@ class MongoConnector @projects_connector ||= Connectors::Project.new(@db) end + def projects_templates_connector + @projects__templates_connector ||= Connectors::ProjectTemplates.new(@db) + end + def servers_connector @servers_connector ||= Connectors::Server.new(@db) end diff --git a/devops-service/db/mongo/mongo_user.rb b/devops-service/db/mongo/mongo_user.rb deleted file mode 100644 index e42cd72..0000000 --- a/devops-service/db/mongo/mongo_user.rb +++ /dev/null @@ -1,18 +0,0 @@ -require "mongo" -require "db/exceptions/record_not_found" - -class MongoUser - - def initialize(db, host, port=27017) - @mongo_client = MongoClient.new(host, port) - @db = @mongo_client.db(db) - @users = @db.collection("users") - end - - def user username, password - u = @users.find("_id" => username, "password" => password).to_a[0] - raise RecordNotFound.new("User '#{username}' does not exist") if u.nil? - u - end - -end diff --git a/devops-service/routes.rb b/devops-service/devops-routes.rb similarity index 100% rename from devops-service/routes.rb rename to devops-service/devops-routes.rb diff --git a/devops-service/devops-service.rb b/devops-service/devops-service.rb index f71a22c..b04698a 100644 --- a/devops-service/devops-service.rb +++ b/devops-service/devops-service.rb @@ -8,16 +8,18 @@ require "db/validators/all" require "db/mongo/mongo_connector" require "providers/provider_factory" -require "loader" -require "devops_db" - -require "sidekiq/web" require "fog" +require "loader" +require "devops_db" require_relative "routes/v2.0" -require_relative "client" -require_relative "report" -require_relative "version" + +require "hooks" +require "app/devops-application" +require "app/devops-client" +require "app/sidekiq_web" +require "app/devops-version" +require "app/devops-api2" require_relative "sinatra/methods_with_headers" @@ -30,11 +32,7 @@ class DevopsService def routes config = DevopsConfig.config Devops::Routes.preffix = config[:url_prefix] - Devops::Routes.route "/version", DevopsVersion - Devops::Routes.route "/v2.0", Devops::Version2_0::Application - Devops::Routes.route "/client", Client.new(config) - Devops::Routes.route "/v2.0/report", ReportRoutes.new(config, "v2") - Devops::Routes.route "/sidekiq", Sidekiq::Web + #require_relative "routes/routes.rb" end # steps: @@ -48,15 +46,34 @@ class DevopsService def init # init database Devops::Db.init - # init plugins + routes + apps = Devops::Application::Application.applications + apps.each do |a| + a.prepare + end + + Devops::Loader.prepare_plugins + apps.each do |a| + a.init + end Devops::Loader.init_plugins + apps.each do |a| + a.routes + end + Devops::Loader.routes +=begin + Devops::Version2_0::Application.load + Devops::Loader.load_plugins # init routes classes Devops::Version2_0::Application.init + # init plugins + Devops::Loader.init_plugins Devops::Version2_0::Application.register_routes # init routes paths routes # add plugins routes Devops::Loader.routes +=end end def debug? diff --git a/devops-service/devops_loader.rb b/devops-service/devops_loader.rb deleted file mode 100644 index 0f63c4c..0000000 --- a/devops-service/devops_loader.rb +++ /dev/null @@ -1,7 +0,0 @@ -class DevopsLoader - - def self.load - #Devops::Routes.route "/version", DevopsVersion - end - -end diff --git a/devops-service/lib/project/handler/generic_type.rb b/devops-service/lib/project/handler/generic_type.rb new file mode 100644 index 0000000..91b3db0 --- /dev/null +++ b/devops-service/lib/project/handler/generic_type.rb @@ -0,0 +1,55 @@ +require_relative "project_type" + +module Devops + class GenericType + + def self.name + "generic" + end + + include ProjectType + + def prepare project + project.before_create create_roles + end + + def create_roles + lambda { + log = DevopsLogger.logger + log.info "Create roles hook" + return {roles: nil} if self.multi? + roles = {:new => [], :error => [], :exist => []} + self.deploy_envs.each do |e| + next if e.use_json_file + role_name = KnifeCommands.role_name(self.id, e.identifier) + chef_env = e.chef_env + if roles[chef_env].nil? + roles[chef_env], r = KnifeCommands.roles(chef_env) + unless r + log.error "Can't get roles list form chef-#{chef_env}: #{roles[chef_env]}" + roles[chef_env] = nil + next + end + end + all_roles = roles[chef_env] + begin + if all_roles.include? role_name + roles[:exist].push role_name + e.run_list << "role[#{role_name}]" + else + KnifeCommands.create_role role_name, self.id, e.identifier, chef_env + roles[:new].push role_name + e.run_list << "role[#{role_name}]" + log.info "Role '#{role_name}' created" + end + rescue => er + roles[:error].push role_name + log.error "Role '#{role_name}' can not be created: #{er.message}" + end + end + {roles: roles} + } + end + + end +end diff --git a/devops-service/lib/project/handler/project_type.rb b/devops-service/lib/project/handler/project_type.rb new file mode 100644 index 0000000..af51ced --- /dev/null +++ b/devops-service/lib/project/handler/project_type.rb @@ -0,0 +1,21 @@ +module Devops + module ProjectType + + class << self + @@types = {} + def included mod + @@types[mod.name] = mod + puts "Project type '#{mod.name}' registered" + end + + def types + @@types + end + end + + def prepare project + + end + + end +end diff --git a/devops-service/lib/project/handler/types_factory.rb b/devops-service/lib/project/handler/types_factory.rb new file mode 100644 index 0000000..5f6773e --- /dev/null +++ b/devops-service/lib/project/handler/types_factory.rb @@ -0,0 +1,16 @@ +require_relative "project_type" +require_relative "generic_type" + +module Devops + class TypesFactory + + def self.type type + t = ProjectType.types + t[type].new + end + + def self.types_names + ProjectType.types.keys + end + end +end diff --git a/devops-service/loader.rb b/devops-service/loader.rb index 4a36eb5..429e299 100644 --- a/devops-service/loader.rb +++ b/devops-service/loader.rb @@ -2,6 +2,12 @@ module Devops module Loader class << self + def prepare_plugins + plugins do |plugin| + plugin.prepare + end + end + def init_plugins plugins do |plugin| plugin.init diff --git a/devops-service/report.rb b/devops-service/report.rb deleted file mode 100644 index 437ad79..0000000 --- a/devops-service/report.rb +++ /dev/null @@ -1,89 +0,0 @@ -require 'sinatra/base' - -class ReportRoutes < Sinatra::Base - - def initialize config, version - super() - @@config = config - end - - enable :inline_templates - - get "/all" do - options = {} - ["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status"].each do |k| - options[k] = params[k] unless params[k].nil? - end - json Devops::Db.connector.reports(options).map{|r| r.to_hash} -=begin - res = {} - uri = URI.parse(request.url) - pref = File.dirname(uri.path) - @paths.each do |key, dir| - files = [] - Dir[File.join(dir, "/**/*")].each do |f| - next if File.directory?(f) - jid = File.basename(f) - uri.path = File.join(pref, key, f[dir.length..-1]) - o = { - "file" => uri.to_s, - "created" => File.ctime(f).to_s, - "status" => task_status(jid) - } - files.push o - end - res[key] = files - end - json res -=end - end - - get "/:id" do - r = Devops::Db.connector.report(params[:id]) - file = r.file - return [404, "Report '#{params[:id]}' does not exist"] unless File.exists? file - @text = File.read(file) - @done = completed?(params[:id]) - erb :index - end - - get "/favicon.ico" do - [404, ""] - end - - def completed? id - r = task_status(id) - r == "completed" or r == "failed" - end - - def task_status id - r = Sidekiq.redis do |connection| - connection.hget("devops", id) - end - end - -end - -__END__ - -@@ layout - -
- <% unless @done %> - - <% end %> - - - <%= yield %> - - - -@@ index --<%= @text %> -diff --git a/devops-service/routes/routes_container.rb b/devops-service/routes/routes_container.rb deleted file mode 100644 index cef0a66..0000000 --- a/devops-service/routes/routes_container.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Devops - module RoutesContainer - - def register_routes - routes = Devops::Version2_0::Routes.constants.collect{|s| Devops::Version2_0::Routes.const_get(s)}.select {|const| const.class == Module} - routes.each do |r| - register r - end - end - - end -end diff --git a/devops-service/routes/v2.0.rb b/devops-service/routes/v2.0.rb index 2826287..dbe1346 100644 --- a/devops-service/routes/v2.0.rb +++ b/devops-service/routes/v2.0.rb @@ -3,113 +3,10 @@ require "sinatra/streaming" require "helpers/version_2" require "json" -require "routes/v2.0/flavor" -require "routes/v2.0/image" -require "routes/v2.0/filter" -require "routes/v2.0/network" -require "routes/v2.0/group" -require "routes/v2.0/deploy" -require "routes/v2.0/project" -require "routes/v2.0/key" -require "routes/v2.0/user" -require "routes/v2.0/provider" -require "routes/v2.0/tag" -require "routes/v2.0/server" -require "routes/v2.0/script" -require "routes/v2.0/status" -require "routes/v2.0/bootstrap_templates" -require "routes/v2.0/stack_template" -require "routes/v2.0/stack_template_presets" -require "routes/v2.0/stack" - -require "routes/v2.0/handlers/provider" -require "routes/v2.0/handlers/bootstrap_templates" -require "routes/v2.0/handlers/deploy" -require "routes/v2.0/handlers/filter" -require "routes/v2.0/handlers/flavor" -require "routes/v2.0/handlers/group" -require "routes/v2.0/handlers/image" -require "routes/v2.0/handlers/network" -require "routes/v2.0/handlers/key" -require "routes/v2.0/handlers/project" -require "routes/v2.0/handlers/script" -require "routes/v2.0/handlers/status" -require "routes/v2.0/handlers/tag" -require "routes/v2.0/handlers/user" -require "routes/v2.0/handlers/server" -require "routes/v2.0/handlers/stack_template" -require "routes/v2.0/handlers/stack_template_preset" -require "routes/v2.0/handlers/stack" -require "routes/routes_container" -require 'lib/stubber' - require "auth/devops_auth" module Devops - module Version2_0 - class Application < Sinatra::Base - - extend Devops::RoutesContainer - - class << self - - @routes = [ - Devops::Version2_0::Routes::ProviderRoutes, - Devops::Version2_0::Routes::BootstrapTemplatesRoutes, - Devops::Version2_0::Routes::UserRoutes, - Devops::Version2_0::Routes::FilterRoutes, - Devops::Version2_0::Routes::FlavorRoutes, - Devops::Version2_0::Routes::GroupRoutes, - Devops::Version2_0::Routes::ImageRoutes, - Devops::Version2_0::Routes::KeyRoutes, - Devops::Version2_0::Routes::NetworkRoutes, - Devops::Version2_0::Routes::ProjectRoutes, - Devops::Version2_0::Routes::ScriptRoutes, - Devops::Version2_0::Routes::ServerRoutes, - Devops::Version2_0::Routes::StatusRoutes, - Devops::Version2_0::Routes::TagRoutes, - Devops::Version2_0::Routes::DeployRoutes, - Devops::Version2_0::Routes::StackTemplateRoutes, - Devops::Version2_0::Routes::StackRoutes - ] - - def init - config = DevopsConfig.config - set :devops_home, config[:devops_dir] - #set :config, config - - set :keys_dir, (config[:keys_dir] || File.join(config[:devops_dir], "files/keys")) - set :scripts_dir, (config[:scripts_dir] || File.join(config[:devops_dir], "files/scripts")) - [:keys_dir, :scripts_dir].each {|key| d = settings.send(key); FileUtils.mkdir_p(d) unless File.exists?(d) } - init_mongo - settings.mongo.create_root_user - ::Provider::ProviderFactory.init(config) - #set_up_providers_keys!(::Provider::ProviderFactory.all, mongo) - - if settings.test? || config[:stub_classes] - Stubber.stub_providers! - Stubber.stub_models! - end - end - - def init_mongo - set :mongo, Devops::Db.connector - end - - private - def set_up_providers_keys!(providers, mongo) - providers.each do |provider| - next if provider.certificate_path.nil? - begin - mongo.key provider.ssh_key, Key::SYSTEM - rescue RecordNotFound => e - k = Key.new({"id" => provider.ssh_key, "path" => provider.certificate_path, "scope" => Key::SYSTEM}) - mongo.key_insert k - end - end - end - - end + class Api2 < Sinatra::Base include Sinatra::JSON helpers Sinatra::Streaming @@ -117,22 +14,11 @@ module Devops register Sinatra::DevopsAuth -=begin - use Rack::Auth::Basic do |username, password| - begin - settings.mongo.user_auth(username, password) - true - rescue RecordNotFound => e - false - end - end -=end - configure :production do disable :dump_errors disable :show_exceptions set :logging, Logger::INFO - puts "TODO2" + puts "Production mode" end configure :development do @@ -140,7 +26,7 @@ module Devops disable :raise_errors # disable :dump_errors set :show_exceptions, :after_handler - puts "TODO1" + puts "Development mode" end not_found do @@ -217,5 +103,4 @@ module Devops end end - end end diff --git a/devops-service/routes/v2.0/handlers/project.rb b/devops-service/routes/v2.0/handlers/project.rb index a122742..c3ae310 100644 --- a/devops-service/routes/v2.0/handlers/project.rb +++ b/devops-service/routes/v2.0/handlers/project.rb @@ -51,9 +51,8 @@ module Devops check_array(body["deploy_envs"], "Parameter 'deploy_envs' must be a not empty array of objects", Hash) p = Devops::Model::Project.new(body) halt_response("Project '#{p.id}' already exist") if settings.mongo.is_project_exists?(p) - puts "USER: #{request.env['REMOTE_USER']}" p.add_authorized_user [request.env['REMOTE_USER']] - settings.mongo.project_insert p + p.create roles_res = "" if p.multi? logger.info "Project '#{p.id}' with type 'multi' created" @@ -133,11 +132,12 @@ module Devops deploy_env = unless body.nil? check_string(body["deploy_env"], "Parameter 'deploy_env' should be a not empty string", true) end + project = settings.mongo.project(params[:project]) info = if deploy_env.nil? + project.delete settings.mongo.project_delete(params[:project]) "Project '#{params[:project]}' is deleted" else - project = settings.mongo.project(params[:project]) project.remove_env deploy_env settings.mongo.project_update project "Project '#{params[:project]}'. Deploy environment '#{deploy_env}' has been deleted" diff --git a/devops-service/routes/v2.0/handlers/report.rb b/devops-service/routes/v2.0/handlers/report.rb new file mode 100644 index 0000000..66e2a99 --- /dev/null +++ b/devops-service/routes/v2.0/handlers/report.rb @@ -0,0 +1,63 @@ +module Devops + module Version2_0 + module Handler + class Report + + def self.reports_all + lambda { + options = {} + ["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "max_number", "chef_node_name"].each do |k| + options[k] = params[k] unless params[k].nil? + end + attributes_keys = params.keys.select{|k| k =~ /attributes\.*/} + attributes_keys.each do |ak| + options[ak] = params[ak] + end + json Devops::Db.connector.reports(options).map{|r| r.to_hash} + } + end + + def self.reports_latest + lambda { + options = {} + ["project", "deploy_env", "type", "created_by", "date_from", "date_to", "sort", "status", "chef_node_name"].each do |k| + options[k] = params[k] unless params[k].nil? + end + attributes_keys = params.keys.select{|k| k =~ /attributes\.*/} + attributes_keys.each do |ak| + options[ak] = params[ak] + end + json Devops::Db.connector.latest_reports(options).map{|r| r.to_hash} + } + end + + def self.attributes_all + lambda{ + json Devops::Db.connector.reports_attributes_values(params["name"]) + } + end + + def self.report + lambda{ + begin + r = Devops::Db.connector.report(params[:id]) + file = r.file + return [404, "Report '#{params[:id]}' does not exist"] unless File.exists? file + @text = Rack::Utils.escape_html(File.read(file)) + @done = completed?(params[:id]) + rescue RecordNotFound => e + if task_status(params[:id]) == Worker::STATUS::IN_QUEUE + @text = "Task '#{params[:id]}' has been queued" + @done = false + else + raise e + end + end + erb :index + } + end + end + end + end +end + diff --git a/devops-service/routes/v2.0/report.rb b/devops-service/routes/v2.0/report.rb new file mode 100644 index 0000000..f9c345c --- /dev/null +++ b/devops-service/routes/v2.0/report.rb @@ -0,0 +1,53 @@ +module Devops + module Version2_0 + module Routes + module ReportRoutes + + def self.registered(app) + + app.get_with_headers "/report/all", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_all + app.get_with_headers "/report/all/latest", headers: [:accept], &Devops::Version2_0::Handler::Report.reports_latest + app.get_with_headers "/report/all/attributes/:name", headers: [:accept], &Devops::Version2_0::Handler::Report.attributes_all + app.get_with_headers "/report/:id", headers: [:accept], &Devops::Version2_0::Handler::Report.report + puts "Report routes initialized" + end + + def completed? id + r = task_status(id) + r == "completed" or r == "failed" + end + + def task_status id + r = Sidekiq.redis do |connection| + connection.hget("devops", id) + end + end + + end + end + end +end + +__END__ + +@@ layout + + + <% unless @done %> + + <% end %> + + + <%= yield %> + + + +@@ index +
+<%= @text %> +diff --git a/devops-service/sinatra/methods_with_headers.rb b/devops-service/sinatra/methods_with_headers.rb index 095b421..aeafdf9 100644 --- a/devops-service/sinatra/methods_with_headers.rb +++ b/devops-service/sinatra/methods_with_headers.rb @@ -1,4 +1,5 @@ require "sinatra/base" +require 'rack/accept_media_types' module Sinatra @@ -99,15 +100,11 @@ module Sinatra # # Can client works with JSON? def accept_json - logger.debug(request.accept) - unless request.accept? 'application/json' + types = request.accept_media_types + unless types.include?('application/json') or types.include?("*/*") response.headers['Accept'] = 'application/json' halt_response("Accept header should contains 'application/json' type", 406) end - rescue NoMethodError => e - #error in sinatra 1.4.4 (https://github.com/sinatra/sinatra/issues/844, https://github.com/sinatra/sinatra/pull/805) - response.headers['Accept'] = 'application/json' - halt_response("Accept header should contains 'application/json' type", 406) end # Check Content-Type header diff --git a/devops-service/tests/features/support/env.rb b/devops-service/tests/features/support/env.rb index be854b1..954dccb 100644 --- a/devops-service/tests/features/support/env.rb +++ b/devops-service/tests/features/support/env.rb @@ -24,6 +24,8 @@ class RequestSender require "httpclient" require "yaml" + include Test::Unit::Assertions + @last_res = nil $test_hash = Hash.new