From d7292d9ace467041f983e19ec67f229e1b6dca80 Mon Sep 17 00:00:00 2001 From: amartynov Date: Mon, 26 Jan 2015 15:02:58 +0300 Subject: [PATCH] routes --- devops-service/Gemfile | 6 +- devops-service/Gemfile.lock | 140 +++++++------- devops-service/config.ru | 29 ++- devops-service/devops-service.rb | 176 +++--------------- devops-service/devops_loader.rb | 17 ++ devops-service/loader.rb | 1 + devops-service/routes.rb | 20 ++ devops-service/routes/v2.0.rb | 175 +++++++++++++++-- .../routes/v2.0/bootstrap_templates.rb | 2 +- 9 files changed, 300 insertions(+), 266 deletions(-) create mode 100644 devops-service/devops_loader.rb create mode 100644 devops-service/routes.rb diff --git a/devops-service/Gemfile b/devops-service/Gemfile index ff2ad69..7bbef51 100644 --- a/devops-service/Gemfile +++ b/devops-service/Gemfile @@ -5,8 +5,8 @@ ruby "2.0.0" gem "thin", "~>1.5.1" gem "mime-types", "~>1.25.1" gem "sinatra", "1.4.3" -gem "sinatra-contrib", "1.4.1" -gem "sinatra-websocket", "~>0.3.0" +gem "sinatra-contrib"#, "1.4.1" +gem "sinatra-websocket"#, "~>0.3.0" gem "fog", "~>1.20" gem "mixlib-shellout" gem "chef", ">=11" @@ -16,6 +16,8 @@ gem "multi_json", "1.7.8" gem "rufus-scheduler", "2.0.24" gem "sidekiq", "3.2.6" gem 'wisper' +gem 'rake', '10.0.0' +gem 'rack', '1.5.2' gem "devops-nibr", :path => "/home/amartynov/workspace/github/devops-service/devops-service/plugins/devops-nibr" diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock index 22580b3..acd3219 100644 --- a/devops-service/Gemfile.lock +++ b/devops-service/Gemfile.lock @@ -6,7 +6,7 @@ PATH GEM remote: https://rubygems.org/ specs: - CFPropertyList (2.2.8) + CFPropertyList (2.3.0) addressable (2.3.6) backports (3.6.4) bson (1.11.1) @@ -15,31 +15,22 @@ GEM builder (3.2.2) celluloid (0.15.2) timers (~> 1.1.0) - chef (12.0.3) - chef-zero (~> 3.2) - diff-lcs (~> 1.2, >= 1.2.4) - erubis (~> 2.7) - ffi-yajl (~> 1.2) - highline (~> 1.6, >= 1.6.9) - mixlib-authentication (~> 1.3) - mixlib-cli (~> 1.4) - mixlib-config (~> 2.0) - mixlib-log (~> 1.3) - mixlib-shellout (>= 2.0.0.rc.0, < 3.0) + chef (11.6.0) + erubis + highline (>= 1.6.9) + json (>= 1.4.4, <= 1.7.7) + mixlib-authentication (>= 1.3.0) + mixlib-cli (~> 1.3.0) + mixlib-config (>= 1.1.2) + mixlib-log (>= 1.3.0) + mixlib-shellout net-ssh (~> 2.6) - net-ssh-multi (~> 1.1) - ohai (~> 8.0) - plist (~> 3.1.0) - pry (~> 0.9) - chef-zero (3.2.1) - ffi-yajl (~> 1.1) - hashie (~> 2.0) - mixlib-log (~> 1.3) - rack - uuidtools (~> 2.1) - coderay (1.1.0) - connection_pool (2.1.0) - cucumber (1.3.17) + net-ssh-multi (~> 1.1.0) + ohai (>= 0.6.0) + rest-client (>= 1.0.4, < 1.7.0) + yajl-ruby (~> 1.1) + connection_pool (2.1.1) + cucumber (1.3.18) builder (>= 2.1.2) diff-lcs (>= 1.1.3) gherkin (~> 2.12) @@ -51,23 +42,21 @@ GEM addressable (>= 2.1.1) eventmachine (>= 0.12.9) erubis (2.7.0) - eventmachine (1.0.3) - excon (0.42.1) - ffi (1.9.6) - ffi-yajl (1.3.1) - ffi (~> 1.5) - libyajl2 (~> 1.2) + eventmachine (1.0.4) + excon (0.43.0) fission (0.5.0) CFPropertyList (~> 2.2) - fog (1.26.0) + fog (1.27.0) fog-atmos + fog-aws (~> 0.0) fog-brightbox (~> 0.4) - fog-core (~> 1.27, >= 1.27.1) + fog-core (~> 1.27, >= 1.27.3) fog-ecloud fog-json fog-profitbricks fog-radosgw (>= 0.0.2) fog-sakuracloud (>= 0.0.4) + fog-serverlove fog-softlayer fog-storm_on_demand fog-terremark @@ -79,11 +68,16 @@ GEM fog-atmos (0.1.0) fog-core fog-xml + fog-aws (0.0.7) + fog-core (~> 1.27) + fog-json (~> 1.0) + fog-xml (~> 0.1) + ipaddress (~> 0.8) fog-brightbox (0.7.1) fog-core (~> 1.22) fog-json inflecto (~> 0.0.2) - fog-core (1.27.2) + fog-core (1.27.3) builder excon (~> 0.38) formatador (~> 0.2) @@ -103,10 +97,13 @@ GEM fog-core (>= 1.21.0) fog-json fog-xml (>= 0.0.1) - fog-sakuracloud (0.1.1) + fog-sakuracloud (1.0.0) fog-core fog-json - fog-softlayer (0.3.25) + fog-serverlove (0.1.1) + fog-core + fog-json + fog-softlayer (0.3.30) fog-core fog-json fog-storm_on_demand (0.1.0) @@ -127,65 +124,56 @@ GEM formatador (0.2.5) gherkin (2.12.2) multi_json (~> 1.3) - hashie (2.1.2) highline (1.6.21) - httpclient (2.5.3.3) + httpclient (2.6.0.1) inflecto (0.0.2) ipaddress (0.8.0) - json (1.8.1) - libyajl2 (1.2.0) - method_source (0.8.2) + json (1.7.7) mime-types (1.25.1) - mini_portile (0.6.1) + mini_portile (0.6.2) mixlib-authentication (1.3.0) mixlib-log - mixlib-cli (1.5.0) + mixlib-cli (1.3.0) mixlib-config (2.1.0) mixlib-log (1.6.0) - mixlib-shellout (2.0.0) + mixlib-shellout (2.0.1) mongo (1.11.1) bson (= 1.11.1) multi_json (1.7.8) multi_test (0.1.1) - net-dhcp (1.3.2) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (2.9.1) + net-ssh (2.9.2) net-ssh-gateway (1.2.0) net-ssh (>= 2.6.5) - net-ssh-multi (1.2.0) - net-ssh (>= 2.6.5) - net-ssh-gateway (>= 1.2.0) - nokogiri (1.6.5) + net-ssh-multi (1.1) + net-ssh (>= 2.1.4) + net-ssh-gateway (>= 0.99.0) + nokogiri (1.6.6.2) mini_portile (~> 0.6.0) - ohai (8.0.1) - ffi (~> 1.9) - ffi-yajl (~> 1.1) + ohai (6.24.2) ipaddress - mime-types (~> 1.16) mixlib-cli - mixlib-config (~> 2.0) + mixlib-config mixlib-log - mixlib-shellout (~> 2.0) - net-dhcp - rake (~> 10.1) - systemu (~> 2.6.4) - wmi-lite (~> 1.0) - plist (3.1.0) + mixlib-shellout + systemu (~> 2.5.2) + yajl-ruby power_assert (0.2.2) - pry (0.10.1) - coderay (~> 1.1.0) - method_source (~> 0.8.1) - slop (~> 3.4) - rack (1.6.0) + rack (1.5.2) rack-protection (1.5.3) rack - rack-test (0.6.2) + rack-test (0.6.3) rack (>= 1.0) - rake (10.4.2) + rake (10.0.0) + rdoc (4.2.0) + json (~> 1.4) redis (3.2.0) redis-namespace (1.5.1) redis (~> 3.0, >= 3.0.4) + rest-client (1.6.8) + mime-types (~> 1.16) + rdoc (>= 2.4.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) sidekiq (3.2.6) @@ -198,7 +186,7 @@ GEM rack (~> 1.4) rack-protection (~> 1.4) tilt (~> 1.3, >= 1.3.4) - sinatra-contrib (1.4.1) + sinatra-contrib (1.4.2) backports (>= 2.0) multi_json rack-protection @@ -209,9 +197,8 @@ GEM em-websocket (~> 0.3.6) eventmachine thin (>= 1.3.1, < 2.0.0) - slop (3.6.0) - systemu (2.6.4) - test-unit (3.0.8) + systemu (2.5.2) + test-unit (3.0.9) power_assert thin (1.5.1) daemons (>= 1.0.9) @@ -222,9 +209,8 @@ GEM timers (1.1.0) tzinfo (1.2.2) thread_safe (~> 0.1) - uuidtools (2.1.5) wisper (1.6.0) - wmi-lite (1.0.0) + yajl-ruby (1.2.1) PLATFORMS ruby @@ -240,11 +226,13 @@ DEPENDENCIES mixlib-shellout mongo multi_json (= 1.7.8) + rack (= 1.5.2) + rake (= 10.0.0) rufus-scheduler (= 2.0.24) sidekiq (= 3.2.6) sinatra (= 1.4.3) - sinatra-contrib (= 1.4.1) - sinatra-websocket (~> 0.3.0) + sinatra-contrib + sinatra-websocket test-unit thin (~> 1.5.1) wisper diff --git a/devops-service/config.ru b/devops-service/config.ru index f7aff87..5938bd7 100644 --- a/devops-service/config.ru +++ b/devops-service/config.ru @@ -1,14 +1,13 @@ # To run devops you can use command # `bundle exec thin -R $devops_home/config.ru -e $env -d -p $port -t 600 -u $user --pid $pid_file --log $log_file start` -require 'rubygems' -require 'bundler/setup' -require "sidekiq/web" +require "rubygems" +require "bundler/setup" + +require_relative "devops-service" +require_relative "devops_loader" +require_relative "routes" root = File.dirname(__FILE__) -require File.join(root, "devops-service") -require File.join(root, "client") -require File.join(root, "report") -require File.join(root, "version") # Read configuration file config_file = File.join(root, "config.rb") @@ -20,16 +19,16 @@ else end config[:devops_dir] = File.join(ENV["HOME"], ".devops") if config[:devops_dir].nil? +puts "Devops home: #{config[:devops_dir]}" +unless File.exists?(config[:devops_dir]) + FileUtils.mkdir_p config[:devops_dir] + puts "Directory '#{config[:devops_dir]}' has been created" +end config[:report_dir_v2] = File.expand_path(File.join(config[:devops_dir], "report", "v2")) unless config[:report_dir_v2] [ :report_dir_v2 ].each {|key| d = config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } -# URL map for API v2.0 -run Rack::URLMap.new({ - "#{config[:url_prefix]}/v2.0" => DevopsService.new(config), - "#{config[:url_prefix]}/client" => Client.new(config), - "#{config[:url_prefix]}/v2.0/report" => ReportRoutes.new(config, "v2"), - "#{config[:url_prefix]}/sidekiq" => Sidekiq::Web, - "#{config[:url_prefix]}/version" => DevopsVersion.new -}) + +DevopsService.init(config) +run Rack::URLMap.new(Devops::Routes.routes) diff --git a/devops-service/devops-service.rb b/devops-service/devops-service.rb index 512cef7..8ba94b1 100644 --- a/devops-service/devops-service.rb +++ b/devops-service/devops-service.rb @@ -1,8 +1,5 @@ #!/usr/bin/env ruby -require "rubygems" -require "sinatra/base" -require "sinatra/streaming" require "fileutils" require "wisper" @@ -16,66 +13,38 @@ require "providers/provider_factory" require "loader" -require "routes/v2.0" -require "helpers/version_2" +require "sidekiq/web" require "test_subscriber" require "test_subscriber_2" -class DevopsService < Sinatra::Base +require_relative "routes/v2.0" +require_relative "client" +require_relative "report" +require_relative "version" - helpers Sinatra::Streaming +class DevopsService include Wisper::Publisher - helpers Devops::Version2_0::Helpers - include Devops::Loader - register Devops::Version2_0::Routes +# include Devops::Loader - def initialize config - super() - puts "Devops home: #{config[:devops_dir]}" - unless File.exists?(config[:devops_dir]) - FileUtils.mkdir_p config[:devops_dir] - puts "Directory '#{config[:devops_dir]}' has been created" + class << self + + def routes config + #Devops::Routes.preffix = "" + Devops::Routes.route "/version", DevopsVersion + Devops::Routes.route "/v2.0", Devops::Version2_0::Application + Devops::Routes.route "/client", Client.new(config) + Devops::Routes.route "/v2.0/report", ReportRoutes.new(config, "v2") + Devops::Routes.route "/sidekiq", Sidekiq::Web end - self.class.set :devops_home, config[:devops_dir] - self.class.set :config, config - @@config = config - root = File.dirname(__FILE__) - @@config[:keys_dir] = File.join(root, "../.devops_files/keys") - if @@config[:scripts_dir].nil? - #default scripts dir - @@config[:scripts_dir] = File.join(root, "../.devops_files/scripts") - end - [:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } - mongo = DevopsService.mongo - self.class.set :mongo, mongo - mongo.create_root_user - ::Provider::ProviderFactory.init(config) - #set_up_providers_keys!(::Provider::ProviderFactory.all, mongo) - Wisper::GlobalListeners.subscribe(TestSubscriber2.new) - Wisper.subscribe(TestSubscriber.new) - end - - @@mongo - # Returns mongo connector - def self.mongo - @@mongo ||= MongoConnector.new(@@config[:mongo_db], @@config[:mongo_host], @@config[:mongo_port], @@config[:mongo_user], @@config[:mongo_password]) - end - - # Returns config hash - def self.config - @@config - end - - use Rack::Auth::Basic do |username, password| - begin - settings.mongo.user_auth(username, password) - true - rescue RecordNotFound => e - false + def init config + routes config +Devops::Version2_0::Application.init config + DevopsLoader.load + DevopsLoader.routes end end @@ -83,109 +52,6 @@ class DevopsService < Sinatra::Base send(:generate_method, name, &block) end - include Sinatra::JSON - - configure :production do - disable :dump_errors - disable :show_exceptions - set :logging, Logger::INFO - puts "TODO2" - end - - configure :development do - set :logging, Logger::DEBUG - disable :raise_errors -# disable :dump_errors - set :show_exceptions, :after_handler - puts "TODO1" - end - - not_found do - "Not found" - end - - error RecordNotFound do - e = env["sinatra.error"] - logger.warn e.message - halt_response(e.message, 404) - end - - error InvalidRecord do - e = env["sinatra.error"] - logger.warn e.message - logger.warn "Request body: #{request.body.read}" - halt_response(e.message, 400) - end - - error InvalidCommand do - e = env["sinatra.error"] - logger.warn e.message - halt_response(e.message, 400) - end - - error DependencyError do - e = env["sinatra.error"] - logger.warn e.message - halt_response(e.message, 400) - end - - error InvalidPrivileges do - e = env["sinatra.error"] - logger.warn e.message - halt_response(e.message, 401) - end - - error Excon::Errors::Unauthorized do - e = env["sinatra.error"] - resp = e.response - ct = resp.headers["Content-Type"] - msg = unless ct.nil? - if ct.include?("application/json") - json = ::Chef::JSONCompat.from_json(resp.body) - m = "ERROR: Unauthorized (#{json['error']['code']}): #{json['error']['message']}" - logger.error(m) - else - end - m - else - "Unauthorized: #{e.inspect}" - end - halt_response(msg, 500) - end - - error Fog::Compute::AWS::Error do - e = env["sinatra.error"] - logger.error e.message - halt_response(e.message, 500) - end - - error do - e = env["sinatra.error"] - logger.error e.message - halt_response(e.message, 500) - end - -# def self.mongo -# DevopsService.mongo -# end - - -# use ::Version2_0::V2_0 - - private - - def set_up_providers_keys!(providers, mongo) - providers.each do |provider| - next if provider.certificate_path.nil? - begin - mongo.key provider.ssh_key, Key::SYSTEM - rescue RecordNotFound => e - k = Key.new({"id" => provider.ssh_key, "path" => provider.certificate_path, "scope" => Key::SYSTEM}) - mongo.key_insert k - end - end - end - end require "wisper_fix" diff --git a/devops-service/devops_loader.rb b/devops-service/devops_loader.rb new file mode 100644 index 0000000..abc4f99 --- /dev/null +++ b/devops-service/devops_loader.rb @@ -0,0 +1,17 @@ +class DevopsLoader + + def self.load + #Devops::Routes.route "/version", DevopsVersion + end + + def self.routes + if defined?(Devops::Plugin) + routes = {} + plugins = Devops::Plugin.constants.collect{|s| Devops::Plugin.const_get(s)}.select {|const| const.class == Module} + puts plugins.inspect + plugins.each do |p| + p.routes + end + end + end +end diff --git a/devops-service/loader.rb b/devops-service/loader.rb index cbe1fc2..8f98566 100644 --- a/devops-service/loader.rb +++ b/devops-service/loader.rb @@ -38,6 +38,7 @@ module Devops puts "#{self} included in #{mod}" if defined?(Devops::Plugin) + routes = {} plugins = Devops::Plugin.constants.collect{|s| Devops::Plugin.const_get(s)}.select {|const| const.class == Module} puts plugins.inspect plugins.each do |p| diff --git a/devops-service/routes.rb b/devops-service/routes.rb new file mode 100644 index 0000000..9ae4825 --- /dev/null +++ b/devops-service/routes.rb @@ -0,0 +1,20 @@ +module Devops + class Routes + + @@routes = {} + @@preffix = nil + + def self.routes + @@routes + end + + def self.preffix= p + @@preffix = p + end + + def self.route path, _class + path = @@preffix + path unless @@preffix.nil? + @@routes[path] = _class + end + end +end diff --git a/devops-service/routes/v2.0.rb b/devops-service/routes/v2.0.rb index f02b095..44b842f 100644 --- a/devops-service/routes/v2.0.rb +++ b/devops-service/routes/v2.0.rb @@ -1,23 +1,164 @@ +require "bundler/setup" +require "sinatra/base" +require "sinatra/streaming" +require "helpers/version_2" + module Devops module Version2_0 - module Routes + class Application < Sinatra::Base - def self.registered(app) - app.register Devops::Version2_0::Routes::ProviderRoutes - app.register Devops::Version2_0::Routes::BootstrapTemplatesRoutes - app.register Devops::Version2_0::Routes::UserRoutes - app.register Devops::Version2_0::Routes::FilterRoutes - app.register Devops::Version2_0::Routes::FlavorRoutes - app.register Devops::Version2_0::Routes::GroupRoutes - app.register Devops::Version2_0::Routes::ImageRoutes - app.register Devops::Version2_0::Routes::KeyRoutes - app.register Devops::Version2_0::Routes::NetworkRoutes - app.register Devops::Version2_0::Routes::ProjectRoutes - app.register Devops::Version2_0::Routes::ScriptRoutes - app.register Devops::Version2_0::Routes::ServerRoutes - app.register Devops::Version2_0::Routes::StatusRoutes - app.register Devops::Version2_0::Routes::TagRoutes - app.register Devops::Version2_0::Routes::DeployRoutes + include Devops::Loader + + class << self + def init config + set :devops_home, config[:devops_dir] + set :config, config + + set :mongo_db, config[:mongo_db] || "devops" + set :mongo_host, config[:mongo_host] || "localhost" + set :mongo_port, config[:mongo_port] || 27017 + set :mongo_user, config[:mongo_user] + set :mongo_password, config[:mongo_password] + + set :keys_dir, (config[:keys_dir] || File.join(config[:devops_dir], "files/keys")) + set :scripts_dir, (config[:scripts_dir] || File.join(config[:devops_dir], "files/scripts")) + [:keys_dir, :scripts_dir].each {|key| d = settings.send(key); FileUtils.mkdir_p(d) unless File.exists?(d) } + init_mongo + settings.mongo.create_root_user + ::Provider::ProviderFactory.init(config) + #set_up_providers_keys!(::Provider::ProviderFactory.all, mongo) + Wisper::GlobalListeners.subscribe(TestSubscriber2.new) + Wisper.subscribe(TestSubscriber.new) + + end + + def init_mongo + mongo = MongoConnector.new(settings.mongo_db, settings.mongo_host, settings.mongo_port, settings.mongo_user, settings.mongo_password) + set :mongo, mongo + end + + private + def set_up_providers_keys!(providers, mongo) + providers.each do |provider| + next if provider.certificate_path.nil? + begin + mongo.key provider.ssh_key, Key::SYSTEM + rescue RecordNotFound => e + k = Key.new({"id" => provider.ssh_key, "path" => provider.certificate_path, "scope" => Key::SYSTEM}) + mongo.key_insert k + end + end + end + + end + + include Sinatra::JSON + helpers Sinatra::Streaming + helpers Devops::Version2_0::Helpers + + register Devops::Version2_0::Routes::ProviderRoutes + register Devops::Version2_0::Routes::BootstrapTemplatesRoutes + register Devops::Version2_0::Routes::UserRoutes + register Devops::Version2_0::Routes::FilterRoutes + register Devops::Version2_0::Routes::FlavorRoutes + register Devops::Version2_0::Routes::GroupRoutes + register Devops::Version2_0::Routes::ImageRoutes + register Devops::Version2_0::Routes::KeyRoutes + register Devops::Version2_0::Routes::NetworkRoutes + register Devops::Version2_0::Routes::ProjectRoutes + register Devops::Version2_0::Routes::ScriptRoutes + register Devops::Version2_0::Routes::ServerRoutes + register Devops::Version2_0::Routes::StatusRoutes + register Devops::Version2_0::Routes::TagRoutes + register Devops::Version2_0::Routes::DeployRoutes + + use Rack::Auth::Basic do |username, password| + begin + settings.mongo.user_auth(username, password) + true + rescue RecordNotFound => e + false + end + end + + configure :production do + disable :dump_errors + disable :show_exceptions + set :logging, Logger::INFO + puts "TODO2" + end + + configure :development do + set :logging, Logger::DEBUG + disable :raise_errors + # disable :dump_errors + set :show_exceptions, :after_handler + puts "TODO1" + end + + not_found do + "Not found" + end + + error RecordNotFound do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 404) + end + + error InvalidRecord do + e = env["sinatra.error"] + logger.warn e.message + logger.warn "Request body: #{request.body.read}" + halt_response(e.message, 400) + end + + error InvalidCommand do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 400) + end + + error DependencyError do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 400) + end + + error InvalidPrivileges do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 401) + end + + error Excon::Errors::Unauthorized do + e = env["sinatra.error"] + resp = e.response + ct = resp.headers["Content-Type"] + msg = unless ct.nil? + if ct.include?("application/json") + json = ::Chef::JSONCompat.from_json(resp.body) + m = "ERROR: Unauthorized (#{json['error']['code']}): #{json['error']['message']}" + logger.error(m) + else + end + m + else + "Unauthorized: #{e.inspect}" + end + halt_response(msg, 500) + end + + error Fog::Compute::AWS::Error do + e = env["sinatra.error"] + logger.error e.message + halt_response(e.message, 500) + end + + error do + e = env["sinatra.error"] + logger.error e.message + halt_response(e.message, 500) end end diff --git a/devops-service/routes/v2.0/bootstrap_templates.rb b/devops-service/routes/v2.0/bootstrap_templates.rb index a7c16ee..501c12e 100644 --- a/devops-service/routes/v2.0/bootstrap_templates.rb +++ b/devops-service/routes/v2.0/bootstrap_templates.rb @@ -10,7 +10,7 @@ module Devops app.before "/templates" do check_headers :accept check_privileges("templates", "r") - broadcast(:cancel_order_failed, "hello") +# broadcast(:cancel_order_failed, "hello") end # Get list of available bootstrap templates