commit 02bf8b1bbad50ac2c6bf9cb622b7abec11575ee1 Author: GGA Software Services LLC Date: Thu May 8 15:34:26 2014 +0400 initial commit diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3990fe3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 GGA Software Services LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..8119803 --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +# Devops services# + +Copyright (c) 2009-2014 GGA Software Services LLC + +Authors: Anton Martynov, Mikhail Mirolyubov, Alexey Lukashin + +##Introduction## + +This software was developed for supporting development and operations activities. This service allows managing servers and deployments in hybrid computing environment such as combination of Amazon EC2, VPC, and OpenStack clouds as well as bare metal servers. Deployment is performed by using Opscode Chef Server. The general idea is to put all software dependencies and deployment scripts into chef recipes and apply these procedures to the server. + +##Devops-service installation## + +Devops service is a REST web service which incapsulates all apllication logic. + +Setup server: + yum install ruby + yum install ruby-devel + yum install libxml2-devel + yum install libxslt-devel + yum install gcc make + yum install wget + wget http://production.cf.rubygems.org/rubygems/rubygems-1.8.24.tgz + tar xvf rubygems-1.8.24.tgz + cd rubygems-1.8.24 + ruby setup.rb + gem install knife-openstack -sinatra thin --no-ri --no-rdoc + +Run server: + ruby -rubygems devops-service.rb + +The deep configuration of Devops Service is performed by Chef cookbook. + +##Devops-client installation## + +Devops client is a ruby gem, which provides CLI application for interaction with Devops Service. + +Dependencies: + gems: + httpclient >= 2.3 + json + terminal-table + +gem install devops-client.gem + + +## License + +Devops-service software is released under the [MIT License](http://www.opensource.org/licenses/MIT). diff --git a/devops-client/.gitignore b/devops-client/.gitignore new file mode 100644 index 0000000..d87d4be --- /dev/null +++ b/devops-client/.gitignore @@ -0,0 +1,17 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp diff --git a/devops-client/LICENSE.txt b/devops-client/LICENSE.txt new file mode 100644 index 0000000..2ef7784 --- /dev/null +++ b/devops-client/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2013 amartynov + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/devops-client/README.md b/devops-client/README.md new file mode 100644 index 0000000..639dfe3 --- /dev/null +++ b/devops-client/README.md @@ -0,0 +1,987 @@ + + + + + + + Devops client + + + + +Devops client +============= + +Devops client is a ruby gem. + +## Table of contents + +* [Installation](#install) +* [First run](#first_run) +* [Client commands](#commands) + * [Templates](#templates) + * [Deploy](#deploy) + * [Filters](#filters) + * [Flavor](#flavor) + * [Group](#group) + * [Image](#image) + * [Key](#key) + * [Network](#network) + * [Project](#project) + * [Provider](#provider) + * [Script](#script) + * [Server](#server) + * [Tag](#tag) + * [User](#user) +* [HOWTO](#howto) + * [Create user](#howto_user) + * [Create image](#howto_image) + * [Create project](#howto_project) + * [Launch new server](#howto_server) + +

Installation

+ +Devops client requirements: + +* ruby v1.9.3 or higher + +Client can be installed by following command + + $ sudo gem install devops-client.gem --no-ri --no-rdoc + +After gem installation new command will be available in your system + + $ devops + +If command wasn't found then necessary to check ruby environment + + $ gem environment + +And add "EXECUTABLE DIRECTORY" into $PATH + +Devops shows help if invoked without parameters: + + $ devops + + Usage: /usr/bin/devops command [options] + + Commands: + Bootsrap templates: + templates list + + Deploy: + deploy NODE_NAME [NODE_NAME ...] + + Filters: + filter image add ec2|openstack IMAGE [IMAGE ...] + filter image delete ec2|openstack IMAGE [IMAGE ...] + filter image list ec2|openstack + + Flavor: + flavor list PROVIDER + + Group: + group list PROVIDER + + Image: + image create + image delete IMAGE + image list [provider] [ec2|openstack] + image show IMAGE + image update IMAGE FILE + + Key: + key add KEY_NAME FILE + key delete KEY_NAME + key list + + Network: + network list PROVIDER + + Project: + project create PROJECT_ID + project delete PROJECT_ID [DEPLOY_ENV] + project deploy PROJECT_ID [DEPLOY_ENV] + project list + project multi create PROJECT_ID + project servers PROJECT_ID [DEPLOY_ENV] + project set run_list PROJECT_ID DEPLOY_ENV [(recipe[mycookbook::myrecipe])|(role[myrole]) ...] + project show PROJECT_ID + project update PROJECT_ID FILE + project user add PROJECT_ID USER_NAME [USER_NAME ...] + project user delete PROJECT_ID USER_NAME [USER_NAME ...] + + Provider: + provider list + + Script: + script list + script add SCRIPT_NAME FILE + script delete SCRIPT_NAME + script run SCRIPT_NAME NODE_NAME [NODE_NAME ... ] + script command NODE_NAME 'sh command' + + Server: + server add PROJECT_ID DEPLOY_ENV IP SSH_USER KEY_ID + server bootstrap INSTANCE_ID + server create PROJECT_ID DEPLOY_ENV + server delete NODE_NAME [NODE_NAME ...] + server list [chef|ec2|openstack] + server pause NODE_NAME + server show NODE_NAME + server unpause NODE_NAME + + Tag: + tag create NODE_NAME TAG_NAME [TAG_NAME ...] + tag delete NODE_NAME TAG_NAME [TAG_NAME ...] + tag list NODE_NAME + + User: + user create USER_NAME + user delete USER_NAME + user grant USER_NAME [COMMAND] [PRIVILEGES] + user list + user password USER_NAME + +Detailed help for each command can be shown by passing --help to command line. + +

First run

+ +During first, run devops will detect that its configuration file is absent and will show warning and ask for required parameters: +First step is to enter server's host and port: + + WARN: File '~/.devops/devops-client.conf' does not exist + Language: ru + Devops service host: :7070 + Default API version (v2.0): + Username: my_user + Password: my_password + Configuration file '~/.devops/devops-client.conf' is created + +Also necessary to enter API version (current is v2.0) and credentials. +After these questions configuration file will be created. + +

Commands

+ +After running some commands, devops client might show information in JSON format and ask for confirmation. User can approve or decline operation. + +Any command has additional options: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionDesciption
-h, --helpShow help
-c, --config FILESpecify devops client config file (/home/my_user/.devops/devops-client.conf)
-v, --versiondevops client version
--host HOSTdevops service host address (devops-server-host:devops-server-port)
--api VERdevops service API version (v2.0)
--user USERNAMEuse USERNAME for authentication
--format FORMATOutput format: 'table', 'json' (table)
--completionInitialize bash completion script
+ +

Templates

+ + $ devops templates + + Usage: /usr/bin/devops command [options] + + Commands: + Bootsrap templates: + templates list + +**devops templates list** - command will list available templates for bootstrapping virtual machines by Chef + +

Deploy

+ +Command performs deployment operation by running Chef client on remote server + + $ devops deploy + + Usage: /usr/bin/devops command [options] + + Commands: + Deploy: + deploy NODE_NAME [NODE_NAME ...] + +**devops deploy** - deploys everything on server + +Options: + + + + + + + + + + +
OptionDescription
--tag TAG1,TAG2...Chef tag names, comma separated list of a tags which will be temporary applied to servers.
+ +

Filters

+ +Filters allows to specify cloud VM images and restrict devops to use only them. It is helpful in case of EC2 which has hungreds of images. + + $ devops filter + + Usage: /usr/bin/devops command [options] + + Commands: + Filters: + filter image add ec2|openstack IMAGE [IMAGE ...] + filter image delete ec2|openstack IMAGE [IMAGE ...] + filter image list ec2|openstack + +**devops filter image add** - adds image id to filters +**devops filter image delete** - removes image id (ids) from filters +**devops filter image list** - shows list of available images + +

Flavor

+ + $ devops flavor + + Usage: /usr/bin/devops command [options] + + Commands: + Flavor: + flavor list PROVIDER + +**devops flavor list** - lists available virtual machine configurations + +

Group

+ + $ devops group + + Usage: /usr/bin/devops command [options] + + Commands: + Group: + group list PROVIDER + +**devops group list** - displays list of security groups + +

Image

+ +Command allows managing virtual machine images. + + $ devops image + + Usage: /usr/bin/devops command [options] + + Commands: + Image: + image create + image delete IMAGE + image list [provider] [ec2|openstack] + image show IMAGE + image update IMAGE FILE + +**devops image create** - creates image. Client will ask several questions: + + Provider: # select cloud provider (e.g., openstack, ec2) + Choose image: # enter image number from a list + The ssh username: # give ssh username for logging in + Bootstrap template (optional): # select bootstrap template + +Options: + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionDescription
--provider PROVIDERImage provider
--image IMAGE_IDImage identifier
--ssh_user USERSSH user name
--bootstrap_template TEMPLATEBootstrap template
--no_bootstrap_templateDo not specify bootstrap template
+ +**devops delete** - delete image by ID + +**devops image list** - list available images + +**devops image list provider ec2|openstack** - list available cloud images (filtered by devops) + +**devops image list ec2|openstack** - list available images + +**devops image show** - show image information + +**devops image update** - update image from provided JSON file + +

Key

+ +Manage keys (SSH certificates) servers. + + Key: + key add KEY_NAME FILE + key delete KEY_NAME + key list + +**devops key add** - adds new key with given name KEY_NAME from file FILE + +**devops key delete** - remove key with name KEY_NAME + +**devops key list** - lists available keys + +There is at least one system key which cannot be deleted by user. System keys are registered during devops server configuration and not manageable by user) + +

Network

+ + $ devops network + + Usage: /usr/bin/devops command [options] + + Commands: + Network: + network list PROVIDER + +**devops network list PROVIDER** - list available cloud networks for given PROVIDER + +

Project

+ +Command allows to manage projects + + $ devops project + + Usage: /usr/bin/devops command [options] + + Commands: + Project: + project create PROJECT_ID + project delete PROJECT_ID [DEPLOY_ENV] + project deploy PROJECT_ID [DEPLOY_ENV] + project list + project servers PROJECT_ID [DEPLOY_ENV] + project set run_list PROJECT_ID DEPLOY_ENV [(recipe[mycookbook::myrecipe])|(role[myrole]) ...] + project show PROJECT_ID + project update PROJECT_ID FILE + project user add PROJECT_ID USER_NAME [USER_NAME ...] + project user delete PROJECT_ID USER_NAME [USER_NAME ...] + +**devops project create** - create a new project + +Client will ask several questions: + Deploy environment identifier: # which environment will be created (dev, test, my_env...) At least one environment required for project. + Provider: # Cloud provider (openstack, amazon ec2) + Security groups (comma separated), like 1,2,3, or empty for 'default': # List of security groups which will be assigned to new VMs in given environment/ + Users, you will be added automatically (comma separated), like 1,2,3, or empty: # list of users + Flavor: # server configuration + Image: # image for virtual machine + Subnets (comma separated), like 1,2,3, or empty: # cloud subnets (openstack or Amazon VPC requires at least one) + Run list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]: role[test_dev], # roles and cookbooks which will be assigned to virtual machines + Enter expires time if necessary (5m, 3h, 2d, 1w, etc): # virtual machine life time (by default forever) + +*If project already exists then new environment will be added to it* + +Options: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionDescription
--groups GROUP_1,GROUP_2...Security groups (comma separated list)
--deploy_env DEPLOY_IDDeploy enviroment identifier
--subnets SUBNET,SUBNET...Subnets identifier for deploy enviroment (ec2 - only one sybnet, openstack - comma separated list)
--flavor FLAVORSpecify flavor for the project
--image IMAGE_IDSpecify image identifier for the project
--run_list RUN_LISTRun list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]:
--users USER,USER...Users for deploy environment control
--provider PROVIDERProvider identifier 'ec2' or 'openstack'
--no_expiresWithout expires time
--expires EXPIRESExpires time (5m, 3h, 2d, 1w, etc)
+ +**devops project delete** - removes project or its environment + +**devops project deploy** - deploys to all servers in a project or in given environment + +Options: + + + + + + + + + + +
OptionDesciption
--servers SERVERSServers list (comma separated)
+ +**devops project list** - list all available projects + +**devops project servers** - list all running servers in a project + +**devops project set run_list** - update run-list for a project's environment + +**devops project show** - display project info + +**devops project update** - update project from JSON file + +**devops project user delete** - add user to project + +Options: + + + + + + + + + + +
OptionDescription
--deploy_env ENVAdd user to deploy enviroment
+ +**devops project user delete** - remove user(s) from a project + +Options: + + + + + + + + + + +
OptionDescription
--deploy_env ENVAdd user to deploy enviroment
+ +

Provider

+ + $ devops provider + + Usage: /usr/bin/devops command [options] + + Commands: + Provider: + provider list + +**devops provider list** - Lists available cloud providers registered on devops server + +

Script

+ +Manages shell scrips for running on servers + + $ devops script + + Usage: /usr/bin/devops command [options] + + Commands: + Script: + script list + script add SCRIPT_NAME FILE + script delete SCRIPT_NAME + script run SCRIPT_NAME NODE_NAME [NODE_NAME ...] + script command NODE_NAME 'sh command' + +**devops script list** - lists available scripts + +**devops script add** - adds new script with name SCRIPT_NAME from file FILE + +**devops script delete** - removes script SCRIPT_NAME + +**devops script run** - runs script with name SCRIPT_NAME on server with node name (on Chef server) NODE_NAME + +Options: + + + + + + + + + + +
OptionDescription
--params PARAMSComma separated scipt parameters
+ +**devops script command** - run shell command on remote server (bash interpreter is used) + +

Server

+ + $ devops server + + Usage: /usr/bin/devops command [options] + + Commands: + Server: + server add PROJECT_ID DEPLOY_ENV IP SSH_USER KEY_ID + server bootstrap INSTANCE_ID + server create PROJECT_ID DEPLOY_ENV + server delete NODE_NAME [NODE_NAME ...] + server list [chef|ec2|openstack] + server pause NODE_NAME + server show NODE_NAME + server unpause NODE_NAME + +**devops server add** - adds new server (bare metal, existing,...) to a project with name PROJECT_ID + +**devops server bootstrap** - bootstraps chef on server and runs Chef client with project run list + +Options: + + + + + + + + + + + + + + +
OptionDescription
-N, --name NAMESet chef name
--bootstrap_template [TEMPLATE]Bootstrap template (optional)
+ +**devops server create** - launches new server in a cloud with project PROJECT_ID and environment DEPLOY_ENV + +Options: + + + + + + + + + + +
OptionDescription
-N, --name NAMESet chef name
+ +**devops server delete** - terminates server + +Options: + + + + + + + + + + +
OptionDescription
--instanceDelete node by instance id
+ +**devops server list** - list servers + +**devops server pause** - put server on pause (only if cloud provider supports it) + +**devops server show** - show detailed information + +**server unpause** - unpause server + +

Tag

+ +Manages tags on Chef servers. This functionality can be used for changing deploy behavior according to given tags. + + $ devops tag + + Usage: /usr/bin/devops command [options] + + Commands: + Tag: + tag create NODE_NAME TAG_NAME [TAG_NAME ...] + tag delete NODE_NAME TAG_NAME [TAG_NAME ...] + tag list NODE_NAME + +**devops tag create** - create new tag on chef node with name NODE_NAME + +**devops tag delete** - removes tag from chef node with name NODE_NAME + +**devops tag list** - lists all tags on a chef node with name NODE_NAME + +

User

+ +User management + + $ devops user + + Usage: /usr/bin/devops command [options] + + Commands: + User: + user create USER_NAME + user delete USER_NAME + user grant USER_NAME [COMMAND] [PRIVILEGES] + user list + user password USER_NAME + +**devops user create** - create user with name USER_NAME + +Options: + + + + + + + + + + +
OptionDescription
--password PASSWORDNew user password
+ +**devops user delete** - remove user with name USER_NAME + +**devops user grant** - grants permissions for user + +Available subcommands: + +* all +* flavor +* group +* image +* project +* server +* key +* user +* filter +* network +* provider +* script + +Available privileges: + +* r +* w +* rw + +If privileges are not specified then user is not allowed to run command. + +If command and privileges are not specified then user's permissions are set to default values. + +**devops user list** - list all users + +**devops user password** - change user's password + +

Mini HOWTO

+ +Mostly used scenarios described below. + + +

User management

+ +After clean install root user has empty password, lets set it: + + $ devops user password root -u root + Enter password for 'root': + Updated + +Let's create user test and grant some permissions for working with filters, images, projects and servers: + +If system doesn't have users then let's use root user: + + $ devops user create test -u root + Password for root: + Enter password for 'test': + Created + +By default user has read permissions for filter, image, project, and server operations. Lets give him write permissions: + + $ devops user grant test filter rw -u root + Password for root: + Updated + + $ devops user grant test image rw -u root + Password for root: + Updated + + $ devops user grant test project rw -u root + Password for root: + Updated + + $ devops user grant test server rw -u root + Password for root: + Updated + + $ devops user grant test user r -u root + Password for root: + Updated + + +

Image management

+ +First step is to add required images to filter. For OpenStack it is OpenStack image id, for EC2 it is AMI. + + devops filter image add openstack 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 + +Next step is to create image and specify required metadata: + + $ devops image create + +--------+-----------+ + | API version: v2.0 | + | Provider | + +--------+-----------+ + | Number | Provider | + +--------+-----------+ + | 1 | ec2 | + | 2 | openstack | + +--------+-----------+ + Provider: 2 + +--------+---------------------------+--------------------------------------+--------+ + | API version: v2.0 | + | Images | + +--------+---------------------------+--------------------------------------+--------+ + | Number | Name | ID | Status | + +--------+---------------------------+--------------------------------------+--------+ + | 1 | centos-6.4-amd64-20130707 | 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 | ACTIVE | + +--------+---------------------------+--------------------------------------+--------+ + Image: 1 + The ssh username: root + Bootstrap template (optional): + { + "provider": "openstack", + "name": "centos-6.4-amd64-20130707", + "id": "78665e7b-5123-4fa8-b39b-d7643ecd8ed7", + "remote_user": "root" + } + Create image? (y/n): + +

Project management

+ +Let's create new project 'my_project' with environment 'test' + + $ devops project create my_project + Deploy environment identifier: test + +--------+-----------+ + | API version: v2.0 | + | Provider | + +--------+-----------+ + | Number | Provider | + +--------+-----------+ + | 1 | ec2 | + | 2 | openstack | + +--------+-----------+ + Provider: 2 + +System will show security groups. We are selecting what is needed: + + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | API version: v2.0 | + | Groups | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | Number | Name | Protocol | From | To | CIDR | Description | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | 1 | default | udp | 1 | 65535 | 0.0.0.0/0 | default | + | | | tcp | 1 | 65535 | 0.0.0.0/0 | | + | | | icmp | -1 | -1 | 0.0.0.0/0 | | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | 2 | webports | tcp | 8080 | 8080 | 0.0.0.0/0 | web ports | + | | | tcp | 80 | 80 | 0.0.0.0/0 | | + | | | tcp | 8089 | 8089 | 0.0.0.0/0 | | + | | | tcp | 8443 | 8443 | 0.0.0.0/0 | | + | | | tcp | 443 | 443 | 0.0.0.0/0 | | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + Security groups (comma separated), like 1,2,3, or empty for 'default': + +Next step is to users which can work with a project: + + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | API version: v2.0 | + | Users | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | | | Privileges | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | Number | User ID | Image | Key | Project | Server | User | Script | Filter | Flavor | Group | Network | Provider | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | 1 | test | rw | r | rw | rw | r | r | rw | r | r | r | r | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | 2 | root | rw | rw | rw | rw | rw | rw | rw | rw | rw | rw | rw | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + Users, you will be added automatically (comma separated), like 1,2,3, or empty: + +Flavor for environment: + + +--------+-----------+--------------+------+-------+ + | API version: v2.0 | + | Flavors | + +--------+-----------+--------------+------+-------+ + | Number | ID | Virtual CPUs | Disk | RAM | + +--------+-----------+--------------+------+-------+ + | 1 | c1.large | 8 | 50 | 8192 | + | 2 | c1.medium | 2 | 50 | 2048 | + | 3 | c1.small | 2 | 20 | 1024 | + | 4 | c2.long | 2 | 120 | 4096 | + | 5 | m1.large | 4 | 80 | 8192 | + | 6 | m1.medium | 2 | 40 | 4096 | + | 7 | m1.small | 1 | 20 | 2048 | + | 8 | m1.tiny | 1 | 3 | 512 | + | 9 | m1.xlarge | 8 | 160 | 16384 | + | 10 | m2.long | 2 | 60 | 2048 | + | 11 | snapshot | 2 | 42 | 2048 | + +--------+-----------+--------------+------+-------+ + Flavor: 7 + +Image for virtual machines: + + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | API version: v2.0 | + | Images | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | Number | ID | Name | Bootstrap template | Remote user | Provider | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | 1 | 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 | centos-6.4-amd64-20130707 | | root | openstack | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + Image: 1 + +Network for a virtual machine: + + +--------+--------------+-----------------+ + | API version: v2.0 | + | Subnets | + +--------+--------------+-----------------+ + | Number | Name | CIDR | + +--------+--------------+-----------------+ + | 1 | 172.16.223.0 | 172.16.223.0/24 | + | 2 | 172.16.227.0 | 172.16.227.0/24 | + | 3 | LocalNetwork | 172.16.37.0/24 | + | 4 | LocalNetwork | 10.1.98.0/24 | + | 5 | private | 10.0.0.0/24 | + +--------+--------------+-----------------+ + Subnets (comma separated), like 1,2,3, or empty: 5 + +Chef roles for project and environment. By default will be created new role with name PROJECT-ENV and added to runlist. Additional roles and recipes can be specified here. + + Run list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]: role[my_project_test], + +Just press enter if server lifetime should be infinite. + + Enter expires time if necessary (5m, 3h, 2d, 1w, etc): + +Assume that we do not need second environment. Just press 'n' here. + + Add deploy environment? (y/n): n + { + "deploy_envs": [ + { + "identifier": "test", + "provider": "openstack", + "groups": [ + "default" + ], + "users": [ + "test" + ], + "flavor": "m1.small", + "image": "78665e7b-5123-4fa8-b39b-d7643ecd8ed7", + "subnets": [ + "private" + ], + "run_list": [ + "role[my_project_test]" + ], + "expires": null + } + ], + "name": "my_project" + } + Create project? (y/n): + + +Last question allows reviewing details and confirming for project creation. + +

Starting new instance

+ +After that we can create servers and apply chef roles: + + devops server create my_project test -N my_server_1 + +'-N' parameter allows to specify chef node name. By default node name will be generated automatically. diff --git a/devops-client/README_ru.md b/devops-client/README_ru.md new file mode 100644 index 0000000..52d7c14 --- /dev/null +++ b/devops-client/README_ru.md @@ -0,0 +1,1038 @@ + + + + + + Devops клиент + + + + +Devops клиент +============= + +Клиент реализован в виде гема. + +## Оглавление + +* [Установка](#install) +* [Первый запуск](#first_run) +* [Команды](#commands) + * [Templates](#templates) + * [Deploy](#deploy) + * [Filters](#filters) + * [Flavor](#flavor) + * [Group](#group) + * [Image](#image) + * [Key](#key) + * [Network](#network) + * [Project](#project) + * [Provider](#provider) + * [Script](#script) + * [Server](#server) + * [Tag](#tag) + * [User](#user) +* [HOWTO](#howto) + * [Создание пользователя](#howto_user) + * [Создание образа](#howto_image) + * [Создание проекта](#howto_project) + * [Запуск сервера](#howto_server) + +

Установка

+ +Для правильной работы devops необхлдимо наличие: + +* ruby v1.9.3 + +Установить devops можно командой + + $ sudo gem install devops-client.gem --no-ri --no-rdoc + +После установки гема devops-client, будет доступна команда + + $ devops + +Если команда не доступна, тогда нужно запустить + + $ gem environment + +и добавить путь "EXECUTABLE DIRECTORY" в $PATH + +При запуске команды будет выдана короткая справка по ее использованию. + + $ devops + + Usage: /usr/bin/devops command [options] + + Commands: + Bootsrap templates: + templates list + + Deploy: + deploy NODE_NAME [NODE_NAME ...] + + Filters: + filter image add ec2|openstack IMAGE [IMAGE ...] + filter image delete ec2|openstack IMAGE [IMAGE ...] + filter image list ec2|openstack + + Flavor: + flavor list PROVIDER + + Group: + group list PROVIDER + + Image: + image create + image delete IMAGE + image list [provider] [ec2|openstack] + image show IMAGE + image update IMAGE FILE + + Key: + key add KEY_NAME FILE + key delete KEY_NAME + key list + + Network: + network list PROVIDER + + Project: + project create PROJECT_ID + project delete PROJECT_ID [DEPLOY_ENV] + project deploy PROJECT_ID [DEPLOY_ENV] + project list + project multi create PROJECT_ID + project servers PROJECT_ID [DEPLOY_ENV] + project set run_list PROJECT_ID DEPLOY_ENV [(recipe[mycookbook::myrecipe])|(role[myrole]) ...] + project show PROJECT_ID + project update PROJECT_ID FILE + project user add PROJECT_ID USER_NAME [USER_NAME ...] + project user delete PROJECT_ID USER_NAME [USER_NAME ...] + + Provider: + provider list + + Script: + script list + script add SCRIPT_NAME FILE + script delete SCRIPT_NAME + script run SCRIPT_NAME NODE_NAME [NODE_NAME ... ] + script command NODE_NAME 'sh command' + + Server: + server add PROJECT_ID DEPLOY_ENV IP SSH_USER KEY_ID + server bootstrap INSTANCE_ID + server create PROJECT_ID DEPLOY_ENV + server delete NODE_NAME [NODE_NAME ...] + server list [chef|ec2|openstack] + server pause NODE_NAME + server show NODE_NAME + server unpause NODE_NAME + + Tag: + tag create NODE_NAME TAG_NAME [TAG_NAME ...] + tag delete NODE_NAME TAG_NAME [TAG_NAME ...] + tag list NODE_NAME + + User: + user create USER_NAME + user delete USER_NAME + user grant USER_NAME [COMMAND] [PRIVILEGES] + user list + user password USER_NAME + +Для каждой команды можно посмотреть более подробную справку с помощью параметра --help + +

Первый запуск

+ +При первом запуске (либо запуске, когда программа не сможет найти конфигурационный файл ~/.devops/devops-client.conf) необходимо будет ввести данные для настройки клиента. +На первый вопрос следует указать адрес и порт devops-сервиса: + + WARN: File '~/.devops/devops-client.conf' does not exist + Language: ru + Devops service host: :7070 + Default API version (v2.0): + Username: my_user + Password: my_password + Configuration file '~/.devops/devops-client.conf' is created + +В результате будет создан конфигурационный файл. + +

Команды

+ +В конце выполнения некоторых команд будет выводиться информация о проделанной работе в формате JSON. И выдаваться вопрос с подтверждением. Если все параметры верны, необходимо подтвердить результат команды, нажав на клавишу "y", либо отменить операцию, нажав на клавишу "n". Данная особенность в тексте упомянаться больше не будет. + +Для всех команд доступны опции (в скобках указаны текущие значения): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
-h, --helpShow helpпоказать справку
-c, --config FILESpecify devops client config file (/home/my_user/.devops/devops-client.conf)указать полный путь к конфигурационному файлу
-v, --versiondevops client versionвывести версию клиента
--host HOSTdevops service host address (devops-server-host:devops-server-port)указать к какому devops серверу стоит обращаться (в формате host:port)
--api VERdevops service API version (v2.0)указать версию API
--user USERNAMEdevops username (my_user)сделать запрос к devops от пользователя USERNAME
--format FORMATOutput format: 'table', 'json' (table)формат вывода ответа от сервера: table - в таблице, json - текст в формате JSON
--completionInitialize bash completion scriptинициализировать скрипт автодополнения команд (только linux, интерпретатор bash)
+ +

Templates

+ + $ devops templates + + Usage: /usr/bin/devops command [options] + + Commands: + Bootsrap templates: + templates list + +**devops templates list** - посмотреть список доступных шаблонов для бутстрапа + +

Deploy

+ +Команда предназначена для деплоя приложений на сервера. + + $ devops deploy + + Usage: /usr/bin/devops command [options] + + Commands: + Deploy: + deploy NODE_NAME [NODE_NAME ...] + +**devops deploy** - деплой приложений на указанные сервера + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--tag TAG1,TAG2...Tag names, comma separated listaУказвается список тегов, которые немходимо применить к серверу при деплое
+ +

Filters

+ +Команда предназначена для управления списком доступных образов. + + $ devops filter + + Usage: /usr/bin/devops command [options] + + Commands: + Filters: + filter image add ec2|openstack IMAGE [IMAGE ...] + filter image delete ec2|openstack IMAGE [IMAGE ...] + filter image list ec2|openstack + +**devops filter image add** - добавить образ(ы) в список фильтров для провайдера +**devops filter image delete** - удалить образ(ы) из списока фильтров для провайдера +**devops filter image list** - посмотреть список доступных образов для провайдера + +

Flavor

+ + $ devops flavor + + Usage: /usr/bin/devops command [options] + + Commands: + Flavor: + flavor list PROVIDER + +**devops flavor list** - посмотреть список доступных конфигураций виртуальных машин провайдера + +

Group

+ + $ devops group + + Usage: /usr/bin/devops command [options] + + Commands: + Group: + group list PROVIDER + +**devops group list** - посмотреть список доступных групп безопасности провайдера + +

Image

+ +Команда предназначена для управления образами + + $ devops image + + Usage: /usr/bin/devops command [options] + + Commands: + Image: + image create + image delete IMAGE + image list [provider] [ec2|openstack] + image show IMAGE + image update IMAGE FILE + +**devops image create** - создать образ, для выполнения команды необходимо ответить на несколько вопросов: + + Provider: # выбрать одного из доступных провыйдеров в списке + Choose image: # ввести номер образа из списка, который необходимо использовать + The ssh username: # имя пользователя для доступа по ssh + Bootstrap template (optional): # название скрипта для бутстрапа + +Опции: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--provider PROVIDERImage providerуказать провайдера в опции, а не в интерактивном режиме
--image IMAGE_IDImage identifierуказать идентификатор образа в опции, а не в интерактивном режиме
--ssh_user USERSSH user nameуказать имя пользователя для доступа по ssh в опции, а не в интерактивном режиме
--bootstrap_template TEMPLATEBootstrap templateуказать шаблон в опции, а не в интерактивном режиме
--no_bootstrap_templateDo not specify bootstrap templateИспользовать шаблон по умолчанию
+ +**devops delete** - удалить образ по ID + +**devops image list** - посмотреть созданные образы + +**devops image list provider ec2|openstack** - посмотреть доступные образы провайдера (с учетом фильтров) + +**devops image list ec2|openstack** - посмотреть созданные образы для провайдера + +**devops image show** - посмотреть информацию об одном образе + +*Команда предусмотрена, однако, может быть лишней, т.к. команда image list ее перекрывает. Команда image show выдает информацию по одному образу. Можно сделать так, что эта команда будет выдавать информацию по нескольким образам (чтобы легче и наглядней было сравнивать, чем с командой image list).* + +**devops image update** - обновить образ из файла, файл должен содержать все необходимые параметры в формате JSON + +

Key

+ +Управление ключами для доступа к серверам + + Key: + key add KEY_NAME FILE + key delete KEY_NAME + key list + +**devops key add** - добавить ключ с именем KEY_NAME из файла FILE + +**devops key delete** - удалить ключ с именем KEY_NAME + +**devops key list** - показать список доступных ключей + +Все ключи можно разделить на два типа: системные и пользовательские. Системные - те, которые были созданы при настройке сервера (их нельзя удалять). Пользовательские - добавленные пользователем. + +

Network

+ + $ devops network + + Usage: /usr/bin/devops command [options] + + Commands: + Network: + network list PROVIDER + +**devops network list PROVIDER** - посмотреть список доступных сетей провайдера + +

Project

+ +Управление проектами + + $ devops project + + Usage: /usr/bin/devops command [options] + + Commands: + Project: + project create PROJECT_ID + project delete PROJECT_ID [DEPLOY_ENV] + project deploy PROJECT_ID [DEPLOY_ENV] + project list + project servers PROJECT_ID [DEPLOY_ENV] + project set run_list PROJECT_ID DEPLOY_ENV [(recipe[mycookbook::myrecipe])|(role[myrole]) ...] + project show PROJECT_ID + project update PROJECT_ID FILE + project user add PROJECT_ID USER_NAME [USER_NAME ...] + project user delete PROJECT_ID USER_NAME [USER_NAME ...] + +**devops project create** - создать проект + +После запуска команды клиент будет собирать необходимую информацию (будут возникать небольшие паузы между вопросами). Для создания проекта необходимо ответить на предлагаемые вопросы: + + Deploy environment identifier: # идентификатор среды окружения (dev, test, my_env...) + Provider: # провайдер, где будет развернут проект + Security groups (comma separated), like 1,2,3, or empty for 'default': # список групп безопасности + Users, you will be added automatically (comma separated), like 1,2,3, or empty: # список пользователей, которые могут работать с проектом + Flavor: # параметры сервера, на котором будет развернут проект + Image: # образ ОС для проекта + Subnets (comma separated), like 1,2,3, or empty: # список подсетей, которые должны быть доступны проекту (для openstack не может быть пустым) + Run list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]: role[test_dev], # список ролей и кукбук для развертывания проекта + Enter expires time if necessary (5m, 3h, 2d, 1w, etc): # время через которое сервер для проекта будет удален (пусто если обция не нужна) + +*Если проект с указанным именем существует, то будет добавлена новое окружение к проекту* + +Опции: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--groups GROUP_1,GROUP_2...Security groups (comma separated list)Указать список групп в опции, а не в интерактивном режиме
--deploy_env DEPLOY_IDDeploy enviroment identifierУказать окружение в опции, а не в интерактивном режиме
--subnets SUBNET,SUBNET...Subnets identifier for deploy enviroment (ec2 - only one sybnet, openstack - comma separated list)Указать подсети в опции, а не в интерактивном режиме
--flavor FLAVORSpecify flavor for the projectУказать конфигурацию в опции, а не в интерактивном режиме
--image IMAGE_IDSpecify image identifier for the projectУказать идентификатор образа в опции, а не в интерактивном режиме
--run_list RUN_LISTRun list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]:Указать список кукбук и ролей в опции, а не в интерактивном режиме
--users USER,USER...Users for deploy environment controlУказать список пользователей в опции, а не в интерактивном режиме
--provider PROVIDERProvider identifier 'ec2' or 'openstack'Указать провайдера в опции, а не в интерактивном режиме
--no_expiresWithout expires timeЕсли не нужно указывать время жизни запускаемых серверов
--expires EXPIRESExpires time (5m, 3h, 2d, 1w, etc)Указать время жизни сервера в опции, а не в интерактивном режиме
+ +**devops project delete** - удалить проект или окружение + +**devops project deploy** - деплоить все сервера проекта или окружения проекта + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--servers SERVERSServers list (comma separated)Список имен серверов, разделенный запятыми
+ +**devops project list** - вывести список созданных проектов + +**devops project servers** - вывести список запущенных машин для проекта или окружения + +**devops project set run_list** - изменить run-list для окружения проекта + +**devops project show** - показать информацию о проекте + +**devops project update** - обновить конфигурацию проекта из файла (файл должен содержать все необходимые параметры в формате JSON) + +**devops project user delete** - добавить пользователя (пользователей) в список доступных пользователей для проекта + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--deploy_env ENVAdd user to deploy enviromentЕсли опция не используется, то пользователь будет добавлен ко всем окружениям проекта
+ +**devops project user delete** - удалить пользователя (пользователей) из списока доступных пользователей для проекта + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--deploy_env ENVAdd user to deploy enviromentЕсли опция не используется, то пользователь будет удален из всех окружений проекта
+ +

Provider

+ + $ devops provider + + Usage: /usr/bin/devops command [options] + + Commands: + Provider: + provider list + +**devops provider list** - посмотреть список доступных провайдеров, доступны провайдеры ec2 и openstack (в зависимости от настроек сервера) + +

Script

+ +Управление скриптами, которые могут быть запущены на сервере (сервер должен быть под управлением devops) + + $ devops script + + Usage: /usr/bin/devops command [options] + + Commands: + Script: + script list + script add SCRIPT_NAME FILE + script delete SCRIPT_NAME + script run SCRIPT_NAME NODE_NAME [NODE_NAME ...] + script command NODE_NAME 'sh command' + +**devops script list** - посмотреть список доступных скриптов (файлов) + +**devops script add** - добавить скрипт с именем SCRIPT_NAME из файла FILE на devops-сервер + +**devops script delete** - удалить скрипт с именем SCRIPT_NAME с devops-сервера + +**devops script run** - запустить скрипт с именем SCRIPT_NAME на серверах NODE_NAME + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--params PARAMSПараметры скрипта (список разделенный запятой)
+ +**devops script command** - запустить команду на сервере + +*Скрипт запускается интерпретатором bash* + +

Server

+ + $ devops server + + Usage: /usr/bin/devops command [options] + + Commands: + Server: + server add PROJECT_ID DEPLOY_ENV IP SSH_USER KEY_ID + server bootstrap INSTANCE_ID + server create PROJECT_ID DEPLOY_ENV + server delete NODE_NAME [NODE_NAME ...] + server list [chef|ec2|openstack] + server pause NODE_NAME + server show NODE_NAME + server unpause NODE_NAME + +**devops server add** - добавить сервер под управление devops и зарегистрировать его в проекте PROJECT_ID + +**devops server bootstrap** - развернуть на сервере инфраструктуру chef и развернуть проект, в котором сервер зарегистрирован + +Опции: + + + + + + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
-N, --name NAMESet chef nameЗадает серверу имя, если не указано, то имя будет сгенерировано автоматически
--bootstrap_template [TEMPLATE]Bootstrap template (optional)Если опция не указана, используется шаблон по умолчанию
+ +**devops server create** - создать сервер для проекта PROJECT_ID и окружения DEPLOY_ENV + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
-N, --name NAMESet chef nameЗадает серверу имя, если не указано, то имя будет сгенерировано автоматически
+ +**devops server delete** - удалить сервер + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--instanceDelete node by instance idУдалить сервер по идентификатору, а не по имени
+ +**devops server list** - получить список доступных серверов + +**devops server pause** - приостановить работу сервера (если сервер запущен в облаке) + +**devops server show** - показать детальную информацию о сервере + +**server unpause** - возобновить работу сервера + +

Tag

+ +Управление тегами на chef-server для указанной сервера. Сервер должен быть создан командами 'server create' или 'server bootstrap' + + $ devops tag + + Usage: /usr/bin/devops command [options] + + Commands: + Tag: + tag create NODE_NAME TAG_NAME [TAG_NAME ...] + tag delete NODE_NAME TAG_NAME [TAG_NAME ...] + tag list NODE_NAME + +**devops tag create** - создать теги на сервере NODE_NAME + +**devops tag delete** - удалить теги с сервера NODE_NAME + +**devops tag list** - вывести список созданных тегов на сервере NODE_NAME + +

User

+ +Управление пользователями + + $ devops user + + Usage: /usr/bin/devops command [options] + + Commands: + User: + user create USER_NAME + user delete USER_NAME + user grant USER_NAME [COMMAND] [PRIVILEGES] + user list + user password USER_NAME + +**devops user create** - создать пользователя с именем USER_NAME + +Опции: + + + + + + + + + + + + +
ОпцияОписаниеКомментарий
--password PASSWORDNew user passwordУказать пароль в опции
+ +**devops user delete** - удалить пользователя с именем USER_NAME + +**devops user grant** - назначить права доступа пользователю + +Доступны команды: + +* all +* flavor +* group +* image +* project +* server +* key +* user +* filter +* network +* provider +* script + +Привилегии: + +* r +* w +* rw + +Если привилегии не указаны, то пользователю запрещается выполнять команду + +Если не указана и команда и привилегии, то права пользователя сбрасываются к правам по умолчанию + +**devops user list** - вывести список всех пользователей + +**devops user password** - изменить пароль пользователю USER_NAME + +

Mini HOWTO

+ +Опишем основные действия, необходимые для создания проекта и сервера. + +

Создание пользователя

+ +По умолчанию, у пользователя root нет пароля, давайте зададим его. + + $ devops user password root -u root + Enter password for 'root': + Updated + +Создадим пользователя 'test' и назначим ему права, необходимые для создания фильтров, образов, проектов и серверов. + +Если в системе еще нет ни одного пользователя, то будем действовать от пользователя root. + + $ devops user create test -u root + Password for root: + Enter password for 'test': + Created + +Допустим, пользователю 'test' мы задали пароль 'test', эти параметры надо прописать в конфигурационный файл. + +При создании нового пользователя, ему назначаются почти все права только на чтение, добавим права на запись для filter, image, project, server: + + $ devops user grant test filter rw -u root + Password for root: + Updated + + $ devops user grant test image rw -u root + Password for root: + Updated + + $ devops user grant test project rw -u root + Password for root: + Updated + + $ devops user grant test server rw -u root + Password for root: + Updated + + $ devops user grant test user r -u root + Password for root: + Updated + +После выполненных действий *в конфигурационном файле должен быть прописан пользователь 'test'* + +

Создание образа

+ +Прежде всего необходимо узнать идентификаторы нужных образов и добавить их в фильтр. + + devops filter image add openstack 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 + +Теперь можно создать образ + + $ devops image create + +--------+-----------+ + | API version: v2.0 | + | Provider | + +--------+-----------+ + | Number | Provider | + +--------+-----------+ + | 1 | ec2 | + | 2 | openstack | + +--------+-----------+ + Provider: 2 + +--------+---------------------------+--------------------------------------+--------+ + | API version: v2.0 | + | Images | + +--------+---------------------------+--------------------------------------+--------+ + | Number | Name | ID | Status | + +--------+---------------------------+--------------------------------------+--------+ + | 1 | centos-6.4-amd64-20130707 | 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 | ACTIVE | + +--------+---------------------------+--------------------------------------+--------+ + Image: 1 + The ssh username: root + Bootstrap template (optional): + { + "provider": "openstack", + "name": "centos-6.4-amd64-20130707", + "id": "78665e7b-5123-4fa8-b39b-d7643ecd8ed7", + "remote_user": "root" + } + Create image? (y/n): + +Если все параметры верны, то можно нажать на 'y' и проект будет создан. + +

Создание проекта

+ +Создадим проект 'my_project' с окружением 'test' + + $ devops project create my_project + Deploy environment identifier: test + +--------+-----------+ + | API version: v2.0 | + | Provider | + +--------+-----------+ + | Number | Provider | + +--------+-----------+ + | 1 | ec2 | + | 2 | openstack | + +--------+-----------+ + Provider: 2 + +Выбираем группы безопасности или нажимаем Enter, чтобы использовать группу по умолчанию + + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | API version: v2.0 | + | Groups | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | Number | Name | Protocol | From | To | CIDR | Description | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | 1 | default | udp | 1 | 65535 | 0.0.0.0/0 | default | + | | | tcp | 1 | 65535 | 0.0.0.0/0 | | + | | | icmp | -1 | -1 | 0.0.0.0/0 | | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + | 2 | webports | tcp | 8080 | 8080 | 0.0.0.0/0 | web ports | + | | | tcp | 80 | 80 | 0.0.0.0/0 | | + | | | tcp | 8089 | 8089 | 0.0.0.0/0 | | + | | | tcp | 8443 | 8443 | 0.0.0.0/0 | | + | | | tcp | 443 | 443 | 0.0.0.0/0 | | + +--------+-------------------------------------+----------+------+-------+-----------+-----------------------------+ + Security groups (comma separated), like 1,2,3, or empty for 'default': + +Указываем пользователей, которые могут работать с окружением проекта, пользователь, который создает окружение будет автоматически добавлен, можно нажать Enter. + + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | API version: v2.0 | + | Users | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | | | Privileges | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | Number | User ID | Image | Key | Project | Server | User | Script | Filter | Flavor | Group | Network | Provider | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | 1 | test | rw | r | rw | rw | r | r | rw | r | r | r | r | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + | 2 | root | rw | rw | rw | rw | rw | rw | rw | rw | rw | rw | rw | + +--------+------------------+-------+-----+---------+--------+------+--------+--------+--------+-------+---------+----------+ + Users, you will be added automatically (comma separated), like 1,2,3, or empty: + +Выбмраем параметры сервера. Например, нам надо чтобы проект был развернут на сервере с характеристиками: одно ядро, 20Гб диск и 2Гб RAM, выбирает flavor с номером 7 + + +--------+-----------+--------------+------+-------+ + | API version: v2.0 | + | Flavors | + +--------+-----------+--------------+------+-------+ + | Number | ID | Virtual CPUs | Disk | RAM | + +--------+-----------+--------------+------+-------+ + | 1 | c1.large | 8 | 50 | 8192 | + | 2 | c1.medium | 2 | 50 | 2048 | + | 3 | c1.small | 2 | 20 | 1024 | + | 4 | c2.long | 2 | 120 | 4096 | + | 5 | m1.large | 4 | 80 | 8192 | + | 6 | m1.medium | 2 | 40 | 4096 | + | 7 | m1.small | 1 | 20 | 2048 | + | 8 | m1.tiny | 1 | 3 | 512 | + | 9 | m1.xlarge | 8 | 160 | 16384 | + | 10 | m2.long | 2 | 60 | 2048 | + | 11 | snapshot | 2 | 42 | 2048 | + +--------+-----------+--------------+------+-------+ + Flavor: 7 + +Выбираем ранее созданый образ + + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | API version: v2.0 | + | Images | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | Number | ID | Name | Bootstrap template | Remote user | Provider | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + | 1 | 78665e7b-5123-4fa8-b39b-d7643ecd8ed7 | centos-6.4-amd64-20130707 | | root | openstack | + +--------+--------------------------------------+---------------------------+--------------------+-------------+-----------+ + Image: 1 + +Если нам надо, чтобы проект был развернут в подсети 10.0.0.0/24, тогда выбираем подсеть с номером 5 + + +--------+--------------+-----------------+ + | API version: v2.0 | + | Subnets | + +--------+--------------+-----------------+ + | Number | Name | CIDR | + +--------+--------------+-----------------+ + | 1 | 172.16.223.0 | 172.16.223.0/24 | + | 2 | 172.16.227.0 | 172.16.227.0/24 | + | 3 | LocalNetwork | 172.16.37.0/24 | + | 4 | LocalNetwork | 10.1.98.0/24 | + | 5 | private | 10.0.0.0/24 | + +--------+--------------+-----------------+ + Subnets (comma separated), like 1,2,3, or empty: 5 + +Укажем роли и рецепты необходимые для развертывания проекта. При создании окружения, на chef-сервере будет создана роль с именем 'my_project_test', эта же роль по умолчанию включается в список. Роль нужно настроить на chef-сервере. + + Run list (comma separated), like recipe[mycookbook::myrecipe], role[myrole]: role[my_project_test], + +Нам не нужно уничтожать сервер через заданный промежуток времени, поэтому просто жмем Enter. + + Enter expires time if necessary (5m, 3h, 2d, 1w, etc): + +Второго окружения мы создавать не будем, поэтому жмем 'n' + + Add deploy environment? (y/n): n + { + "deploy_envs": [ + { + "identifier": "test", + "provider": "openstack", + "groups": [ + "default" + ], + "users": [ + "test" + ], + "flavor": "m1.small", + "image": "78665e7b-5123-4fa8-b39b-d7643ecd8ed7", + "subnets": [ + "private" + ], + "run_list": [ + "role[my_project_test]" + ], + "expires": null + } + ], + "name": "my_project" + } + Create project? (y/n): + +Если все правильно, жмем 'y' и проект будет создан. + +

Запуск сервера

+ +Теперь запустить новый сервер очень просто, нужно выполнить команду + + devops server create my_project test -N my_server_1 + +Параметр '-N' говорит о том, что серверу нужно задать имя. Если параметр не указывать, то имя будет сгенерировано автоматически. diff --git a/devops-client/Rakefile b/devops-client/Rakefile new file mode 100644 index 0000000..2995527 --- /dev/null +++ b/devops-client/Rakefile @@ -0,0 +1 @@ +require "bundler/gem_tasks" diff --git a/devops-client/bin/devops b/devops-client/bin/devops new file mode 100644 index 0000000..92f45ad --- /dev/null +++ b/devops-client/bin/devops @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require 'rubygems' +require 'devops-client' + +DevopsClient.run diff --git a/devops-client/completion/devops_complete.sh b/devops-client/completion/devops_complete.sh new file mode 100644 index 0000000..a35475b --- /dev/null +++ b/devops-client/completion/devops_complete.sh @@ -0,0 +1,256 @@ +_devops() +{ + + local cur prev cmds cmd + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + + PROVIDERS="ec2 openstack" + + grant="" + project="create delete list servers set show update add_user remove_user" + server="bootstrap create delete list pause show unpause add" + + declare -A commands=( [flavor]=1 [group]=1 [image]=1 [project]=0 [server]=0 [deploy]=1 [key]=1 [user]=1 [grant]=0 [tag]=1 [provider]=1 [network]=1 [script]=1 ) + + case "${COMP_CWORD}" in + 1) + #cmds="${!commands[@]}" + #cmds="--help --version --completion" + cmds="" + if [[ "$cur" =~ ^-.* ]]; then + _devops_options + else + for i in "${!commands[@]}" + do + if [ ${commands[$i]} -eq 1 ]; then + cmds="$cmds $i" + fi + done + _set_devops_params + fi + ;; + *) + if [ ${commands[${COMP_WORDS[1]}]} -ne 1 ]; then + # invalid command + return + fi + eval _devops_${COMP_WORDS[1]} ${COMP_WORDS[@]:2} + ;; + esac + +# case "$cmds" in +# PROVIDERS) +# cmds=$PROVIDERS +# ;; +# FILE) +# COMPREPLY=($(compgen -f "${COMP_WORDS[${COMP_CWORD}]}" )) +# return 0 +# ;; +# esac + +# COMPREPLY=( $(compgen -W "${cmds}" -- ${cur}) ) + return 0 +} + +_devops_flavor() +{ + case "$1" in + list) + case "$2" in + ec2|openstack) + _devops_options "" + ;; + *) + _set_devops_params_providers + ;; + esac + ;; + *) + cmds="list" + _set_devops_params + ;; + esac +} +alias _devops_group=_devops_flavor +alias _devops_network=_devops_flavor + +_devops_provider() +{ + case "$1" in + list) + case "$2" in + *) + _devops_options "" + ;; + esac + ;; + *) + cmds="list" + _set_devops_params + ;; + esac +} + +_devops_deploy() +{ + cmds="NODE_NAME" + _set_devops_params +} + +_devops_user() +{ + case "$1" in + list) + ;; + create) + ;; + delete) + ;; + grant) + ;; + password) + ;; + *) + cmds="create delete grant list password" + _set_devops_params + ;; + esac +} + +_devops_tag() +{ + case "$1" in + list) + ;; + create) + ;; + delete) + ;; + *) + cmds="create delete list" + _set_devops_params + ;; + esac +} + +_devops_key() +{ + case "$1" in + list) + ;; + add) + ;; + delete) + ;; + *) + cmds="add delete list" + _set_devops_params + ;; + esac +} + +_devops_image() +{ + case "$1" in + list) + case "$2" in + provider) + case "$3" in + ec2|openstack) + _devops_options "" + ;; + *) + _set_devops_params_providers + ;; + esac + ;; + *) + cmds="provider" + _set_devops_params + ;; + esac + ;; + create) + _devops_options + ;; + update) + if [[ "$2" == "" ]]; then + cmds="IMAGE" + _set_devops_params + else + if [[ $COMP_CWORD -eq 4 ]]; then + _set_devops_params_file + else + _devops_options "" + fi + fi + ;; + delete|show) + if [[ "$2" == "" ]]; then + cmds="IMAGE" + _set_devops_params + else + _devops_options "" + fi + ;; + *) + cmds="create delete list show update" + _set_devops_params + ;; + esac +} + +_devops_script() +{ + case "$1" in + list) + ;; + add) + ;; + run) + ;; + delete) + ;; + command) + ;; + *) + cmds="list add delete run command" + _set_devops_params + ;; + esac +} + +_devops_options() +{ + declare -A common_options=([--help]="" [--version]="" [--host]="HOST" [--api]="API" [--user]="USER" [--format]="table json" [--completion]="") + val="${common_options[${COMP_WORDS[COMP_CWORD - 1]}]}" + if [ -z "$val" ]; then + cmds="${!common_options[@]}" + else + cmds="$val" + fi + _set_devops_params +} + +# set copmletion providers +_set_devops_params_providers() +{ + cmds="ec2 openstack" + _set_devops_params +} + +# set copmletion from $cmds +_set_devops_params() +{ + COMPREPLY=( $(compgen -W "${cmds}" -- ${cur}) ) +} + +# set copmletion if type is FILE +_set_devops_params_file() +{ + COMPREPLY=($(compgen -f "${COMP_WORDS[${COMP_CWORD}]}" )) +} +complete -o filenames -o bashdefault -F _devops devops diff --git a/devops-client/devops-client.gemspec b/devops-client/devops-client.gemspec new file mode 100644 index 0000000..2e38f8c --- /dev/null +++ b/devops-client/devops-client.gemspec @@ -0,0 +1,25 @@ +# -*- encoding: utf-8 -*- +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'devops-client/version' +require 'devops-client/name' + +Gem::Specification.new do |gem| + #devops-client + gem.name = DevopsClient::NAME + gem.version = DevopsClient::VERSION + gem.authors = ["amartynov"] + gem.email = ["amartynov@ggasoftware.com"] + gem.description = %q{This is client for devops service} + gem.summary = %q{This is client for devops service} + gem.homepage = "" + + gem.files = Dir['{bin,lib,completion,locales}/**/*', 'README*', 'LICENSE*'] + gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } +# gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) + gem.require_paths = ["lib"] + + gem.add_dependency("httpclient", ">= 2.3") + gem.add_dependency("json") + gem.add_dependency("terminal-table") +end diff --git a/devops-client/lib/devops-client.rb b/devops-client/lib/devops-client.rb new file mode 100644 index 0000000..db1d738 --- /dev/null +++ b/devops-client/lib/devops-client.rb @@ -0,0 +1,181 @@ +require 'devops-client/name' +require "devops-client/version" +require "devops-client/handler/handler_factory" +require "exceptions/not_found" +require "exceptions/invalid_query" +require "exceptions/devops_exception" +require "optparse" +require "devops-client/i18n" + +module DevopsClient + + DEVOPS_HOME = "#{ENV["HOME"]}/.devops/" + # properties file key=value + @@config_file = File.join(DEVOPS_HOME, "devops-client.conf") + #CONFIG_FILE="#{ENV["HOME"]}/.devops/devops-client.conf" + + def self.config_file + @@config_file + end + + def self.run + + DevopsClient::get_config_file_option + config = DevopsClient::read_config(@@config_file) + + I18n.language=(config[:locale] || "en") + + if ARGV.include? "--completion" + init_completion + exit + end + + if config[:host].nil? + abort(I18n.t("config.invalid.host"), :file => @@config_file) + end + [:api, :username, :password].each do |key| + if config[key].nil? + abort(I18n.t("config.invalid.empty", :file => @@config_file, :key => key)) + end + end + configure_proxy config + + host = config[:host] + default = {:username => config[:username], :api => config[:api], :host => config[:host]} + auth = {:username => config[:username], :password => config[:password], :type => "basic"} + + handler = HandlerFactory.create(ARGV[0], host, auth, default) + result = handler.handle + if result.is_a?(Hash) + puts result["message"] + else + puts result + end + rescue OptionParser::InvalidOption => e + puts e.message + exit(11) + rescue NotFound => e + puts "Not found: #{e.message}" + exit(12) + rescue InvalidQuery => e + puts "Invalid query: #{e.message}" + exit(13) + rescue DevopsException => e + puts I18n.t("log.error", :msg => e.message) + exit(14) + rescue => e + puts I18n.t("log.error", :msg => e.message) + raise e + rescue Interrupt + puts "\nInterrupted" + exit(15) + end + + PROXY_TYPE_NONE = "none" + PROXY_TYPE_SYSTEM = "system" + PROXY_TYPE_CUSTOM = "custom" + PROXY_TYPES = [PROXY_TYPE_NONE, PROXY_TYPE_SYSTEM, PROXY_TYPE_CUSTOM] + PROXY_ENV = ["all_proxy", "ALL_PROXY", "proxy", "PROXY", "http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"] + def self.configure_proxy config + config[:proxy_type] = PROXY_TYPE_NONE if config[:proxy_type].nil? + case config[:proxy_type] + when PROXY_TYPE_SYSTEM + nil + when PROXY_TYPE_NONE + PROXY_ENV.each {|k| ENV[k] = nil} + when PROXY_TYPE_CUSTOM + ["http_proxy", "HTTP_PROXY"].each {|k| ENV[k] = config[:http_proxy]} + else + abort(I18n.t("config.invalid.proxy_type", :file => @@config_file, :values => PROXY_TYPES.join(", "))) + end + end + + def self.read_config file + config = {} + if File.exists? file + File.open(file, "r") do |f| + f.each_line do |line| + line.strip! + next if line.empty? or line.start_with?("#") + buf = line.split("=") + config[buf[0].strip.to_sym] = buf[1].strip if !(buf[1].nil? or buf[1].empty?) + end + end + else + puts I18n.t("log.warn", :msg => I18n.t("config.not_exist", :file => file)) + config = set_default_config(file) + end + config + end + + def self.set_default_config file + locales = I18n.locales + config = {:api => "v2.0", :locale => "en"} + I18n.language = config[:locale] + config[:locale] = begin + l = get_config_parameter(I18n.t("config.property.lang", :langs => locales.join(", ")), config[:locale]) + raise ArgumentError unless locales.include?(l) + I18n.language = l + l + rescue ArgumentError + retry + end + config[:host] = get_config_parameter(I18n.t("config.property.host")) + config[:api] = get_config_parameter(I18n.t("config.property.api"), config[:api]) + config[:username] = get_config_parameter(I18n.t("config.property.username")) + config[:password] = get_config_parameter(I18n.t("config.property.password")) + begin + config[:proxy_type] = get_config_parameter(I18n.t("config.property.proxy_type")) + raise ArgumentError unless PROXY_TYPES.include?(config[:proxy_type]) + rescue ArgumentError + retry + end + if config[:proxy_type] == PROXY_TYPE_CUSTOM + config[:http_proxy] = get_config_parameter(I18n.t("config.property.http_proxy")) + end + + dir = File.dirname(@@config_file) + require "fileutils" + FileUtils.mkdir(dir) unless File.exists? dir + File.open(file, "w") do |f| + config.each do |k,v| + f.puts "#{k.to_s}=#{v}" + end + end + puts I18n.t("config.created", :file => file) + config + end + + def self.get_config_parameter msg, default=nil + print(msg + (default.nil? ? ": " : "(#{default}): ")) + p = STDIN.gets.strip + return (p.empty? ? default : p) + end + + def self.get_config_file_option + ARGV.each_index do |i| + if ARGV[i] == "-c" or ARGV[i] == "--config" + if ARGV[i+1] !~ /^-.*/ and ARGV[i+i] !~ /^--.*/ + @@config_file = ARGV[i+1] + ARGV.delete_at(i) + ARGV.delete_at(i) + else + puts I18n.t("log.error", :msg => I18n.t("config.invalid.parameter")) + exit(3) + end + end + end + end + + def self.init_completion + spec = Gem::Specification.find_by_name(DevopsClient::NAME) + gem_root = spec.gem_dir + path = File.join(gem_root, "completion", "devops_complete.sh") + require "fileutils" + FileUtils.cp(path, DEVOPS_HOME) + file = File.join(DEVOPS_HOME, "devops_complete.sh") + puts I18n.t("completion.message", :file => file) + puts "\n\e[32m#{I18n.t("completion.put", :file => file)}\e[0m" + end + +end diff --git a/devops-client/lib/devops-client/handler/bootstrap_templates.rb b/devops-client/lib/devops-client/handler/bootstrap_templates.rb new file mode 100644 index 0000000..c00c6d5 --- /dev/null +++ b/devops-client/lib/devops-client/handler/bootstrap_templates.rb @@ -0,0 +1,32 @@ +require "devops-client/handler/handler" +require "devops-client/options/bootstrap_templates_options" +require "json" +require "devops-client/output/bootstrap_templates" + +class BootstrapTemplates < Handler + + include Output::BootstrapTemplates + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = BootstrapTemplatesOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler args + @list = get("/templates") + end + +end + diff --git a/devops-client/lib/devops-client/handler/deploy.rb b/devops-client/lib/devops-client/handler/deploy.rb new file mode 100644 index 0000000..b35ba97 --- /dev/null +++ b/devops-client/lib/devops-client/handler/deploy.rb @@ -0,0 +1,31 @@ +require "devops-client/handler/handler" +require "devops-client/options/deploy_options" + +class Deploy < Handler + + def initialize(host, def_options={}) + self.host = host +# self.def_options = def_options + @options_parser = DeployOptions.new(ARGV, def_options) + end + + def handle + if ARGV.size > 1 + self.options = @options_parser.deploy_options + deploy_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def deploy_handler args + tags = options[:tags] + names = args[1..-1] + if names.empty? + @options_parser.invalid_deploy_command + abort(r) + end + post_chunk("/deploy", :names => names, :tags => tags) + end + +end diff --git a/devops-client/lib/devops-client/handler/filter.rb b/devops-client/lib/devops-client/handler/filter.rb new file mode 100644 index 0000000..d22bb83 --- /dev/null +++ b/devops-client/lib/devops-client/handler/filter.rb @@ -0,0 +1,68 @@ +require "devops-client/handler/handler" +require "devops-client/options/filter_options" +require "json" +require "devops-client/output/filters" + +class Filter < Handler + + attr_accessor :def_options + + def initialize(host, def_options) + self.host = host + self.def_options = def_options + @options_parser = FilterOptions.new(ARGV, def_options) + end + + include Output::Filters + + def handle + case ARGV[1] + when "image" + provider = ARGV[3] + case ARGV[2] + when "list" + self.options = @options_parser.image_list_options + check_provider provider + @list = get("/filter/#{provider}/images") + output + when "add" + self.options = @options_parser.image_add_options + check_provider provider + @list = put_body("/filter/#{provider}/image", get_images(ARGV).to_json) + @list = @list["images"] unless @list.nil? + output + when "delete" + self.options = @options_parser.image_delete_options + check_provider provider + images = get_images(ARGV) + if question(I18n.t("handler.filter.question.delete", :name => images.join("', '"))) + @list = delete_body("/filter/#{provider}/image", images.to_json) + @list = @list["images"] unless @list.nil? + output + end + else + @options_parser.invalid_image_command + abort("Invalid image parameter: #{ARGV[2]}, it should be 'add' or 'delete' or 'list'") + end + else + @options_parser.invalid_command + end + end + + def check_provider provider + if provider != "ec2" and provider != "openstack" + @options_parser.invalid_image_command + abort("Invalid image parameter: provider '#{provider}', it should be 'ec2' or 'openstack'") + end + end + + def get_images args + images = args[4..-1] + if images.empty? + @options_parser.invalid_image_command + abort("Images list is empty") + end + images + end + +end diff --git a/devops-client/lib/devops-client/handler/flavor.rb b/devops-client/lib/devops-client/handler/flavor.rb new file mode 100644 index 0000000..b618e12 --- /dev/null +++ b/devops-client/lib/devops-client/handler/flavor.rb @@ -0,0 +1,37 @@ +require "devops-client/handler/handler" +require "devops-client/options/flavor_options" +require "json" +require "devops-client/output/flavors" + +class Flavor < Handler + + include Output::Flavors + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = FlavorOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler args + r = inspect_parameters @options_parser.list_params, args[2] + unless r.nil? + @options_parser.invalid_list_command + abort(r) + end + @provider = args[2] + @list = get("/flavors/#{args[2]}").sort!{|x,y| x["id"] <=> y["id"]} + end + +end diff --git a/devops-client/lib/devops-client/handler/group.rb b/devops-client/lib/devops-client/handler/group.rb new file mode 100644 index 0000000..da7af35 --- /dev/null +++ b/devops-client/lib/devops-client/handler/group.rb @@ -0,0 +1,39 @@ +require "devops-client/handler/handler" +require "devops-client/options/group_options" +require "json" +require "devops-client/output/groups" + +class Group < Handler + + include Output::Groups + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = GroupOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler args + r = inspect_parameters @options_parser.list_params, args[2] + unless r.nil? + @options_parser.invalid_list_command + abort(r) + end + @provider = args[2] + p = {} + p["vpc-id"] = args[3] unless args[3].nil? + @list = get("/groups/#{args[2]}", p) + end + +end diff --git a/devops-client/lib/devops-client/handler/handler.rb b/devops-client/lib/devops-client/handler/handler.rb new file mode 100644 index 0000000..27ab4fa --- /dev/null +++ b/devops-client/lib/devops-client/handler/handler.rb @@ -0,0 +1,336 @@ +require "httpclient" +require "exceptions/devops_exception" +require "exceptions/not_found" +require "exceptions/invalid_query" +require "devops-client/options/common_options" +require "uri" +require "json" +require "devops-client/i18n" + +class Handler + + attr_reader :options + attr_writer :host + attr_accessor :auth + + def host + "http://#{@host}" + end + + #TODO: only basic auth now + def username + self.options[:username] || self.auth[:username] + end + + def password + self.options[:password] || self.auth[:password] + end + + def options= o + self.host = o.delete(:host) if o.has_key? :host + @options = o + end + + def get_chunk path, params={} + submit do |http| + http.get(create_url(path), convert_params(params)) do |chunk| + puts chunk + end + end + "" + end + + def get path, params={} + get_with_headers path, params, self.headers("Content-Type") + end + + def get_with_headers path, params={}, headers={} + submit do |http| + http.get(create_url(path), convert_params(params), headers) + end + end + + def post path, params={} + self.post_body(path, params.to_json) + end + + def post_body path, body + post_body_with_headers path, body, self.headers + end + + def post_chunk_body path, body, json=true + h = (json ? self.headers : self.headers("Content-Type", "Accept")) + submit do |http| + buf = "" + resp = http.post(create_url(path), body, h) do |chunk| + puts chunk + buf = chunk + end + if resp.ok? + status = check_status(buf) + exit(status) unless status == 0 + end + resp + end + "" + end + + def post_chunk path, params={} + self.post_chunk_body path, params.to_json + end + + def post_body_with_headers path, body='', headers={} + submit do |http| + http.post(create_url(path), body, headers) + end + end + + def delete path, params={} + delete_body path, params.to_json + end + + def delete_body path, body + submit do |http| + http.delete(create_url(path), body, self.headers) + end + end + + def put path, params={} + put_body path, params.to_json + end + + def put_body path, body + submit do |http| + http.put(create_url(path), body, self.headers) + end + end + +protected + def puts_warn msg + puts "\e[33m#{msg}\e[0m" + end + + def puts_error msg + puts "\e[31m#{msg}\e[0m" + end + + def question str + return true if self.options[:no_ask] + if block_given? + yield + end + res = false + #system("stty raw -echo") + begin + print "#{str} (y/n): " + s = STDIN.gets.strip + if s == "y" + res = true + elsif s == "n" + res == false + else + raise ArgumentError.new + end + rescue ArgumentError + retry + end + #print "#{s}\n\r" + #system("stty -raw echo") + res + end + + def choose_image_cmd images, t=nil + abort(I18n.t("handler.error.list.empty", :name => "Image")) if images.empty? + if options[:image_id].nil? + images[ choose_number_from_list(I18n.t("headers.image"), images, t) ] + else + i = images.detect { |i| i["name"] == options[:image_id]} + abort("No such image") if i.nil? + return i + end + end + + def get_comma_separated_list msg + print msg + STDIN.gets.strip.split(",").map{|e| e.strip} + end + + def enter_parameter msg + str = enter_parameter_or_empty(msg) + raise ArgumentError.new if str.empty? + str + rescue ArgumentError + retry + end + + def enter_parameter_or_empty msg + print msg + return STDIN.gets.strip + end + + def choose_number_from_list title, list, table=nil, default=nil + i = 0 + if table.nil? + puts I18n.t("handler.message.choose", :name => title.downcase) + "\n" + list.map{|p| i += 1; "#{i}. #{p}"}.join("\n") + else + puts table + end + begin + print "#{title}: " + buf = STDIN.gets.strip + if buf.empty? and !default.nil? + return default + end + i = Integer buf + rescue ArgumentError + retry + end until i > 0 and i <= list.size + return i - 1 + end + + def choose_indexes_from_list title, list, table=nil, default=nil, defindex=nil + abort(I18n.t("handler.error.list.empty", :name => title)) if list.empty? + ar = nil + if table.nil? + i = 0 + print I18n.t("handler.message.choose", :name => title.downcase) + "\n#{list.map{|p| i += 1; "#{i}. #{p}"}.join("\n")}\n" + else + puts table + end + msg = if default.nil? + I18n.t("handler.message.choose_list", :name => title) + else + I18n.t("handler.message.choose_list_default", :name => title, :default => default) + end + begin + ar = get_comma_separated_list(msg).map do |g| + n = Integer g.strip + raise ArgumentError.new(I18n.t("handler.error.number.invalid")) if n < 1 or n > list.size + n + end + if ar.empty? and !default.nil? + return [ defindex ] + end + rescue ArgumentError + retry + end + ar.map{|i| i - 1} + end + + def output + case self.options[:format] + when CommonOptions::TABLE_FORMAT + table + when CommonOptions::JSON_FORMAT + json + when CommonOptions::CSV_FORMAT + csv + end + end + + def update_object_from_file object_class, object_id, file + unless File.exists?(file) + @options_parser.invalid_update_command + abort I18n.t("handler.error.file.not_exist", :file => file) + end + update_object_from_json object_class, object_id, File.read(file) + end + + def update_object_from_json object_class, object_id, json + put_body "/#{object_class}/#{object_id}", json + rescue NotFound => e + post_body "/#{object_class}", json + end + + def create_url path + URI.join(self.host, self.options[:api] + path).to_s + end + + def submit + http = HTTPClient.new + http.receive_timeout = 0 + http.send_timeout = 0 + http.set_auth(nil, self.username, self.password) + res = yield http + if res.ok? + return (res.contenttype.include?("application/json") ? JSON.parse(res.body) : res.body) + end + case res.status + when 404 + raise NotFound.new(extract_message(res)) + when 400 + raise InvalidQuery.new(extract_message(res)) + when 401 + e = extract_message(res) + e = I18n.t("handler.error.unauthorized") if (e.nil? or e.strip.empty?) + raise DevopsException.new(e) + else + raise DevopsException.new(extract_message(res)) + end + end + + def extract_message result + return nil if result.body.nil? + result.contenttype.include?("application/json") ? JSON.parse(result.body)["message"] : result.body + end + + def convert_params params + params_filter(params.select{|k,v| k != :cmd and !v.nil?}).join("&") + end + + def params_filter params + r = [] + return params if params.kind_of?(String) + params.each do |k,v| + key = k.to_s + if v.kind_of?(Array) + v.each do |val| + r.push "#{key}[]=#{val}" + end + elsif v.kind_of?(Hash) + buf = {} + v.each do |k1,v1| + buf["#{key}[#{k1}]"] = v1 + end + r = r + params_filter(buf) + else + r.push "#{key}=#{v}" + end + end + r + end + + def inspect_parameters names, *args + names.each_with_index do |name, i| + next if name.start_with? "[" and name.end_with? "]" + if args[i].nil? or args[i].empty? + return "\n" + I18n.t("handler.error.parameter.undefined", :name => name) + end + end + nil + end + + def headers *exclude + h = { + "Accept" => "application/json", + "Content-Type" => "application/json; charset=UTF-8" + } + + h["Accept-Language"] = I18n.lang + exclude.each do |key| + h.delete(key) + end + h + end + + def check_status status + r = status.scan(/--\sStatus:\s([0-9]{1,5})\s--/i)[0] + if r.nil? + puts "WARN: status undefined" + -1 + else + r[0].to_i + end + end + +end diff --git a/devops-client/lib/devops-client/handler/handler_factory.rb b/devops-client/lib/devops-client/handler/handler_factory.rb new file mode 100644 index 0000000..d938ede --- /dev/null +++ b/devops-client/lib/devops-client/handler/handler_factory.rb @@ -0,0 +1,56 @@ +class HandlerFactory + + def self.create cmd, host, auth, def_options + klass = case cmd + when "flavor" + require "devops-client/handler/flavor" + Flavor + when "image" + require "devops-client/handler/image" + Image + when "filter" + require "devops-client/handler/filter" + Filter + when "group" + require "devops-client/handler/group" + Group + when "deploy" + require "devops-client/handler/deploy" + Deploy + when "project" + require "devops-client/handler/project" + Project + when "network" + require "devops-client/handler/network" + Network + when "key" + require "devops-client/handler/key" + Key + when "user" + require "devops-client/handler/user" + User + when "provider" + require "devops-client/handler/provider" + Provider + when "tag" + require "devops-client/handler/tag" + Tag + when "server" + require "devops-client/handler/server" + Server + when "script" + require "devops-client/handler/script" + Script + when "templates" + require "devops-client/handler/bootstrap_templates" + BootstrapTemplates + else + require "devops-client/options/main" + Main.new(ARGV, def_options).info + exit(10) + end + service = klass.new(host, def_options) + service.auth = auth + service + end +end diff --git a/devops-client/lib/devops-client/handler/image.rb b/devops-client/lib/devops-client/handler/image.rb new file mode 100644 index 0000000..0238f80 --- /dev/null +++ b/devops-client/lib/devops-client/handler/image.rb @@ -0,0 +1,139 @@ +require "devops-client/handler/provider" +require "devops-client/handler/handler" +require "devops-client/options/image_options" +require "devops-client/output/image" +require "devops-client/handler/bootstrap_templates" + +class Image < Handler + + include Output::Image + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = ImageOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + when "show" + self.options = @options_parser.show_options + show_handler @options_parser.args + output + when "create" + self.options = @options_parser.create_options + create_handler + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + when "update" + self.options = @options_parser.update_options + update_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def get_providers + p = Provider.new(@host, self.options) + p.auth = self.auth + return p.list_handler(["provider", "list"]), p.table + end + + def get_templates + bt = BootstrapTemplates.new(@host, self.options) + bt.auth = self.auth + return bt.list_handler(["templates", "list"]), bt.table + end + + def create_handler + providers, table = get_providers + provider = (self.options[:provider].nil? ? providers[ choose_number_from_list(I18n.t("headers.provider"), providers, table) ] : self.options[:provider]) + provider_images provider + q = { "provider" => provider } + + image = nil + if options[:image_id].nil? + image = choose_image_cmd(@list, self.table) + else + image = @list.detect{|i| i["id"] == options[:image_id]} + abort("Invalid image id '#{options[:image_id]}'") if image.nil? + end + q["name"] = image["name"] + q["id"] = image["id"] + + if options[:ssh_username].nil? + q["remote_user"] = enter_parameter(I18n.t("handler.image.create.ssh_user") + ": ") + else + q["remote_user"] = options[:ssh_username] + end + + q["bootstrap_template"] = if options[:bootstrap_template].nil? and options[:no_bootstrap_template] == false + bt, bt_t = get_templates + i = choose_number_from_list(I18n.t("handler.image.create.template"), bt, bt_t, -1) + if i == -1 + nil + else + bt[i] + end + else + nil + end + json = JSON.pretty_generate(q) + post_body "/image", json if question(I18n.t("handler.image.question.create")){puts json} + end + + def list_handler args + if args[2].nil? + @provider = false + @list = get("/images") + elsif args[2] == "provider" and (args[3] == "ec2" || args[3] == "openstack") + provider_images args[3] + elsif args[2] == "ec2" || args[2] == "openstack" + @provider = false + @list = get("/images", :provider => args[2]) + else + @options_parser.invalid_list_command + abort() + end + end + + def provider_images p + @provider = true + @list = get("/images/provider/#{p}") + end + + def show_handler args + r = inspect_parameters @options_parser.show_params, args[2] + unless r.nil? + @options_parser.invalid_show_command + abort(r) + end + @show = get "/image/#{args[2]}" + end + + def delete_handler args + r = inspect_parameters @options_parser.delete_params, args[2] + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + if question(I18n.t("handler.image.question.delete", :name => args[2])) + delete "/image/#{args[2]}" + end + end + + def update_handler args + r = inspect_parameters @options_parser.update_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_update_command + abort(r) + end + update_object_from_file "image", args[2], args[3] + end + +end diff --git a/devops-client/lib/devops-client/handler/key.rb b/devops-client/lib/devops-client/handler/key.rb new file mode 100644 index 0000000..41443ab --- /dev/null +++ b/devops-client/lib/devops-client/handler/key.rb @@ -0,0 +1,63 @@ +require "devops-client/handler/handler" +require "devops-client/options/key_options" +require "json" +require "devops-client/output/key" + +class Key < Handler + include Output::Key + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = KeyOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler + output + when "add" + self.options = @options_parser.add_options + add_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def add_handler args + r = inspect_parameters @options_parser.add_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_add_command + abort(r) + end + + content = File.read(args[3]) + q = { + "key_name" => args[2], + "file_name" => File.basename(args[3]), + "content" => content + } + post "/key", q + end + + def delete_handler args + r = inspect_parameters @options_parser.delete_params, args[2] + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + if question(I18n.t("handler.key.question.delete", :name => args[2])) + delete "/key/#{args[2]}" + end + end + + def list_handler + @list = get("/keys") + end + +end diff --git a/devops-client/lib/devops-client/handler/network.rb b/devops-client/lib/devops-client/handler/network.rb new file mode 100644 index 0000000..a973502 --- /dev/null +++ b/devops-client/lib/devops-client/handler/network.rb @@ -0,0 +1,38 @@ +require "devops-client/handler/handler" +require "devops-client/options/network_options" +require "json" +require "devops-client/output/network" + +class Network < Handler + + include Output::Network + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = NetworkOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler args + r = inspect_parameters @options_parser.list_params, args[2] + unless r.nil? + @options_parser.invalid_list_command + abort(r) + end + @provider = args[2] + @list = get("/networks/#{args[2]}").sort!{|x,y| x["name"] <=> y["name"]} + end + +end + diff --git a/devops-client/lib/devops-client/handler/project.rb b/devops-client/lib/devops-client/handler/project.rb new file mode 100644 index 0000000..020f72e --- /dev/null +++ b/devops-client/lib/devops-client/handler/project.rb @@ -0,0 +1,578 @@ +require "devops-client/handler/provider" +require "devops-client/handler/image" +require "devops-client/handler/flavor" +require "devops-client/handler/network" +require "devops-client/handler/group" +require "devops-client/handler/user" +require "devops-client/options/project_options" +require "json" +require "set" +require "devops-client/output/project" + +class Project < Handler + + attr_accessor :def_options + + include Output::Project + + def initialize(host, def_options) + self.host = host + self.def_options = def_options + @options_parser = ProjectOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "create" + self.options = @options_parser.create_options + create_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + when "deploy" + self.options = @options_parser.deploy_options + deploy_handler @options_parser.args + when "list" + self.options = @options_parser.list_options + list_handler + output + when "multi" + case ARGV[2] + when "create" + self.options = @options_parser.multi_create_options + multi_create_handler @options_parser.args + else + @options_parser.invalid_multi_command + abort(I18n.t("handler.project.invalid_subcommand", :cmd => ARGV[1], :scmd => ARGV[2])) + end + when "servers" + self.options = @options_parser.servers_options + servers_handler @options_parser.args + output + when "set" + case ARGV[2] + when "run_list" + self.options = @options_parser.set_run_list_options + set_run_list_handler @options_parser.args + else + @options_parser.invalid_set_command + abort(I18n.t("handler.project.invalid_subcommand", :cmd => ARGV[1], :scmd => ARGV[2])) + end + when "show" + self.options = @options_parser.show_options + show_handler @options_parser.args + output + when "update" + self.options = @options_parser.update_options + update_handler @options_parser.args + when "user" + case ARGV[2] + when "add" + self.options = @options_parser.user_add_options + user_add_handler @options_parser.args + when "delete" + self.options = @options_parser.user_delete_options + user_delete_handler @options_parser.args + else + @options_parser.invalid_user_command + abort(I18n.t("handler.project.invalid_subcommand", :cmd => ARGV[1], :scmd => ARGV[2])) + end + when "test" + self.options = @options_parser.test_options + test_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler + @list = get "/projects" + end + + def delete_handler args + r = inspect_parameters @options_parser.delete_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + o = {} + o[:deploy_env] = args[3] unless args[3].nil? + + message = args[2] + message += ".#{args[3]}" unless args[3].nil? + if question(I18n.t("handler.project.question.delete", :name => message)) + delete "/project/#{args[2]}", o + end + end + + def show_handler args + r = inspect_parameters @options_parser.show_params, args[2] + unless r.nil? + @options_parser.invalid_show_command + abort(r) + end + @show = get_project_info_obj(args[2]) + end + + def update_handler args + r = inspect_parameters @options_parser.update_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_update_command + abort(r) + end + update_object_from_file "project", args[2], args[3] + end + + def create_handler args + file = self.options[:file] + unless file.nil? + json = File.read(file) + begin + JSON.parse(json) + rescue JSON::ParserError => e + abort(I18n.t("handler.project.create.invalid_json", :file => file)) + end + post_body("/project", json) + else + r = inspect_parameters @options_parser.create_params, args[2] + unless r.nil? + @options_parser.invalid_create_command + abort(r) + end + unless self.options[:username].nil? || self.options[:password].nil? + self.auth[:username] = self.options[:username] + self.auth[:password] = self.options[:password] + self.def_options[:username] = self.auth[:username] + end + create_project args, :create_project_deploy_env_cmd + end + end + + def servers_handler args + r = inspect_parameters @options_parser.servers_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_servers_command + abort(r) + end + o = {} + unless args[3].nil? + o[:deploy_env] = args[3] + end + @servers = get "/project/#{args[2]}/servers", o + end + + def user_add_handler args + r = inspect_parameters @options_parser.user_add_params, args[3], args[4] + unless r.nil? + @options_parser.invalid_user_add_command + abort(r) + end + q = {:users => args[4..-1]} + q[:deploy_env] = options[:deploy_env] unless options[:deploy_env].nil? + put "/project/#{args[3]}/user", q + end + + def user_delete_handler args + r = inspect_parameters @options_parser.user_delete_params, args[3], args[4] + unless r.nil? + @options_parser.invalid_user_delete_command + abort(r) + end + q = {:users => args[4..-1]} + q[:deploy_env] = options[:deploy_env] unless options[:deploy_env].nil? + delete_body "/project/#{args[3]}/user", q.to_json + end + + def multi_create_handler args + r = inspect_parameters @options_parser.multi_create_params, args[3] + unless r.nil? + @options_parser.invalid_multi_create_command + abort(r) + end + + create_project args, :create_project_multi_deploy_env_cmd, :multi + + i = Image.new(@host, self.def_options) + images, ti = i.list_handler, i.table + f = Flavor.new(@host, self.def_options) + flavors, tf = f.list_handler, f.table + g = Group.new(@host, self.def_options) + groups, tg = g.list_handler, g.table + + list = list_handler + info, multi = {}, {:type => "multi", :name => args[3], :deploy_envs => []} + begin # Add environment + nodes, projects, servers = [], [], {} + deploy_env = {:identifier => enter_parameter("Deploy environment identifier: ")} + begin # Add server + server_name = args[3] + "_" + enter_parameter("Server name: " + args[3] + "_") + s = servers[server_name] = {} + s[:groups] = choose_indexes_from_list("Security groups", list, tg, "default", list.index("default")).map{|i| list[i]} + s[:flavor] = choose_flavor_cmd(flavors, tf)["name"] + s[:image] = choose_image_cmd(images, ti)["id"] + subprojects = s[:subprojects] = [] + + begin # Add project + o = {} + o[:project_id] = project_id = choose_project(list, table) + info[project_id] = get_project_info_obj(project_id) unless info.has_key?(project_id) + envs = info[project_id]["deploy_envs"].map{|de| de["identifier"]} + o[:project_env] = ( envs.size > 1 ? choose_project_env(envs) : envs[0] ) + subprojects.push o + end while question("Add project?") + + end while question("Add server?") + + deploy_env[:servers] = servers + multi[:deploy_envs].push deploy_env + end while question(I18n.t("handler.project.question.add_env")) + puts JSON.pretty_generate(multi) + post "/project", :json => multi.to_json if question(I18n.t("handler.project.question.create")) + end + + def set_run_list_handler args + r = inspect_parameters @options_parser.set_run_list_params, args[3], args[4], args[5] + unless r.nil? + @options_parser.invalid_set_run_list_command + abort(r) + end + run_list = [] + args[5..args.size].each do |e| + run_list += e.split(",") + end + if run_list.empty? + exit unless question(I18n.t("handler.project.run_list.empty")) + else + exit unless Project.validate_run_list(run_list) + end + put "/project/#{args[3]}/#{args[4]}/run_list", run_list + end + + def deploy_handler args + r = inspect_parameters @options_parser.deploy_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_deploy_command + abort(r) + end + q = {} + q[:servers] = options[:servers] unless options[:servers].nil? + q[:deploy_env] = args[3] unless args[3].nil? + post_chunk "/project/#{args[2]}/deploy", q + end + + def test_handler args + r = inspect_parameters @options_parser.test_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_test_command + abort(r) + end + @test = post "/project/test/#{args[2]}/#{args[3]}" + end + +protected + def get_project_info_obj project_id + get("/project/#{project_id}") + end + + def get_providers + p = Provider.new(@host, self.def_options) + p.auth = self.auth + return p.list_handler(["provider", "list"]), p.table + end + + def get_images provider + img = Image.new(@host, self.def_options) + img.auth = self.auth + return img.list_handler(["image", "list", provider]), img.table + end + + def get_flavors provider + f = Flavor.new(@host, self.def_options) + f.auth = self.auth + return f.list_handler(["flavor", "list", provider]), f.table + end + + def get_groups provider, vpcId + g = Group.new(@host, self.def_options) + g.auth = self.auth + p = ["group", "list", provider] + p.push vpcId if !vpcId.nil? and provider == "ec2" + return g.list_handler(p), g.table + end + + def get_networks provider + n = Network.new(@host, self.def_options) + n.auth = self.auth + return n.list_handler(["network", "list", provider]), n.table + end + + def get_users + u = User.new(@host, self.def_options) + u.auth = self.auth + return u.list_handler, u.table + end + + def create_project args, env_method_name, type=nil + project_name = args[2] + providers = {} + begin + project = get_project_info_obj(project_name) + puts_warn I18n.t("handler.project.exist", :project => project_name) + names = project["deploy_envs"].map{|de| de["identifier"]} + while question(I18n.t("handler.project.question.add_env")) + d = method(env_method_name).call(project_name, providers, names) + project["deploy_envs"].push d + break if self.options[:no_ask] + end + puts json = JSON.pretty_generate(project) + update_object_from_json("project", project_name, json) if question(I18n.t("handler.project.question.update")) + rescue NotFound => e + project = create_project_cmd(project_name, providers, env_method_name) + project[:name] = args[2] + puts json = JSON.pretty_generate(project) + post_body("/project", json) if question(I18n.t("handler.project.question.create")) + end + end + + def create_project_cmd project_name, providers, env_method + project = {:deploy_envs => []} + names = [] + begin + d = method(env_method).call(project_name, providers, names) + project[:deploy_envs].push d + break if self.options[:no_ask] + end while question(I18n.t("handler.project.question.add_env")) + project + end + + def create_project_deploy_env_cmd project, providers, names + d = {} + set_identifier(d, names) + + set_provider(d, providers) + buf = providers[d[:provider]] + + set_flavor(d, buf) + set_image(d, buf) + vpc_id = set_subnets(d, buf) + set_groups(d, buf, vpc_id) + set_users(d, buf) + + unless self.options[:run_list].nil? + self.options[:run_list] = self.options[:run_list].split(",").map{|e| e.strip} + abort("Invalid run list: '#{self.options[:run_list].join(",")}'") unless Project.validate_run_list(self.options[:run_list]) + end + set_parameter d, :run_list do + set_run_list_cmd project, d[:identifier] + end + + unless self.options[:no_expires] + set_parameter d, :expires do + s = enter_parameter_or_empty(I18n.t("options.project.create.expires") + ": ").strip + s.empty? ? nil : s + end + end + d + end + + def create_project_multi_deploy_env_cmd project, providers, names + d = {} + set_identifier(d, names) + + set_provider(d, providers) + buf = providers[d[:provider]] + + set_flavor(d, buf) + set_image(d, buf) + vpc_id = set_subnets(d, buf) + set_groups(d, buf, vpc_id) + set_users(d, buf) + + unless self.options[:run_list].nil? + self.options[:run_list] = self.options[:run_list].split(",").map{|e| e.strip} + abort("Invalid run list: '#{self.options[:run_list].join(",")}'") unless Project.validate_run_list(self.options[:run_list]) + end + set_parameter d, :run_list do + set_run_list_cmd project, d[:identifier] + end + + unless self.options[:no_expires] + set_parameter d, :expires do + s = enter_parameter_or_empty(I18n.t("options.project.create.expires") + ": ").strip + s.empty? ? nil : s + end + end + d + end + + def set_identifier d, names + set_parameter d, :identifier do + begin + n = enter_parameter I18n.t("handler.project.create.env") + ": " + if names.include?(n) + puts I18n.t("handler.project.create.env_exist", :env => n) + raise ArgumentError + else + names.push n + n + end + rescue ArgumentError + retry + end + end + end + + def set_provider d, providers + if providers[:obj].nil? + providers[:obj], providers[:table] = get_providers + providers[:obj].each{|p| providers[p] = {}} + end + + set_parameter d, :provider do + providers[:obj][ choose_number_from_list(I18n.t("headers.provider"), providers[:obj], providers[:table]) ] + end + end + + def set_flavor d, buf + flavors, tf = nil, nil + if buf[:flavors].nil? + flavors, tf = get_flavors(d[:provider]) + add_object buf, :flavors, flavors, tf + else + flavors, tf = buf[:flavors][:obj], buf[:flavors][:table] + end + unless self.options[:flavor].nil? + f = flavors.detect { |f| f["id"] == self.options[:flavor] } + abort(I18n.t("handler.project.create.flavor.not_found")) if f.nil? + end + set_parameter d, :flavor do + choose_flavor_cmd(flavors, tf)["id"] + end + end + + def set_image d, buf + images, ti = nil, nil + if buf[:images].nil? + images, ti = get_images(d[:provider]) + add_object buf, :images, images, ti + else + images, ti = buf[:images][:obj], buf[:images][:table] + end + set_parameter d, :image do + choose_image_cmd(images, ti)["id"] + end + end + + def set_subnets d, buf + networks, tn = nil, nil + if buf[:networks].nil? + networks, tn = get_networks(d[:provider]) + add_object buf, :networks, networks, tn + else + networks, tn = buf[:networks][:obj], buf[:networks][:table] + end + unless self.options[:subnets].nil? + if "ec2" == d[:provider] + self.options[:subnets] = [ self.options[:subnets][0] ] + end + end + vpc_id = nil + set_parameter d, :subnets do + if "ec2" == d[:provider] + if networks.any? + num = choose_number_from_list(I18n.t("handler.project.create.subnet.ec2"), networks, tn, -1) + vpc_id = networks[num]["vpcId"] unless num == -1 + num == -1 ? [] : [ networks[num]["subnetId"] ] + else + [] + end + else + s = [] + begin + s = choose_indexes_from_list(I18n.t("handler.project.create.subnet.openstack"), networks, tn).map{|i| networks[i]["name"]} + end while s.empty? + s + end + end + return vpc_id + end + + def set_groups d, buf, vpc_id + groups, tg = nil, nil + if buf[:groups].nil? + groups, tg = get_groups(d[:provider], vpc_id) + add_object buf, :groups, groups, tg + else + groups, tg = buf[:groups][:obj], buf[:groups][:table] + end + set_parameter d, :groups do + list = groups.keys + choose_indexes_from_list(I18n.t("options.project.create.groups"), list, tg, "default", list.index("default")).map{|i| list[i]} + end + end + + def set_users d, buf + users, tu = nil, nil + if buf[:users].nil? + users, tu = get_users + add_object buf, :users, users, tu + else + users, tu = buf[:users][:obj], buf[:users][:table] + end + set_parameter d, :users do + list = users.map{|u| u["id"]} + Set.new choose_indexes_from_list(I18n.t("handler.project.create.user"), list, tu).map{|i| list[i]} + end + d[:users].add(self.options[:username]) + d[:users] = d[:users].to_a + end + + def add_object tec, key, obj, table + tec[key] = {:obj => obj, :table => table} + end + + def set_parameter obj, key + if self.options[key].nil? + obj[key] = yield + else + obj[key] = self.options[key] + end + end + + # returns flavor hash + def choose_flavor_cmd flavors, table=nil + abort(I18n.t("handler.flavor.list.empty")) if flavors.empty? + flavors[ choose_number_from_list(I18n.t("headers.flavor"), flavors.map{|f| "#{f["id"]}. #{f["name"]} - #{f["ram"]}, #{f["disk"]}, #{f["v_cpus"]} CPU"}.join("\n"), table) ] + end + + # returns project id + def choose_project projects, table=nil + abort(I18n.t("handler.project.list.empty")) if projects.empty? + projects[ choose_number_from_list(I18n.t("headers.project"), projects, table) ] + end + + # returns project env + def choose_project_env project_envs, table=nil + abort(I18n.t("handler.project.env.list.empty")) if project_envs.empty? + project_envs[ choose_number_from_list(I18n.t("headers.project_env"), project_envs, table) ] + end + + def set_run_list_cmd project, env + res = nil + begin + res = get_comma_separated_list(I18n.t("options.project.create.run_list") + ": ") + end until Project.validate_run_list(res) + res + end + + def self.validate_run_list run_list + return true if run_list.empty? + rl = /\Arole|recipe\[[\w-]+(::[\w-]+)?\]\Z/ + e = run_list.select {|l| (rl =~ l).nil?} + res = e.empty? + puts I18n.t("handler.project.create.run_list.invalid", :list => e.join(", ")) unless res + res + end + +end diff --git a/devops-client/lib/devops-client/handler/provider.rb b/devops-client/lib/devops-client/handler/provider.rb new file mode 100644 index 0000000..fdd74fb --- /dev/null +++ b/devops-client/lib/devops-client/handler/provider.rb @@ -0,0 +1,35 @@ +require "devops-client/handler/handler" +require "devops-client/options/provider_options" +require "devops-client/output/provider" + +class Provider < Handler + + include Output::Provider + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = ProviderOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + else + @options_parser.invalid_command + end + end + + def list_handler args + r = inspect_parameters @options_parser.list_params + unless r.nil? + @options_parser.invalid_list_command + abort(r) + end + @list = get("/providers").sort!{|x,y| x["id"] <=> y["id"]} + end + +end diff --git a/devops-client/lib/devops-client/handler/script.rb b/devops-client/lib/devops-client/handler/script.rb new file mode 100644 index 0000000..cb87f8a --- /dev/null +++ b/devops-client/lib/devops-client/handler/script.rb @@ -0,0 +1,84 @@ +require "devops-client/handler/handler" +require "devops-client/options/script_options" +require "devops-client/output/script" + +class Script < Handler + include Output::Script + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = ScriptOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + when "add" + self.options = @options_parser.add_options + add_handler @options_parser.args + when "run" + self.options = @options_parser.run_options + run_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + when "command" + self.options = @options_parser.command_options + command_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def command_handler args + r = inspect_parameters @options_parser.command_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_command_command + abort(r) + end + post_chunk_body "/script/command/#{args[2]}", args[3], false + end + + def list_handler args + @list = get("/scripts") + end + + def add_handler args + r = inspect_parameters @options_parser.add_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_add_command + abort(r) + end + abort("File '#{args[3]}' does not exist") unless File.exists?(args[3]) + put_body "/script/#{args[2]}", File.read(args[3]) + end + + def delete_handler args + r = inspect_parameters @options_parser.delete_params, args[2] + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + if question(I18n.t("handler.script.question.delete", :name => args[2])) + delete "/script/#{args[2]}" + end + end + + def run_handler args + r = inspect_parameters @options_parser.run_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_run_command + abort(r) + end + q = { + :nodes => args[3..-1] + } + q[:params] = self.options[:params] unless self.options[:params].nil? + post_chunk "/script/run/#{args[2]}", q + end + +end diff --git a/devops-client/lib/devops-client/handler/server.rb b/devops-client/lib/devops-client/handler/server.rb new file mode 100644 index 0000000..1fb6e89 --- /dev/null +++ b/devops-client/lib/devops-client/handler/server.rb @@ -0,0 +1,167 @@ +require "devops-client/handler/handler" +require "devops-client/options/server_options" +require "devops-client/output/server" +require "devops-client/handler/project" + +class Server < Handler + + include Output::Server + + def initialize(host, def_options={}) + self.host = host + @options_parser = ServerOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + when "bootstrap" + self.options = @options_parser.bootstrap_options + bootstrap_handler @options_parser.args + when "create" + self.options = @options_parser.create_options + create_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + when "show" + self.options = @options_parser.show_options + show_handler @options_parser.args + output + when "sync" + self.options = @options_parser.sync_options + sync_handler + when "pause" + self.options = @options_parser.pause_options + pause_handler @options_parser.args + when "unpause" + self.options = @options_parser.unpause_options + unpause_handler @options_parser.args + when "add" + self.options = @options_parser.add_options + add_static_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def list_handler args + if args[2].nil? + @list = get("/servers") + return @list + end + self.options[:type] = args[2] + @list = case args[2] + when "chef" + get("/servers/chef").map {|l| {"chef_node_name" => l}} + when "ec2", "openstack" + get("/servers/#{args[2]}") + else + @options_parser.invalid_list_command + abort("Invlid argument '#{args[2]}'") + end + end + + def create_handler args + r = inspect_parameters @options_parser.create_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_create_command + abort(r) + end + + q = { + :project => args[2], + :deploy_env => args[3] + } + + [:key, :without_bootstrap, :name, :groups, :force].each do |k| + q[k] = self.options[k] unless self.options[k].nil? + end + + post_chunk "/server", q + end + + def delete_handler args + args[2..-1].each do |name| + r = inspect_parameters @options_parser.delete_params, name + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + if question(I18n.t("handler.server.question.delete", :name => name)) + puts "Server '#{name}', deleting..." + o = delete("/server/#{name}", options) + ["server", "chef_node", "chef_client", "message"].each do |k| + puts o[k] unless o[k].nil? + end + end + end + "" + end + + def show_handler args + r = inspect_parameters @options_parser.show_params, args[2] + unless r.nil? + @options_parser.invalid_show_command + abort r + end + @show = get("/server/#{args[2]}") + end + + def bootstrap_handler args + r = inspect_parameters @options_parser.bootstrap_params, args[2] + unless r.nil? + @options_parser.invalid_bootstrap_command + abort(r) + end + q = { + :instance_id => args[2] + } + [:name, :bootstrap_template, :run_list].each do |k| + q[k] = self.options[k] unless self.options[k].nil? + end + if q.has_key?(:run_list) + abort unless Project.validate_run_list(q[:run_list]) + end + post_chunk "/server/bootstrap", q + end + + def add_static_handler args # add --public-ip -k + r = inspect_parameters @options_parser.add_params, args[2], args[3], args[4], args[5], args[6] + unless r.nil? + @options_parser.invalid_add_command + abort(r) + end + q = { + :project => args[2], + :deploy_env => args[3], + :private_ip => args[4], + :remote_user => args[5], + :key => args[6] + } + q[:public_ip] = self.options[:public_ip] unless self.options[:public_ip].nil? + post_chunk "/server/add", q + end + + def pause_handler args + r = inspect_parameters @options_parser.pause_params, args[2] + unless r.nil? + @options_parser.invalid_pause_command + abort(r) + end + post "/server/#{args[2]}/pause" + end + + def unpause_handler args + r = inspect_parameters @options_parser.unpause_params, args[2] + unless r.nil? + @options_parser.invalid_unpause_command + abort(r) + end + post "/server/#{args[2]}/unpause" + end + +end diff --git a/devops-client/lib/devops-client/handler/tag.rb b/devops-client/lib/devops-client/handler/tag.rb new file mode 100644 index 0000000..f4ebb75 --- /dev/null +++ b/devops-client/lib/devops-client/handler/tag.rb @@ -0,0 +1,64 @@ +require "devops-client/handler/handler" +require "devops-client/options/tag_options" +require "json" +require "devops-client/output/tag" + +class Tag < Handler + include Output::Tag + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = TagOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler @options_parser.args + output + when "create" + self.options = @options_parser.create_options + create_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def list_handler args + r = inspect_parameters @options_parser.list_params, args[2] + unless r.nil? + @options_parser.invalid_list_command + abort(r) + end + @list = get("/tags/#{args[2]}") + end + + def create_handler args + if args.length == 3 + @options_parser.invalid_create_command + abort() + end + node = args[2] + tags = args[3..-1] + + post "/tags/#{node}", tags + end + + def delete_handler args + if args.length == 3 + @options_parser.invalid_delete_command + abort() + end + node = args[2] + tags = args[3..-1] + + if question(I18n.t("handler.user.question.delete", :name => tags.join("', '"), :node => node)) + delete "/tags/#{node}", tags + end + end +end diff --git a/devops-client/lib/devops-client/handler/user.rb b/devops-client/lib/devops-client/handler/user.rb new file mode 100644 index 0000000..36a4a89 --- /dev/null +++ b/devops-client/lib/devops-client/handler/user.rb @@ -0,0 +1,129 @@ +require "devops-client/handler/handler" +require "devops-client/options/user_options" +require "devops-client/output/user" + +class User < Handler + include Output::User + + def initialize(host, def_options={}) + self.host = host + self.options = def_options + @options_parser = UserOptions.new(ARGV, def_options) + end + + def handle + case ARGV[1] + when "list" + self.options = @options_parser.list_options + list_handler + output + when "create" + self.options = @options_parser.create_options + create_handler @options_parser.args + when "delete" + self.options = @options_parser.delete_options + delete_handler @options_parser.args + when "grant" + self.options = @options_parser.grant_options + grant_handler @options_parser.args + when "password" + self.options = @options_parser.password_options + password_handler @options_parser.args + when "email" + self.options = @options_parser.email_options + email_handler @options_parser.args + else + @options_parser.invalid_command + end + end + + def list_handler + @list = get("/users") + end + + def create_handler args + r = inspect_parameters @options_parser.create_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_create_command + abort(r) + end + + password = self.options[:new_password] || enter_password(args[2]) + + q = { + "username" => args[2], + "password" => password, + "email" => args[3] + } + post "/user", q + end + + def delete_handler args + r = inspect_parameters @options_parser.delete_params, args[2] + unless r.nil? + @options_parser.invalid_delete_command + abort(r) + end + + if question(I18n.t("handler.user.question.delete", :name => args[2])) + delete "/user/#{args[2]}" + end + end + + def password_handler args + r = inspect_parameters @options_parser.password_params, args[2] + unless r.nil? + @options_parser.invalid_password_command + abort(r) + end + + password = enter_password(args[2]) + q = { + "password" => password + } + + put "/user/#{args[2]}/password", q + end + + def email_handler args + r = inspect_parameters @options_parser.email_params, args[2], args[3] + unless r.nil? + @options_parser.invalid_email_command + abort(r) + end + q = { + "email" => args[3] + } + put "/user/#{args[2]}/email", q + end + + def grant_handler args + r = inspect_parameters @options_parser.grant_params, args[2], args[3], args[4] + unless r.nil? + @options_parser.invalid_grant_command + abort(r) + end + + args[3] = '' if args[3].nil? + q = { + 'cmd' => args[3], + 'privileges' => args[4] + } + + put "/user/#{args[2]}", q + end + + def enter_password user + print "Enter password for '#{user}': " + password = "" + begin + system("stty -echo") + password = STDIN.gets.strip + puts + ensure + system("stty echo") + end + password + end + +end diff --git a/devops-client/lib/devops-client/i18n.rb b/devops-client/lib/devops-client/i18n.rb new file mode 100644 index 0000000..7efcf67 --- /dev/null +++ b/devops-client/lib/devops-client/i18n.rb @@ -0,0 +1,53 @@ +module I18n + + @@lang = {} + + def self.language= locale + spec = Gem::Specification.find_by_name(DevopsClient::NAME) + gem_root = spec.gem_dir + path = File.join(gem_root, "locales", "#{locale}.yml") + raise ArgumentError.new("Invalid locale '#{locale}'") unless File.exist?(path) + require 'yaml' + begin + @@lang = YAML.load_file(path)[locale] + rescue + raise ArgumentError.new("Invalid file '#{locale}.yml'") + end + end + + def self.t label, options={} + path = label.split(".") + buf = @@lang + begin + path.each do |index| + buf = buf[index] + end + raise ArgumentError unless buf.is_a?(String) + rescue + return "Translation missing" + end + options.each do |k,v| + buf.gsub!("%{#{k.to_s}}", v.to_s) + end + buf + end + + def self.locales + spec = Gem::Specification.find_by_name(DevopsClient::NAME) + gem_root = spec.gem_dir + path = File.join(gem_root, "locales") + locales = [] + Dir.foreach(path) do |item| + next if item.start_with? '.' + if item.end_with? ".yml" + locales.push item.split(".")[0] + end + end + locales + end + + def self.lang + @@lang + end + +end diff --git a/devops-client/lib/devops-client/name.rb b/devops-client/lib/devops-client/name.rb new file mode 100644 index 0000000..07f879e --- /dev/null +++ b/devops-client/lib/devops-client/name.rb @@ -0,0 +1,3 @@ +module DevopsClient + NAME = "devops-client" +end diff --git a/devops-client/lib/devops-client/options/bootstrap_templates_options.rb b/devops-client/lib/devops-client/options/bootstrap_templates_options.rb new file mode 100644 index 0000000..6e76fd8 --- /dev/null +++ b/devops-client/lib/devops-client/options/bootstrap_templates_options.rb @@ -0,0 +1,13 @@ +require "devops-client/options/common_options" + +class BootstrapTemplatesOptions < CommonOptions + + commands :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.template") + self.banner_header = "templates" + end + +end diff --git a/devops-client/lib/devops-client/options/common_options.rb b/devops-client/lib/devops-client/options/common_options.rb new file mode 100644 index 0000000..a44b8d7 --- /dev/null +++ b/devops-client/lib/devops-client/options/common_options.rb @@ -0,0 +1,170 @@ +require "optparse" +require "devops-client/version" + +class CommonOptions + + attr_accessor :header, :args, :default_options + attr_writer :banner_header + + TABLE_FORMAT = "table" + JSON_FORMAT = "json" + CSV_FORMAT = "csv" + OUTPUT_FROMATS = [TABLE_FORMAT, JSON_FORMAT, CSV_FORMAT] + + def initialize args, def_options + self.args = args + self.default_options = def_options + end + + def self.commands *cmds + cmds.each do |cmd| + if cmd.is_a?(Hash) + key = cmd.keys[0] + cmd[key].each do |subcmd| + create_command key.to_s, subcmd.to_s + end + invalid_command_method = "invalid_#{key}_command" + banner_method = "#{key}_banner" + + define_method invalid_command_method do + puts "#{self.header}:\n#{self.send(banner_method)}" + end + + define_method banner_method do + cmd[key].map{|sc| self.send("#{key}_#{sc}_banner")}.join("") + "\n" + end + else + create_command cmd.to_s + end + end + + define_method "banners" do + r = [] + cmds.each do |cmd| + if cmd.is_a?(Hash) + key = cmd.keys[0] + cmd[key].each do |subcmd| + r.push self.send("#{key}_#{subcmd}_banner") + end + else + r.push self.send("#{cmd.to_s}_banner") + end + end + r + end + + end + + def self.create_command cmd, subcmd=nil + name = (subcmd.nil? ? cmd : "#{cmd}_#{subcmd}") + banner = (subcmd.nil? ? cmd : "#{cmd} #{subcmd}") + + invalid_command_method = "invalid_#{name}_command" + banner_method = "#{name}_banner" + + define_method invalid_command_method do + puts "#{self.header}:\n#{self.send(banner_method)}" + end + + params_method = "#{name}_params" + define_method banner_method do + self.banner_header + " #{banner} #{(self.send(params_method) || []).join(" ")}\n" + end + + options_method = "#{name}_options" + define_method options_method do + self.options do |opts, options| + opts.banner << self.send(banner_method) + end + end + + attr_accessor params_method + + end + + def options + o = {} + optparse = OptionParser.new do |opts| + + opts.banner = "\n" + I18n.t("options.usage", :cmd => $0) + "\n\n" + I18n.t("options.commands") + ":\n" + + if block_given? + opts.separator(I18n.t("options.options") + ":\n") + yield opts, o + end + + opts.separator("\n" + I18n.t("options.common_options") + ":\n") + opts.on("-h", "--help", I18n.t("options.common.help")) do + opts.banner << "\n" + puts opts + exit + end + + o[:no_ask] = false + opts.on("-y", "--assumeyes", I18n.t("options.common.confirmation")) do + o[:no_ask] = true; + end + + #Not used, just for banner purposes. This should be fixed when we find how to deal with options separetely + opts.on("-c", "--config CONFIG", I18n.t("options.common.config", :file => DevopsClient.config_file)) do + puts "Not implemented yet" + exit + end + + opts.on("-v", "--version", I18n.t("options.common.version")) do + puts I18n.t("options.common.version") + ": #{DevopsClient::VERSION}" + exit + end + + opts.on("--host HOST", I18n.t("options.common.host", :host => default_options[:host])) do |h| + o[:host] = h + end + + o[:api] = default_options[:api] + opts.on("--api VER", I18n.t("options.common.api", :api => o[:api])) do |a| + o[:api] = a + end + + o[:username] = default_options[:username] + opts.on("--user USERNAME", I18n.t("options.common.username", :username => o[:username])) do |u| + o[:username] = u.strip + print I18n.t("handler.user.password_for", :user => o[:username]) + begin + system("stty -echo") + o[:password] = STDIN.gets.strip + ensure + system("stty echo") + end + puts + end + + o[:format] = TABLE_FORMAT + opts.on("--format FORMAT", I18n.t("options.common.format", :formats => OUTPUT_FROMATS.join("', '"), :format => TABLE_FORMAT)) do |f| + o[:format] = f if OUTPUT_FROMATS.include?(f) + end + + # should be handled in lib/devops-client.rb + opts.on("", "--completion", I18n.t("options.common.completion")) + + end + optparse.parse!(self.args) + o + end + + def invalid_command + options do |opts, options| + opts.banner << self.error_banner + puts opts.banner + exit(2) + end + end + + def error_banner + "\t#{self.header}:\n\t#{self.banners.join("\t")}\n" + end + + def banner_header + "\t" + @banner_header + end + +end diff --git a/devops-client/lib/devops-client/options/deploy_options.rb b/devops-client/lib/devops-client/options/deploy_options.rb new file mode 100644 index 0000000..6991e27 --- /dev/null +++ b/devops-client/lib/devops-client/options/deploy_options.rb @@ -0,0 +1,36 @@ +require "devops-client/options/common_options" + +class DeployOptions < CommonOptions + + attr_accessor :deploy_params + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.deploy") + # self.deploy_params = ["PROJECT_ID", "DEPLOY_ENV"] + end + + def deploy_options + options do |opts, options| + opts.banner << self.banner + + options[:tag] = nil + opts.on("--tag TAG1,TAG2...", "Tag names, comma separated list") do |tags| + options[:tags] = tags.split(",") + end + end + end + + def banners + [ self.banner ] + end + + def banner + "\tdeploy NODE_NAME [NODE_NAME ...]\n" + end + + def invalid_deploy_command + puts "#{self.header}:\n#{self.banner}" + end + +end diff --git a/devops-client/lib/devops-client/options/filter_options.rb b/devops-client/lib/devops-client/options/filter_options.rb new file mode 100644 index 0000000..0992eb8 --- /dev/null +++ b/devops-client/lib/devops-client/options/filter_options.rb @@ -0,0 +1,18 @@ +require "optparse" +require "devops-client/options/common_options" + +class FilterOptions < CommonOptions + + commands :image => [:add, :delete, :list] + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.filters") + self.banner_header = "filter" + p = "PROVIDER" + self.image_list_params = [p] + i = "IMAGE [IMAGE ...]" + self.image_add_params = [p, i] + self.image_delete_params = [p, i] + end +end diff --git a/devops-client/lib/devops-client/options/flavor_options.rb b/devops-client/lib/devops-client/options/flavor_options.rb new file mode 100644 index 0000000..b977ffc --- /dev/null +++ b/devops-client/lib/devops-client/options/flavor_options.rb @@ -0,0 +1,15 @@ +require "optparse" +require "devops-client/options/common_options" + +class FlavorOptions < CommonOptions + + commands :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.flavor") + self.banner_header = "flavor" + self.list_params = ["PROVIDER"] + end + +end diff --git a/devops-client/lib/devops-client/options/group_options.rb b/devops-client/lib/devops-client/options/group_options.rb new file mode 100644 index 0000000..5e1e91a --- /dev/null +++ b/devops-client/lib/devops-client/options/group_options.rb @@ -0,0 +1,15 @@ +require "optparse" +require "devops-client/options/common_options" + +class GroupOptions < CommonOptions + + commands :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.group") + self.banner_header = "group" + self.list_params = ["PROVIDER", "[VPC-ID]"] + end + +end diff --git a/devops-client/lib/devops-client/options/image_options.rb b/devops-client/lib/devops-client/options/image_options.rb new file mode 100644 index 0000000..3408df5 --- /dev/null +++ b/devops-client/lib/devops-client/options/image_options.rb @@ -0,0 +1,50 @@ +require "devops-client/options/common_options" + +class ImageOptions < CommonOptions + + commands :create, :delete, :list, :show, :update + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.image") + self.banner_header = "image" + self.list_params = ["[provider]", "[ec2|openstack]"] + self.show_params = ["IMAGE"] + self.delete_params = ["IMAGE"] + self.update_params = ["IMAGE", "FILE"] + end + + def create_options + + self.options do |opts, options| + opts.banner << self.create_banner + + options[:provider] = nil + opts.on("--provider PROVIDER", "Image provider") do |provider| + options[:provider] = provider + end + + options[:image_id] = nil + opts.on("--image IMAGE_ID", "Image identifier") do |image_id| + options[:image_id] = image_id + end + + options[:ssh_username] = nil + opts.on("--ssh_user USER", "SSH user name") do |username| + options[:ssh_username] = username + end + + options[:bootstrap_template] = nil + opts.on("--bootstrap_template TEMPLATE", "Bootstrap template") do |template| + options[:bootstrap_template] = template + end + + options[:no_bootstrap_template] = false + opts.on("--no_bootstrap_template", "Do not specify bootstrap template") do + options[:no_bootstrap_template] = true + end + + end + end + +end diff --git a/devops-client/lib/devops-client/options/key_options.rb b/devops-client/lib/devops-client/options/key_options.rb new file mode 100644 index 0000000..f1cf490 --- /dev/null +++ b/devops-client/lib/devops-client/options/key_options.rb @@ -0,0 +1,14 @@ +require "devops-client/options/common_options" + +class KeyOptions < CommonOptions + commands :add, :delete, :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.key") + self.banner_header = "key" + self.add_params = ["KEY_NAME", "FILE"] + self.delete_params = ["KEY_NAME"] + end + +end diff --git a/devops-client/lib/devops-client/options/main.rb b/devops-client/lib/devops-client/options/main.rb new file mode 100644 index 0000000..04811b8 --- /dev/null +++ b/devops-client/lib/devops-client/options/main.rb @@ -0,0 +1,46 @@ +require "optparse" +require "devops-client/options/server_options" +require "devops-client/options/image_options" +require "devops-client/options/project_options" +require "devops-client/options/provider_options" +require "devops-client/options/flavor_options" +require "devops-client/options/common_options" +require "devops-client/options/group_options" +require "devops-client/options/deploy_options" +require "devops-client/options/key_options" +require "devops-client/options/user_options" +require "devops-client/options/tag_options" +require "devops-client/options/script_options" +require "devops-client/options/filter_options" +require "devops-client/options/network_options" +require "devops-client/options/bootstrap_templates_options" + +class Main < CommonOptions + + def initialize args, def_options + super(args, def_options) + end + + def info + o = nil + options do |opts, options| + opts.banner << BootstrapTemplatesOptions.new(ARGV, default_options).error_banner + opts.banner << DeployOptions.new(ARGV, default_options).error_banner + opts.banner << FilterOptions.new(ARGV, default_options).error_banner + opts.banner << FlavorOptions.new(ARGV, default_options).error_banner + opts.banner << GroupOptions.new(ARGV, default_options).error_banner + opts.banner << ImageOptions.new(ARGV, default_options).error_banner + opts.banner << KeyOptions.new(ARGV, default_options).error_banner + opts.banner << NetworkOptions.new(ARGV, default_options).error_banner + opts.banner << ProjectOptions.new(ARGV, default_options).error_banner + opts.banner << ProviderOptions.new(ARGV, default_options).error_banner + opts.banner << ScriptOptions.new(ARGV, default_options).error_banner + opts.banner << ServerOptions.new(ARGV, default_options).error_banner + opts.banner << TagOptions.new(ARGV, default_options).error_banner + opts.banner << UserOptions.new(ARGV, default_options).error_banner + o = opts + end + puts(o.banner + "\n") + end + +end diff --git a/devops-client/lib/devops-client/options/network_options.rb b/devops-client/lib/devops-client/options/network_options.rb new file mode 100644 index 0000000..1a1a508 --- /dev/null +++ b/devops-client/lib/devops-client/options/network_options.rb @@ -0,0 +1,16 @@ +require "optparse" +require "devops-client/options/common_options" + +class NetworkOptions < CommonOptions + + commands :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.network") + self.banner_header = "network" + self.list_params = ["PROVIDER"] + end + +end + diff --git a/devops-client/lib/devops-client/options/project_options.rb b/devops-client/lib/devops-client/options/project_options.rb new file mode 100644 index 0000000..34fdcca --- /dev/null +++ b/devops-client/lib/devops-client/options/project_options.rb @@ -0,0 +1,119 @@ +require "devops-client/options/common_options" +require "set" + +class ProjectOptions < CommonOptions + + commands :create, :delete, :deploy, :list, {:multi => [:create]}, :servers, {:set => [:run_list]}, :show, :test, :update, {:user => [:add, :delete]} + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.project") + self.banner_header = "project" + id = "PROJECT_ID" + env = "DEPLOY_ENV" + self.show_params = [id] + self.create_params = [id] + self.delete_params = [id, "[#{env}]"] + self.deploy_params = [id, "[#{env}]"] + self.set_run_list_params = [id, env, "[(recipe[mycookbook::myrecipe])|(role[myrole]) ...]"] + self.servers_params = [id, "[#{env}]"] + self.multi_create_params = [id] + self.update_params = [id, "FILE"] + self.user_add_params = [id, "USER_NAME", "[USER_NAME ...]"] + self.user_delete_params = [id, "USER_NAME", "[USER_NAME ...]"] + self.test_params = [id, env] + end + + def create_options + self.options do |opts, options| + opts.banner << self.create_banner + options[:groups] = nil + opts.on("--groups GROUP_1,GROUP_2...", I18n.t("options.project.create.groups")) do |groups| + options[:groups] = groups.split(",") + end + + options[:identifier] = nil + opts.on("--deploy_env DEPLOY_ID", I18n.t("options.project.create.deploy_env")) do |identifier| + options[:identifier] = identifier + end + + options[:file] = nil + opts.on("-f", "--file FILE", I18n.t("options.project.create.file")) do |file| + abort("File '#{file}' does not exist") unless File.exist?(file) + options[:file] = file + end + + options[:subnets] = nil + opts.on("--subnets SUBNET,SUBNET...", I18n.t("options.project.create.subnets")) do |subnet| + options[:subnets] = subnet.split(",") + end + + options[:flavor] = nil + opts.on("--flavor FLAVOR", I18n.t("options.project.create.flavor")) do |flavor| + options[:flavor] = flavor + end + + options[:image] = nil + opts.on("--image IMAGE_ID", I18n.t("options.project.create.image")) do |image| + options[:image] = image + end + + options[:run_list] = nil + opts.on("--run_list RUN_LIST", I18n.t("options.project.create.run_list")) do |run_list| + options[:run_list] = run_list + end + + options[:users] = nil + opts.on("--users USER,USER...", I18n.t("options.project.create.users")) do |users| + options[:users] = Set.new(users.split(",")) + end + + options[:provider] = nil + opts.on("--provider PROVIDER", I18n.t("options.project.create.provider")) do |provider| + options[:provider] = provider + end + + options[:no_expires] = false + opts.on("--no_expires", I18n.t("options.project.create.no_expires")) do + options[:no_expires] = true + end + + options[:expires] = nil + opts.on("--expires EXPIRES", I18n.t("options.project.create.expires")) do |e| + options[:expires] = e + end + end + + end + + def user_add_options + self.options do |opts, options| + opts.banner << self.user_add_banner + options[:deploy_env] = nil + opts.on("--deploy_env ENV", I18n.t("options.project.user_add.deploy_env")) do |env| + options[:deploy_env] = env + end + end + end + + def user_delete_options + self.options do |opts, options| + opts.banner << self.user_delete_banner + options[:deploy_env] = nil + opts.on("--deploy_env ENV", I18n.t("options.project.user_delete.deploy_env")) do |env| + options[:deploy_env] = env + end + end + end + + def deploy_options + options do |opts, options| + opts.banner << self.deploy_banner + options[:servers] = nil + opts.on("--servers SERVERS", I18n.t("options.project.deploy.servers")) do |l| + options[:servers] = l.split(",") + end + end + end + +end diff --git a/devops-client/lib/devops-client/options/provider_options.rb b/devops-client/lib/devops-client/options/provider_options.rb new file mode 100644 index 0000000..72eb74b --- /dev/null +++ b/devops-client/lib/devops-client/options/provider_options.rb @@ -0,0 +1,15 @@ +require "optparse" +require "devops-client/options/common_options" + +class ProviderOptions < CommonOptions + + commands :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.provider") + self.banner_header = "provider" + self.list_params = [] + end + +end diff --git a/devops-client/lib/devops-client/options/script_options.rb b/devops-client/lib/devops-client/options/script_options.rb new file mode 100644 index 0000000..f1a36f9 --- /dev/null +++ b/devops-client/lib/devops-client/options/script_options.rb @@ -0,0 +1,27 @@ +require "devops-client/options/common_options" + +class ScriptOptions < CommonOptions + + commands :list, :add, :delete, :run, :command + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.script") + self.banner_header = "script" + sname = "SCRIPT_NAME" + self.add_params = [sname, "FILE"] + self.delete_params = [sname] + self.run_params = [sname, "NODE_NAME", "[NODE_NAME ...]"] + self.command_params = ["NODE_NAME", "'sh command'"] + end + + def run_options + options do |opts, options| + opts.banner << self.delete_banner + opts.on("--params PARAMS", I18n.t("options.script.run.params")) do |p| + options[:params] = p.split(",") + end + end + end + +end diff --git a/devops-client/lib/devops-client/options/server_options.rb b/devops-client/lib/devops-client/options/server_options.rb new file mode 100644 index 0000000..ee6ce09 --- /dev/null +++ b/devops-client/lib/devops-client/options/server_options.rb @@ -0,0 +1,101 @@ +require "devops-client/options/common_options" + +class ServerOptions < CommonOptions + + commands :add, :bootstrap, :create, :delete, :list, :pause, :show, :unpause # :sync, + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.server") + self.banner_header = "server" + self.list_params = ["[chef|ec2|openstack]"] + self.create_params = ["PROJECT_ID", "DEPLOY_ENV"] + node_params = ["NODE_NAME"] + self.delete_params = node_params + self.show_params = node_params + self.pause_params = node_params + self.unpause_params = node_params + self.bootstrap_params = ["INSTANCE_ID"] + self.add_params = ["PROJECT_ID", "DEPLOY_ENV", "IP", "SSH_USER", "KEY_ID"] + end + + def delete_options + options do |opts, options| + opts.banner << self.delete_banner + options[:key] = "node" + opts.on('--instance', "Delete node by instance id") do + options[:key] = "instance" + end + + options[:no_ask] = false + opts.on("--no_ask", "Don't ask for permission for server deletion") do + options[:no_ask] = true + end + end + end + + def create_options + options do |opts, options| + opts.banner << self.create_banner + opts.on('--without-bootstrap', "Run server without bootsraping phase") do + options[:without_bootstrap] = true + end + +=begin + opts.on('--public-ip', "Associate public IP with server") do + options[:public_ip] = true + end +=end + + opts.on("-N", "--name NAME", "Set node name") do |n| + options[:name] = n + end + + opts.on("-G", "--groups X,Y,Z", "The security groups for this server") do |g| + options[:groups] = g.split(",") + end + + opts.on("-f", "--force", "Cancel rollback operation on error") do |f| + options[:force] = true + end + + opts.on("--key KEY", "Use another key for the server") do |k| + options[:key] = k + end + + end + end + + def bootstrap_options + options do |opts, options| + opts.banner << self.bootstrap_banner + + opts.on("-N", "--name NAME", "Set chef name") do |n| + options[:name] = n + end + + opts.on("--bootstrap_template TEMPLATE", "Bootstrap template") do |template| + options[:bootstrap_template] = template + end + + opts.on("--run_list LIST", "Comma separated list like 'role[my_role],recipe[my_recipe]'") do |list| + options[:run_list] = list.split(",") + end + end + end + + def add_options + options do |opts, options| + opts.banner << self.add_banner + + opts.on('--public-ip PUBLIC_IP', "Specify public IP for the server") do |ip| + options[:public_ip] = ip + end + end + end + + def delete_banner + self.banner_header + " delete NODE_NAME [NODE_NAME ...]\n" + end + +end diff --git a/devops-client/lib/devops-client/options/tag_options.rb b/devops-client/lib/devops-client/options/tag_options.rb new file mode 100644 index 0000000..5494071 --- /dev/null +++ b/devops-client/lib/devops-client/options/tag_options.rb @@ -0,0 +1,14 @@ +require "devops-client/options/common_options" + +class TagOptions < CommonOptions + commands :create, :delete, :list + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.tag") + self.banner_header = "tag" + self.create_params = ["NODE_NAME", "TAG_NAME", "[TAG_NAME ...]"] + self.delete_params = ["NODE_NAME", "TAG_NAME", "[TAG_NAME ...]"] + self.list_params = ["NODE_NAME"] + end +end diff --git a/devops-client/lib/devops-client/options/user_options.rb b/devops-client/lib/devops-client/options/user_options.rb new file mode 100644 index 0000000..5b23460 --- /dev/null +++ b/devops-client/lib/devops-client/options/user_options.rb @@ -0,0 +1,29 @@ +require "devops-client/options/common_options" + +class UserOptions < CommonOptions + commands :create, :delete, :grant, :list, :password, :email + + def initialize args, def_options + super(args, def_options) + self.header = I18n.t("headers.user") + self.banner_header = "user" + self.create_params = ["USER_NAME", "EMAIL"] + self.delete_params = ["USER_NAME"] + self.password_params = ["USER_NAME"] + self.email_params = ["USER_NAME", "EMAIL"] + self.grant_params = ["USER_NAME", "[COMMAND]", "[PRIVILEGES]"] + end + + def create_options + self.options do |opts, options| + opts.banner << self.create_banner + + options[:new_password] = nil + opts.on("--password PASSWORD", "New user password") do |p| + options[:new_password] = p + end + + end + end + +end diff --git a/devops-client/lib/devops-client/output/base.rb b/devops-client/lib/devops-client/output/base.rb new file mode 100644 index 0000000..9b1fc61 --- /dev/null +++ b/devops-client/lib/devops-client/output/base.rb @@ -0,0 +1,41 @@ +require "terminal-table" +require "csv" +require "json" + +module Output + module Base + + def create_table headers, rows, title=nil, with_num=true, separator=false + return nil if headers.nil? or rows.nil? + if with_num + headers.unshift(I18n.t("output.table_header.number")) + rows.each_with_index {|row, i| row.unshift(i + 1)} + end + table = Terminal::Table.new do |t| + titles = ["#{I18n.t("output.table_header.api_version")}: #{self.options[:api]}", + "#{title}" + ] + t.title = titles.join( "\n" ) + t.headings = headers + t.add_row rows[0] + rows[1..-1].each do |r| + t.add_separator if separator + t.add_row r + end + end + table + end + + def create_csv headers, rows, with_num=true, separator=":" + if with_num + headers.unshift(I18n.t("output.table_header.number")) + rows.each_with_index {|row, i| row.unshift(i + 1)} + end + c = CSV.new("", {col_sep: separator, headers: true}) + c << headers + rows.each{|r| c << r} + c.string + end + + end +end diff --git a/devops-client/lib/devops-client/output/bootstrap_templates.rb b/devops-client/lib/devops-client/output/bootstrap_templates.rb new file mode 100644 index 0000000..712878e --- /dev/null +++ b/devops-client/lib/devops-client/output/bootstrap_templates.rb @@ -0,0 +1,34 @@ +require "devops-client/output/base" + +module Output + module BootstrapTemplates + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.bootstrap_template.list")) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list + abort I18n.t("output.not_found.bootstrap_template.list") if list.nil? or list.empty? + headers = [ I18n.t("output.table_header.name") ] + rows = [] + list.each do |l| + rows.push [ l ] + end + return headers, rows + end + + end +end + diff --git a/devops-client/lib/devops-client/output/filters.rb b/devops-client/lib/devops-client/output/filters.rb new file mode 100644 index 0000000..f39b9d6 --- /dev/null +++ b/devops-client/lib/devops-client/output/filters.rb @@ -0,0 +1,32 @@ +require "devops-client/output/base" + +module Output + module Filters + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.filter.list")) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list + abort(I18n.t("output.not_found.filter.list")) if list.nil? or list.empty? + headers = [ I18n.t("output.table_header.image_id") ] + rows = [] + list.each do |l| + rows.push [ l ] + end + return headers, rows + end + end +end diff --git a/devops-client/lib/devops-client/output/flavors.rb b/devops-client/lib/devops-client/output/flavors.rb new file mode 100644 index 0000000..00cc9a8 --- /dev/null +++ b/devops-client/lib/devops-client/output/flavors.rb @@ -0,0 +1,51 @@ +require "devops-client/output/base" + +module Output + module Flavors + include Base + + def table + headers, rows = create(@list, @provider) + create_table(headers, rows, I18n.t("output.title.flavor.list")) + end + + def csv + headers, rows = create(@list, @provider) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list, provider + abort(I18n.t("output.not_found.flavor.list")) if list.nil? or list.empty? + headers = nil + rows = [] + if provider == "openstack" + headers = [ + I18n.t("output.table_header.id"), + I18n.t("output.table_header.virtual_cpus"), + I18n.t("output.table_header.disk"), + I18n.t("output.table_header.ram") + ] + list.each do |l| + rows << [ l["id"], l["v_cpus"], l["disk"], l["ram"] ] + end + elsif provider == "ec2" + headers = [ + I18n.t("output.table_header.name"), + I18n.t("output.table_header.id"), + I18n.t("output.table_header.virtual_cpus"), + I18n.t("output.table_header.disk"), + I18n.t("output.table_header.ram") + ] + list.each do |l| + rows << [ l["name"], l["id"], l["cores"], l["disk"], l["ram"] ] + end + end + return headers, rows + end + end +end diff --git a/devops-client/lib/devops-client/output/groups.rb b/devops-client/lib/devops-client/output/groups.rb new file mode 100644 index 0000000..da393a6 --- /dev/null +++ b/devops-client/lib/devops-client/output/groups.rb @@ -0,0 +1,48 @@ +require "devops-client/output/base" + +module Output + module Groups + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.group.list"), true, true) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list + abort(I18n.t("output.not_found.group.list")) if list.nil? or list.empty? + headers = [ + I18n.t("output.table_header.name"), + I18n.t("output.table_header.protocol"), + I18n.t("output.table_header.from"), + I18n.t("output.table_header.to"), + I18n.t("output.table_header.cidr"), + I18n.t("output.table_header.description") + ] + rows = [] + list.each do |name, v| + next if v.nil? or v.empty? + p, f, t, c = [], [], [], [] + v["rules"].map do |l| + p.push l["protocol"] + f.push l["from"] + t.push l["to"] + c.push l["cidr"] + end + rows.push [ name, p.join("\n"), f.join("\n"), t.join("\n"), c.join("\n"), v["description"] ] + end + return headers, rows + end + + end +end diff --git a/devops-client/lib/devops-client/output/image.rb b/devops-client/lib/devops-client/output/image.rb new file mode 100644 index 0000000..27779d2 --- /dev/null +++ b/devops-client/lib/devops-client/output/image.rb @@ -0,0 +1,79 @@ +require "devops-client/output/base" + +module Output + module Image + include Base + + def table + title, headers, rows = nil, nil, nil + with_num = if !@list.nil? + title = I18n.t("output.title.image.list") + headers, rows = create_list(@list, @provider) + true + elsif !@show.nil? + title = I18n.t("output.title.image.show", :id => @show["id"]) + headers, rows = create_show @show + false + end + create_table headers, rows, title, with_num + end + + def csv + title, headers, rows = nil, nil, nil + with_num = if !@list.nil? + headers, rows = create_list(@list, @provider) + true + elsif !@show.nil? + headers, rows = create_show @show + false + end + create_csv headers, rows, with_num + end + + def json + JSON.pretty_generate( case ARGV[1] + when "list" + @list + when "show" + @show + end) + end + + private + def create_list list, provider + abort(I18n.t("output.not_found.image.list")) if list.empty? + rows = [] + headers = if provider + list.each {|l| rows.push [ l["name"], l["id"], l["status"] ]} + [ + I18n.t("output.table_header.name"), + I18n.t("output.table_header.id"), + I18n.t("output.table_header.status") + ] + else + list.each {|l| rows.push [ l["id"], l["name"], l["bootstrap_template"], l["remote_user"], l["provider"] ] } + [ + I18n.t("output.table_header.id"), + I18n.t("output.table_header.name"), + I18n.t("output.table_header.template"), + I18n.t("output.table_header.remote_user"), + I18n.t("output.table_header.provider") + ] + end + return headers, rows + end + + def create_show show + rows = [ [ show["id"], show["name"], show["bootstrap_template"], show["remote_user"], show["provider"] ] ] + headers = [ + I18n.t("output.table_header.id"), + I18n.t("output.table_header.name"), + I18n.t("output.table_header.template"), + I18n.t("output.table_header.remote_user"), + I18n.t("output.table_header.provider") + ] + return headers, rows + end + + end +end diff --git a/devops-client/lib/devops-client/output/key.rb b/devops-client/lib/devops-client/output/key.rb new file mode 100644 index 0000000..9865c55 --- /dev/null +++ b/devops-client/lib/devops-client/output/key.rb @@ -0,0 +1,33 @@ +require "devops-client/output/base" + +module Output + module Key + include Base + + def table + title = I18n.t("output.title.key.list") + headers, rows = create(@list) + create_table headers, rows, title + end + + def csv + headers, rows = create(@list) + create_csv headers, rows + end + + def json + JSON.pretty_generate( case ARGV[1] + when "list" + @list + end) + end + + private + def create list + abort(I18n.t("output.not_found.key.list")) if list.nil? or list.empty? + rows = [] + list.each {|l| rows.push [ l["id"], l["scope"] ] } + return [ I18n.t("output.table_header.id"), I18n.t("output.table_header.scope") ], rows + end + end +end diff --git a/devops-client/lib/devops-client/output/network.rb b/devops-client/lib/devops-client/output/network.rb new file mode 100644 index 0000000..7609391 --- /dev/null +++ b/devops-client/lib/devops-client/output/network.rb @@ -0,0 +1,50 @@ +require "devops-client/output/base" + +module Output + module Network + include Base + + def table + headers, rows = create(@list, @provider) + create_table(headers, rows, I18n.t("output.title.network.list")) + end + + def csv + headers, rows = create(@list, @provider) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list, provider + headers = nil + rows = [] + if provider == "openstack" + abort(I18n.t("output.not_found.network.list")) if list.nil? or list.empty? + headers = [ I18n.t("output.table_header.name"), I18n.t("output.table_header.cidr") ] + list.each do |l| + rows.push [ l["name"], l["cidr"] ] + end + elsif provider == "ec2" + if list.nil? or list.empty? + puts(I18n.t("output.not_found.network.list")) + return nil, nil + end + headers = [ + I18n.t("output.table_header.subnet"), + I18n.t("output.table_header.vpc_id"), + I18n.t("output.table_header.cidr"), + I18n.t("output.table_header.zone") + ] + list.each do |l| + rows.push [ l["subnetId"], l["vpcId"], l["cidr"], l["zone"] ] + end + end + return headers, rows + end + end +end + diff --git a/devops-client/lib/devops-client/output/project.rb b/devops-client/lib/devops-client/output/project.rb new file mode 100644 index 0000000..2acfb4c --- /dev/null +++ b/devops-client/lib/devops-client/output/project.rb @@ -0,0 +1,146 @@ +require "devops-client/output/base" + +module Output + module Project + include Base + + NODE_HEADER = "Node number" + SUBPROJECT_HEADER = "Subproject" + + def table + title, = nil + with_num, with_separator = true, false + headers, rows = if !@list.nil? + title = I18n.t("output.title.project.list") + create_list(@list) + elsif !@show.nil? + title = I18n.t("output.title.project.show", :name => @show["name"]) + with_num = false + with_separator = true + create_show(@show) + elsif !@servers.nil? + title = ARGV[2] + title += " " + ARGV[3] unless ARGV[3].nil? + title = I18n.t("output.title.project.servers", :title => title) + create_servers(@servers) + elsif !@test.nil? + with_num = false + title = I18n.t("output.title.project.test", :project => ARGV[2], :env => ARGV[3]) + create_test(@test) + end + create_table(headers, rows, title, with_num, with_separator) + end + + def csv + with_num = true + headers, rows = if !@list.nil? + create_list(@list) + elsif !@show.nil? + with_num = false + create_show(@show) + elsif !@servers.nil? + create_servers(@servers) + elsif !@test.nil? + with_num = false + create_test(@test) + end + create_csv(headers, rows, with_num) + end + + def json + JSON.pretty_generate(case ARGV[1] + when "list" + @list + when "show" + @show + when "servers" + @servers + when "test" + @test + end) + end + + private + def create_list list + abort(I18n.t("output.not_found.project.list")) if list.empty? + rows = list.map {|l| [l]} + headers = [ I18n.t("output.table_header.id") ] + return headers, rows + end + + def create_show show + rows = [] + headers = if show["type"] == "multi" + show["deploy_envs"].each do |de| + subprojects = [] + nodes = [] + de["servers"].each do |s| + s["subprojects"].each do |sp| + subprojects.push "#{sp["name"]} - #{sp["env"]}" + nodes.push sp["node"] + end + end + rows.push [ de["identifier"], subprojects.join("\n"), nodes.join("\n"), de["users"].join("\n") ] + end + [ + I18n.t("output.table_header.deploy_env"), + I18n.t("output.table_header.subproject") + " - " + I18n.t("output.table_header.deploy_env"), + I18n.t("output.table_header.node_number"), + I18n.t("output.table_header.users") + ] + else + show["deploy_envs"].each do |de| + rows.push [ show["name"], de["identifier"], de["image"], de["flavor"], de["run_list"].join("\n"), de["groups"].join("\n"), de["subnets"].join("\n"), de["users"].join("\n") ] + end + [ + I18n.t("output.table_header.id"), + I18n.t("output.table_header.deploy_env"), + I18n.t("output.table_header.image_id"), + I18n.t("output.table_header.flavor"), + I18n.t("output.table_header.run_list"), + I18n.t("output.table_header.groups"), + I18n.t("output.table_header.subnets"), + I18n.t("output.table_header.users") + ] + end + return headers, rows + end + + def create_servers servers + abort(I18n.t("output.not_found.project.servers")) if servers.empty? + rows = [] + servers.each do |s| + rows.push [ s["project"], s["deploy_env"], s["chef_node_name"], s["remote_user"], s["provider"], s["id"] ] + end + headers = [ + I18n.t("output.table_header.id"), + I18n.t("output.table_header.deploy_env"), + I18n.t("output.table_header.node_name"), + I18n.t("output.table_header.remote_user"), + I18n.t("output.table_header.provider"), + I18n.t("output.table_header.instance_id") + ] + return headers, rows + end + + def create_test test + rows = [] + headers = [ + I18n.t("output.table_header.server"), + I18n.t("output.table_header.node_name"), + I18n.t("output.table_header.creation"), + I18n.t("output.table_header.bootstrap"), + I18n.t("output.table_header.deletion") + ] + test["servers"].each do |s| + rows.push [ s["id"], + s["chef_node_name"], + "#{s["create"]["status"]}\n#{s["create"]["time"]}", + "#{s["bootstrap"]["status"]}\n#{s["bootstrap"]["time"]}", + "#{s["delete"]["status"]}\n#{s["delete"]["time"]}" ] + end + return headers, rows + end + + end +end diff --git a/devops-client/lib/devops-client/output/provider.rb b/devops-client/lib/devops-client/output/provider.rb new file mode 100644 index 0000000..bbbc9fb --- /dev/null +++ b/devops-client/lib/devops-client/output/provider.rb @@ -0,0 +1,32 @@ +require "devops-client/output/base" + +module Output + module Provider + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.provider.list")) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list + abort(I18n.t("output.not_found.provider.list")) if list.empty? + headers = [ I18n.t("output.table_header.provider") ] + rows = [] + list.each do |l| + rows.push [ l ] + end + return headers, rows + end + end +end diff --git a/devops-client/lib/devops-client/output/script.rb b/devops-client/lib/devops-client/output/script.rb new file mode 100644 index 0000000..8dc0669 --- /dev/null +++ b/devops-client/lib/devops-client/output/script.rb @@ -0,0 +1,35 @@ +require "devops-client/output/base" + +module Output + module Script + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.script.list")) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate( case ARGV[1] + when "list" + @list + end) + end + + private + def create list + rows = [] + abort(I18n.t("output.not_found.script.list")) if list.nil? or list.empty? + list.each do |l| + rows.push [ l ] + end + headers = [I18n.t("output.table_header.name")] + return headers, rows + end + end +end diff --git a/devops-client/lib/devops-client/output/server.rb b/devops-client/lib/devops-client/output/server.rb new file mode 100644 index 0000000..97b4691 --- /dev/null +++ b/devops-client/lib/devops-client/output/server.rb @@ -0,0 +1,121 @@ +require "devops-client/output/base" + +module Output + module Server + include Base + + def table + title = nil + headers, rows = if !@list.nil? + case options[:type] + when "chef" + title = I18n.t("output.title.server.chef") + when "openstack" + title = I18n.t("output.title.server.openstack") + when "ec2" + title = I18n.t("output.title.server.ec2") + else + title = I18n.t("output.title.server.list") + end + create_list(@list) + elsif !@show.nil? + title = I18n.t("output.title.server.show", :name => @show["chef_node_name"]) + create_show(@show) + end + create_table headers, rows, title + end + + def csv + headers, rows = if !@list.nil? + create_list(@list) + elsif !@show.nil? + create_show(@show) + end + create_csv headers, rows + end + + def json + JSON.pretty_generate(case ARGV[1] + when "list" + @list + when "show" + @show + end) + end + + private + def create_list list + abort(I18n.t("output.not_found.server.list")) if list.empty? + rows, keys = [], nil + headers = case options[:type] + when "chef" + keys = ["chef_node_name"] + title = "Chef servers" + [I18n.t("output.table_header.node_name")] + when "openstack" + keys = ["instance_id", "name", "public_ip", "private_ip", "keypair", "flavor", "image", "state"] + title = "Openstack servers" + [ + I18n.t("output.table_header.instance_id"), + I18n.t("output.table_header.node_name"), + I18n.t("output.table_header.public_ip"), + I18n.t("output.table_header.private_ip"), + I18n.t("output.table_header.keypair"), + I18n.t("output.table_header.flavor"), + I18n.t("output.table_header.image"), + I18n.t("output.table_header.state") + ] + when "ec2" + keys = ["instance_id", "name", "ip", "private_ip", "dns_name", "keypair", "flavor", "image", "zone", "state", "launched_at"] + title = "Ec2 servers" + [ + I18n.t("output.table_header.instance_id"), + I18n.t("output.table_header.node_name"), + I18n.t("output.table_header.public_ip"), + I18n.t("output.table_header.private_ip"), + I18n.t("output.table_header.dns"), + I18n.t("output.table_header.keypair"), + I18n.t("output.table_header.flavor"), + I18n.t("output.table_header.image"), + I18n.t("output.table_header.zone"), + I18n.t("output.table_header.state"), + I18n.t("output.table_header.created_at") + ] + else + keys = ["id", "chef_node_name"] + title = "Servers" + [ + I18n.t("output.table_header.instance_id"), + I18n.t("output.table_header.node_name") + ] + end + list.each do |l| + row = [] + keys.each{|k| row.push l[k]} + rows.push row + end + return headers, rows + end + + def create_show show + rows = [] + headers = [ + I18n.t("output.table_header.instance_id"), + I18n.t("output.table_header.node_name"), + I18n.t("output.table_header.project"), + I18n.t("output.table_header.deploy_env"), + I18n.t("output.table_header.provider"), + I18n.t("output.table_header.remote_user"), + I18n.t("output.table_header.private_ip"), + I18n.t("output.table_header.created_at"), + I18n.t("output.table_header.created_by") + ] + keys = ["id", "chef_node_name", "project", "deploy_env", "provider", "remote_user", "private_ip", "created_at", "created_by"] + row = [] + keys.each{|k| row.push show[k]} + rows.push row + return headers, rows + end + + end +end diff --git a/devops-client/lib/devops-client/output/tag.rb b/devops-client/lib/devops-client/output/tag.rb new file mode 100644 index 0000000..4c26fc5 --- /dev/null +++ b/devops-client/lib/devops-client/output/tag.rb @@ -0,0 +1,29 @@ +require "devops-client/output/base" + +module Output + module Tag + include Base + + def table + headers, rows = create(@list) + create_table(headers, rows, I18n.t("output.title.tag.list")) + end + + def csv + headers, rows = create(@list) + create_csv(headers, rows) + end + + def json + JSON.pretty_generate @list + end + + private + def create list + abort(I18n.t("output.not_found.tag.list")) if list.empty? + headers = [I18n.t("output.table_header.tag")] + rows = list.map {|l| [ l ]} + return headers, rows + end + end +end diff --git a/devops-client/lib/devops-client/output/user.rb b/devops-client/lib/devops-client/output/user.rb new file mode 100644 index 0000000..df9ebe2 --- /dev/null +++ b/devops-client/lib/devops-client/output/user.rb @@ -0,0 +1,79 @@ +require "devops-client/output/base" + +module Output + module User + include Base + + def table + title, headers = nil, nil + rows, with_num = create_subheader, false + rows += create_rows(@list) + headers = [ + "", + "", + "", + {:value => I18n.t("output.table_header.privileges"), :colspan => 12, :alignment => :center } + ] + + create_table headers, rows, I18n.t("output.title.user.list"), with_num, true + end + + def csv + rows = create_rows(@list) + headers = create_subheader + create_csv headers, rows + end + + def json + JSON.pretty_generate( case ARGV[1] + when "list" + @list + end) + end + + private + def create_subheader + [ [ + I18n.t("output.table_header.number"), + I18n.t("output.table_header.id"), + I18n.t("output.table_header.email"), + I18n.t("output.table_header.image"), + I18n.t("output.table_header.key"), + I18n.t("output.table_header.project"), + I18n.t("output.table_header.server"), + I18n.t("output.table_header.users"), + I18n.t("output.table_header.script"), + I18n.t("output.table_header.filter"), + I18n.t("output.table_header.flavor"), + I18n.t("output.table_header.group"), + I18n.t("output.table_header.network"), + I18n.t("output.table_header.provider"), + I18n.t("output.table_header.templates") + ] ] + end + + def create_rows list + abort(I18n.t("output.not_found.user.list")) if list.nil? or list.empty? + rows = [] + list.each_with_index do |l, i| + next if l["privileges"].nil? + + flavor = "#{l["privileges"]["flavor"]}" + group = "#{l["privileges"]["group"]}" + image = "#{l["privileges"]["image"]}" + project = "#{l["privileges"]["project"]}" + server = "#{l["privileges"]["server"]}" + key = "#{l["privileges"]["key"]}" + user = "#{l["privileges"]["user"]}" + filter = "#{l["privileges"]["filter"]}" + network = "#{l["privileges"]["network"]}" + provider = "#{l["privileges"]["provider"]}" + script = "#{l["privileges"]["script"]}" + templates = "#{l["privileges"]["templates"]}" + + rows.push [ (i + 1).to_s, l["id"], l["email"], image, key, project, server, user, script, filter, flavor, group, network, provider, templates] + end + rows + end + end +end diff --git a/devops-client/lib/devops-client/version.rb b/devops-client/lib/devops-client/version.rb new file mode 100644 index 0000000..d01f705 --- /dev/null +++ b/devops-client/lib/devops-client/version.rb @@ -0,0 +1,3 @@ +module DevopsClient + VERSION = "2.1.29" +end diff --git a/devops-client/lib/exceptions/devops_exception.rb b/devops-client/lib/exceptions/devops_exception.rb new file mode 100644 index 0000000..931991b --- /dev/null +++ b/devops-client/lib/exceptions/devops_exception.rb @@ -0,0 +1,4 @@ +class DevopsException < StandardError + +end + diff --git a/devops-client/lib/exceptions/invalid_query.rb b/devops-client/lib/exceptions/invalid_query.rb new file mode 100644 index 0000000..4dafc13 --- /dev/null +++ b/devops-client/lib/exceptions/invalid_query.rb @@ -0,0 +1,4 @@ +class InvalidQuery < StandardError + +end + diff --git a/devops-client/lib/exceptions/not_found.rb b/devops-client/lib/exceptions/not_found.rb new file mode 100644 index 0000000..0d88620 --- /dev/null +++ b/devops-client/lib/exceptions/not_found.rb @@ -0,0 +1,3 @@ +class NotFound < StandardError + +end diff --git a/devops-client/locales/en.yml b/devops-client/locales/en.yml new file mode 100644 index 0000000..936bf4c --- /dev/null +++ b/devops-client/locales/en.yml @@ -0,0 +1,268 @@ +en: + config: + invalid: + host: "Empty or invalid property 'host' in configuration file '%{file}'\n\nCheck your configuration '%{file}' file or use --host option" + empty: "Empty or undefined property '%{key}' in configuration file '%{file}'" + proxy_type: "Invalid 'proxy_type' property in configuration file '%{file}', available values: %{values}" + parameter: "Wrong config file parameter: -c file" + property: + lang: "Language (%{langs})" + host: "Devops server host and port" + api: "API version" + username: "Username" + password: "Password" + proxy_type: "Proxy type (none, system, custom)" + http_proxy: "HTTP proxy" + created: "Configuration file '%{file}' has been created" + not_exist: "File '%{file}' does not exist" + completion: + message: "Bash completion file has been copied to '%{file}'" + put: "Put '. %{file}' to your .bashrc file" + log: + info: "INFO: %{msg}" + warn: "WARN: %{msg}" + error: "ERROR: %{msg}" + headers: + flavor: "Flavor" + template: "Bootstrap template" + deploy: "Deploy" + filters: "Filters" + group: "Security groups" + image: "Image" + key: "Key" + network: "Network" + project: "Project" + project_env: "Project environment" + provider: "Provider" + server: "Server" + script: "Script" + tag: "Tag" + user: "User" + handler: + flavor: + list: + empty: "Flavors list is empty" + env: + list: + empty: "Project environment list is empty" + image: + question: + delete: "Are you sure to delete image '%{name}'?" + create: "Create image?" + create: + ssh_user: "The ssh username" + template: "Bootstrap template or empty value" + filter: + question: + delete: "Are you sure to delete image filter(s) '%{name}'?" + key: + question: + delete: "Are you sure to delete key '%{name}'?" + project: + list: + empty: "Project list is empty" + question: + delete: "Are you sure to delete project '%{name}'?" + create: "Create project?" + update: "Update project?" + add_env: "Add environment?" + parameter: + run_list: + empty: "WARN: run list is empty, continue?" + exist: "Project '%{project}' already exist" + create: + invalid_json: "Invalid JSON in file '%{file}'" + env: "Deploy environment identifier" + env_exist: "Deploy environment '%{env}' already exist" + flavor: + not_found: "No such flavor" + subnet: + ec2: "Subnet (or Enter)" + openstack: "Subnets" + user: "Users, you will be added automatically" + run_list: + invalid: "ERROR: invalid run list elements: %{list}" + invalid_subcommand: "Invalid subcommand for '%{cmd}': '%{scmd}'" + script: + question: + delete: "Are you sure to delete script '%{name}'?" + tag: + question: + delete: "Are you sure to delete tag(s) '%{name}' from node '%{node}'?" + user: + question: + delete: "Are you sure to delete user '%{name}'?" + password_for: "Password for %{user}:" + server: + question: + delete: "Are you sure to delete server '%{name}'?" + message: + choose_list_default: "Choose %{name} (comma separated), like 1,2,3 or empty for default value '%{default}': " + choose_list: "Choose %{name} (comma separated), like 1,2,3: " + choose: "Choose '%{name}': " + error: + parameter: + undefined: "ERROR: parameter '%{name}' is undefined" + unauthorized: "401 - Unauthorized" + file: + not_exist: "File '%{file}' does not exist" + number: + invalid: "Invalid number" + list: + empty: "%{name} list is empty" + output: + table_header: + api_version: "API version" + number: "Number" + id: "Id" + name: "Name" + provider: "Provider" + remote_user: "Remote user" + disk: "Disk" + virtual_cpus: "Virtual CPUs" + ram: "RAM" + image_id: "Image Id" + protocol: "Protocol" + from: "From" + to: "To" + cidr: "CIDR" + description: "Description" + status: "Status" + template: "Bootstrap template" + scope: "Scope" + subnet: "Subnet" + vpc_id: "VPC Id" + zone: "Zone" + deploy_env: "Environment" + node_name: "Node name" + image: "Image" + flavor: "Flavor" + group: "Group" + key: "Keys" + templates: "Templates" + run_list: "Run list" + groups: "Security groups" + subnets: "Subnets" + users: "Users" + server: "Server" + project: "Project" + script: "Script" + network: "Network" + filter: "Filter" + creation: "Creation" + bootatrap: "Bootstrap" + deletion: "Deletion" + instance_id: "Instance Id" + subproject: "Subproject" + node_number: "Node number" + tag: "Tag" + privileges: "Privileges" + email: "E-mail" + state: "State" + public_ip: "Public IP" + private_ip: "Private IP" + dns: "DNS" + keypair: "Keypair" + created_at: "Created at" + created_by: "Created by" + title: + flavor: + list: "Flavors" + bootstrap_template: + list: "Bootstrap templates" + filter: + list: "Filters" + group: + list: "Security groups" + image: + list: "Images" + show: "Image '%{id}' information" + key: + list: "Keys" + network: + list: "Networks" + project: + list: "Projects" + show: "Project '%{name}' information" + servers: "Project '%{title}' servers" + test: "Project test: %{project} - %{env}" + provider: + list: "Providers" + script: + list: "Scripts" + server: + list: "Servers" + chef: "Chef servers" + ec2: "Ec2 servers" + openstack: "Openstack servers" + show: "Server '%{name}'" + tag: + list: "Tags" + users: + list: "Users" + not_found: + flavor: + list: "No flavor found" + bootstrap_template: + list: "No bootstrap templates found" + filter: + list: "No filters found" + group: + list: "No security groups found" + image: + list: "No images found" + key: + list: "No keys found" + network: + list: "No networks found" + project: + list: "No project found" + servers: "No servers for project '%{name}' found" + provider: + list: "Empty providers list" + script: + list: "No scripts uploaded" + server: + list: "No servers found" + tag: + list: "No tags found" + user: + list: "No users found" + options: + usage: "Usage: %{cmd} command [options]" + commands: "Commands" + options: "Options" + common_options: "Common options" + common: + help: "Show help" + confirmation: "Answer 'yes' for all questions" + config: "Specify devops client config file (%{file})" + version: "devops client version" + host: "devops service host address (%{host})" + api: "devops service API version (%{api})" + username: "devops username (%{username})" + format: "Output format: '%{formats}' (%{format})" + completion: "Initialize bash completion script" + project: + header: "Project" + create: + groups: "Security groups (comma separated list)" + deploy_env: "Deploy environment identifier" + file: "File in JSON with project settings" + subnets: "Subnets identifiers for deploy environment (ec2 - only one subnet, openstack - comma separated list)" + flavor: "Specify flavor for the project environment" + image: "Specify image identifier for the project environment" + run_list: "Additional recipes and roles (comma separated), like recipe[mycookbook::myrecipe],role[myrole]" + users: "Users list (comma separated) for deploy environment control" + provider: "Provider - 'ec2' or 'openstack'" + no_expires: "Without expiry time" + expires: "Expiry time (5m, 3h, 2d, 1w, etc)" + user_add: + deploy_env: "Add user to deploy environment" + user_delete: + deploy_env: "Delete user from deploy environment" + deploy: + servers: "Servers list (comma separated) for deploy" + script: + run: + params: "Script arguments (comma separated list)" diff --git a/devops-client/locales/ru.yml b/devops-client/locales/ru.yml new file mode 100644 index 0000000..e798698 --- /dev/null +++ b/devops-client/locales/ru.yml @@ -0,0 +1,268 @@ +ru: + config: + invalid: + host: "Пустой или неверный параметр 'host' в конфигурационном файле '%{file}'\n\nПроверьте конфигурационный файл '%{file}' или используйте опцию --host" + empty: "Параметр '%{key}' в конфигурационном файле '%{file}' пуст или не определен" + proxy_type: "Неверный параметр 'proxy_type' в конфигурационном файле '%{file}', допустимые значения: %{values}" + parameter: "Неверный параметр конфигурационного файла: -c file" + property: + lang: "Язык (%{langs})" + host: "Адрес и порт сервера Devops" + api: "Версия API" + username: "Имя пользователя" + password: "Пароль" + proxy_type: "Тип прокси (none, system, custom)" + http_proxy: "HTTP прокси" + created: "Конфигурационный файл '%{file}' создан" + not_exist: "Файл '%{file}' не существует" + completion: + message: "Файл дополнений bash скопирован в '%{file}'" + put: "Добавьте '. %{file}' в Ваш .bashrc файл" + log: + info: "Информация: %{msg}" + warn: "Предупреждение: %{msg}" + error: "Ошибка: %{msg}" + headers: + flavor: "Конфигурация" + deploy: "Деплой" + template: "Шаблон загрузки" + filters: "Фильтры" + group: "Группы безопасности" + image: "Образ" + key: "Ключ" + network: "Сеть" + project: "Проект" + project_env: "Окружение проект" + provider: "Провайдер" + server: "Сервер" + script: "Скрипт" + tag: "Тег" + user: "Пользователь" + handler: + flavor: + list: + empty: "Список конфигураций пуст" + image: + question: + delete: "Вы уверены, что хотите удалить образ '%{name}'?" + create: "Создать образ?" + create: + ssh_user: "Имя пользоватлея ssh" + template: "Шаблон загрузки или пустое значение" + filter: + question: + delete: "Вы уверены, что хотите удалить фильтр(ы) образа '%{name}'?" + key: + question: + delete: "Вы уверены, что хотите удалить ключ '%{name}'?" + project: + list: + empty: "Список проектов пуст" + env: + list: + empty: "Список окружений проекта пуст" + question: + delete: "Вы уверены, что хотите удалить проект '%{name}'?" + create: "Создать проект?" + update: "Обновить проект?" + add_env: "Добавить окружение?" + parameter: + run_list: + empty: "Предупреждение: список запуска пуст, продолжить?" + exist: "Проект '%{project}' уже существует" + create: + invalid_json: "Неверный JSON в файле '%{file}'" + env: "Идентификатор окружения" + env_exist: "Окружение '%{env}' уже существует" + flavor: + not_found: "Конфигурация не надена" + subnet: + ec2: "Подсеть (или Enter)" + openstack: "Подсети" + user: "Пользователи, Вы будете добалены автоматически" + run_list: + invalid: "ОШИБКА: неверные элементы run_list: %{list}" + invalid_subcommand: "Неверная подкомманда для '%{cmd}': '%{scmd}'" + script: + question: + delete: "Вы уверены, что хотите удалить скрипт '%{name}'?" + tag: + question: + delete: "Вы уверены, что хотите удалить тэг(и) '%{name}' из ноды '%{node}'?" + user: + question: + delete: "Вы уверены, что хотите удалить пользователя '%{name}'?" + password_for: "Пароль для %{user}:" + server: + question: + delete: "Вы уверены, что хотите удалить сервер '%{name}'?" + message: + choose_list_default: "Выберите значения из списка (разделенные запятой), например 1,2,3 или оставте пустым для значеия по умолчанию '%{default}': " + choose_list: "Выберите значения из списка (разделенные запятой), например 1,2,3: " + choose: "Выберите '%{name}': " + error: + parameter: + undefined: "Ошибка: параметр '%{name}' не указан" + unauthorized: "401 - Не авторизован" + file: + not_exist: "Файл '%{file}' не существует" + number: + invalid: "Неверный номер" + list: + empty: "%{name} список пуст" + output: + table_header: + api_version: "Версия API" + number: "Номер" + id: "Id" + name: "Имя" + provider: "Провайдер" + remote_user: "Пользователь" + disk: "Диск" + virtual_cpus: "Виртуальные ЦПУ" + ram: "RAM" + image_id: "Id образа" + protocol: "Протокол" + from: "Из" + to: "В" + cidr: "CIDR" + description: "Описание" + status: "Статус" + template: "Шаблон" + scope: "Область" + subnet: "Подсеть" + vpc_id: "VPC Id" + zone: "Зона" + deploy_env: "Окружение" + node_name: "Имя ноды" + image: "Образ" + flavor: "Конфигурация" + group: "Группы" + key: "Ключи" + templates: "Шаблоны" + run_list: "Список запуска" + groups: "Группы безоп." + subnets: "Подсети" + users: "Пользователи" + server: "Сервер" + project: "Проект" + script: "Скрипт" + network: "Сеть" + filter: "Филтер" + creation: "Создание" + bootatrap: "Бутстрап" + deletion: "Удаление" + instance_id: "Id объекта" + subproject: "Подпроект" + node_number: "Номер ноды" + tag: "Тэг" + privileges: "Привилегии" + email: "E-mail" + state: "Состояние" + public_ip: "Публичный IP" + private_ip: "Приватный IP" + dns: "DNS" + keypair: "Ключ" + created_at: "Создан" + created_by: "Создал" + title: + flavor: + list: "Конфигурации" + bootstrap_template: + list: "Шаблоны" + filter: + list: "Фильтры" + group: + list: "Группы безопасности" + image: + list: "Образы" + show: "Информация об образе '%{id}'" + key: + list: "Ключи" + network: + list: "Сети" + project: + list: "Проекты" + show: "Информация о проекте '%{name}'" + servers: "Сервера проекта '%{title}'" + test: "Тестирование проекта: %{project} - %{env}" + provider: + list: "Провайдеры" + script: + list: "Скрипты" + server: + list: "Сервера" + chef: "Сервера Chef" + ec2: "Сервера Ec2" + openstack: "Сервера Openstack" + show: "Сервер '%{name}'" + tag: + list: "Теги" + users: + list: "Пользователи" + not_found: + flavor: + list: "Ни одной конфигурации не найдено" + bootstrap_template: + list: "Ни одного шаблона не найдено" + filter: + list: "Ни одного фильтра не найдено" + group: + list: "Ни одной группы безопасности не найдено" + image: + list: "Ни одного образа не найдено" + key: + list: "Ни одного ключа не найдено" + network: + list: "Ни одной сети не найдено" + project: + list: "Ни одного проекта не найдено" + servers: "Ни одного сервера для проекта '%{name}' не найдено" + provider: + list: "Ни одного провайдера не найдено" + script: + list: "Не загружено ни одного скрипта" + server: + list: "Ни одного сервера не найдено" + tag: + list: "Ни одного тега не найдено" + user: + list: "Ни одного пользователя не найдено" + options: + usage: "Использование: %{cmd} команда [опции]" + commands: "Команды" + options: "Опции" + common_options: "Общие опции" + common: + help: "Показать помощь" + confirmation: "Ответить 'да' на все вопросы" + config: "Указать конфигурационный файл для devops клиента (%{file})" + version: "Версия devops клиента" + host: "Адрес хоста devops сервера (%{host})" + api: "Версия API devops сервера (%{api})" + username: "Имя пользователя devops (%{username})" + format: "Формат вывода: '%{formats}' (%{format})" + completion: "Инициализировать bash скрипт автодополнений" + project: + header: "Проект" + create: + groups: "Группы безопасности (список разделенный запятой)" + deploy_env: "Идентификатор окружения" + file: "Файл в формате JSON, содержащий настройки проекта" + subnets: "Идентификаторы подсетей для окружения (ec2 - только одна подсеть, openstack - список разделенный запятой)" + flavor: "Указать flavor для окружения проекта" + image: "Указать идентификатор образа для окружения проекта" + run_list: "Дополнительные рецепты и роли (разделенный запятой), например recipe[mycookbook::myrecipe],role[myrole]" + users: "Список пользователей (разделенный запятой) для управления окружением проекта" + provider: "Провайдер - 'ec2' или 'openstack'" + no_expires: "Без времени жизни" + expires: "Время жизни (5m, 3h, 2d, 1w, etc)" + user_add: + deploy_env: "Добавить пользователя к окружению проекта" + user_delete: + deploy_env: "Удалить пользователя из окружения проекта" + deploy: + servers: "Список серверов (разделенный запятой) для деплоя" + script: + run: + params: "Аргументы скрипта (список разделенный запятой)" diff --git a/devops-client/tests/base_test.rb b/devops-client/tests/base_test.rb new file mode 100644 index 0000000..cad3244 --- /dev/null +++ b/devops-client/tests/base_test.rb @@ -0,0 +1,104 @@ +require "json" +class BaseTest + + COMMAND = "devops" + SUCCESS = "\e[32msuccess\e[0m" + FAILED = "\e[31mfailed\e[0m" + CONFIGS = ["./devops-client-test.conf"] + #, "-c ./test_conf1.conf"] + + TITLE_SEPARATOR = "-" * 80 + END_SEPARATOR = "*" * 80 + "\n" + + def title= title + @title = title + end + + def run_tests cmds, check=true + puts + puts @title + puts TITLE_SEPARATOR + cmds.each do |cmd| + command = create_cmd(cmd) + s = `#{command}` + if check + if $?.success? + print SUCCESS + else + print FAILED + puts_error s + exit(1) + end + end + puts + end + puts END_SEPARATOR + end + + def run_test_with_block cmd + puts + puts @title + puts TITLE_SEPARATOR + command = create_cmd(cmd) + s = `#{command}` + if $?.success? + puts SUCCESS + if block_given? + print "Validation block...\t" + res = yield s + if res + puts SUCCESS + else + puts FAILED + puts_error("Validation block returns 'false'") + exit(-1) + end + end + else + puts FAILED + puts_error s + exit(1) + end + puts END_SEPARATOR + end + + def run_tests_invalid cmds + puts + puts @title + puts TITLE_SEPARATOR + cmds.each do |cmd| + command = create_cmd(cmd) + s = `#{command}` + if $?.success? + puts FAILED + exit(1) + else + puts SUCCESS + end + end + puts END_SEPARATOR + end + + def puts_error str + puts "\e[31m#{str}\e[0m" + end + + def puts_warn str + puts "\e[33m#{str}\e[0m" + end + + def config= conf + @config = conf + end + + def create_cmd cmd + command = if @config.nil? + "#{COMMAND} #{cmd}" + else + "#{COMMAND} -c #{@config} #{cmd}" + end + print "#{command}...\t" + command + end + +end diff --git a/devops-client/tests/create_server.rb b/devops-client/tests/create_server.rb new file mode 100644 index 0000000..8ca545f --- /dev/null +++ b/devops-client/tests/create_server.rb @@ -0,0 +1,284 @@ +require "./base_test" +require "json" + +class CreateServer < BaseTest + TITLE = "Create server tests. " + + def run + openstack = { + :name => "openstack", + :image => "36dc7618-4178-4e29-be43-286fbfe90f50", + :flavor => "m1.small", + :ssh_user => "root", + :server_name => "test_create_server_openstack", + :states => {:pause => "PAUSED", :unpause => "ACTIVE"} + } + ec2 = { + :name => "ec2", + :image => "ami-83e4bcea", + :flavor => "m1.small", + :ssh_user => "ec2-user", + :server_name => "test_create_server_ec2", + :states => {:pause => "stopped", :unpause => "running"} + } + + project = { + "name" => "project_test", + "deploy_envs" => [ + { + "flavor" => openstack[:flavor], + "groups" => [ + "default" + ], + "identifier" => "openstack", + "image" => openstack[:image], + "provider" => "openstack", + "run_list" => [ + "role[project_test_openstack]" + ], + "subnets" => [ "private" ], + "users" => [ "user_for_testing" ], + "expires" => nil + }, + { + "flavor" => ec2[:flavor], + "groups" => [ + "default" + ], + "identifier" => "ec2", + "image" => ec2[:image], + "provider" => "ec2", + "run_list" => [ + "role[project_test_ec2]" + ], + "subnets" => [], + "users" => [ "user_for_testing" ], + "expires" => nil + } + + ] + } + + self.config = CONFIGS[0] + + prepare openstack + prepare ec2 + + env_os = project["deploy_envs"][0] + env_ec2 = project["deploy_envs"][1] + self.title = TITLE + "Create project '#{project["name"]}'" + run_tests [ + "project create #{project["name"]} --groups #{env_os["groups"].join(",")} --deploy_env #{env_os["identifier"]} --subnets #{env_os["subnets"].join(",")} --flavor #{env_os["flavor"]} --image #{env_os["image"]} --run_list role[#{project["name"]}_#{env_os["identifier"]}] --users #{env_os["users"].join(",")} --provider openstack -y --no_expires", + "project create #{project["name"]} --groups #{env_ec2["groups"].join(",")} --deploy_env #{env_ec2["identifier"]} --flavor #{env_ec2["flavor"]} --image #{env_ec2["image"]} --run_list role[#{project["name"]}_#{env_ec2["identifier"]}] --users #{env_ec2["users"].join(",")} --provider ec2 -y --no_expires" + ] + + self.title = TITLE + "Project list" + run_test_with_block "project list --format json" do |l| + projects = JSON.parse(l) + projects.include? project["name"] + end + + self.title = TITLE + "Show project '#{project["name"]}'" + run_test_with_block "project show #{project["name"]} --format json" do |p| + pr = JSON.parse(p) + name = (project["name"] == pr["name"]) + envs = (project["deploy_envs"].size == pr["deploy_envs"].size) + o = pr["deploy_envs"].detect{|e| e["identifier"] == "openstack"} + po = project["deploy_envs"][0] + e = pr["deploy_envs"].detect{|e| e["identifier"] == "ec2"} + pe = project["deploy_envs"][1] + unless name + puts "Project name is not a '#{project["name"]}'" + end + unless envs + puts "Project environments not equals #{project["deploy_envs"].size}" + end + name and envs and check_envs(po, o) and check_envs(pe, e) + end + + self.title = TITLE + "Add user 'root' to project '#{project["name"]}'" + run_tests [ "project user add #{project["name"]} root" ] + + self.title = TITLE + "Show project '#{project["name"]}' with user 'root'" + run_test_with_block "project show #{project["name"]} --format json" do |p| + pr = JSON.parse(p) + envs = true + pr["deploy_envs"].each {|e| envs = (envs and e["users"].include?("root"))} + envs + end + + self.title = TITLE + "Delete user 'root' from project '#{project["name"]}'" + run_tests [ "project user delete #{project["name"]} root -y" ] + + self.title = TITLE + "Show project '#{project["name"]}' without user 'root'" + run_test_with_block "project show #{project["name"]} --format json" do |p| + pr = JSON.parse(p) + envs = true + pr["deploy_envs"].each {|e| envs = (envs and !e["users"].include?("root"))} + envs + end + + create_server project["name"], env_os["identifier"], openstack + create_server project["name"], env_ec2["identifier"], ec2 + + self.title = TITLE + "Delete project '#{project["name"]}'" + run_tests [ "project delete #{project["name"]} -y" ] + + clear openstack + clear ec2 + + end + + def check_envs origin, created + r = true + %w(flavor groups identifier image provider run_list subnets users expires).each do |key| + flag = (origin[key] == created[key]) + unless flag + puts "Environments params '#{key}' not equals ('#{origin[key].inspect}' and '#{created[key].inspect}')" + end + r = r and flag + end + r + end + + def prepare conf + name = conf[:name] + self.title = TITLE + "Check #{name} flavor" + run_test_with_block "flavor list #{name} --format json" do |f| + flavors = JSON.parse(f) + !flavors.detect{|o| o["id"] == conf[:flavor]}.nil? + end + + image_in_filter = false + self.title = TITLE + "Check #{name} filter" + run_test_with_block "filter image list #{name} --format json" do |i| + images = JSON.parse(i) + image_in_filter = !images.index(conf[:image]).nil? + true + end + + if image_in_filter + puts_warn "Image '#{conf[:image]}' for '#{name}' already in filter" + else + self.title = TITLE + "Add #{name} filter" + run_tests [ "filter image add #{name} #{conf[:image]}" ] + end + + image_created = false + self.title = TITLE + "Check image for #{name}" + run_test_with_block "image list #{name} --format json" do |s| + images = JSON.parse s + image_created = !images.detect{|i| i["id"] == conf[:image]}.nil? + true + end + + if image_created + puts_warn "Image '#{conf[:image]}' for '#{name}' already created" + else + self.title = TITLE + "Create image for #{name}" + run_tests [ "image create --image #{conf[:image]} --ssh_user #{conf[:ssh_user]} --provider #{name} --no_bootstrap_template -y" ] + end + + end + + def create_server project, env, conf + + self.title = TITLE + "Create server '#{conf[:server_name]}'" + run_tests [ "server create #{project} #{env} -N #{conf[:server_name]}" ] + + self.title = TITLE + "Is server '#{conf[:server_name]}' created" + run_test_with_block "server list --format json" do |l| + servers = JSON.parse l + !servers.detect{|s| s["chef_node_name"] == conf[:server_name].to_s }.nil? + end + + self.title = TITLE + "Pause server '#{conf[:server_name]}'" + run_tests [ "server pause #{conf[:server_name]}" ] + delay = (conf[:name] == "openstack" ? 5 : 90) + puts "Sleeping for #{delay} seconds" + sleep(delay) + + self.title = TITLE + "Check server '#{conf[:server_name]}' state" + run_test_with_block "server list #{conf[:name]} --format json" do |s| + servers = JSON.parse s + state = servers.detect{|o| o["name"] == conf[:server_name]}["state"] + if state == conf[:states][:pause] + true + else + puts_error "State should be '#{conf[:states][:pause]}' but it is '#{state}'" + false + end + end + + self.title = TITLE + "Unpause server '#{conf[:server_name]}'" + run_tests [ "server unpause #{conf[:server_name]}" ] + delay = (conf[:name] == "openstack" ? 5 : 90) + puts "Sleeping for #{delay} seconds" + sleep(delay) + + self.title = TITLE + "Check server '#{conf[:server_name]}' state" + run_test_with_block "server list #{conf[:name]} --format json" do |s| + servers = JSON.parse s + state = servers.detect{|o| o["name"] == conf[:server_name]}["state"] + if state == conf[:states][:unpause] + true + else + puts_error "State should be '#{conf[:states][:unpause]}' but it is '#{state}'" + false + end + end + + tag = "tag_" + conf[:name] + self.title = TITLE + "Add tag '#{tag}' to server '#{conf[:server_name]}'" + run_tests [ + "tag create #{conf[:server_name]} #{tag}", + "tag create #{conf[:server_name]} #{tag}" + ] + self.title = TITLE + "Check tag '#{tag}' for server '#{conf[:server_name]}'" + run_test_with_block "tag list #{conf[:server_name]} --format json" do |t| + JSON.parse(t).include?(tag) + end + + tag2 = tag + "_2" + self.title = TITLE + "Check deploy with tag '#{tag2}' for server '#{conf[:server_name]}'" + run_tests ["deploy #{conf[:server_name]} -t #{tag2}"] + + self.title = TITLE + "Check tag '#{tag}' for server '#{conf[:server_name]}'" + run_test_with_block "tag list #{conf[:server_name]} --format json" do |t| + JSON.parse(t).include?(tag) + !JSON.parse(t).include?(tag2) + end + + self.title = TITLE + "Delete tag '#{tag}' from server '#{conf[:server_name]}'" + run_tests [ + "tag delete #{conf[:server_name]} #{tag} -y", + "tag delete #{conf[:server_name]} #{tag} -y" + ] + + self.title = TITLE + "Delete server '#{conf[:server_name]}'" + run_tests [ "server delete #{conf[:server_name]} -y" ] + + end + + def clear conf + name = conf[:name] + self.title = TITLE + "Delete image for #{name}" + run_tests [ "image delete #{conf[:image]} -y" ] + + self.title = TITLE + "Check image for #{name}" + run_test_with_block "image list #{name} --format json" do |s| + images = JSON.parse s + images.detect{|i| i["id"] == conf[:image]}.nil? + end + + self.title = TITLE + "Delete #{name} filter" + run_tests [ "filter image delete #{name} #{conf[:image]} -y" ] + + self.title = TITLE + "Check #{name} filter" + run_test_with_block "filter image list #{name} --format json" do |i| + images = JSON.parse(i) + images.index(conf[:image]).nil? + end + end + +end diff --git a/devops-client/tests/devops-client-test.conf b/devops-client/tests/devops-client-test.conf new file mode 100644 index 0000000..ef4714d --- /dev/null +++ b/devops-client/tests/devops-client-test.conf @@ -0,0 +1,5 @@ +api=v2.0 +host= +username= +password= + diff --git a/devops-client/tests/flavor.rb b/devops-client/tests/flavor.rb new file mode 100644 index 0000000..1220dcd --- /dev/null +++ b/devops-client/tests/flavor.rb @@ -0,0 +1,21 @@ +require "./base_test" + +class Flavor < BaseTest + TITLE = "Flavor tests" + + def run + self.title = TITLE + run_tests [ + "flavor list ec2", + "flavor list openstack", + "flavor list ec2 --format json", + "flavor list openstack --format json" + ] + self.title = TITLE + " invalid " + run_tests_invalid [ + "flavor list", + "flavor" + ] + end + +end diff --git a/devops-client/tests/group.rb b/devops-client/tests/group.rb new file mode 100644 index 0000000..b08bfd9 --- /dev/null +++ b/devops-client/tests/group.rb @@ -0,0 +1,20 @@ +class Group < BaseTest + TITLE = "Group tests" + + def run + self.title = TITLE + run_tests [ + "group list ec2", + "group list openstack", + "group list ec2 --format json", + "group list openstack --format json" + ] + self.title = TITLE + " invalid " + run_tests_invalid [ + "group list", + "group" + ] + end + +end + diff --git a/devops-client/tests/image.rb b/devops-client/tests/image.rb new file mode 100644 index 0000000..c5d1acf --- /dev/null +++ b/devops-client/tests/image.rb @@ -0,0 +1,25 @@ +require "./base_test" +class Image < BaseTest + TITLE = "Image tests" + + def run + self.title = TITLE + image_o = { + :provider => "openstack", + :id => "89ecfe3f-9f25-4982-a0cf-b9b3814c02d6" + } + run_tests [ "image list", + "image list ec2", + "image list openstack", + "image list provider ec2", + "image list provider openstack" + # "image create --image RHEL-6.4_GA-x86_64-7-Hourly2 --ssh_user root --no_bootstrap_template -y --provider ec2", + # "image create --image ubuntu-12.04-qcow-amd64 --ssh_user root --no_bootstrap_template -y --provider openstack", + # "image show cirros", + # "image update cirros ./image_update_test_file", + # "image delete cirros" + ] + end +end + +# test/test diff --git a/devops-client/tests/image_update_test_file b/devops-client/tests/image_update_test_file new file mode 100644 index 0000000..17e775a --- /dev/null +++ b/devops-client/tests/image_update_test_file @@ -0,0 +1,6 @@ +{ + "provider": "openstack", + "image_id": "89ecfe3f-9f25-4982-a0cf-b9b3814c02d6", + "remote_user": "root", + "id": "cirros" +} \ No newline at end of file diff --git a/devops-client/tests/key.rb b/devops-client/tests/key.rb new file mode 100644 index 0000000..fd3c010 --- /dev/null +++ b/devops-client/tests/key.rb @@ -0,0 +1,44 @@ +require "./base_test" + +class Key < BaseTest + TITLE = "Key tests - " + + def run + self.title = TITLE + run_tests [ + "key list" + ] + + key = "test_key" + self.title = TITLE + "add" + run_tests [ + "key add #{key} key_file" + ] + self.title = TITLE + "add, invalid" + run_tests_invalid [ + "key add #{key} key_file" + ] + self.title = TITLE + "check" + run_test_with_block "key list --format json" do |k| + !JSON.parse(k).detect{|jk| jk["id"] == key and jk["scope"] == "user"}.nil? + end + + self.title = TITLE + "delete" + run_tests [ + "key delete #{key} -y" + ] + self.title = TITLE + "delete, invalid" + run_tests_invalid [ + "key delete #{key} -y" + ] + self.title = TITLE + "invalid" + run_tests_invalid [ + "key", + "key add", + "key add #{key}", + "key delete" + ] + end + +end + diff --git a/devops-client/tests/key_file b/devops-client/tests/key_file new file mode 100644 index 0000000..9e47d10 --- /dev/null +++ b/devops-client/tests/key_file @@ -0,0 +1 @@ +this is the test file diff --git a/devops-client/tests/network.rb b/devops-client/tests/network.rb new file mode 100644 index 0000000..3637b95 --- /dev/null +++ b/devops-client/tests/network.rb @@ -0,0 +1,23 @@ +require "./base_test" + +class Network < BaseTest + TITLE = "Network tests" + + def run + self.title = TITLE + run_tests [ + "network list openstack", + "network list ec2", + "network list openstack --format json", + "network list ec2 --format json" + ] + self.title = TITLE + " invalid " + run_tests_invalid [ + "network list", + "network" + ] + end + +end + + diff --git a/devops-client/tests/output.rb b/devops-client/tests/output.rb new file mode 100644 index 0000000..b82c224 --- /dev/null +++ b/devops-client/tests/output.rb @@ -0,0 +1,35 @@ +require "./base_test" + +class Output < BaseTest + TITLE = "Output tests" + + def run + tests = { + :server => ["list"], + :flavor => ["list ec2", "list openstack"], + :network => ["list ec2", "list openstack"], + :group => ["list ec2", "list openstack"], + :templates => ["list"], + :provider => ["list"], + :filter => ["image list ec2", "image list openstack"], + :image => ["list", "list provider", "list provider ec2", "list provider openstack"], + :key => ["list"], + :project => ["list"], + :script => ["list"], + :server => ["list"], + :tag => ["list"], + :user => ["list"] + } + ["table", "json", "csv"].each do |f| + self.title = TITLE + ", format '#{f}'" + c = [] + tests.each do |k,v| + v.each do |cmd| + c.push "#{k} #{cmd}" + end + end + run_tests c, false + end + end + +end diff --git a/devops-client/tests/project.rb b/devops-client/tests/project.rb new file mode 100644 index 0000000..140832b --- /dev/null +++ b/devops-client/tests/project.rb @@ -0,0 +1,16 @@ +require "./base_test" + +class Project < BaseTest + TITLE = "Project tests" + + def run + self.title = TITLE + run_tests ["project list"] + run_tests ["project create endtest --groups default --deploy_env dev --flavor c1.small --image cirros --run_list role[devops_service_dev] -y"] + run_tests ["project show test"] + run_tests ["project servers test"] + run_tests ["project set run_list test dev role[devops_service_dev]"] + run_tests ["project update test project_update_test_file"] + run_tests ["project delete test"] + end +end diff --git a/devops-client/tests/provider.rb b/devops-client/tests/provider.rb new file mode 100644 index 0000000..c2ea908 --- /dev/null +++ b/devops-client/tests/provider.rb @@ -0,0 +1,19 @@ +require "./base_test" + +class Provider < BaseTest + TITLE = "Provider tests" + + def run + self.title = TITLE + run_tests [ + "provider list", + "provider list --format json" + ] + self.title = TITLE + " invalid " + run_tests_invalid [ + "provider" + ] + end + +end + diff --git a/devops-client/tests/run_tests.rb b/devops-client/tests/run_tests.rb new file mode 100644 index 0000000..02fcb27 --- /dev/null +++ b/devops-client/tests/run_tests.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby + +dir = File.dirname(__FILE__) +tests = nil +if ARGV.empty? + tests = ["flavor", "group", "network", "provider", "user", "key", "script", "image", "server", "project", "create_server"] +else + tests = ARGV +end + +classes = [] +tests.each do |f| + require "#{dir}/#{f}.rb" + case f + when "flavor" + classes.push Flavor.new + when "group" + classes.push Group.new + when "network" + classes.push Network.new + when "provider" + classes.push Provider.new + when "user" + classes.push User.new + when "key" + classes.push Key.new + when "script" + classes.push Script.new + when "image" + classes.push Image.new + when "project" + classes.push Project.new + when "server" + classes.push Server.new + when "output" + classes.push Output.new + when "create_server" + classes.push CreateServer.new + end +end + +classes.each do |c| + c.run +end diff --git a/devops-client/tests/script.rb b/devops-client/tests/script.rb new file mode 100644 index 0000000..16f628e --- /dev/null +++ b/devops-client/tests/script.rb @@ -0,0 +1,47 @@ +require "./base_test" + +class Script < BaseTest + TITLE = "Script tests - " + + def run + self.title = TITLE + run_tests [ + "script list" + ] + + script = "test_script" + self.title = TITLE + "add" + run_tests [ + "script add #{script} script_file.sh" + ] + self.title = TITLE + "add, invalid" + run_tests_invalid [ + "script add #{script} script_file.sh" + ] + self.title = TITLE + "check" + run_test_with_block "script list --format json" do |s| + JSON.parse(s).include?(script) + end + + self.title = TITLE + "delete" + run_tests [ + "script delete #{script} -y" + ] + self.title = TITLE + "delete, invalid" + run_tests_invalid [ + "script delete #{script} -y" + ] + + self.title = TITLE + "invalid" + run_tests_invalid [ + "script", + "script create", + "script create #{script}", + "script delete", + "script run", + "script run #{script}" + ] + end + +end + diff --git a/devops-client/tests/script_file.sh b/devops-client/tests/script_file.sh new file mode 100644 index 0000000..f1cdc46 --- /dev/null +++ b/devops-client/tests/script_file.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Hello! I'm a test file" diff --git a/devops-client/tests/server.rb b/devops-client/tests/server.rb new file mode 100644 index 0000000..2a00e93 --- /dev/null +++ b/devops-client/tests/server.rb @@ -0,0 +1,12 @@ +class Server < BaseTest + TITLE = "Image tests" + + def run + self.title = TITLE + run_tests [ + "server list", + "server list openstack", + "server list ec2" + ] + end +end diff --git a/devops-client/tests/user.rb b/devops-client/tests/user.rb new file mode 100644 index 0000000..ba40d5e --- /dev/null +++ b/devops-client/tests/user.rb @@ -0,0 +1,74 @@ +require "./base_test" + +class User < BaseTest + TITLE = "User tests - " + + def run + user = "test_user" + psw = "test" + self.title = TITLE + "list" + run_tests [ + "user list" + ] + self.title = TITLE + "create" + run_tests [ + "user create #{user} --password #{psw}" + ] + self.title = TITLE + "create, invalid" + run_tests_invalid [ + "user create #{user} --password #{psw}" + ] + + run_test_with_block "user list --format json" do |o| + !JSON.parse(o).detect{|u| u["id"] == user}.nil? + end + + self.title = TITLE + "grant" + cmds = %w{flavor group image project server key user filter network provider script} + p = %w{r w rw} + cmds.each do |c| + p.each do |pr| + self.title = TITLE + "grant #{c} #{pr}" + run_tests ["user grant #{user} #{c} #{pr}"] + run_test_with_block "user list --format json" do |o| + u = JSON.parse(o).detect{|u| u["id"] == user} + u["privileges"][c] == pr + end + end + end + p.push("") + p.each do |pr| + self.title = TITLE + "grant all #{pr}" + run_tests ["user grant #{user} all #{pr}"] + run_test_with_block "user list --format json" do |o| + u = JSON.parse(o).detect{|u| u["id"] == user} + u["privileges"].each do |cmd, p| + unless p == pr + puts_error "#{cmd} should be equals '#{pr}' but it is '#{p}'" + end + true + end + end + end + + self.title = TITLE + "delete" + run_tests [ + "user delete #{user} -y" + ] + self.title = TITLE + "delete invalid" + run_tests_invalid [ + "user delete #{user} -y" + ] + + self.title = TITLE + "invalid" + run_tests_invalid [ + "user", + "user create", + "user delete", + "user grant", + "user password" + ] + end + +end + diff --git a/devops-service/Gemfile b/devops-service/Gemfile new file mode 100644 index 0000000..1a11bb8 --- /dev/null +++ b/devops-service/Gemfile @@ -0,0 +1,14 @@ +source 'https://rubygems.org' + +gem "thin", "~>1.5.1" +gem "mime-types", "~>1.25.1" +gem "sinatra", "1.4.3" +gem "sinatra-contrib", "1.4.1" +gem "sinatra-websocket", "~>0.3.0" +gem "fog", "~>1.20" +gem "mixlib-shellout" +gem "chef", ">=11" +gem "mongo" +gem "bson_ext" +gem "multi_json" +gem "rufus-scheduler", "2.0.24" diff --git a/devops-service/Gemfile.lock b/devops-service/Gemfile.lock new file mode 100644 index 0000000..50f24f3 --- /dev/null +++ b/devops-service/Gemfile.lock @@ -0,0 +1,148 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.3.5) + atomic (1.1.14) + backports (3.6.0) + bson (1.9.2) + bson_ext (1.9.2) + bson (~> 1.9.2) + builder (3.2.2) + chef (11.10.4) + chef-zero (~> 1.7, >= 1.7.2) + diff-lcs (~> 1.2, >= 1.2.4) + erubis (~> 2.7) + highline (~> 1.6, >= 1.6.9) + json (>= 1.4.4, <= 1.8.1) + mime-types (~> 1.16) + mixlib-authentication (~> 1.3) + mixlib-cli (~> 1.4) + mixlib-config (~> 2.0) + mixlib-log (~> 1.3) + mixlib-shellout (~> 1.3) + net-ssh (~> 2.6) + net-ssh-multi (~> 1.1) + ohai (~> 6.0) + pry (~> 0.9) + puma (~> 1.6) + rest-client (>= 1.0.4, < 1.7.0) + yajl-ruby (~> 1.1) + chef-zero (1.7.3) + hashie (~> 2.0) + json + mixlib-log (~> 1.3) + moneta (< 0.7.0) + rack + coderay (1.1.0) + daemons (1.1.9) + diff-lcs (1.2.5) + em-websocket (0.3.8) + addressable (>= 2.1.1) + eventmachine (>= 0.12.9) + erubis (2.7.0) + eventmachine (1.0.3) + excon (0.31.0) + fog (1.20.0) + builder + excon (~> 0.31.0) + formatador (~> 0.2.0) + mime-types + multi_json (~> 1.0) + net-scp (~> 1.1) + net-ssh (>= 2.1.3) + nokogiri (>= 1.5.11) + formatador (0.2.4) + hashie (2.0.5) + highline (1.6.20) + ipaddress (0.8.0) + json (1.8.1) + method_source (0.8.2) + mime-types (1.25.1) + mini_portile (0.5.2) + mixlib-authentication (1.3.0) + mixlib-log + mixlib-cli (1.4.0) + mixlib-config (2.1.0) + mixlib-log (1.6.0) + mixlib-shellout (1.3.0) + moneta (0.6.0) + mongo (1.9.2) + bson (~> 1.9.2) + multi_json (1.8.4) + net-scp (1.1.2) + net-ssh (>= 2.6.5) + net-ssh (2.8.0) + net-ssh-gateway (1.2.0) + net-ssh (>= 2.6.5) + net-ssh-multi (1.2.0) + net-ssh (>= 2.6.5) + net-ssh-gateway (>= 1.2.0) + nokogiri (1.6.1) + mini_portile (~> 0.5.0) + ohai (6.20.0) + ipaddress + mixlib-cli + mixlib-config + mixlib-log + mixlib-shellout + systemu (~> 2.5.2) + yajl-ruby + pry (0.9.12.6) + coderay (~> 1.0) + method_source (~> 0.8) + slop (~> 3.4) + puma (1.6.3) + rack (~> 1.2) + rack (1.5.2) + rack-protection (1.5.2) + rack + rack-test (0.6.2) + rack (>= 1.0) + rest-client (1.6.7) + mime-types (>= 1.16) + rufus-scheduler (2.0.24) + tzinfo (>= 0.3.22) + sinatra (1.4.3) + rack (~> 1.4) + rack-protection (~> 1.4) + tilt (~> 1.3, >= 1.3.4) + sinatra-contrib (1.4.1) + backports (>= 2.0) + multi_json + rack-protection + rack-test + sinatra (~> 1.4.0) + tilt (~> 1.3) + sinatra-websocket (0.3.0) + em-websocket (~> 0.3.6) + eventmachine + thin (>= 1.3.1) + slop (3.4.7) + systemu (2.5.2) + thin (1.5.1) + daemons (>= 1.0.9) + eventmachine (>= 0.12.6) + rack (>= 1.0.0) + thread_safe (0.1.3) + atomic + tilt (1.4.1) + tzinfo (1.1.0) + thread_safe (~> 0.1) + yajl-ruby (1.2.0) + +PLATFORMS + ruby + +DEPENDENCIES + bson_ext + chef (>= 11) + fog (~> 1.20) + mime-types (~> 1.25.1) + mixlib-shellout + mongo + multi_json + rufus-scheduler (= 2.0.24) + sinatra (= 1.4.3) + sinatra-contrib (= 1.4.1) + sinatra-websocket (~> 0.3.0) + thin (~> 1.5.1) diff --git a/devops-service/client.rb b/devops-service/client.rb new file mode 100644 index 0000000..c676b69 --- /dev/null +++ b/devops-service/client.rb @@ -0,0 +1,32 @@ +require 'sinatra/base' + +class Client < Sinatra::Base + + def initialize config + super() + @@config = config + end + + # Route to download devops client + get "/devops-client.gem" do + begin + send_file @@config[:client_file] + rescue + msg = "No file '#{@@config[:client_file]}' found" + logger.error msg + return [404, msg] + end + end + + # Route to get client documentation + get "/ru/index.html" do + file = File.join(@@config[:public_dir], "ru_index.html") + if File.exist? file + File.read(file) + else + logger.error "File '#{file}' does not exist" + return [404, "File '/ru/index.html' does not exist"] + end + end + +end diff --git a/devops-service/commands/deploy.rb b/devops-service/commands/deploy.rb new file mode 100644 index 0000000..e468fef --- /dev/null +++ b/devops-service/commands/deploy.rb @@ -0,0 +1,31 @@ +module DeployCommands + + def deploy_server out, server, cert_path + out << "\nRun chef-client on '#{server.chef_node_name}'" + cmd = (server.remote_user == "root" ? "chef-client" : "sudo chef-client") + ip = if server.public_ip.nil? + server.private_ip + else + out << "Public IP detected\n" + server.public_ip + end + cmd = "ssh -t -i #{cert_path} #{server.remote_user}@#{ip} \"#{cmd}\"" + out << "\nCommand: '#{cmd}'\n" + status = nil + IO.popen(cmd + " 2>&1") do |c| + buf = "" + while line = c.gets do + out << line + buf = line + end + c.close + status = $?.to_i + r = buf.scan(/exit\scode\s([0-9]{1,3})/)[0] + unless r.nil? + status = r[0].to_i + end + end + return status + end + +end diff --git a/devops-service/commands/deploy_env.rb b/devops-service/commands/deploy_env.rb new file mode 100644 index 0000000..c70d8c5 --- /dev/null +++ b/devops-service/commands/deploy_env.rb @@ -0,0 +1,49 @@ +require "db/exceptions/invalid_record" +require "commands/image" + +module DeployEnvCommands + + include ImageCommands + + def check_expires! val + raise InvalidRecord.new "Parameter 'expires' is invalid" if val.match(/^[0-9]+[smhdw]$/).nil? + end + + def check_flavor! p, val + f = p.flavors.detect{|f| f["id"] == val} + raise InvalidRecord.new "Invalid flavor '#{val}'" if f.nil? + end + + def check_image! p, val + images = get_images(DevopsService.mongo, p.name) + raise InvalidRecord.new "Invalid image '#{val}'" unless images.map{|i| i["id"]}.include?(val) + end + + def check_subnets_and_groups! p, subnets, groups + networks = p.networks + n = subnets - networks.map{|n| n["name"]} + raise InvalidRecord.new "Invalid networks '#{n.join("', '")}'" unless n.empty? + + filter = nil + if p.name == ::Version2_0::Provider::Ec2::PROVIDER + unless subnets.empty? + subnets = [ subnets[0] ] if subnets.size > 1 + filter = {"vpc-id" => networks.detect{|n| n["name"] == subnets[0]}["vpcId"] } + end + elsif p.name == ::Version2_0::Provider::Openstack::PROVIDER + if subnets.empty? + raise InvalidRecord.new "Subnets array can not be empty" + end + end + + g = groups - p.groups(filter).keys + raise InvalidRecord.new "Invalid groups '#{g.join("', '")}'" unless g.empty? + end + + def check_users! val + users = DevopsService.mongo.users_names(val) + buf = val - users + raise InvalidRecord.new("Invalid users: '#{buf.join("', '")}'") unless buf.empty? + end + +end diff --git a/devops-service/commands/image.rb b/devops-service/commands/image.rb new file mode 100644 index 0000000..da21645 --- /dev/null +++ b/devops-service/commands/image.rb @@ -0,0 +1,13 @@ +require "providers/provider_factory" + +module ImageCommands + + def get_images mongo, provider + filters = mongo.available_images(provider) + if filters.empty? + [] + else + ::Version2_0::Provider::ProviderFactory.get(provider).images(filters) + end + end +end diff --git a/devops-service/commands/knife_commands.rb b/devops-service/commands/knife_commands.rb new file mode 100644 index 0000000..0e8b419 --- /dev/null +++ b/devops-service/commands/knife_commands.rb @@ -0,0 +1,69 @@ +require "json" + +class KnifeCommands + + def self.chef_node_list + knife("node list")[0].split.map{|c| c.strip} + end + + def self.chef_client_list + knife("client list")[0].split.map{|c| c.strip} + end + + def self.chef_node_delete name + o = knife("node delete #{name} -y")[0] + (o.nil? ? o : o.strip) + end + + def self.chef_client_delete name + o = knife("client delete #{name} -y")[0] + (o.nil? ? o : o.strip) + end + + def self.tags_list name + knife("tag list #{name}")[0].split.map{|c| c.strip} + end + + def self.tags_create name, tagsStr + knife("tag create #{name} #{tagsStr}") + end + + def self.tags_delete name, tagsStr + knife("tag delete #{name} #{tagsStr}") + end + + def self.create_role project, env + file = "/tmp/new_role.json" + File.open(file, "w") do |f| + f.puts <<-EOH +{ + "name" : "#{project}_#{env}", + "description": "", + "json_class": "Chef::Role", + "default_attributes": { + "project": "#{project}", + "env": "#{env}" + }, + "override_attributes": {}, + "chef_type": "role", + "run_list": [], + "env_run_lists": {} +} +EOH + end + out = `knife role from file #{file}` + raise "Cannot create role '#{project}_#{env}': #{out}" unless $?.success? + true + end + + def self.roles + o, s = knife("role list --format json") + return (s ? JSON.parse(o) : nil) + end + + def self.knife cmd + o = `knife #{cmd} 2>&1` + return o, $?.success? + end + +end diff --git a/devops-service/commands/server.rb b/devops-service/commands/server.rb new file mode 100644 index 0000000..925fc7a --- /dev/null +++ b/devops-service/commands/server.rb @@ -0,0 +1,190 @@ +require "commands/knife_commands" +require "db/exceptions/record_not_found" + +module ServerCommands + + def extract_servers provider, project, env, params, user, mongo + flavors = provider.flavors + projects = {} + env_name = env.identifier + project_name = project.id + servers_info = [] + if project.multi? + #TODO: fix multi project + images = {} + env.servers.each do |name, server| + images[server["image"]] = mongo.image(server["image"]) unless images.has_key?(server["image"]) + flavor = flavors.detect {|f| f["name"] == server["flavor"]} + raise RecordNotFound.new("Flavor with name '#{server["flavor"]}' not found") if flavor.nil? + run_list = [] + project_ids = server["subprojects"].map{|sp| sp["project_id"]} + db_subprojects = mongo.projects project_ids + ids = project_ids - db_subprojects.map{|sp| sp.id} + unless ids.empty? + return [400, "Subproject(s) '#{ids.join("', '")}' is/are not exists"] + end + server["subprojects"].each do |sp| + p = db_subprojects.detect{|db_sp| db_sp.id == sp["project_id"]} + run_list += p.deploy_env(sp["project_env"]).run_list + end + o = { + :image => images[server["image"]], + :name => "#{name}_#{Time.now.to_i}", + :flavor => flavor["id"], + :groups => server["groups"], + :run_list => run_list + } + servers_info.push(o) + end + else + i = mongo.image env.image + flavor = flavors.detect {|f| f["id"] == env.flavor} + raise RecordNotFound.new("Flavor with id '#{env.flavor}' not found") if flavor.nil? + o = { + :image => i, + :name => params["name"], + :flavor => flavor["id"], + :groups => params["groups"] || env.groups, + :run_list => env.run_list, + :subnets => env.subnets, + :key => params["key"] + } + servers_info.push(o) + end + + servers = [] + servers_info.each do |info| + image = info[:image] + s = Server.new + s.provider = provider.name + s.project = project_name + s.deploy_env = env_name + s.remote_user = image.remote_user + s.chef_node_name = info[:name] || "#{provider.ssh_key}-#{project_name}-#{env_name}-#{Time.now.to_i}" + s.key = info[:key] || provider.ssh_key + s.options = { + :image => image.id, + :flavor => info[:flavor], + :name => info[:name], + :groups => info[:groups], + :run_list => info[:run_list], + :bootstrap_template => image.bootstrap_template, + :subnets => info[:subnets] + } + s.created_by = user + servers.push s + end + return servers + end + + def delete_from_chef_server node_name + { + :chef_node => KnifeCommands.chef_node_delete(node_name), + :chef_client => KnifeCommands.chef_client_delete(node_name) + } + end + + def check_server s + KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name) + end + + def bootstrap s, out, cert_path, logger + if s.private_ip.nil? + out << "Error: Private IP is null" + return false + end + ja = { + :provider => s.provider, + :devops_host => `hostname`.strip + } + bootstrap_options = [ + "-x #{s.remote_user}", + "-i #{cert_path}", + "--json-attributes '#{ja.to_json}'" + ] + bootstrap_options.push "--sudo" unless s.remote_user == "root" + bootstrap_options.push "-N #{s.chef_node_name}" if s.chef_node_name + bootstrap_options.push "-d #{s.options[:bootstrap_template]}" if s.options[:bootstrap_template] + bootstrap_options.push "-r #{s.options[:run_list].join(",")}" unless s.options[:run_list].empty? + ip = s.private_ip + unless s.public_ip.nil? || s.public_ip.strip.empty? + ip = s.public_ip + out << "\nPublic IP is present\n" + end + out << "\nWaiting for SSH..." + i = 0 + begin + sleep(1) + `ssh -i #{cert_path} -q #{s.remote_user}@#{ip} exit` + i += 1 + if i == 300 + res = `ssh -i #{cert_path} #{s.remote_user}@#{ip} "exit" 2>&1` + out << "\nCan not connect to #{s.remote_user}@#{ip}" + out << "\n" + res + logger.error "Can not connect with command 'ssh -i #{cert_path} #{s.remote_user}@#{ip}':\n#{res}" + return false + end + raise unless $?.success? + rescue + retry + end + + bootstrap_cmd = "knife bootstrap #{bootstrap_options.join(" ")} #{ip}" + out << "\nExecuting '#{bootstrap_cmd}' \n\n" + status = nil + IO.popen(bootstrap_cmd + " 2>&1") do |bo| + while line = bo.gets do + out << line + end + bo.close + status = $?.to_i + end + return status + end + + def unbootstrap s, cert_path + i = 0 + begin + `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef` + raise unless $?.success? + rescue => e + logger.error "Unbootstrap eeror: " + e.message + i += 1 + sleep(1) + retry unless i == 5 + end + end + + def delete_server s, mongo, logger + if s.chef_node_name.nil? + mongo.server_delete s.id + msg = "Added server '#{s.id}' is removed" + logger.info msg + return msg, nil + end + r = delete_from_chef_server(s.chef_node_name) + info = if s.static + cert = mongo.key(s.key).path + unbootstrap(s, cert) + mongo.server_delete s.id + msg = "Static server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" + logger.info msg + msg + else + provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) + begin + r[:server] = provider.delete_server s.id + rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound + r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" + logger.warn r[:server] + end + mongo.server_delete s.id + msg = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed" + logger.info msg + msg + end + r.each{|key, log| logger.info("#{key} - #{log}")} + return info, r + end + +end diff --git a/devops-service/commands/status.rb b/devops-service/commands/status.rb new file mode 100644 index 0000000..a440891 --- /dev/null +++ b/devops-service/commands/status.rb @@ -0,0 +1,30 @@ +module StatusCommands + + def create_status status + s = if status.empty? + 1 + else + b = 0 + status.each{|s| b |= s} + b + end + return "\n-- Status: #{s} --" + end + + def time_diff_milli start, finish + ((finish - start) * 1000.0).to_i + end + + def time_diff_milli_s start, finish + time_diff_milli(start, finish).to_s + "ms" + end + + def time_diff start, finish + (finish - start).to_i + end + + def time_diff_s start, finish + time_diff(start, finish).to_s + "s" + end + +end diff --git a/devops-service/config.rb b/devops-service/config.rb new file mode 100644 index 0000000..3097f9b --- /dev/null +++ b/devops-service/config.rb @@ -0,0 +1,36 @@ +# path to log file +config[:log_file] = "/path/to/log" +# path to chef knife.rb file +config[:knife_config_file] = "/path/to/.chef/knife.rb" +# role name separator +config[:role_separator] = "_" + +# mongodb settings +config[:mongo_host] = "localhost" +config[:mongo_port] = 27017 +config[:mongo_db] = "devops" +config[:mongo_user] = "user" +config[:mongo_password] = "pass" + +# devops port +config[:port] = 7070 + +# path to devops-client.gem file +config[:client_file] = "/path/to/public/devops-client.gem" +# path to devops public directory +config[:public_dir] = "/path/to/public" + +# openstack settings +config[:openstack_username] = "openstack_username" +config[:openstack_api_key] = "openstack_pass" +config[:openstack_auth_url] = "http://openstack.host:5000/v2.0/tokens" +config[:openstack_tenant] = "tenant" +config[:openstack_ssh_key] = "ssh_key" +config[:openstack_certificate] = "/path/to/.ssh/openstack.pem" + +# aws settings +config[:aws_access_key_id] = "access_key_id" +config[:aws_secret_access_key] = "secret_access_key" +config[:aws_ssh_key] = "ssh_key" +config[:aws_certificate] = "/path/to/.ssh/ec2.pem" +config[:aws_availability_zone] = "aws_zone" diff --git a/devops-service/config.ru b/devops-service/config.ru new file mode 100644 index 0000000..edd6711 --- /dev/null +++ b/devops-service/config.ru @@ -0,0 +1,23 @@ +# To run devops you can use command +# `bundle exec thin -R $devops_home/config.ru -e $env -d -p $port -t 600 -u $user --pid $pid_file --log $log_file start` +require 'rubygems' +require 'bundler/setup' + +root = File.dirname(__FILE__) +require File.join(root, "devops-service") +require File.join(root, "client") + +# Read configuration file +config_file = File.join(root, "config.rb") +config = {} +if File.exists? config_file + eval File.read config_file +else + raise "No config file '#{config_file}' found" +end + +# URL map for API v2.0 +run Rack::URLMap.new({ + "/v2.0" => DevopsService.new(config), + "/client" => Client.new(config) +}) diff --git a/devops-service/db/exceptions/invalid_record.rb b/devops-service/db/exceptions/invalid_record.rb new file mode 100644 index 0000000..c94a1c4 --- /dev/null +++ b/devops-service/db/exceptions/invalid_record.rb @@ -0,0 +1,3 @@ +class InvalidRecord < Exception + +end diff --git a/devops-service/db/exceptions/record_not_found.rb b/devops-service/db/exceptions/record_not_found.rb new file mode 100644 index 0000000..775e2bb --- /dev/null +++ b/devops-service/db/exceptions/record_not_found.rb @@ -0,0 +1,3 @@ +class RecordNotFound < Exception#StandardError + +end diff --git a/devops-service/db/mongo/models/deploy_env.rb b/devops-service/db/mongo/models/deploy_env.rb new file mode 100644 index 0000000..7103a34 --- /dev/null +++ b/devops-service/db/mongo/models/deploy_env.rb @@ -0,0 +1,85 @@ +require "db/mongo/models/mongo_model" +require "db/exceptions/invalid_record" +require "providers/provider_factory" +require "commands/deploy_env" + +class DeployEnv < MongoModel + + include DeployEnvCommands + + attr_accessor :identifier, :flavor, :image, :run_list, :subnets, :expires, :provider, :groups, :users + + types :identifier => {:type => String, :empty => false}, + :image => {:type => String, :empty => false}, + :flavor => {:type => String, :empty => false}, + :provider => {:type => String, :empty => false}, + :expires => {:type => String, :empty => false, :nil => true}, + :run_list => {:type => Array, :empty => true}, + :users => {:type => Array, :empty => true}, + :subnets => {:type => Array, :empty => true}, + :groups => {:type => Array, :empty => false} + + def initialize d={} + self.identifier = d["identifier"] + self.flavor = d["flavor"] + self.image = d["image"] + b = d["subnets"] || [] + self.subnets = (b.is_a?(Array) ? b.uniq : b) + b = d["run_list"] || [] + self.run_list = (b.is_a?(Array) ? b.uniq : b) + self.expires = d["expires"] + self.provider = d["provider"] + b = d["groups"] || ["default"] + self.groups = (b.is_a?(Array) ? b.uniq : b) + b = d["users"] || [] + self.users = (b.is_a?(Array) ? b.uniq : b) + end + + def validate! + super + e = DeployEnv.validate_run_list(self.run_list) + raise InvalidRecord.new "Invalid run list elements: '#{e.join("', '")}'" unless e.empty? + + unless self.expires.nil? + check_expires!(self.expires) + end + + p = ::Version2_0::Provider::ProviderFactory.get(self.provider) + check_flavor!(p, self.flavor) + check_image!(p, self.image) + check_subnets_and_groups!(p, self.subnets, self.groups) + check_users!(self.users) + + true + rescue InvalidRecord => e + raise InvalidRecord.new "Deploy environment '#{self.identifier}'. " + e.message + end + + def to_hash + { + "flavor" => self.flavor, + "identifier" => self.identifier, + "image" => self.image, + "run_list" => self.run_list, + "subnets" => self.subnets, + "expires" => self.expires, + "provider" => self.provider, + "groups" => self.groups, + "users" => self.users + } + end + + def self.create_from_bson d + DeployEnv.new(d) + end + + def self.create hash + DeployEnv.new(hash) + end + + def self.validate_run_list list + rl = /\Arole|recipe\[[\w-]+(::[\w-]+)?\]\Z/ + list.select {|l| (rl =~ l).nil?} + end + +end diff --git a/devops-service/db/mongo/models/deploy_env_multi.rb b/devops-service/db/mongo/models/deploy_env_multi.rb new file mode 100644 index 0000000..edafdbb --- /dev/null +++ b/devops-service/db/mongo/models/deploy_env_multi.rb @@ -0,0 +1,118 @@ +require "db/mongo/models/mongo_model" +require "db/exceptions/invalid_record" +require "commands/deploy_env" + +class DeployEnvMulti < MongoModel + + include DeployEnvCommands + + attr_accessor :identifier, :servers, :expires, :users + + types :identifier => {:type => String, :empty => false}, + :expires => {:type => String, :empty => false, :nil => true}, + :users => {:type => Array, :empty => true}, + :servers => {:type => Array, :empty => false, :value_type => Hash} + + def initialize d={} + self.identifier = d["identifier"] + self.expires = d["expires"] + self.servers = d["servers"] + b = d["users"] || [] + self.users = (b.is_a?(Array) ? b.uniq : b) + end + + def validate! + super + e = [] + check_users!(self.users) + unless self.expires.nil? + check_expires!(self.expires) + end + self.servers.each_with_index do |server, i| + begin + if server["priority"].nil? + server["priority"] = 100 + else + begin + Integer(server["priority"]) + rescue ArgumentError, TypeError + raise InvalidRecord.new("Parameter 'priority' should be an integer") + end + end + + if !server["subprojects"].is_a?(Array) or server["subprojects"].empty? + raise InvalidRecord.new("Parameter 'subprojects' must be a not empty array") + end + if server["subprojects"].size > 1 + check_provider(server["provider"]) + # strings + %w{image flavor provider}.each do |p| + begin + check_string!(server[p]) + rescue ArgumentError + raise InvalidRecord.new("Parameter '#{p}' must be a not empty string") + end + end + # arrays + %w{subnets groups}.each do |p| + begin + raise ArgumentError if !server[p].is_a?(Array) or server[p].empty? + server[p].each do |v| + raise ArgumentError unless v.is_a?(String) + end + rescue ArgumentError + raise InvalidRecord.new("Parameter '#{p}' must be a not empty array of strings") + end + end + + p = ::Version2_0::Provider::ProviderFactory.get(server["provider"]) + check_flavor!(p, server["flavor"]) + check_image!(p, server["image"]) + check_subnets_and_groups!(p, server["subnets"], server["groups"]) + end + names = {} + server["subprojects"].each_with_index do |sp, spi| + begin + raise InvalidRecord.new("Parameter 'subprojects' must contains objects only") unless sp.is_a?(Hash) + %w{name env}.each do |p| + begin + check_string!(sp[p]) + rescue ArgumentError + raise InvalidRecord.new("Parameter '#{p}' must be a not empty string") + end + end + rescue InvalidRecord => e + raise InvalidRecord.new("Subproject '#{spi}'. #{e.message}") + end + end + pdb = DevopsService.mongo.project_names_with_envs(server["subprojects"].map{|sp| sp["name"]}) + server["subprojects"].each_with_index do |sp, spi| + raise InvalidRecord.new("Subproject '#{spi}'. Project '#{sp["name"]}' with env '#{sp["env"]}' does not exist") if pdb[sp["name"]].nil? or !pdb[sp["name"]].include?(sp["env"]) + end + rescue InvalidRecord => e + raise InvalidRecord.new("Server '#{i}'. #{e.message}") + end + end + true + rescue InvalidRecord => e + raise InvalidRecord.new "Deploy environment '#{self.identifier}'. " + e.message + end + + def to_hash + { + "identifier" => self.identifier, + "expires" => self.expires, + "users" => self.users, + "servers" => self.servers + } + end + + def self.create_from_bson d + DeployEnvMulti.new(d) + end + + def self.create hash + DeployEnvMulti.new(hash) + end + +end diff --git a/devops-service/db/mongo/models/image.rb b/devops-service/db/mongo/models/image.rb new file mode 100644 index 0000000..1a7713e --- /dev/null +++ b/devops-service/db/mongo/models/image.rb @@ -0,0 +1,50 @@ +require "db/exceptions/invalid_record" +require "db/mongo/models/mongo_model" +require "commands/image" + +class Image < MongoModel + + include ImageCommands + + attr_accessor :id, :provider, :remote_user, :name, :bootstrap_template + types :id => {:type => String, :empty => false}, + :provider => {:type => String, :empty => false}, + :remote_user => {:type => String, :empty => false}, + :name => {:type => String, :empty => true}, + :bootstrap_template => {:type => String, :empty => false, :nil => true} + + def validate! + super + images = get_images(DevopsService.mongo, self.provider) + raise InvalidRecord.new "Invalid image id '#{self.id}' for provider '#{self.provider}', please check image filters" unless images.map{|i| i["id"]}.include?(self.id) + end + + def initialize p={} + self.id = p["id"] + self.provider = p["provider"] + self.remote_user = p["remote_user"] + self.name = p["name"] || "" + self.bootstrap_template = p["bootstrap_template"] + end + + def self.create_from_bson args + image = Image.new(args) + image.id = args["_id"] + image + end + + def to_hash_without_id + o = { + "provider" => self.provider, + "name" => self.name, + "remote_user" => self.remote_user + } + o["bootstrap_template"] = self.bootstrap_template + o + end + + def self.create_from_json! json + Image.new( JSON.parse(json) ) + end + +end diff --git a/devops-service/db/mongo/models/key.rb b/devops-service/db/mongo/models/key.rb new file mode 100644 index 0000000..507e858 --- /dev/null +++ b/devops-service/db/mongo/models/key.rb @@ -0,0 +1,50 @@ +require "db/exceptions/invalid_record" +require "db/mongo/models/mongo_model" +require "json" + +class Key < MongoModel + + SYSTEM = "system" + USER = "user" + + attr_accessor :id, :path, :scope + types :id => {:type => String, :empty => false}, + :path => {:type => String, :empty => false}, + :scope => {:type => String, :empty => false} + + def initialize p={} + self.id = p["id"] + self.path = p["path"] + self.scope = p["scope"] || USER + end + + def self.create_from_bson s + key = Key.new s + key.id = s["_id"] + key + end + + def self.create_from_json json + Key.new( JSON.parse(json) ) + end + + def filename + File.basename(self.path) + end + + def to_hash_without_id + o = { + "path" => self.path, + "scope" => self.scope + } + o + end + + def validate! + super + raise InvalidRecord.new "File does not exist" unless File.exist?(self.path) + raise InvalidRecord.new "Key parameter 'scope' is invalid" unless [SYSTEM, USER].include?(self.scope) + true + end + +end diff --git a/devops-service/db/mongo/models/mongo_model.rb b/devops-service/db/mongo/models/mongo_model.rb new file mode 100644 index 0000000..5fd374d --- /dev/null +++ b/devops-service/db/mongo/models/mongo_model.rb @@ -0,0 +1,105 @@ +require "providers/provider_factory" +require "db/exceptions/invalid_record" +require "json" + +class MongoModel + + def to_json + JSON.pretty_generate self.to_hash + end + + def to_hash + h = to_hash_without_id + h["id"] = self.id + h + end + + def to_mongo_hash + h = to_hash_without_id + h["_id"] = self.id + h + end + + def is_empty? val + val.nil? or val.strip.empty? + end + + def check_string! val + raise ArgumentError unless val.is_a?(String) + val.strip! + raise ArgumentError if val.empty? + end + + def check_array! val, type, empty=false + raise ArgumentError unless val.is_a?(Array) + raise ArgumentError if !empty and val.empty? + val.each do |v| + raise ArgumentError unless v.is_a?(type) + end + end + + def check_name_value val + raise ArgumentError.new "Invalid name, it should contains 0-9, a-z, A-Z, _, - symbols only" if val.match(/^[0-9a-zA-Z_\-]+$/).nil? + end + + def check_provider provider=self.provider + unless ::Version2_0::Provider::ProviderFactory.providers.include?(provider) or provider == "static" + raise InvalidRecord.new "Invalid provider '#{provider}'" + end + end + + # types - Hash + # key - param name + # value - Hash + # :type - param type + # :empty - can param be empty? (false) + # :nil - can param be nil? (false) + # :value_type - type of array element (String) + def self.types types + define_method :validate do + t = types.keys + e = types.keys + n = types.keys + types.each do |name, value| + if value[:nil] + n.delete(name) + if self.send(name).nil? + e.delete(name) + t.delete(name) + next + end + else + n.delete(name) unless self.send(name).nil? + end + if self.send(name).is_a? value[:type] + t.delete(name) + self.send(name).strip! if value[:type] == String + if value[:type] == Array + unless value[:value_type] == false + type = value[:value_type] || String + self.send(name).each do |e| + unless e.is_a?(type) + t.push(name) + break + end + end + end + end + e.delete(name) if value[:empty] or !self.send(name).empty? + end + end + raise InvalidRecord.new "Parameter(s) '#{n.join("', '")}' can not be undefined" unless n.empty? + raise InvalidRecord.new "Parameter(s) '#{t.join("', '")}' have invalid type(s)" unless t.empty? + raise InvalidRecord.new "Parameter(s) '#{e.join("', '")}' can not be empty" unless e.empty? + if types.has_key? :provider + self.send("check_provider") + end + true + end + end + + def validate! + self.validate + end + +end diff --git a/devops-service/db/mongo/models/project.rb b/devops-service/db/mongo/models/project.rb new file mode 100644 index 0000000..f08dde9 --- /dev/null +++ b/devops-service/db/mongo/models/project.rb @@ -0,0 +1,127 @@ +require "db/exceptions/invalid_record" +require "db/mongo/models/deploy_env" +require "db/mongo/models/user" +require "db/mongo/models/deploy_env_multi" +require "db/mongo/models/mongo_model" +require "json" + +class Project < MongoModel + + attr_accessor :id, :deploy_envs, :type + + types :id => {:type => String, :empty => false}, + :deploy_envs => {:type => Array, :value_type => false, :empty => false} + + MULTI_TYPE = "multi" + + def initialize p={} + self.id = p["name"] + raise InvalidRecord.new "No deploy envirenments for project #{self.id}" if p["deploy_envs"].nil? or p["deploy_envs"].empty? + self.deploy_envs = [] + self.type = p["type"] + env_class = ( self.multi? ? DeployEnvMulti : DeployEnv ) + + p["deploy_envs"].each do |e| + env = env_class.create(e) + self.deploy_envs.push env + end + end + + def deploy_env env + de = self.deploy_envs.detect {|e| e.identifier == env} + raise InvalidRecord.new("Project '#{self.id}' does not have deploy environment '#{env}'") if de.nil? + de + end + + def add_authorized_user user, env=nil + return if user.nil? + new_users = ( user.is_a?(Array) ? user : [ user ] ) + if env.nil? + self.deploy_envs.each do |e| + return unless e.users.is_a?(Array) + e.users = (e.users + new_users).uniq + end + else + e = self.deploy_env(env) + return unless e.users.is_a?(Array) + e.users = (e.users + new_users).uniq + end + end + + def remove_authorized_user user, env=nil + return if user.nil? + users = ( user.is_a?(Array) ? user : [ user ] ) + if env.nil? + self.deploy_envs.each do |e| + return unless e.users.is_a?(Array) + e.users = e.users - users + end + else + e = self.deploy_env(env) + return unless e.users.is_a?(Array) + e.users = e.users - users + end + end + + def check_authorization user_id, env + e = self.deploy_env(env) + return true if user_id == User::ROOT_USER_NAME + return e.users.include? user_id + rescue RecordNotFound => e + return false + end + + def validate! + super + check_name_value(self.id) + envs = self.deploy_envs.map {|d| d.identifier} + non_uniq = envs.uniq.select{|u| envs.count(u) > 1} + raise InvalidRecord.new "Deploy environment(s) '#{non_uniq.join("', '")}' is/are not unique" unless non_uniq.empty? + self.deploy_envs.each do |d| + d.validate! + unless self.multi? + rn = "#{self.id}#{DevopsService.config[:role_separator] || "_"}#{d.identifier}" + role = "role[#{rn}]" + d.run_list = d.run_list - [rn, role] + d.run_list.unshift(role) + end + end + + true + rescue InvalidRecord, ArgumentError => e + raise InvalidRecord.new "Project '#{self.id}'. #{e.message}" + end + + def remove_env env + self.deploy_envs.delete_if {|e| e.identifier == env} + end + + def add_env env + raise InvalidRecord.new "Deploy environment '#{env.identifier}' for project '#{self.id}' already exist" unless self.deploy_env(env.identifier).nil? + self.deploy_envs.push env + end + + def to_hash + h = self.to_hash_without_id + h["name"] = self.id + h + end + + def to_hash_without_id + h = {"deploy_envs" => self.deploy_envs.map {|e| e.to_hash}} + if self.multi? + h["type"] = MULTI_TYPE + end + h + end + + def multi? + self.type == MULTI_TYPE + end + + def self.create_from_bson p + p["name"] = p["_id"] + Project.new p + end + +end diff --git a/devops-service/db/mongo/models/server.rb b/devops-service/db/mongo/models/server.rb new file mode 100644 index 0000000..a8ca217 --- /dev/null +++ b/devops-service/db/mongo/models/server.rb @@ -0,0 +1,80 @@ +require "db/exceptions/invalid_record" +require "db/mongo/models/mongo_model" + +class Server < MongoModel + + attr_accessor :provider, :chef_node_name, :id, :remote_user, :project, :deploy_env, :private_ip, :public_ip, :created_at, :without_bootstrap, :created_by + attr_accessor :options, :static, :key + + types :id => {:type => String, :empty => false}, + :provider => {:type => String, :empty => false}, + :remote_user => {:type => String, :empty => false}, + :project => {:type => String, :empty => false}, + :deploy_env => {:type => String, :empty => false}, + :private_ip => {:type => String, :empty => false}, + :public_ip => {:type => String, :empty => true, :nil => true}, + :key => {:type => String, :empty => false}, + :created_by => {:type => String, :empty => false}, + :chef_node_name => {:type => String, :empty => true} + + def initialize + self.static = false + end + + def validate! + super + true + end + + def to_hash_without_id + { + "provider" => self.provider, + "chef_node_name" => self.chef_node_name, + "remote_user" => self.remote_user, + "project" => self.project, + "deploy_env" => self.deploy_env, + "private_ip" => self.private_ip, + "public_ip" => self.public_ip, + "created_at" => self.created_at, + "created_by" => self.created_by, + "static" => self.static, + "key" => self.key + } + end + + def to_list_hash + { + "id" => self.id, + "chef_node_name" => self.chef_node_name + } + end + + def self.create_from_bson s + server = Server.new + server.provider = s["provider"] + server.chef_node_name = s["chef_node_name"] + server.id = s["_id"] + server.remote_user = s["remote_user"] + server.project = s["project"] + server.deploy_env = s["deploy_env"] + server.public_ip = s["public_ip"] + server.private_ip = s["private_ip"] + server.created_at = s["created_at"] + server.created_by = s["created_by"] + server.static = s["static"] || false + server.key = s["key"] + server + end + + def info + str = "Instance Name: #{self.chef_node_name}\n" + str << "Instance ID: #{self.id}\n" + str << "Private IP: #{self.private_ip}\n" + str << "Public IP: #{self.public_ip}\n" unless self.public_ip.nil? + str << "Remote user: #{self.remote_user}\n" + str << "Project: #{self.project} - #{self.deploy_env}\n" + str << "Created by: #{self.created_by}" + str + end + +end diff --git a/devops-service/db/mongo/models/user.rb b/devops-service/db/mongo/models/user.rb new file mode 100644 index 0000000..9a35f98 --- /dev/null +++ b/devops-service/db/mongo/models/user.rb @@ -0,0 +1,115 @@ +require "db/exceptions/invalid_record" +require "exceptions/invalid_command" +require "db/mongo/models/mongo_model" + +#require "common/fog" + +class User < MongoModel + + ROOT_USER_NAME = 'root' + ROOT_PASSWORD = '' + + PRIVILEGES = ["r", "w", "rw", ""] + + attr_accessor :id, :password, :privileges, :email + types :id => {:type => String, :empty => false}, + :email => {:type => String, :empty => false}, + :password => {:type => String, :empty => true} + + def initialize p={} + self.id = p['username'] + self.email = p['email'] + self.password = p['password'] + self.privileges = p["privileges"] || self.default_privileges + end + + def all_privileges + privileges_with_value("rw") + end + + def default_privileges + privileges_with_value("r", "user" => "") + end + + def grant cmd, priv='' + raise InvalidCommand.new "Invalid privileges '#{priv}'. Available values are '#{PRIVILEGES.join("', '")}'" unless PRIVILEGES.include?(priv) + raise InvalidCommand.new "Can't grant privileges to root" if self.id == ROOT_USER_NAME + + case cmd + when "all" + self.privileges.each_key do |key| + self.privileges[key] = priv + end + when "" + self.privileges = self.default_privileges + else + raise InvalidCommand.new "Unsupported command #{cmd}" unless self.all_privileges.include?(cmd) + self.privileges[cmd] = priv + end + end + + def self.create_from_bson s + user = User.new s + user.id = s["_id"] + user + end + + def self.create_from_json json + User.new( JSON.parse(json) ) + end + + def to_hash_without_id + o = { + "email" => self.email, + "password" => self.password, + "privileges" => self.privileges + } + o + end + + def check_privilege cmd, priv + p = self.privileges[cmd] + return false if p.nil? + return p.include?(priv) + end + + def check_privilege_read cmd + check_privilege_r_w cmd, "r" + end + + def check_privilege_write cmd + check_privilege_r_w cmd, "w" + end + + def check_privilege_r_w cmd, flag + p = self.privileges[cmd] + return false if p.nil? + return p == flag || p == 'rw' + end + + def self.create_root + root = User.new({'username' => ROOT_USER_NAME, 'password' => ROOT_PASSWORD}) + root.privileges = root.all_privileges + root.email = "#{ROOT_USER_NAME}@host" + root + end + + private + def privileges_with_value v, options={} + { + "flavor" => v, + "group" => v, + "image" => v, + "project" => v, + "server" => v, + "key" => v, + "user" => v, + "filter" => v, + "network" => v, + "provider" => v, + "script" => v, + "templates" => v + }.merge(options) + end + +end diff --git a/devops-service/db/mongo/mongo_connector.rb b/devops-service/db/mongo/mongo_connector.rb new file mode 100644 index 0000000..850543c --- /dev/null +++ b/devops-service/db/mongo/mongo_connector.rb @@ -0,0 +1,364 @@ +require "mongo" + +require "db/exceptions/record_not_found" +require "db/exceptions/invalid_record" +require "exceptions/invalid_privileges" + +require "db/mongo/models/project" +require "db/mongo/models/image" +require "db/mongo/models/project" +require "db/mongo/models/server" +require "db/mongo/models/user" + +include Mongo + +class MongoConnector + + def initialize(db, host, port=27017, user=nil, password=nil) + @mongo_client = MongoClient.new(host, port) + @db = @mongo_client.db(db) + @db.authenticate(user, password) unless user.nil? or password.nil? + @projects = @db.collection("projects") + @images = @db.collection("images") + @servers = @db.collection("servers") + @filters = @db.collection("filters") + @keys = @db.collection("keys") + @users = @db.collection("users") + @statistic = @db.collection("statistic") + end + + def images provider=nil + q = (provider.nil? ? {} : {"provider" => provider}) + @images.find(q).to_a.map {|bi| Image.create_from_bson bi} + end + + def image id + i = @images.find(create_query("_id" => id)).to_a[0] + raise RecordNotFound.new("Image '#{id}' does not exist") if i.nil? + Image.create_from_bson i + end + + def image_insert image + image.validate! + @images.insert(image.to_mongo_hash) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: image with id '#{image.id}'") + end + end + + def image_update image + image.validate! + @images.update(create_query({"_id" => image.id}), create_query(image.to_mongo_hash)) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: image with id '#{image.id}'") + end + end + + def image_delete id + r = @images.remove(create_query("_id" => id)) + raise RecordNotFound.new("Image '#{id}' not found") if r["n"] == 0 + r + end + + def available_images provider + f = @filters.find(create_query("type" => "image", "provider" => provider)).to_a[0] + return [] if f.nil? + f["images"] + end + + def add_available_images images, provider + return unless images.is_a?(Array) + f = @filters.find(create_query("type" => "image", "provider" => provider)).to_a[0] + if f.nil? + @filters.insert(create_query({"type" => "image", "provider" => provider, "images" => images})) + return images + else + f["images"] |= images + @filters.update({"_id" => f["_id"]}, f) + return f["images"] + end + end + + def delete_available_images images, provider + return unless images.is_a?(Array) + f = @filters.find(create_query("type" => "image", "provider" => provider)).to_a[0] + unless f.nil? + f["images"] -= images + @filters.update({"_id" => f["_id"]}, f) + return f["images"] + end + [] + end + + def is_project_exists? project + self.project project.id + return true + rescue RecordNotFound => e + return false + end + + def project_insert project + project.validate! + @projects.insert(create_query(project.to_mongo_hash)) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: project with id '#{project.id}'") + end + end + + def project name + p = @projects.find(create_query("_id" => name)).to_a[0] + raise RecordNotFound.new("Project '#{name}' does not exist") if p.nil? + Project.create_from_bson p + end + + def projects_all + p = @projects.find() + p.to_a.map {|bp| Project.create_from_bson bp} + end + + def projects list=nil, type=nil + q = (list.nil? ? {} : {"_id" => {"$in" => list}}) + case type + when :multi + q["type"] = "multi" + #else + # q["type"] = {"$exists" => false} + end + res = @projects.find(create_query(q)) + a = res.to_a + a.map {|bp| Project.create_from_bson bp} + end + + # names - array of project names + def project_names_with_envs names=nil + # db.projects.aggregate({$unwind:"$deploy_envs"}, {$project:{"deploy_envs.identifier":1}}, {$group:{_id:"$_id", envs: {$addToSet: "$deploy_envs.identifier"}}}) + q = [] + unless names.nil? + q.push({ + "$match" => { + "_id" => { + "$in" => names + } + } + }) + end + q.push({ + "$unwind" => "$deploy_envs" + }) + q.push({ + "$project" => { + "deploy_envs.identifier" => 1 + } + }) + q.push({ + "$group" => { + "_id" => "$_id", + "envs" => { + "$addToSet" => "$deploy_envs.identifier" + } + } + }) + res = @projects.aggregate(q) + r = {} + res.each do |ar| + r[ar["_id"]] = ar["envs"] + end + return r + end + + def projects_by_image image + @projects.find(create_query("deploy_envs.image" => image)).to_a.map {|bp| Project.create_from_bson bp} + end + + def projects_by_user user + @projects.find(create_query("deploy_envs.users" => user)).to_a.map {|bp| Project.create_from_bson bp} + end + + def project_delete name + r = @projects.remove(create_query("_id" => name)) + raise RecordNotFound.new("Project '#{name}' not found") if r["n"] == 0 + end + + def project_update project + project.validate! + @projects.update(create_query({"_id" => project.id}), project.to_mongo_hash) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: project with id '#{project.id}'") + end + end + + def servers p=nil, env=nil, names=nil + q = {} + q["project"] = p unless p.nil? or p.empty? + q["deploy_env"] = env unless env.nil? or env.empty? + q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array + @servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs} + end + + def servers_by_names names + q = {} + q["chef_node_name"] = {"$in" => names} unless names.nil? or names.class != Array + @servers.find(create_query(q)).to_a.map{|bs| Server.create_from_bson bs} + end + + def server_by_instance_id id + find_server "_id" => id + end + + def server_by_chef_node_name name + find_server "chef_node_name" => name + end + + def servers_by_key key_name + @servers.find(create_query("key" => key_name)).to_a.map {|bs| Server.create_from_bson bs} + end + + def server_insert s + #s.validate! + s.created_at = Time.now + @servers.insert(create_query(s.to_mongo_hash)) + end + + def server_delete id + @servers.remove(create_query("_id" => id)) + end + + def server_update server + @servers.update({"_id" => server.id}, server.to_hash_without_id) + end + + def keys + @keys.find(create_query).to_a.map {|bi| Key.create_from_bson bi} + end + + def key id, scope=nil + q = { + "_id" => id + } + q["scope"] = scope unless scope.nil? + k = @keys.find(create_query(q)).to_a[0] + raise RecordNotFound.new("Key '#{id}' does not exist") if k.nil? + Key.create_from_bson k + end + + def key_insert key + key.validate! + @keys.insert(create_query(key.to_mongo_hash)) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: key with id '#{key.id}'") + end + end + + def key_delete id + r = @keys.remove(create_query("_id" => id, "scope" => Key::USER)) + raise RecordNotFound.new("Key '#{id}' not found") if r["n"] == 0 + r + end + + def user_auth user, password + u = @users.find("_id" => user, "password" => password).to_a[0] + raise RecordNotFound.new("Invalid username or password") if u.nil? + end + + def user id + u = @users.find("_id" => id).to_a[0] + raise RecordNotFound.new("User '#{id}' does not exist") if u.nil? + User.create_from_bson u + end + + def users array=nil + q = {} + q["_id"] = {"$in" => array} if array.is_a?(Array) + @users.find(q).to_a.map {|bi| User.create_from_bson bi} + end + + def users_names array=nil + q = {} + q["_id"] = {"$in" => array} if array.is_a?(Array) + @users.find({}, :fields => ["_id"]).to_a.map{|u| u["_id"]} + end + + def user_insert user + user.validate! + @users.insert(user.to_mongo_hash) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: user with id '#{user.id}'") + end + end + + def user_delete id + r = @users.remove("_id" => id) + raise RecordNotFound.new("User '#{id}' not found") if r["n"] == 0 + r + end + + def user_update user + user.validate! + @users.update({"_id" => user.id}, user.to_mongo_hash) + rescue Mongo::OperationFailure => e + if e.message =~ /^11000/ + raise InvalidRecord.new("Duplicate key error: user with id '#{user.id}'") + end + end + + def create_root_user + begin + u = user("root") + rescue RecordNotFound => e + root = User.create_root + @users.insert(root.to_mongo_hash) + end + end + + def check_user_privileges id, cmd, priv + user = self.user(id) + case priv + when "r" + raise InvalidPrivileges.new("Access denied for '#{user.id}'") unless user.check_privilege_read cmd + when "w" + raise InvalidPrivileges.new("Access denied for '#{user.id}'") unless user.check_privilege_write cmd + else + raise InvalidPrivileges.new("Access internal problem with privilege '#{priv}'") + end + end + + def check_project_auth project_id, env, user_id + p = @projects.find(create_query("_id" => project_id)).to_a[0] + raise RecordNotFound.new("Project '#{project_id}' does not exist") if p.nil? + project = Project.create_from_bson p + raise InvalidPrivileges.new("User '#{user_id}' unauthorized to work with project '#{project_id}'") unless project.check_authorization(user_id, env) + project + end + + def statistic user, path, method, body, response_code + @statistic.insert({:user => user, :path => path, :method => method, :body => body, :response_code => response_code, :date => Time.now}) + end + +private + def find_server params + s = @servers.find(create_query(params)).to_a[0] + if s.nil? + if params.has_key? "_id" + raise RecordNotFound.new("No server by instance id '#{params["_id"]}' found") + elsif params.has_key? "chef_node_name" + raise RecordNotFound.new("No server by node name '#{params["chef_node_name"]}' found") + end + end + Server.create_from_bson s + end + + def create_query q={} + q + end + + def create_query_with_provider provider, q={} + q["provider"] = provider + q + end +end diff --git a/devops-service/db/mongo/mongo_user.rb b/devops-service/db/mongo/mongo_user.rb new file mode 100644 index 0000000..e42cd72 --- /dev/null +++ b/devops-service/db/mongo/mongo_user.rb @@ -0,0 +1,18 @@ +require "mongo" +require "db/exceptions/record_not_found" + +class MongoUser + + def initialize(db, host, port=27017) + @mongo_client = MongoClient.new(host, port) + @db = @mongo_client.db(db) + @users = @db.collection("users") + end + + def user username, password + u = @users.find("_id" => username, "password" => password).to_a[0] + raise RecordNotFound.new("User '#{username}' does not exist") if u.nil? + u + end + +end diff --git a/devops-service/devops-service.rb b/devops-service/devops-service.rb new file mode 100644 index 0000000..38402d0 --- /dev/null +++ b/devops-service/devops-service.rb @@ -0,0 +1,64 @@ +#!/usr/bin/env ruby + +require "rubygems" +require "sinatra/base" +require "sinatra/streaming" +require "fileutils" + +$:.push File.dirname(__FILE__) +require "db/exceptions/invalid_record" +require "db/exceptions/record_not_found" +require "db/mongo/mongo_connector" +require "providers/provider_factory" + +require "routes/v2.0" + +class DevopsService < Sinatra::Base + + helpers Sinatra::Streaming + + def initialize config + super() + @@config = config + root = File.dirname(__FILE__) + @@config[:keys_dir] = File.join(root, "../.devops_files/keys") + if @@config[:scripts_dir].nil? + #default scripts dir + @@config[:scripts_dir] = File.join(root, "../.devops_files/scripts") + end + [:keys_dir, :scripts_dir].each {|key| d = @@config[key]; FileUtils.mkdir_p(d) unless File.exists?(d) } + mongo = DevopsService.mongo + mongo.create_root_user + ::Version2_0::Provider::ProviderFactory.all.each do |p| + begin + mongo.key p.ssh_key, Key::SYSTEM + rescue RecordNotFound => e + k = Key.new({"id" => p.ssh_key, "path" => p.certificate_path, "scope" => Key::SYSTEM}) + mongo.key_insert k + end + end + end + + @@mongo + # Returns mongo connector + def self.mongo + @@mongo ||= MongoConnector.new(@@config[:mongo_db], @@config[:mongo_host], @@config[:mongo_port], @@config[:mongo_user], @@config[:mongo_password]) + end + + # Returns config hash + def self.config + @@config + end + + use Rack::Auth::Basic do |username, password| + begin + mongo.user_auth(username, password) + true + rescue RecordNotFound => e + false + end + end + + use ::Version2_0::V2_0 + +end diff --git a/devops-service/exceptions/dependency_error.rb b/devops-service/exceptions/dependency_error.rb new file mode 100644 index 0000000..b7215c6 --- /dev/null +++ b/devops-service/exceptions/dependency_error.rb @@ -0,0 +1,3 @@ +class DependencyError < Exception + +end \ No newline at end of file diff --git a/devops-service/exceptions/invalid_command.rb b/devops-service/exceptions/invalid_command.rb new file mode 100644 index 0000000..2c219df --- /dev/null +++ b/devops-service/exceptions/invalid_command.rb @@ -0,0 +1,3 @@ +class InvalidCommand < Exception + +end diff --git a/devops-service/exceptions/invalid_privileges.rb b/devops-service/exceptions/invalid_privileges.rb new file mode 100644 index 0000000..16aaa03 --- /dev/null +++ b/devops-service/exceptions/invalid_privileges.rb @@ -0,0 +1,3 @@ +class InvalidPrivileges < Exception + +end diff --git a/devops-service/providers/base_provider.rb b/devops-service/providers/base_provider.rb new file mode 100644 index 0000000..8388c6f --- /dev/null +++ b/devops-service/providers/base_provider.rb @@ -0,0 +1,28 @@ +require "fog" + +module Version2_0 + module Provider + class BaseProvider + + attr_accessor :ssh_key, :certificate_path, :connection_options + + protected + def connection_compute options + Fog::Compute.new( options ) + end + + def connection_network options + Fog::Network.new( options ) + end + + def configured? + !(empty_param?(self.ssh_key) or empty_param?(self.certificate_path)) + end + + def empty_param? param + param.nil? or param.empty? + end + + end + end +end diff --git a/devops-service/providers/ec2.rb b/devops-service/providers/ec2.rb new file mode 100644 index 0000000..89b0518 --- /dev/null +++ b/devops-service/providers/ec2.rb @@ -0,0 +1,243 @@ +require "providers/base_provider" +#require 'xml' + +module Version2_0 + module Provider + # Provider for Amazon EC2 + class Ec2 < BaseProvider + + PROVIDER = "ec2" + + attr_accessor :availability_zone + + def initialize config + self.certificate_path = config[:aws_certificate] + self.ssh_key = config[:aws_ssh_key] + self.connection_options = { + :provider => "aws", + :aws_access_key_id => config[:aws_access_key_id], + :aws_secret_access_key => config[:aws_secret_access_key] + } + self.availability_zone = config[:aws_availability_zone] || "us-east-1a" + end + + def configured? + o = self.connection_options + super and !(empty_param?(o[:aws_access_key_id]) or empty_param?(o[:aws_secret_access_key])) + end + + def name + PROVIDER + end + + def compute + connection_compute(connection_options) + end + + def network + nil + end + + def flavors + self.compute.flavors.all.map do |f| + { + "id" => f.id, + "cores" => f.cores, + "disk" => f.disk, + "name" => f.name, + "ram" => f.ram + } + end + end + + def groups filters=nil + buf = {} + buf = filters.select{|k,v| ["vpc-id"].include?(k)} unless filters.nil? + g = if buf.empty? + self.compute.describe_security_groups + else + self.compute.describe_security_groups(buf) + end + convert_groups(g.body["securityGroupInfo"]) + end + + def images filters + self.compute.describe_images({"image-id" => filters}).body["imagesSet"].map do |i| + { + "id" => i["imageId"], + "name" => i["name"], + "status" => i["imageState"] + } + end + end + + def networks_detail + self.networks + end + + def networks + self.compute.describe_subnets.body["subnetSet"].select{|n| n["state"] == "available"}.map do |n| + { + "cidr" => n["cidrBlock"], + "vpcId" => n["vpcId"], + "subnetId" => n["subnetId"], + "name" => n["subnetId"], + "zone" => n["availabilityZone"] + } + end + end + + def servers + list = self.compute.describe_instances.body["reservationSet"] + list.select{|l| l["instancesSet"][0]["instanceState"]["name"].to_s != "terminated"}.map do |server| + convert_server server["instancesSet"][0] + end + end + + def server id + list = self.compute.describe_instances('instance-id' => [id]).body["reservationSet"] + convert_server list[0]["instancesSet"][0] + end + + def create_server s, out + out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" + options = { + "InstanceType" => s.options[:flavor], + "Placement.AvailabilityZone" => s.options[:availability_zone], + "KeyName" => self.ssh_key + } + vpcId = nil + unless s.options[:subnets].empty? + options["SubnetId"] = s.options[:subnets][0] + vpcId = self.networks.detect{|n| n["name"] == options["SubnetId"]}["vpcId"] + if vpcId.nil? + out << "Can not get 'vpcId' by subnet name '#{options["SubnetId"]}'\n" + return false + end + end + options["SecurityGroupId"] = extract_group_ids(s.options[:groups], vpcId).join(",") + + aws_server = nil + compute = self.compute + begin + aws_server = compute.run_instances(s.options[:image], 1, 1, options) + rescue Excon::Errors::Unauthorized => ue + #root = XML::Parser.string(ue.response.body).parse.root + #msg = root.children.find { |node| node.name == "Message" } + #code = root.children.find { |node| node.name == "Code" } + code = "TODO" + msg = ue.response.body + out << "\nERROR: Unauthorized (#{code}: #{msg})" + return false + rescue Fog::Compute::AWS::Error => e + out << e.message + return false + end + + abody = aws_server.body + instance = abody["instancesSet"][0] + s.id = instance["instanceId"] + + out << "\nInstance Name: #{s.chef_node_name}" + out << "\nInstance ID: #{s.id}\n" + out << "\nWaiting for server..." + + details, state = nil, instance["instanceState"]["name"] + until state == "running" + sleep(2) + details = compute.describe_instances("instance-id" => [s.id]).body["reservationSet"][0]["instancesSet"][0] + state = details["instanceState"]["name"].to_s + next if state == "pending" or state == "running" + out << "Server returns state '#{state}'" + return false + end + s.public_ip = details["ipAddress"] + s.private_ip = details["privateIpAddress"] + compute.create_tags(s.id, {"Name" => s.chef_node_name}) + out << "\nDone\n\n" + out << s.info + + true + end + + def delete_server id + r = self.compute.terminate_instances(id) + i = r.body["instancesSet"][0] + old_state = i["previousState"]["name"] + state = i["currentState"]["name"] + return r.status == 200 ? "Server with id '#{id}' changed state '#{old_state}' to '#{state}'" : r.body + end + + def pause_server id + s = self.server id + if s["state"] == "running" + self.compute.stop_instances [ id ] + return nil + else + return s["state"] + end + end + + def unpause_server id + s = self.server id + if s["state"] == "stopped" + self.compute.start_instances [ id ] + return nil + else + return s["state"] + end + end + + private + def convert_groups list + res = {} + list.each do |g| + res[g["groupName"]] = { + "description" => g["groupDescription"], + "id" => g["groupId"] + } + rules = [] + g["ipPermissions"].each do |r| + cidr = r["ipRanges"][0] || {} + rules.push({ + "protocol" => r["ipProtocol"], + "from" => r["fromPort"], + "to" => r["toPort"], + "cidr" => cidr["cidrIp"] + }) + end + res[g["groupName"]]["rules"] = rules + end + res + end + + def convert_server s + { + "state" => s["instanceState"]["name"], + "name" => s["tagSet"]["Name"], + "image" => s["imageId"], + "flavor" => s["instanceType"], + "keypair" => s["keyName"], + "instance_id" => s["instanceId"], + "dns_name" => s["dnsName"], + "zone" => s["placement"]["availabilityZone"], + "private_ip" => s["privateIpAddress"], + "public_ip" => s["ipAddress"], + "launched_at" => s["launchTime"] + } + end + + def extract_group_ids names, vpcId + return [] if names.nil? + p = nil + p = {"vpc-id" => vpcId} unless vpcId.nil? + groups = self.groups(p) + r = names.map do |name| + groups[name]["id"] + end + r + end + + end + end +end diff --git a/devops-service/providers/openstack.rb b/devops-service/providers/openstack.rb new file mode 100644 index 0000000..f1043ac --- /dev/null +++ b/devops-service/providers/openstack.rb @@ -0,0 +1,207 @@ +require "providers/base_provider" + +module Version2_0 + module Provider + # Provider for 'openstack' + class Openstack < BaseProvider + + PROVIDER = "openstack" + + def initialize config + self.certificate_path = config[:openstack_certificate] + self.ssh_key = config[:openstack_ssh_key] + self.connection_options = { + :provider => PROVIDER, + :openstack_username => config[:openstack_username], + :openstack_api_key => config[:openstack_api_key], + :openstack_auth_url => config[:openstack_auth_url], + :openstack_tenant => config[:openstack_tenant] + } + end + + # Returns 'true' if all parameters defined + def configured? + o = self.connection_options + super and !(empty_param?(o[:openstack_username]) or empty_param?(o[:openstack_api_key]) or empty_param?(o[:openstack_auth_url]) or empty_param?(o[:openstack_tenant])) + end + + def name + PROVIDER + end + + def compute + connection_compute(self.connection_options) + end + + def network + connection_network(self.connection_options) + end + + def groups filter=nil + convert_groups(compute.list_security_groups.body["security_groups"]) + end + + def flavors + self.compute.list_flavors_detail.body["flavors"].map do |f| + { + "id" => f["name"], + "v_cpus" => f["vcpus"], + "ram" => f["ram"], + "disk" => f["disk"] + } + end + end + + def images filters + self.compute.list_images_detail.body["images"].select{|i| filters.include?(i["id"]) and i["status"] == "ACTIVE"}.map do |i| + { + "id" => i["id"], + "name" => i["name"], + "status" => i["status"] + } + end + end + + def networks_detail + net = self.network + subnets = net.list_subnets.body["subnets"].select{|s| net.current_tenant["id"] == s["tenant_id"]} + net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| + sn = subnets.detect{|s| n["subnets"][0] == s["id"]} + { + "cidr" => sn["cidr"], + "name" => n["name"], + "id" => n["id"] + } + } + end + + def networks + net = self.network + net.list_networks.body["networks"].select{|n| n["router:external"] == false and n["status"] == "ACTIVE" and net.current_tenant["id"] == n["tenant_id"]}.map{|n| + { + "name" => n["name"], + "id" => n["id"] + } + } + end + + def servers + list = self.compute.list_servers_detail.body["servers"] +puts list[0].inspect + list.map do |s| + o = {"state" => s["status"], "name" => s["name"], "image" => s["image"]["id"], "flavor" => s["flavor"]["name"], "keypair" => s["key_name"], "instance_id" => s["id"]} + s["addresses"].each_value do |a| + a.each do |addr| + o["private_ip"] = addr["addr"] if addr["OS-EXT-IPS:type"] == "fixed" + end + end + o + end + end + + def create_server s, out + out << "Creating server for project '#{s.project} - #{s.deploy_env}'\n" + networks = self.networks.select{|n| s.options[:subnets].include?(n["name"])} + buf = s.options[:subnets] - networks.map{|n| n["name"]} + unless buf.empty? + out << "No networks with names '#{buf.join("', '")}' found" + return false + end + s.options[:flavor] = self.compute.list_flavors_detail.body["flavors"].detect{|f| f["name"] == s.options[:flavor]}["id"] + out << "Creating server with name '#{s.chef_node_name}', image '#{s.options[:image]}', flavor '#{s.options[:flavor]}', key '#{s.key}' and networks '#{networks.map{|n| n["name"]}.join("', '")}'...\n\n" + compute = self.compute + begin + o_server = compute.create_server(s.chef_node_name, s.options[:image], s.options[:flavor], + "nics" => networks.map{|n| {"net_id" => n["id"]}}, + "security_groups" => s.options[:groups], + "key_name" => s.key) + rescue Excon::Errors::BadRequest => e + response = ::Chef::JSONCompat.from_json(e.response.body) + if response['badRequest']['code'] == 400 + if response['badRequest']['message'] =~ /Invalid flavorRef/ + out << "\nERROR: Bad request (400): Invalid flavor id specified: #{s.options[:flavor]}" + elsif response['badRequest']['message'] =~ /Invalid imageRef/ + out << "\nERROR: Bad request (400): Invalid image specified: #{s.options[:image]}" + else + out << "\nERROR: Bad request (400): #{response['badRequest']['message']}" + end + return false + else + out << "\nERROR: Unknown server error (#{response['badRequest']['code']}): #{response['badRequest']['message']}" + return false + end + rescue => e2 + out << "Error: Unknown error: " + e.message + return false + end + sbody = o_server.body + s.id = sbody["server"]["id"] + + out << "\nInstance Name: #{s.chef_node_name}" + out << "\nInstance ID: #{s.id}\n" + out << "\nWaiting for server..." + + details, status = nil, nil + until status == "ACTIVE" + sleep(1) + details = compute.get_server_details(s.id).body + status = details["server"]["status"].upcase + if status == "ERROR" + out << "Server returns status 'ERROR'" + return false + end + end + network = networks[0]["name"] + s.private_ip = details["server"]["addresses"][network][0]["addr"] + out << "\nDone\n\n" + out << s.info + true + end + + def delete_server id + r = self.compute.delete_server(id) + return r.status == 204 ? "Server with id '#{id}' terminated" : r.body + end + + def pause_server id + begin + self.compute.pause_server id + rescue Excon::Errors::Conflict => e + return "pause" + end + return nil + end + + def unpause_server id + begin + self.compute.unpause_server id + rescue Excon::Errors::Conflict => e + return "unpause" + end + return nil + end + + private + def convert_groups list + res = {} + list.map do |g| + res[g["name"]] = { + "description" => g["description"] + } + rules = [] + g["rules"].each do |r| + rules.push({ + "protocol" => r["ip_protocol"], + "from" => r["from_port"], + "to" => r["to_port"], + "cidr" => r["ip_range"]["cidr"] + }) + end + res[g["name"]]["rules"] = rules + end + res + end + + end + end +end diff --git a/devops-service/providers/provider_factory.rb b/devops-service/providers/provider_factory.rb new file mode 100644 index 0000000..ca37f77 --- /dev/null +++ b/devops-service/providers/provider_factory.rb @@ -0,0 +1,42 @@ +module Version2_0 + module Provider + class ProviderFactory + + @@providers = nil + + def self.providers + @@providers.keys + end + + def self.get provider + p = @@providers[provider] + raise ::Sinatra::NotFound.new("Provider #{provider} not found") if p.nil? + p + end + + def self.all + if @@providers.nil? + ProviderFactory.init + end + @@providers.values + end + + def self.init + conf = DevopsService.config + @@providers = {} + ["ec2", "openstack"].each do |p| + begin + require "providers/#{p}" + o = Version2_0::Provider.const_get(p.capitalize).new(conf) + @@providers[p] = o if o.configured? + rescue => e + next + rescue LoadError + next + end + end + end + + end + end +end diff --git a/devops-service/routes/v2.0.rb b/devops-service/routes/v2.0.rb new file mode 100644 index 0000000..8be3e34 --- /dev/null +++ b/devops-service/routes/v2.0.rb @@ -0,0 +1,34 @@ +require "routes/v2.0/flavor" +require "routes/v2.0/image" +require "routes/v2.0/filter" +require "routes/v2.0/network" +require "routes/v2.0/group" +require "routes/v2.0/deploy" +require "routes/v2.0/project" +require "routes/v2.0/key" +require "routes/v2.0/user" +require "routes/v2.0/provider" +require "routes/v2.0/tag" +require "routes/v2.0/server" +require "routes/v2.0/script" +require "routes/v2.0/bootstrap_templates" + +module Version2_0 + class V2_0 + + # Initialize modules of devops API v2.0 + def initialize app + stack = Rack::Builder.new + [FlavorRoutes, ImageRoutes, FilterRoutes, NetworkRoutes, GroupRoutes, DeployRoutes, + ProjectRoutes, KeyRoutes, UserRoutes, ProviderRoutes, TagRoutes, ServerRoutes, ScriptRoutes, BootstrapTemplatesRoutes].each do |m| + stack.use m + end + stack.run app + @app = stack.to_app + end + + def call(env) + @app.call env + end + end +end diff --git a/devops-service/routes/v2.0/base_routes.rb b/devops-service/routes/v2.0/base_routes.rb new file mode 100644 index 0000000..29f9c5e --- /dev/null +++ b/devops-service/routes/v2.0/base_routes.rb @@ -0,0 +1,223 @@ +require "json" +require "db/exceptions/record_not_found" +require "db/exceptions/invalid_record" +require "exceptions/dependency_error" +require "exceptions/invalid_privileges" +require "fog" +require "logger" +require "providers/provider_factory" +require "sinatra/json" +require "sinatra/base" + +module Version2_0 + # Basic class for devops routes classes + class BaseRoutes < Sinatra::Base + + helpers do + def create_response msg, obj=nil, rstatus=200 + logger.info(msg) + status rstatus + obj = {} if obj.nil? + obj[:message] = msg + json(obj) + end + + def halt_response msg, rstatus=400 + obj = {:message => msg} + halt(rstatus, json(obj)) + end + + def check_privileges cmd, p=nil + if p != "r" and p != "w" + p = request.get? ? "r" : "w" + end + BaseRoutes.mongo.check_user_privileges(request.env['REMOTE_USER'], cmd, p) + end + + # Check request headers + def check_headers *headers + ha = (headers.empty? ? [:accept, :content_type] : headers) + ha.each do |h| + case h + when :accept, "accept" + accept_json + when :content_type, "content_type" + request_json + end + end + end + + # Check Accept header + # + # Can client works with JSON? + def accept_json + logger.debug(request.accept) + unless request.accept? 'application/json' + response.headers['Accept'] = 'application/json' + halt_response("Accept header should contains 'application/json' type", 406) + end + rescue NoMethodError => e + #error in sinatra 1.4.4 (https://github.com/sinatra/sinatra/issues/844, https://github.com/sinatra/sinatra/pull/805) + response.headers['Accept'] = 'application/json' + halt_response("Accept header should contains 'application/json' type", 406) + end + + # Check Content-Type header + def request_json + halt_response("Content-Type should be 'application/json'", 415) if request.media_type.nil? or request.media_type != 'application/json' + end + + def check_provider provider + list = ::Version2_0::Provider::ProviderFactory.providers + halt_response("Invalid provider '#{provider}', available providers: '#{list.join("', '")}'", 404) unless list.include?(provider) + end + + def create_object_from_json_body type=Hash, empty_body=false + json = request.body.read.strip + return nil if json.empty? and empty_body + @body_json = begin + JSON.parse(json) + rescue => e + logger.error e.message + logger.debug(json) + halt_response("Invalid JSON") + end + halt_response("Invalid JSON, it should be an #{type == Array ? "array" : "object"}") unless @body_json.is_a?(type) + @body_json + end + + def check_string val, msg, _nil=false, empty=false + check_param val, String, msg, _nil, empty + end + + def check_array val, msg, vals_type=String, _nil=false, empty=false + check_param val, Array, msg, _nil, empty + val.each {|v| halt_response(msg) unless v.is_a?(vals_type)} unless val.nil? + val + end + + def check_filename file_name, not_string_msg, json_resp=true + check_string file_name, not_string_msg + r = Regexp.new("^[\\w _\\-.]{1,255}$", Regexp::IGNORECASE) + if r.match(file_name).nil? + msg = "Invalid file name '#{file_name}'. Expected name with 'a'-'z', '0'-'9', ' ', '_', '-', '.' symbols with length greate then 0 and less then 256 " + if json_resp + halt_response(msg) + else + halt(400, msg) + end + end + file_name + end + + def check_param val, type, msg, _nil=false, empty=false + if val.nil? + if _nil + return val + else + halt_response(msg) + end + end + if val.is_a?(type) + halt_response(msg) if val.empty? and !empty + val + else + halt_response(msg) + end + end + + # Save information about requests with methods POST, PUT, DELETE + def statistic msg=nil + unless request.get? + BaseRoutes.mongo.statistic request.env['REMOTE_USER'], request.path, request.request_method, @body_json, response.status + end + end + + end + + include Sinatra::JSON + + configure :production do + disable :dump_errors + disable :show_exceptions + set :logging, Logger::INFO + end + + configure :development do + set :logging, Logger::DEBUG + disable :raise_errors +# disable :dump_errors + set :show_exceptions, :after_handler + end + + not_found do + "Not found" + end + + error RecordNotFound do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 404) + end + + error InvalidRecord do + e = env["sinatra.error"] + logger.warn e.message + logger.warn "Request body: #{request.body.read}" + halt_response(e.message, 400) + end + + error InvalidCommand do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 400) + end + + error DependencyError do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 400) + end + + error InvalidPrivileges do + e = env["sinatra.error"] + logger.warn e.message + halt_response(e.message, 401) + end + + error Excon::Errors::Unauthorized do + e = env["sinatra.error"] + resp = e.response + ct = resp.headers["Content-Type"] + msg = unless ct.nil? + if ct.include?("application/json") + json = ::Chef::JSONCompat.from_json(resp.body) + m = "ERROR: Unauthorized (#{json['error']['code']}): #{json['error']['message']}" + logger.error(m) + else + end + m + else + "Unauthorized: #{e.inspect}" + end + halt_response(msg, 500) + end + + error Fog::Compute::AWS::Error do + e = env["sinatra.error"] + logger.error e.message + halt_response(e.message, 500) + end + + error do + e = env["sinatra.error"] + logger.error e.message + halt_response(e.message, 500) + end + + def self.mongo + DevopsService.mongo + end + + end +end diff --git a/devops-service/routes/v2.0/bootstrap_templates.rb b/devops-service/routes/v2.0/bootstrap_templates.rb new file mode 100644 index 0000000..c6d2156 --- /dev/null +++ b/devops-service/routes/v2.0/bootstrap_templates.rb @@ -0,0 +1,34 @@ +require "json" +require "routes/v2.0/base_routes" +require "providers/provider_factory" + +module Version2_0 + class BootstrapTemplatesRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Bootstrap templates routes initialized" + end + + # Get list of available bootstrap templates + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of strings + # [ + # "omnibus" + # ] + get "/templates" do + check_headers :accept + check_privileges("templates", "r") + res = [] + Dir.foreach("#{ENV["HOME"]}/.chef/bootstrap/") {|f| res.push(f[0..-5]) if f.end_with?(".erb")} if File.exists? "#{ENV["HOME"]}/.chef/bootstrap/" + json res + end + + end +end + diff --git a/devops-service/routes/v2.0/deploy.rb b/devops-service/routes/v2.0/deploy.rb new file mode 100644 index 0000000..b9ea528 --- /dev/null +++ b/devops-service/routes/v2.0/deploy.rb @@ -0,0 +1,102 @@ +require "commands/knife_commands" +require "routes/v2.0/base_routes" +require "providers/provider_factory" +require "commands/deploy" +require "commands/status" + +module Version2_0 + class DeployRoutes < BaseRoutes + + include DeployCommands + include StatusCommands + + def initialize wrapper + super wrapper + puts "Deploy routes initialized" + end + + after "/deploy" do + statistic + end + + # Run chef-client on some instances + # + # * *Request* + # - method : POST + # - headers : + # - Content-Type: application/json + # - body : + # { + # "names": [], -> array of servers names to run chef-client + # "tags": [] -> array of tags to apply on each server before running chef-client + # } + # + # * *Returns* : text stream + post "/deploy" do + check_headers :content_type + check_privileges("server", "w") + r = create_object_from_json_body + names = check_array(r["names"], "Parameter 'names' should be a not empty array of strings") + tags = check_array(r["tags"], "Parameter 'tags' should be an array of strings", String, true) || [] + + servers = BaseRoutes.mongo.servers_by_names(names) + halt(404, "No servers found for names '#{names.join("', '")}'") if servers.empty? + keys = {} + servers.sort_by!{|s| names.index(s.chef_node_name)} + stream() do |out| + status = [] + servers.each do |s| + begin + + begin + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + rescue InvalidPrivileges, RecordNotFound => e + out << e.message + "\n" + status.push 2 + next + end + + old_tags_str = nil + new_tags_str = nil + unless tags.empty? + old_tags_str = KnifeCommands.tags_list(s.chef_node_name).join(" ") + out << "Server tags: #{old_tags_str}\n" + KnifeCommands.tags_delete(s.chef_node_name, old_tags_str) + + new_tags_str = tags.join(" ") + out << "Server new tags: #{new_tags_str}\n" + cmd = KnifeCommands.tags_create(s.chef_node_name, new_tags_str) + unless cmd[1] + m = "Error: Cannot add tags '#{new_tags_str}' to server '#{s.chef_node_name}'" + logger.error(m) + out << m + "\n" + status.push 3 + next + end + logger.info("Set tags for '#{s.chef_node_name}': #{new_tags_str}") + end + + unless keys.key? s.key + k = BaseRoutes.mongo.key s.key + keys[s.key] = k.path + end + status.push(deploy_server out, s, keys[s.key]) + + unless tags.empty? + out << "Restore tags\n" + cmd = KnifeCommands.tags_delete(s.chef_node_name, new_tags_str) + logger.info("Deleted tags for #{s.chef_node_name}: #{new_tags_str}") + cmd = KnifeCommands.tags_create(s.chef_node_name, old_tags_str) + logger.info("Set tags for #{s.chef_node_name}: #{old_tags_str}") + end + out << create_status(status) + rescue IOError => e + logger.error e.message + break + end + end + end + end + + end +end diff --git a/devops-service/routes/v2.0/filter.rb b/devops-service/routes/v2.0/filter.rb new file mode 100644 index 0000000..ca58534 --- /dev/null +++ b/devops-service/routes/v2.0/filter.rb @@ -0,0 +1,83 @@ +require "routes/v2.0/base_routes" + +module Version2_0 + class FilterRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Filter routes initialized" + end + + before "/filter/:provider/image" do + check_headers :accept, :content_type + check_privileges("filter", "w") + check_provider(params[:provider]) + @images = create_object_from_json_body(Array) + halt_response("Request body should not be an empty array") if @images.empty? + end + + after "/filter/:provider/image" do + statistic + end + + # Get list of images filters for :provider + # + # Devops can works with images from filters list only + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of strings + # - ec2: + # [ + # "ami-83e4bcea" + # ] + # - openstack: + # [ + # "36dc7618-4178-4e29-be43-286fbfe90f50" + # ] + get "/filter/:provider/images" do + check_headers :accept + check_privileges("filter", "r") + check_provider(params[:provider]) + json BaseRoutes.mongo.available_images(params[:provider]) + end + + # Add image ids to filter for :provider + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # [ + # "image_id" + # ] -> array of image ids to add to filter + # + # * *Returns* : list of images filters for :provider + put "/filter/:provider/image" do + create_response("Updated", {:images => BaseRoutes.mongo.add_available_images(@images, params[:provider])}) + end + + # Delete image ids from filter for :provider + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # [ + # "image_id" + # ] -> array of image ids to delete from filter + # + # * *Returns* : list of images filters for :provider + delete "/filter/:provider/image" do + create_response("Deleted", {:images => BaseRoutes.mongo.delete_available_images(@images, params[:provider])}) + end + + end +end diff --git a/devops-service/routes/v2.0/flavor.rb b/devops-service/routes/v2.0/flavor.rb new file mode 100644 index 0000000..efb72aa --- /dev/null +++ b/devops-service/routes/v2.0/flavor.rb @@ -0,0 +1,49 @@ +require "json" +require "routes/v2.0/base_routes" +require "providers/provider_factory" + +module Version2_0 + class FlavorRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Flavor routes initialized" + end + + # Get list of flavors for :provider + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of objects + # - ec2: + # [ + # { + # "id": "t1.micro", + # "cores": 2, + # "disk": 0, + # "name": "Micro Instance", + # "ram": 613 + # } + # ] + # - openstack: + # [ + # { + # "id": "m1.small", + # "v_cpus": 1, + # "ram": 2048, + # "disk": 20 + # } + # ] + get "/flavors/:provider" do + check_headers :accept + check_privileges("flavor", "r") + check_provider(params[:provider]) + p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + json p.flavors + end + + end +end diff --git a/devops-service/routes/v2.0/group.rb b/devops-service/routes/v2.0/group.rb new file mode 100644 index 0000000..6c27d0a --- /dev/null +++ b/devops-service/routes/v2.0/group.rb @@ -0,0 +1,61 @@ +# encoding: UTF-8 +require "json" +require "routes/v2.0/base_routes" +require "providers/provider_factory" + +module Version2_0 + class GroupRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Group routes initialized" + end + + # Get security groups for :provider + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # - ec2: + # { + # "default": { + # "description": "default group", + # "id": "sg-565cf93f", + # "rules": [ + # { + # "protocol": "tcp", + # "from": 22, + # "to": 22, + # "cidr": "0.0.0.0/0" + # } + # ] + # } + # } + # - openstack: + # { + # "default": { + # "description": "default", + # "rules": [ + # { + # "protocol": null, + # "from": null, + # "to": null, + # "cidr": null + # } + # ] + # } + # } + # TODO: vpc support for ec2 + get "/groups/:provider" do + check_headers :accept + check_privileges("group", "r") + check_provider(params[:provider]) + p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + json p.groups(params) + end + + end +end diff --git a/devops-service/routes/v2.0/image.rb b/devops-service/routes/v2.0/image.rb new file mode 100644 index 0000000..11fc852 --- /dev/null +++ b/devops-service/routes/v2.0/image.rb @@ -0,0 +1,181 @@ +require "providers/provider_factory" +require "routes/v2.0/base_routes" +require "commands/image" + +module Version2_0 + class ImageRoutes < BaseRoutes + + include ImageCommands + + def initialize wrapper + super wrapper + puts "Image routes initialized" + end + + before "/image/:image_id" do + if request.get? or request.delete? + check_headers :accept + else + check_headers + end + check_privileges("image") + end + + after %r{\A/image(/[\w]+)?\z} do + statistic + end + + # Get devops images list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # - parameters: + # - provider=ec2|openstack -> return images for provider + # + # * *Returns* : + # [ + # { + # "provider": "openstack", + # "name": "centos-6.4-x86_64", + # "remote_user": "root", + # "bootstrap_template": null, + # "id": "36dc7618-4178-4e29-be43-286fbfe90f50" + # } + # ] + get "/images" do + check_headers :accept + check_privileges("image", "r") + check_provider(params[:provider]) if params[:provider] + images = BaseRoutes.mongo.images(params[:provider]) + json(images.map {|i| i.to_hash}) + end + + # Get raw images for :provider + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # - ec2 + # [ + # { + # "id": "ami-83e4bcea", + # "name": "amzn-ami-pv-2013.09.1.x86_64-ebs", + # "status": "available" + # } + # ] + # - openstack + # [ + # { + # "id": "36dc7618-4178-4e29-be43-286fbfe90f50", + # "name": "centos-6.4-x86_64", + # "status": "ACTIVE" + # } + # ] + get "/images/provider/:provider" do + check_headers :accept + check_privileges("image", "r") + check_provider(params[:provider]) + json get_images(BaseRoutes.mongo, params[:provider]) + end + + # Get devops image by id + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # { + # "provider": "openstack", + # "name": "centos-6.4-x86_64", + # "remote_user": "root", + # "bootstrap_template": null, + # "id": "36dc7618-4178-4e29-be43-286fbfe90f50" + # } + get "/image/:image_id" do + json BaseRoutes.mongo.image(params[:image_id]) + end + + # Create devops image + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "id": "image id", + # "provider": "image provider", + # "remote_user": "user", -> the ssh username + # "bootstrap_template": null, -> specific bootstrap template name or nil + # "name": "image name" + # } + # + # * *Returns* : + # 201 - Created + post "/image" do + check_headers + check_privileges("image", "w") + image = create_object_from_json_body + BaseRoutes.mongo.image_insert Image.new(image) + create_response "Created", nil, 201 + end + + # Update devops image + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "id": "image id", + # "provider": "image provider", + # "remote_user": "user" -> the ssh username + # "bootstrap_template": null -> specific bootstrap template name or nil + # "name": "image name" + # } + # + # * *Returns* : + # 200 - Updated + put "/image/:image_id" do + BaseRoutes.mongo.image params[:image_id] + image = Image.new(create_object_from_json_body) + image.id = params[:image_id] + BaseRoutes.mongo.image_update image + create_response("Image '#{params[:image_id]}' updated") + end + + # Delete devops image + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Deleted + delete "/image/:image_id" do + projects = BaseRoutes.mongo.projects_by_image params[:image_id] + unless projects.empty? + ar = [] + projects.each do |p| + ar += p.deploy_envs.select{|e| e.image == params[:image_id]}.map{|e| "#{p.id}.#{e.identifier}"} + end + raise DependencyError.new "Deleting is forbidden: Image is used in #{ar.join(", ")}" + end + + r = BaseRoutes.mongo.image_delete params[:image_id] + create_response("Image '#{params[:image_id]}' removed") + end + + end +end diff --git a/devops-service/routes/v2.0/key.rb b/devops-service/routes/v2.0/key.rb new file mode 100644 index 0000000..2936c4e --- /dev/null +++ b/devops-service/routes/v2.0/key.rb @@ -0,0 +1,110 @@ +require "json" +require "db/exceptions/invalid_record" +require "db/mongo/models/key" +require "fileutils" + +module Version2_0 + class KeyRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Key routes initialized" + end + + before %r{\A/key(/[\w]+)?\z} do + if request.delete? + check_headers :accept + else + check_headers :accept, :content_type + end + check_privileges("key", "w") + end + + after %r{\A/key(/[\w]+)?\z} do + statistic + end + + # Get list of available ssh keys + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of strings + # [ + # { + # "scope": "system", -> 'system' - key was added by server, 'user' - key was added by user + # "id": "devops" + # } + # ] + get "/keys" do + check_headers :accept + check_privileges("key", "r") + keys = BaseRoutes.mongo.keys.map {|i| i.to_hash} + keys.each {|k| k.delete("path")} # We should not return path to the key + json keys + end + + # Create ssh key on devops server + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "file_name": "key file name", + # "key_name": "key name", + # "content": "key content" + # } + # + # * *Returns* : + # 201 - Created + post "/key" do + key = create_object_from_json_body + fname = check_filename(key["file_name"], "Parameter 'file_name' must be a not empty string") + kname = check_string(key["key_name"], "Parameter 'key_name' should be a not empty string") + content = check_string(key["content"], "Parameter 'content' should be a not empty string") + file_name = File.join(DevopsService.config[:keys_dir], fname) + halt(400, "File '#{fname}' already exist") if File.exists?(file_name) + File.open(file_name, "w") do |f| + f.write(content) + f.chmod(0400) + end + + key = Key.new({"path" => file_name, "id" => kname}) + BaseRoutes.mongo.key_insert key + create_response("Created", nil, 201) + end + + # Delete ssh key from devops server + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Deleted + delete "/key/:key" do + servers = BaseRoutes.mongo.servers_by_key params[:key] + unless servers.empty? + s_str = servers.map{|s| s.id}.join(", ") + raise DependencyError.new "Deleting is forbidden: Key is used in servers: #{s_str}" + end + + k = BaseRoutes.mongo.key params[:key] + begin + FileUtils.rm(k.path) + rescue + logger.error "Missing key file for #{params[:key]} - #{k.filename}" + end + r = BaseRoutes.mongo.key_delete params[:key] + return [500, r["err"].inspect] unless r["err"].nil? + create_response("Key '#{params[:key]}' removed") + end + + end +end diff --git a/devops-service/routes/v2.0/network.rb b/devops-service/routes/v2.0/network.rb new file mode 100644 index 0000000..2f8201f --- /dev/null +++ b/devops-service/routes/v2.0/network.rb @@ -0,0 +1,49 @@ +# encoding: UTF-8 +require "json" +require "routes/v2.0/base_routes" +require "providers/provider_factory" + +module Version2_0 + class NetworkRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Network routes initialized" + end + + # Get list of networks for :provider + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : array of strings + # - ec2: + # [ + # { + # "cidr": "0.0.0.0/16", + # "vpcId": "vpc-1", + # "subnetId": "subnet-1", + # "name": "subnet-1", + # "zone": "us-east-1a" + # } + # ] + # - openstack: + # [ + # { + # "cidr": "0.0.0.0/16", + # "name": "private", + # "id": "b14f8df9-ac27-48e2-8d65-f7ef78dc2654" + # } + # ] + get "/networks/:provider" do + check_headers :accept + check_privileges("network", "r") + check_provider(params[:provider]) + p = ::Version2_0::Provider::ProviderFactory.get params[:provider] + json p.networks_detail + end + + end +end diff --git a/devops-service/routes/v2.0/project.rb b/devops-service/routes/v2.0/project.rb new file mode 100644 index 0000000..cfdec35 --- /dev/null +++ b/devops-service/routes/v2.0/project.rb @@ -0,0 +1,574 @@ +require "json" +require "db/mongo/models/project" +require "db/mongo/models/deploy_env" +require "db/exceptions/invalid_record" +require "commands/deploy" +require "commands/status" +require "commands/server" + +module Version2_0 + class ProjectRoutes < BaseRoutes + + include DeployCommands + include StatusCommands + include ServerCommands + + def initialize wrapper + super wrapper + puts "Project routes initialized" + end + + before "/project/:id" do + if request.get? + check_headers :accept + else + check_headers :accept, :content_type + end + check_privileges("project") + end + + before "/project/:id/user" do + check_headers :accept, :content_type + check_privileges("project", "w") + body = create_object_from_json_body + @users = check_array(body["users"], "Parameter 'users' must be a not empty array of strings") + @deploy_env = check_string(body["deploy_env"], "Parameter 'deploy_env' must be a not empty string", true) + @project = BaseRoutes.mongo.project(params[:id]) + end + + after %r{\A/project(/[\w]+(/(user|deploy))?)?\z} do + statistic + end + + after "/project/:id/:env/run_list" do + statistic + end + + # Get projects list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # "project_1" + # ] + # TODO: list with environments + get "/projects" do + check_headers :accept + check_privileges("project", "r") + json BaseRoutes.mongo.projects.map {|p| p.id} + end + + # Get project by id + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # { + # "deploy_envs": [ + # { + # "flavor": "flavor", + # "identifier": "prod", + # "image": "image id", + # "run_list": [ + # "role[project_1-prod]" + # ], + # "subnets": [ + # "private" + # ], + # "expires": null, + # "provider": "openstack", + # "groups": [ + # "default" + # ], + # "users": [ + # "user" + # ] + # } + # ], + # "name": "project_1" + # } + get "/project/:project" do + json BaseRoutes.mongo.project(params[:project]) + end + + # Get project servers + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # - parameters : + # - deploy_env=:env -> show servers with environment :env + # + # * *Returns* : + # [ + # { + # "provider": "openstack", + # "chef_node_name": "project_1_server", + # "remote_user": "root", + # "project": "project_1", + # "deploy_env": "prod", + # "private_ip": "10.8.8.8", + # "public_ip": null, + # "created_at": "2014-04-23 13:35:18 UTC", + # "created_by": "user", + # "static": false, + # "key": "ssh key", + # "id": "nstance id" + # } + # ] + get "/project/:project/servers" do + check_headers :accept + check_privileges("project", "r") + BaseRoutes.mongo.project(params[:project]) + json BaseRoutes.mongo.servers(params[:project], params[:deploy_env]).map{|s| s.to_hash} + end + + # Create project and chef roles + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "deploy_envs": [ + # { + # "identifier": "prod", + # "provider": "openstack", + # "flavor": "m1.small", + # "image": "image id", + # "subnets": [ + # "private" + # ], + # "groups": [ + # "default" + # ], + # "users": [ + # "user" + # ], + # "run_list": [ + # + # ], + # "expires": null + # } + # ], + # "name": "project_1" + # } + # + # * *Returns* : + # 201 - Created + # TODO: multi project + post "/project" do + check_headers :accept, :content_type + check_privileges("project", "w") + body = create_object_from_json_body + check_string(body["name"], "Parameter 'name' must be a not empty string") + check_array(body["deploy_envs"], "Parameter 'deploy_envs' must be a not empty array of objects", Hash) + p = Project.new(body) + halt_response("Project '#{p.id}' already exist") if BaseRoutes.mongo.is_project_exists?(p) + p.add_authorized_user [request.env['REMOTE_USER']] + BaseRoutes.mongo.project_insert p + roles_res = "" + if p.multi? + logger.info "Project '#{p.id}' with type 'multi' created" + else + logger.info "Project '#{p.id}' created" + roles = create_roles p.id, p.deploy_envs, logger + roles_res = ". " + create_roles_response(roles) + end + res = "Created" + roles_res + create_response(res, nil, 201) + end + + # Update project and create chef roles + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "deploy_envs": [ + # { + # "identifier": "dev", + # "provider": "openstack", + # "flavor": "m1.small", + # "image": "image id", + # "subnets": [ + # "private" + # ], + # "groups": [ + # "default" + # ], + # "users": [ + # "user" + # ], + # "run_list": [ + # + # ], + # "expires": null + # } + # ], + # "name": "project_1" + # } + # + # * *Returns* : + # 200 - Updated + # TODO: multi project + put "/project/:id" do + project = Project.new(create_object_from_json_body) + project.id = params[:id] + old_project = BaseRoutes.mongo.project params[:id] + BaseRoutes.mongo.project_update project + roles = create_new_roles(old_project, project, logger) + info = "Project '#{project.id}' has been updated." + create_roles_response(roles) + create_response(info) + end + + # Add users to project environment + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "users": [ + # "user1" + # ], + # "deploy_env": "env" -> if null, users will be added to all environments + # } + # + # * *Returns* : + # 200 - Updated + # TODO: multi project + put "/project/:id/user" do + users = BaseRoutes.mongo.users(@users).map{|u| u.id} + buf = @users - users + @project.add_authorized_user users, @deploy_env + BaseRoutes.mongo.project_update(@project) + info = "Users '#{users.join("', '")}' have been added to '#{params[:id]}' project's authorized users" + info << ", invalid users: '#{buf.join("', '")}'" unless buf.empty? + create_response(info) + end + + # Delete users from project environment + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "users": [ + # "user1" + # ], + # "deploy_env": "env" -> if null, users will be deleted from all environments + # } + # + # * *Returns* : + # 200 - Updated + # TODO: multi project + delete "/project/:id/user" do + @project.remove_authorized_user @users, @deploy_env + BaseRoutes.mongo.project_update @project + info = "Users '#{@users.join("', '")}' have been removed from '#{params[:id]}' project's authorized users" + create_response(info) + end + + # Set run_list to project environment + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # [ + # "role[role_1]", + # "recipe[recipe_1]" + # ] + # + # * *Returns* : + # 200 - Updated + # TODO: multi project + put "/project/:id/:env/run_list" do + check_headers :accept, :content_type + check_privileges("project", "w") + list = create_object_from_json_body(Array) + check_array(list, "Body must contains not empty array of strings") + project = BaseRoutes.mongo.project(params[:id]) + env = project.deploy_env params[:env] + env.run_list = list + BaseRoutes.mongo.project_update project + create_response("Updated environment '#{env.identifier}' with run_list '#{env.run_list.inspect}' in project '#{project.id}'") + end + + # Delete project + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "deploy_env": "env" -> if not null, will be deleted environment only + # } + # + # * *Returns* : + # 200 - Deleted + delete "/project/:id" do + servers = BaseRoutes.mongo.servers params[:id] + raise DependencyError.new "Deleting #{params[:id]} is forbidden: Project has servers" if !servers.empty? + body = create_object_from_json_body(Hash, true) + deploy_env = unless body.nil? + check_string(body["deploy_env"], "Parameter 'deploy_env' should be a not empty string", true) + end + info = if deploy_env.nil? + BaseRoutes.mongo.project_delete(params[:id]) + "Project '#{params[:id]}' is deleted" + else + project = BaseRoutes.mongo.project(params[:id]) + project.remove_env params[:deploy_env] + BaseRoutes.mongo.project_update project + "Project '#{params[:id]}'. Deploy environment '#{params[:deploy_env]}' has been deleted" + end + create_response(info) + end + + # Run chef-client on project servers + # + # * *Request* + # - method : POST + # - headers : + # - Content-Type: application/json + # - body : + # { + # "servers": [ + # "server_1" + # ], -> deploy servers from list, all servers if null + # "deploy_env": "env" -> deploy servers with environment 'env' or all project servers if null + # } + # + # * *Returns* : text stream + post "/project/:id/deploy" do + check_headers :content_type + check_privileges("project", "w") + obj = create_object_from_json_body + check_string(obj["deploy_env"], "Parameter 'deploy_env' should be a not empty string", true) + check_array(obj["servers"], "Parameter 'servers' should be a not empty array of strings", String, true) + project = BaseRoutes.mongo.project(params[:id]) + servers = BaseRoutes.mongo.servers(params[:id], obj["deploy_env"]) + unless obj["servers"].nil? + logger.debug "Servers in params: #{obj["servers"].inspect}\nServers: #{servers.map{|s| s.chef_node_name}.inspect}" + servers.select!{|ps| obj["servers"].include?(ps.chef_node_name)} + end + keys = {} + stream() do |out| + begin + out << (servers.empty? ? "No servers to deploy\n" : "Deploy servers: '#{servers.map{|s| s.chef_node_name}.join("', '")}'\n") + status = [] + servers.each do |s| + + begin + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + rescue InvalidPrivileges, RecordNotFound => e + out << e.message + "\n" + status.push 2 + next + end + unless keys.key? s.key + k = BaseRoutes.mongo.key s.key + keys[s.key] = k.path + end + status.push(deploy_server out, s, keys[s.key]) + end + out << create_status(status) + rescue IOError => e + logger.error e.message + end + end + end + + # Test project environment + # + # Run tests: + # - run server + # - bootstrap server + # - delete server + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # + # * *Returns* : + # 200 - + # { + # "servers": [ + # { + # "id": "132958f0-61c5-4665-8cc3-66e1bacd285b", + # "create": { + # "status": true, + # "time": "155s" + # }, + # "chef_node_name": "chef name", + # "bootstrap": { + # "status": true, + # "log": "\nWaiting for SSH...\n" + # "return_code": 0 + # }, + # "delete": { + # "status": true, + # "time": "2s" + # "log": { + # "chef_node": "Deleted node[chef name]", + # "chef_client": "Deleted client[chef name]", + # "server": "Server with id '132958f0-61c5-4665-8cc3-66e1bacd285b' terminated" + # } + # }, + # } + # ], + # "project": { + # "deploy_envs": [ + # { + # "flavor": "flavor", + # "identifier": "prod", + # "image": "image id", + # "run_list": [ + # "role[prod]" + # ], + # "subnets": [ + # "private" + # ], + # "expires": null, + # "provider": "openstack", + # "groups": [ + # "default" + # ], + # "users": [ + # "root" + # ] + # } + # ], + # "name": "prject_1" + # }, + # "message": "Test project 'project_1' and environment 'prod'" + # } + post "/project/test/:id/:env" do + check_headers :accept, :content_type + check_privileges("project", "r") + project = BaseRoutes.mongo.project(params[:id]) + env = project.deploy_env params[:env] + user = request.env['REMOTE_USER'] + provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) + header = "Test project '#{project.id}' and environment '#{env.identifier}'" + logger.info header + servers = extract_servers(provider, project, env, {}, user, BaseRoutes.mongo) + result = {:servers => []} + project.deploy_envs = [ env ] + result[:project] = project.to_hash + servers.each do |s| + sr = {} + t1 = Time.now + out = "" + if provider.create_server(s, out) + t2 = Time.now + sr[:id] = s.id + sr[:create] = {:status => true} + sr[:create][:time] = time_diff_s(t1, t2) + logger.info "Server with parameters: #{s.to_hash.inspect} is running" + key = BaseRoutes.mongo.key(s.key) + b_out = "" + r = bootstrap(s, b_out, key.path, logger) + t1 = Time.now + sr[:chef_node_name] = s.chef_node_name + if r == 0 + sr[:bootstrap] = {:status => true} + sr[:bootstrap][:time] = time_diff_s(t2, t1) + logger.info "Server with id '#{s.id}' is bootstraped" + if check_server(s) + BaseRoutes.mongo.server_insert s + end + else + sr[:bootstrap] = {:status => false} + sr[:bootstrap][:log] = b_out + sr[:bootstrap][:return_code] = r + end + + t1 = Time.now + r = delete_from_chef_server(s.chef_node_name) + begin + r[:server] = provider.delete_server s.id + rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::Error + r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers" + logger.warn r[:server] + end + BaseRoutes.mongo.server_delete s.id + t2 = Time.now + sr[:delete] = {:status => true} + sr[:delete][:time] = time_diff_s(t1, t2) + sr[:delete][:log] = r + else + sr[:create] = {:status => false} + sr[:create][:log] = out + end + result[:servers].push sr + end + create_response(header, result) + end + + private + def create_roles project_id, envs, logger + all_roles = KnifeCommands.roles + return "Can't get roles list" if all_roles.nil? + roles = {:new => [], :error => [], :exist => []} + envs.each do |e| + role_name = project_id + (DevopsService.config[:role_separator] || "_") + e.identifier + begin + if all_roles.include? role_name + roles[:exist].push role_name + else + KnifeCommands.create_role project_id, e.identifier + roles[:new].push role_name + logger.info "Role '#{role_name}' created" + end + rescue => er + roles[:error].push role_name + logger.error "Role '#{role_name}' can not be created: #{er.message}" + end + end + roles + end + + def create_new_roles old_project, new_project, logger + old_project.deploy_envs.each do |e| + new_project.remove_env(e.identifier) + end + create_roles new_project.id, new_project.deploy_envs, logger + end + + def create_roles_response roles + info = "" + info += " Project roles '#{roles[:new].join("', '")}' have been automaticaly created" unless roles[:new].empty? + info += " Project roles '#{roles[:exist].join("', '")}' weren't created because they exist" unless roles[:exist].empty? + info += " Project roles '#{roles[:error].join("', '")}' weren't created because of internal error" unless roles[:error].empty? + info + end + end +end + diff --git a/devops-service/routes/v2.0/provider.rb b/devops-service/routes/v2.0/provider.rb new file mode 100644 index 0000000..61415d1 --- /dev/null +++ b/devops-service/routes/v2.0/provider.rb @@ -0,0 +1,33 @@ +# encoding: UTF-8 +require "json" +require "routes/v2.0/base_routes" +require "providers/provider_factory" + +module Version2_0 + class ProviderRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Provider routes initialized" + end + + # Get devops providers + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # "ec2", + # "openstack" + # ] + get "/providers" do + check_headers :accept + check_privileges("provider", "r") + json ::Version2_0::Provider::ProviderFactory.providers + end + + end +end diff --git a/devops-service/routes/v2.0/script.rb b/devops-service/routes/v2.0/script.rb new file mode 100644 index 0000000..ec8d9b0 --- /dev/null +++ b/devops-service/routes/v2.0/script.rb @@ -0,0 +1,181 @@ +require "providers/provider_factory" +require "routes/v2.0/base_routes" +require "fileutils" +require "commands/status" + +module Version2_0 + class ScriptRoutes < BaseRoutes + + include StatusCommands + + def initialize wrapper + super wrapper + puts "Script routes initialized" + end + + before "/script/:script_name" do + check_headers :accept + check_privileges("script", "w") + file_name = params[:script_name] + @file = File.join(DevopsService.config[:scripts_dir], check_filename(file_name, "Parameter 'script_name' must be a not empty string")) + if request.put? + halt_response("File '#{file_name}' already exist") if File.exists?(@file) + elsif request.delete? + halt_response("File '#{file_name}' does not exist", 404) unless File.exists?(@file) + end + end + + after %r{\A/script/((command|run)/)?[\w]+\z} do + statistic + end + + # Get scripts names + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # "script_1" + # ] + get "/scripts" do + check_headers :accept + check_privileges("script", "r") + res = [] + Dir.foreach(DevopsService.config[:scripts_dir]) {|f| res.push(f) unless f.start_with?(".")} + json res + end + + # Run command on node :node_name + # + # * *Request* + # - method : POST + # - body : + # command to run + # + # * *Returns* : text stream + post "/script/command/:node_name" do + check_privileges("script", "w") + user = request.env['REMOTE_USER'] + s = BaseRoutes.mongo.server_by_chef_node_name params[:node_name] + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, user + cert = BaseRoutes.mongo.key s.key + cmd = request.body.read + addr = "#{s.remote_user}@#{s.public_ip || s.private_ip}" + ssh_cmd = "ssh -i %s #{addr} '#{cmd}'" + stream() do |out| + begin + out << ssh_cmd % File.basename(cert.path) + out << "\n" + IO.popen((ssh_cmd % cert.path) + " 2>&1") do |so| + while line = so.gets do + out << line + end + end + out << "\nDone" + rescue IOError => e + logger.error e.message + end + end + end + + # Run script :script_name on nodes + # + # * *Request* + # - method : POST + # - headers : + # - Content-Type: application/json + # - body : + # { + # "nodes": [], -> array of nodes names + # "params": [] -> array of script arguments + # } + # + # * *Returns* : text stream + post "/script/run/:script_name" do + check_headers :content_type + check_privileges("script", "w") + file_name = params[:script_name] + @file = File.join(DevopsService.config[:scripts_dir], check_filename(file_name, "Parameter 'script_name' must be a not empty string", false)) + halt(404, "File '#{file_name}' does not exist") unless File.exists?(@file) + body = create_object_from_json_body + nodes = check_array(body["nodes"], "Parameter 'nodes' must be a not empty array of strings") + p = check_array(body["params"], "Parameter 'params' should be a not empty array of strings", String, true) + servers = BaseRoutes.mongo.servers_by_names(nodes) + return [404, "No servers found for names '#{nodes.join("', '")}'"] if servers.empty? + user = request.env['REMOTE_USER'] + servers.each do |s| + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, user + end + stream() do |out| + begin + status = [] + servers.each do |s| + cert = begin + BaseRoutes.mongo.key s.key + rescue + out << "No key found for '#{s.chef_node_name}'" + status.push 2 + next + end + ssh_cmd = "ssh -i #{cert.path} #{s.remote_user}@#{s.public_ip || s.private_ip} 'bash -s' < %s" + out << "\nRun script on '#{s.chef_node_name}'\n" + unless p.nil? + ssh_cmd += " " + p.join(" ") + end + out << (ssh_cmd % [params[:script_name]]) + out << "\n" + + begin + IO.popen( (ssh_cmd % [@file]) + " 2>&1") do |so| + while line = so.gets do + out << line + end + so.close + status.push $?.to_i + end + rescue IOError => e + logger.error e.message + out << e.message + status.push 3 + end + end + out << create_status(status) + rescue IOError => e + logger.error e.message + end + end + end + + # Create script :script_name + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - body : script content + # + # * *Returns* : + # 201 - Created + put "/script/:script_name" do + File.open(@file, "w") {|f| f.write(request.body.read)} + create_response("File '#{params[:script_name]}' created", nil, 201) + end + + # Delete script :script_name + # + # * *Request* + # - method : Delete + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Deleted + delete "/script/:script_name" do + FileUtils.rm(@file) + create_response("File '#{params[:script_name]}' deleted") + end + end +end diff --git a/devops-service/routes/v2.0/server.rb b/devops-service/routes/v2.0/server.rb new file mode 100644 index 0000000..3414bad --- /dev/null +++ b/devops-service/routes/v2.0/server.rb @@ -0,0 +1,448 @@ +require "json" +require "chef" +require "commands/knife_commands" +require 'rufus-scheduler' +require "routes/v2.0/base_routes" +require "providers/provider_factory" +require "db/mongo/models/deploy_env" +require "commands/status" +require "commands/server" + +module Version2_0 + + class ExpireHandler + include ServerCommands + + def initialize server, logger + @server = server + @logger = logger + end + + def call(job) + @logger.info("Removing node '#{@server.chef_node_name}' form project '#{@server.project}' and env '#{@server.deploy_env}'") + begin + delete_server(@server, BaseRoutes.mongo, @logger) + rescue => e + logger.error "ExpiredHandler error: " + e.message + end + end + end + + class ServerRoutes < BaseRoutes + + include StatusCommands + include ServerCommands + + def initialize wrapper + super wrapper + puts "Server routes initialized" + end + + before "/server/:name_or_cmd" do + if request.get? + check_headers :accept + else + check_headers :accept, :content_type + end + check_privileges("server") + end + + before %r{\A/server/[\w]+/(un)?pause\z} do + check_headers :accept, :content_type + check_privileges("server", "w") + end + + before "/servers/:provider" do + check_headers :accept + check_privileges("server", "r") + end + + after %r{\A/server(/[\w]+)?\z | \A/server/(add|bootstrap)\z | \A/server/[\w]+/(un)?pause\z} do + statistic + end + + scheduler = Rufus::Scheduler.new + + # Get devops servers list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # { + # "id": "instance id", + # "chef_node_name": "chef name" + # } + # ] + get "/servers" do + check_headers :accept + check_privileges("server", "r") + json BaseRoutes.mongo.servers.map {|s| s.to_list_hash} + end + + # Get chef nodes list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # { + # "chef_node_name": "chef name" + # } + # ] + get "/servers/chef" do + json KnifeCommands.chef_node_list + end + + # Get provider servers list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # -ec2 + # [ + # { + # "state": "running", + # "name": "name", + # "image": "ami-83e4bcea", + # "flavor": "m1.small", + # "keypair": "ssh key", + # "instance_id": "i-8441bfd4", + # "dns_name": "ec2-204-236-199-49.compute-1.amazonaws.com", + # "zone": "us-east-1d", + # "private_ip": "10.215.217.210", + # "public_ip": "204.236.199.49", + # "launched_at": "2014-04-25 07:56:33 UTC" + # } + # ] + # -openstack + # [ + # { + # "state": "ACTIVE", + # "name": "name", + # "image": "image id", + # "flavor": null, + # "keypair": "ssh key", + # "instance_id": "instance id", + # "private_ip": "172.17.0.1" + # } + # ] + get "/servers/:provider" do + json ::Version2_0::Provider::ProviderFactory.get(params[:provider]).servers + end + + # Get server info by :name + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # { + # "chef_node_name": "chef name" + # } + # ] + get "/server/:name" do + json BaseRoutes.mongo.server_by_chef_node_name(params[:name]) + end + + # Delete devops server + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body + # { + # "key": "instance" -> if key=instance, then :name - instance id, if key=null, then name - node name + # } + # + # * *Returns* : + # 200 - Deleted + delete "/server/:id" do + body = create_object_from_json_body(Hash, true) + key = (body.nil? ? nil : body["key"]) + id = params[:id] + s = (key == "instance" ? BaseRoutes.mongo.server_by_instance_id(id) : BaseRoutes.mongo.server_by_chef_node_name(id)) + ### Authorization + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + info, r = delete_server(s, BaseRoutes.mongo, logger) + create_response(info, r) + end + + # Create devops server + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "project": "project name", -> mandatory parameter + # "deploy_env": "env", -> mandatory parameter + # "name": "server_name", -> if null, name will be generated + # "without_bootstrap": null, -> do not install chef on instance if true + # "force": null, -> do not delete server on error + # "groups": [], -> specify special security groups, overrides value from project env + # "key": "ssh key" -> specify ssh key for server, overrides value from project env + # } + # + # * *Returns* : text stream + post "/server" do + check_headers :content_type + check_privileges("server", "w") + body = create_object_from_json_body + user = request.env['REMOTE_USER'] + project_name = check_string(body["project"], "Parameter 'project' must be a not empty string") + env_name = check_string(body["deploy_env"], "Parameter 'deploy_env' must be a not empty string") + server_name = check_string(body["name"], "Parameter 'name' should be null or not empty string", true) + without_bootstrap = body["without_bootstrap"] + halt_response("Parameter 'without_bootstrap' should be a null or true") unless without_bootstrap.nil? or without_bootstrap == true + force = body["force"] + halt_response("Parameter 'force' should be a null or true") unless force.nil? or force == true + groups = check_array(body["groups"], "Parameter 'groups' should be null or not empty array of string", String, true) + key_name = check_string(body["key"], "Parameter 'key' should be null or not empty string", true) + new_key = BaseRoutes.mongo.key(key_name) unless key_name.nil? + + p = BaseRoutes.mongo.check_project_auth(project_name, env_name, user) + env = p.deploy_env(env_name) + + provider = ::Version2_0::Provider::ProviderFactory.get(env.provider) + check_chef_node_name(server_name, provider) unless server_name.nil? + unless groups.nil? + buf = groups - provider.groups.keys + halt_response("Invalid security groups '#{buf.join("', '")}' for provider '#{provider.name}'") if buf.empty? + end + + servers = extract_servers(provider, p, env, body, user, BaseRoutes.mongo) + stream() do |out| + status = [] + servers.each do |s| + begin + unless provider.create_server(s, out) + status.push 3 + next + end + logger.info "Server with parameters: #{s.to_hash.inspect} is running" + unless without_bootstrap + key = new_key || BaseRoutes.mongo.key(s.key) + bootstrap(s, out, key.path, logger) + logger.info "Server with id '#{s.id}' is bootstraped" + if force or check_server(s) + BaseRoutes.mongo.server_insert s + scheduler.in(env.expires, ExpireHandler.new(s, logger)) unless env.expires.nil? + out << "Server #{s.chef_node_name} is created" + status.push 0 + else + out << roll_back(s, provider) + status.push 5 + end + out << "\n" + else + BaseRoutes.mongo.server_insert s + status.push 0 + end + out << create_status(status) + rescue IOError => e + logger.error e.message + logger.warn roll_back(s, provider) + break + end + end + end + end + + # Pause devops server by name + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Paused + post "/server/:node_name/pause" do + s = BaseRoutes.mongo.server_by_chef_node_name params[:node_name] + ## Authorization + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) + r = provider.pause_server s.id + if r.nil? + create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is paused") + else + halt_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' can not be paused, It in state '#{r}'", 409) + end + end + + # Unpause devops server by name + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Unpaused + post "/server/:node_name/unpause" do + s = BaseRoutes.mongo.server_by_chef_node_name params[:node_name] + ## Authorization + BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + provider = ::Version2_0::Provider::ProviderFactory.get(s.provider) + r = provider.unpause_server s.id + if r.nil? + create_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' is unpaused") + else + halt_response("Server with instance ID '#{s.id}' and node name '#{params[:node_name]}' can not be unpaused, It in state '#{r}'", 409) + end + end + + # Bootstrap devops server + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "instance_id": "instance id", -> mandatory parameter + # "name": "server_name", -> if null, name will be generated + # "run_list": [], -> specify list of roles and recipes + # "bootstrap_template": "template" -> specify ssh key for server, overrides value from project env + # } + # + # * *Returns* : text stream + # TODO: check bootstrap template name + post "/server/bootstrap" do + body = create_object_from_json_body(Hash, true) + id = check_string(body["instance_id"], "Parameter 'instance_id' must be a not empty string") + name = check_string(body["name"], "Parameter 'name' should be a not empty string", true) + rl = check_array(body["run_list"], "Parameter 'run_list' should be a not empty array of string", String, true) + unless rl.nil? + e = DeployEnv.validate_run_list(rl) + halt_response("Invalid run list elements: '#{e.join("', '")}'") unless e.empty? + end + t = check_string(body["bootstrap_template"], "Parameter 'bootstrap_template' should be a not empty string", true) + s = BaseRoutes.mongo.server_by_instance_id(id) + + p = BaseRoutes.mongo.check_project_auth s.project, s.deploy_env, request.env['REMOTE_USER'] + d = p.deploy_env s.deploy_env + + s.options = { + :run_list => rl || d.run_list, + } + s.options[:bootstrap_template] = t unless t.nil? + stream() do |out| + begin + s.chef_node_name = name || "static_#{s.key}-#{Time.now.to_i}" + cert = BaseRoutes.mongo.key s.key + logger.debug "Bootstrap certificate path: #{cert.path}" + bootstrap s, out, cert.path, logger + str = nil + if check_server(s) + BaseRoutes.mongo.server_update s + str = "Server with id '#{s.id}' is bootstraped" + logger.info str + else + str = "Server with id '#{s.id}' is not bootstraped" + logger.warn str + end + out << str + out << "\n" + rescue IOError => e + logger.error e.message + end + end + end + + # Add external server to devops + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "project": "project name", -> mandatory parameter + # "deploy_env": "env", -> mandatory parameter + # "key": "ssh key", -> mandatory parameter + # "remote_user": "ssh user", -> mandatory parameter + # "private_ip": "ip", -> mandatory parameter + # "public_ip": "ip" + # } + # + # * *Returns* : + # 200 - Added + # TODO: should be refactored + post "/server/add" do + body = create_object_from_json_body + project = check_string(body["project"], "Parameter 'project' must be a not empty string") + deploy_env = check_string(body["deploy_env"], "Parameter 'deploy_env' must be a not empty string") + key = check_string(body["key"], "Parameter 'key' must be a not empty string") + remote_user = check_string(body["remote_user"], "Parameter 'remote_user' must be a not empty string") + private_ip = check_string(body["private_ip"], "Parameter 'private_ip' must be a not empty string") + public_ip = check_string(body["public_ip"], "Parameter 'public_ip' should be a not empty string", true) + p = BaseRoutes.mongo.check_project_auth project, deploy_env, request.env['REMOTE_USER'] + + d = p.deploy_env(deploy_env) + + cert = BaseRoutes.mongo.key(key) + s = Server.new + s.provider = "static" + s.project = project + s.deploy_env = deploy_env + s.remote_user = remote_user + s.private_ip = private_ip + s.public_ip = public_ip + s.static = true + s.id = "static_#{cert.id}-#{Time.now.to_i}" + s.key = cert.id + BaseRoutes.mongo.server_insert s + create_response("Server '#{s.id}' has been added") + end + + private + def roll_back s, provider + str = "" + unless s.id.nil? + str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n" + str << delete_from_chef_server(s.chef_node_name).values.join("\n") + begin + str << provider.delete_server(s.id) unless s.static + rescue => e + str << e.message + end + str << "\nRolled back\n" + end + return str + end + + def check_chef_node_name name, provider + BaseRoutes.mongo.server_by_chef_node_name name + halt(400, "Server with name '#{name}' already exist") + rescue RecordNotFound => e + # server not found - OK + s = provider.servers.detect {|s| s["name"] == name} + halt(400, "#{provider.name} node with name '#{name}' already exist") unless s.nil? + s = KnifeCommands.chef_node_list.detect {|n| n == name} + halt(400, "Chef node with name '#{name}' already exist") unless s.nil? + s = KnifeCommands.chef_client_list.detect {|c| c == name} + halt(400, "Chef client with name '#{name}' already exist") unless s.nil? + end + + end +end diff --git a/devops-service/routes/v2.0/tag.rb b/devops-service/routes/v2.0/tag.rb new file mode 100644 index 0000000..b5d13af --- /dev/null +++ b/devops-service/routes/v2.0/tag.rb @@ -0,0 +1,87 @@ +require "commands/knife_commands" + +module Version2_0 + class TagRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "Tag routes initialized" + end + + before "/tags/:node_name" do + if request.get? + check_headers :accept + check_privileges("server", "r") + else + check_headers :accept, :content_type + check_privileges("server", "w") + @tags = create_object_from_json_body(Array) + check_array(@tags, "Request body should be a not empty array of strings") + end + server = BaseRoutes.mongo.server_by_chef_node_name(params[:node_name]) + halt_response("No servers found for name '#{params[:node_name]}'", 404) if server.nil? + @chef_node_name = server.chef_node_name + end + + after "/tags/:node_name" do + statistic + end + + # Get tags list for :node_name + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # "tag_1" + # ] + get "/tags/:node_name" do + json(KnifeCommands.tags_list(@chef_node_name)) + end + + # Set tags list to :node_name + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # [ + # "tag_1" + # ] + # + # * *Returns* : + # 200 + post "/tags/:node_name" do + tagsStr = @tags.join(" ") + cmd = KnifeCommands.tags_create(@chef_node_name, tagsStr) + halt_response("Error: Cannot add tags #{tagsStr} to server #{@chef_node_name}", 500) unless cmd[1] + create_response("Set tags for #{@chef_node_name}: #{tagsStr}") + end + + # Delete tags from :node_name + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # [ + # "tag_1" + # ] + # + # * *Returns* : + # 200 + delete "/tags/:node_name" do + tagsStr = @tags.join(" ") + cmd = KnifeCommands.tags_delete(@chef_node_name, tagsStr) + halt_response("Cannot delete tags #{tagsStr} from server #{@chef_node_name}: #{cmd[0]}", 500) unless cmd[1] + create_response("Deleted tags for #{@chef_node_name}: #{tagsStr}") + end + end +end diff --git a/devops-service/routes/v2.0/user.rb b/devops-service/routes/v2.0/user.rb new file mode 100644 index 0000000..d94fa11 --- /dev/null +++ b/devops-service/routes/v2.0/user.rb @@ -0,0 +1,166 @@ +require "json" +require "db/exceptions/invalid_record" +require "db/mongo/models/user" + +module Version2_0 + class UserRoutes < BaseRoutes + + def initialize wrapper + super wrapper + puts "User routes initialized" + end + + after %r{\A/user(/[\w]+(/password)?)?\z} do + statistic + end + + # Get users list + # + # * *Request* + # - method : GET + # - headers : + # - Accept: application/json + # + # * *Returns* : + # [ + # { + # "email": "test@test.test", + # "privileges": { + # "flavor": "r", + # "group": "r", + # "image": "r", + # "project": "r", + # "server": "r", + # "key": "r", + # "user": "", + # "filter": "r", + # "network": "r", + # "provider": "r", + # "script": "r", + # "templates": "r" + # }, + # "id": "test" + # } + # ] + get "/users" do + check_headers :accept + check_privileges("user", "r") + users = BaseRoutes.mongo.users.map {|i| i.to_hash} + users.each {|u| u.delete("password")} + json users + end + + # Create user + # + # * *Request* + # - method : POST + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "username": "user name", + # "password": "user password" + # } + # + # * *Returns* : + # 201 - Created + post "/user" do + check_headers :accept, :content_type + check_privileges("user", "w") + user = create_object_from_json_body + ["username", "password"].each do |p| + check_string(user[p], "Parameter '#{p}' must be a not empty string") + end + BaseRoutes.mongo.user_insert User.new(user) + create_response("Created", nil, 201) + end + + # Delete user + # + # * *Request* + # - method : DELETE + # - headers : + # - Accept: application/json + # + # * *Returns* : + # 200 - Deleted + delete "/user/:user" do + check_headers :accept + check_privileges("user", "w") + projects = BaseRoutes.mongo.projects_by_user params[:user] + if !projects.empty? + str = "" + projects.each do |p| + p.deploy_envs.each do |e| + str+="#{p.id}.#{e.identifier} " if e.users.include? params[:user] + end + end + logger.info projects + raise DependencyError.new "Deleting is forbidden: User is included in #{str}" + #return [400, "Deleting is forbidden: User is included in #{str}"] + end + + r = BaseRoutes.mongo.user_delete params[:user] + create_response("User '#{params[:user]}' removed") + end + + # Change user privileges + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "cmd": "command or all", -> if empty, set default privileges + # "privileges": "priv" -> 'r', 'rw' or '' + # } + # + # * *Returns* : + # 200 - Updated + put "/user/:user" do + check_headers :accept, :content_type + check_privileges("user", "w") + data = create_object_from_json_body + user = BaseRoutes.mongo.user params[:user] + cmd = check_string(data["cmd"], "Parameter 'cmd' should be a not empty string", true) || "" + privileges = check_string(data["privileges"], "Parameter 'privileges' should be a not empty string", true) || "" + user.grant(cmd, privileges) + BaseRoutes.mongo.user_update user + create_response("Updated") + end + + # Change user email/password + # + # * *Request* + # - method : PUT + # - headers : + # - Accept: application/json + # - Content-Type: application/json + # - body : + # { + # "email/password": "new user email/password", + # } + # + # * *Returns* : + # 200 - Updated + put %r{\A/user/[\w]+/(email|password)\z} do + check_headers :accept, :content_type + action = File.basename(request.path) + u = File.basename(File.dirname(request.path)) + raise InvalidPrivileges.new("Access denied for '#{request.env['REMOTE_USER']}'") if u == User::ROOT_USER_NAME and request.env['REMOTE_USER'] != User::ROOT_USER_NAME + + check_privileges("user", "w") unless request.env['REMOTE_USER'] == u + + body = create_object_from_json_body + p = check_string(body[action], "Parameter '#{action}' must be a not empty string") + user = BaseRoutes.mongo.user u + user.send("#{action}=", p) + BaseRoutes.mongo.user_update user + create_response("Updated") + end + + end +end diff --git a/devops-service/tests/config.rb b/devops-service/tests/config.rb new file mode 100644 index 0000000..e9ae011 --- /dev/null +++ b/devops-service/tests/config.rb @@ -0,0 +1,6 @@ +USERNAME="" +PASSWORD="" +HOST="" +ROOTUSER="root" +ROOTPASS="" +PROVIDERS=["ec2", "openstack"] diff --git a/devops-service/tests/cud_command.rb b/devops-service/tests/cud_command.rb new file mode 100644 index 0000000..0475c5d --- /dev/null +++ b/devops-service/tests/cud_command.rb @@ -0,0 +1,61 @@ +module CudCommand + + HEADERS = { + "Accept" => "application/json", + "Content-Type" => "application/json" + } + + def test_headers path, method="post", accept=true + m = method.downcase + if accept + h = HEADERS.clone + h["Accept"] = "application/xml" + self.send("send_#{m}", path, {}, h, 406) + end + + h = HEADERS.clone + h.delete("Content-Type") + self.send("send_#{m}", path, {}, h, 415) + end + + def test_request path, obj, method="post", exclude=nil + m = method.downcase + [{}, [], ""].each do |o| + next if o.class == exclude + self.send("send_#{m}", path, o, HEADERS, 400) unless exclude.is_a?(Hash) + end + self.send("send_#{m}", path, nil, HEADERS, 400) + return if exclude == obj.class + obj.keys.each do |key| + buf = obj.clone + buf.delete(key) + self.send("send_#{m}", path, buf, HEADERS, 400) + buf[key] = "" + self.send("send_#{m}", path, buf, HEADERS, 400) + buf[key] = {} + self.send("send_#{m}", path, buf, HEADERS, 400) + buf[key] = [] + self.send("send_#{m}", path, buf, HEADERS, 400) + end + end + + def test_auth path, obj, empty_obj_code=400, method="post" + m = method.downcase + read_only_privileges + self.send("send_#{m}", path, {}, HEADERS, 401) + self.send("send_#{m}", path, obj, HEADERS, 401) + + empty_privileges + self.send("send_#{m}", path, {}, HEADERS, 401) + self.send("send_#{m}", path, obj, HEADERS, 401) + + write_only_privileges + self.send("send_#{m}", path, {}, HEADERS, empty_obj_code) + + self.username = ROOTUSER + self.send("send_#{m}", path, {}, HEADERS, empty_obj_code) + + all_privileges + end + +end diff --git a/devops-service/tests/deploy.rb b/devops-service/tests/deploy.rb new file mode 100644 index 0000000..4bbca04 --- /dev/null +++ b/devops-service/tests/deploy.rb @@ -0,0 +1,55 @@ +require "devops_test" +require "cud_command" +class Deploy < DevopsTest + + include CudCommand + + def title + "Deploy test (invalid queries only)" + end + + # tests invalid queries only, valid query in client test + def run + all_privileges + test_headers("deploy", "post", false) + + deploy = { + :names => ["foo"], + :tags => ["foo"] + } + test_auth("deploy", deploy) + cnt = 0 + headers = HEADERS.clone + headers.delete("Accept") + st = 400 + nf_st = 404 + begin + [{}, [], "", nil].each do |p| + self.send_post("deploy", p, headers, st) + d = deploy.clone + d.delete(:tags) + d[:names] = p + self.send_post("deploy", d, headers, st) + unless p.nil? + d = deploy.clone + d[:tags] = p + self.send_post("deploy", d, headers, st) + end + end + deploy[:tags] = nil + self.send_post("deploy", deploy, headers, nf_st) + deploy.delete(:tags) + self.send_post("deploy", deploy, headers, nf_st) + if cnt == 0 + cnt = 1 + write_only_privileges + raise RangeError + end + rescue RangeError + retry + end + + end + +end + diff --git a/devops-service/tests/devops_test.rb b/devops-service/tests/devops_test.rb new file mode 100644 index 0000000..c22bebd --- /dev/null +++ b/devops-service/tests/devops_test.rb @@ -0,0 +1,202 @@ +require "json" +require "httpclient" +require "./config" + +class DevopsTest + + attr_accessor :password, :response, :host + attr_reader :username + + def initialize + puts_head " #{self.title} ".center(80, "=") + self.host = HOST +# self.username = USERNAME +# self.password = PASSWORD + end + + def submit + http = HTTPClient.new + http.receive_timeout = 0 + http.send_timeout = 0 + http.set_auth(nil, self.username, self.password) + self.response = yield http + self.response.body + end + + def username= user + @username = user + self.password = if user == ROOTUSER + ROOTPASS + else + PASSWORD + end + puts_warn(" - User: " + @username + " - ") + end + + def all_privileges + self.username = USERNAME + end + + def read_only_privileges + self.username = USERNAME + "_r" + end + + def write_only_privileges + self.username = USERNAME + "_w" + end + + def empty_privileges + self.username = USERNAME + "_" + end + + def get path, params={}, headers={} + url = create_url(path) + m = "GET #{url.path.ljust(30)}" + m += " HEADERS(#{headers.map{|k,v| "#{k}: #{v}" }.join(", ")})" unless headers.empty? + print m.ljust(99) + submit do |http| + http.get(url, convert_params(params), headers) + end + end + + def post path, params={}, headers={} + url = create_url(path) + m = "POST #{url.path.ljust(30)}" + m += " HEADERS(#{headers.map{|k,v| "#{k}: #{v}" }.join(", ")})" unless headers.empty? + print m.ljust(99) + submit do |http| + http.post(url, params.to_json, headers) + end + end + + def send_post path, params={}, headers={}, status=200 + self.post path, params, headers + self.check_status status + self.check_json_response + end + + def delete path, params={}, headers={} + url = create_url(path) + m = "DELETE #{url.path.ljust(30)}" + m += " HEADERS(#{headers.map{|k,v| "#{k}: #{v}" }.join(", ")})" unless headers.empty? + print m.ljust(99) + b = (params.nil? ? nil : params.to_json) + submit do |http| + http.delete(url, b, headers) + end + end + + def send_delete path, params={}, headers={}, status=200 + self.delete path, params, headers + self.check_status status + self.check_json_response + end + + def post_chunk path, params={}, headers={} + url = create_url(path) + m = "POST #{url.path.ljust(30)}" + m += " HEADERS(#{headers.map{|k,v| "#{k}: #{v}" }.join(", ")})" unless headers.empty? + print m.ljust(99) + submit do |http| + http.post(url, params.to_json, headers) do |c| + end + end + end + + def send_put path, params={}, headers={}, status=200 + self.put path, params, headers + self.check_status status + self.check_json_response + end + + def put path, params={}, headers={} + url = create_url(path) + m = "PUT #{url.path.ljust(30)}" + m += " HEADERS(#{headers.map{|k,v| "#{k}: #{v}" }.join(", ")})" unless headers.empty? + print m.ljust(99) + submit do |http| + http.put(url, params.to_json, headers) + end + end + + def check_status code + if self.response.status == code + self.puts_success + else + self.puts_error "STATUS: #{self.response.status}, but checked with '#{code}'" + end + end + + def check_json_response + return if self.response.status == 404 + j = begin + JSON.parse(self.response.body) + rescue + self.puts_error "Invalid json, response body: '#{self.response.body}'" + end + self.puts_error "Response in Json format, but without parameter 'message'" unless j.key?("message") + end + + def check_type type + res = self.response + if res.ok? + case type + when :json + puts_error("Invalid content-type '#{res.contenttype}'") unless res.contenttype.include?("application/json") + else + puts_error("Unknown type '#{type}'") + end + end + end + + def puts_head str + puts "\e[31m#{str}\e[0m" + end + + def puts_error str + puts "\t\e[31m#{str}\e[0m" + raise str + end + + def puts_warn str + puts "\t\e[33m#{str}\e[0m" + end + + def puts_success str="success" + puts "\t\e[32m#{str}\e[0m" + end + +private + def create_url path + path = "/" + path unless path.start_with? "/" + URI.join("http://" + self.host, "v2.0" + path) + end + + def convert_params params + return nil if params.nil? or params.empty? + params_filter(params).join("&") + end + + def params_filter params + r = [] + return params if params.kind_of?(String) + params.each do |k,v| + key = k.to_s + if v.kind_of?(Array) + v.each do |val| + r.push "#{key}[]=#{val}" + end + elsif v.kind_of?(Hash) + buf = {} + v.each do |k1,v1| + buf["#{key}[#{k1}]"] = v1 + end + r = r + params_filter(buf) + else + r.push "#{key}=#{v}" + end + end + r + end + +end diff --git a/devops-service/tests/filter.rb b/devops-service/tests/filter.rb new file mode 100644 index 0000000..1ff7f03 --- /dev/null +++ b/devops-service/tests/filter.rb @@ -0,0 +1,27 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Filter < DevopsTest + include ListCommand + include CudCommand + + def title + "Filter test" + end + + def run + list_providers("filter/:provider/images") + + filters = [ + "foo" + ] + cmd = "filter/openstack/image" + test_auth cmd, {}, 400, "put" + test_headers cmd, "put" + test_request cmd, filters, "put", Array + + test_auth cmd, {}, 400, "delete" + test_headers cmd, "delete" + test_request cmd, filters, "delete", Array + end +end diff --git a/devops-service/tests/flavor.rb b/devops-service/tests/flavor.rb new file mode 100644 index 0000000..0932281 --- /dev/null +++ b/devops-service/tests/flavor.rb @@ -0,0 +1,14 @@ +require "devops_test" +require "list_command" +class Flavor < DevopsTest + + include ListCommand + + def title + "Flavor test" + end + + def run + list_providers("flavors/:provider") + end +end diff --git a/devops-service/tests/group.rb b/devops-service/tests/group.rb new file mode 100644 index 0000000..7c844a0 --- /dev/null +++ b/devops-service/tests/group.rb @@ -0,0 +1,14 @@ +require "devops_test" +require "list_command" +class Group < DevopsTest + + include ListCommand + + def title + "Group test" + end + + def run + list_providers("groups/:provider") + end +end diff --git a/devops-service/tests/image.rb b/devops-service/tests/image.rb new file mode 100644 index 0000000..3aed7c6 --- /dev/null +++ b/devops-service/tests/image.rb @@ -0,0 +1,64 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Image < DevopsTest + + include ListCommand + include CudCommand + + def title + "Image test" + end + + def run + list("images") + PROVIDERS.each do |p| + list_send("images", 200, :provider => p) + end + ["foo", nil, ["ec2"], {"provider" => "ec2"}].each do |p| + list_send("images", 404, :provider => p) + end + + cmd = "images/provider/:provider" + [USERNAME, USERNAME + "_r", ROOTUSER].each do |u| + self.username = u + PROVIDERS.each do |p| + list_send(cmd.gsub(":provider", p), 200) + end + list_send(cmd.gsub(":provider", "foo"), 404) + self.get("images/provider") + self.check_status 404 + end + list_deny do + PROVIDERS.each do |p| + list_send(cmd.gsub(":provider", p), 401) + end + list_send(cmd.gsub(":provider", "foo"), 401) + self.get("images/provider") + self.check_status 404 + end + + image = { + :id => "foo_image", + :provider => "foo_provider", + :name => "foo_name", + :remote_user => "foo_user", + } + all_privileges + test_headers "image" + test_request "image", image + self.send_post "image", image, HEADERS, 400 + i = image.clone + i[:provider] = "openstack" + self.send_post "image", i, HEADERS, 400 + + test_auth "image", image + + test_auth "image/foo", {}, 404, "delete" + self.send_delete "image/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "image/foo", nil, h, 404 + end + +end diff --git a/devops-service/tests/key.rb b/devops-service/tests/key.rb new file mode 100644 index 0000000..61d0fe6 --- /dev/null +++ b/devops-service/tests/key.rb @@ -0,0 +1,38 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Key < DevopsTest + + include ListCommand + include CudCommand + + def title + "Key test" + end + + def run + list("keys") + + all_privileges + key = { + :content => "content", + :file_name => "key_file.pem", + :key_name => "test_key" + } + + test_headers "key" + test_auth "key", key + test_request "key", key + k = key.clone + k[:file_name] = "key*_file.pem" + self.send_post "key", k, HEADERS, 400 + + test_auth "key/foo", key, 404, "delete" + + self.send_delete "key/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "key/foo", nil, h, 404 + end +end + diff --git a/devops-service/tests/list_command.rb b/devops-service/tests/list_command.rb new file mode 100644 index 0000000..aaa6ea2 --- /dev/null +++ b/devops-service/tests/list_command.rb @@ -0,0 +1,86 @@ +module ListCommand + + def list path, params=nil, status=200 + all_privileges +# list_send(path, status, 406, params) + + cmd = "" + path.split("/").each do |s| + cmd += "/" + s + if cmd == "/" + path + list_send(cmd, status) + else + self.get cmd + self.check_status 404 + end + end + + read_only_privileges + list_send(path, status, params) + write_only_privileges + list_send(path, 401, params) + empty_privileges + list_send(path, 401, params) + self.username = ROOTUSER + list_send(path, status, params) + end + + def list_deny + empty_privileges + cnt = 0 + begin + yield + if cnt == 0 + write_only_privileges + cnt = 1 + raise RangeError + end + rescue RangeError + retry + end + + end + + def list_providers cmd, as_ok=401, as_not_found=401 + all_privileges + check_provider cmd + + read_only_privileges + check_provider cmd + + write_only_privileges + check_provider cmd, as_ok, as_not_found, 401 + + empty_privileges + check_provider cmd, as_ok, as_not_found, 401 + + self.username = ROOTUSER + check_provider cmd + end + + def check_provider cmd, ok_status=200, not_found_status=404, auth_not_found=406 + #js = (ok_status == 200 ? 406 : ok_status) + PROVIDERS.each do |p| + list_send(cmd.gsub(":provider", p), ok_status) + end + path = "" + st = not_found_status + cmd.split("/").each do |s| + path += "/" + s + if path == "/" + cmd + list_send(path, st) + else + self.get path + self.check_status 404 + end + end + end + + def list_send path, status=200, params=nil + self.get path, params, {"Accept" => "application/xml"} + self.check_status 406 + self.get path, params, {"Accept" => "application/json"} + self.check_status status + self.check_type :json if status == 200 + end +end diff --git a/devops-service/tests/network.rb b/devops-service/tests/network.rb new file mode 100644 index 0000000..9f49d39 --- /dev/null +++ b/devops-service/tests/network.rb @@ -0,0 +1,14 @@ +require "devops_test" +require "list_command" +class Network < DevopsTest + + include ListCommand + + def title + "Network test" + end + + def run + list_providers("networks/:provider") + end +end diff --git a/devops-service/tests/project.rb b/devops-service/tests/project.rb new file mode 100644 index 0000000..1399cda --- /dev/null +++ b/devops-service/tests/project.rb @@ -0,0 +1,100 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Project < DevopsTest + + include ListCommand + include CudCommand + + def title + "Project test" + end + + def run + list("projects") + list("project/foo", nil, 404) + list_send("project/foo/servers", 404) + + project = { + :deploy_envs => [ + { + :flavor => "c1.large", + :identifier => "test", + :image => "e6f44159-f50a-49a5-bfd5-865d0f68779d", + :run_list => [ + "role[solr_test]" + ], + :subnets => [ + "private" + ], + :expires => nil, + :provider => "openstack", + :groups => [ + "default" + ], + :users => [ + USERNAME + ] + } + ], + :name => "test" + } + + test_auth "project", project + test_headers "project" + test_request "project", project + ["openstack", "ec2"].each do |provider| + project[:deploy_envs][0].keys.each do |k| + p = project.clone + d = p[:deploy_envs][0] + d[:provider] = provider + if k == :expires + ["foo", "", [], {}].each do |v| + d[k] = v + send_post "project", p, HEADERS, 400 + end + elsif k == :run_list or k == :groups or k == :users + ["", {}, nil].each do |v| + d[k] = v + send_post "project", p, HEADERS, 400 + end + elsif k == :subnets and provider == "ec2" + ["", {}].each do |v| + d[k] = v + send_post "project", p, HEADERS, 400 + end + else + d.delete(k) + send_post "project", p, HEADERS, 400 + [nil, "", [], {}].each do |v| + d[k] = v + send_post "project", p, HEADERS, 400 + end + end + end + end + + test_auth "project/foo", project, 404, "delete" + self.send_delete "project/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "project/foo", nil, h, 415 + self.send_delete "project/foo", nil, HEADERS, 404 + self.send_delete "project/foo", {:deploy_env => ""}, HEADERS, 400 + + deploy = { + :servers => ["foo"], + :deploy_env => "foo" + } + test_headers "project/foo/deploy", "post", false + deploy.keys.each do |k| + d = deploy.clone + ["", [], {}].each do |v| + d[k] = v + send_post "project/foo/deploy", p, HEADERS, 400 + end + end +# test_auth "project/foo/deploy", deploy, 404 + end + +end diff --git a/devops-service/tests/provider.rb b/devops-service/tests/provider.rb new file mode 100644 index 0000000..2cb2d5f --- /dev/null +++ b/devops-service/tests/provider.rb @@ -0,0 +1,14 @@ +require "devops_test" +require "list_command" +class Provider < DevopsTest + + include ListCommand + + def title + "Provider test" + end + + def run + list("providers") + end +end diff --git a/devops-service/tests/run_tests.rb b/devops-service/tests/run_tests.rb new file mode 100644 index 0000000..d4cebc5 --- /dev/null +++ b/devops-service/tests/run_tests.rb @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +dir = File.dirname(__FILE__) +$:.push dir +tests = nil +if ARGV.empty? + tests = %w{flavor group network provider deploy image key script tag filter project user server} +else + tests = ARGV +end + +classes = [] +tests.each do |f| + require "#{dir}/#{f}.rb" + case f + when "flavor" + classes.push Flavor + when "group" + classes.push Group + when "network" + classes.push Network + when "provider" + classes.push Provider + when "user" + classes.push User + when "key" + classes.push Key + when "script" + classes.push Script + when "image" + classes.push Image + when "project" + classes.push Project + when "server" + classes.push Server + when "deploy" + classes.push Deploy + when "tag" + classes.push Tag + when "filter" + classes.push Filter + end +end + +classes.each do |c| + c.new.run +end diff --git a/devops-service/tests/script.rb b/devops-service/tests/script.rb new file mode 100644 index 0000000..5fbd1ad --- /dev/null +++ b/devops-service/tests/script.rb @@ -0,0 +1,33 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Script < DevopsTest + + include ListCommand + include CudCommand + + def title + "Script test" + end + + def run + list("scripts") + + self.username = USERNAME +# test_headers "script/run/foo" + + script = { + :nodes => ["foo"] + } + +# test_request "script/run/foo", script, "post", Hash + + test_auth "script/run/foo", script, 404 + test_auth "script/command/foo", {}, 404 + + self.send_delete "script/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "script/foo", nil, h, 404 + end +end diff --git a/devops-service/tests/server.rb b/devops-service/tests/server.rb new file mode 100644 index 0000000..d9c24b8 --- /dev/null +++ b/devops-service/tests/server.rb @@ -0,0 +1,134 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Server < DevopsTest + + include ListCommand + include CudCommand + + def title + "Server test" + end + + def run + list("servers") + + cmd = "servers/:provider" + p = PROVIDERS.clone + p.push "chef" + [USERNAME, USERNAME + "_r", ROOTUSER].each do |u| + self.username = u + p.each do |p| + list_send(cmd.gsub(":provider", p), 200) + end + list_send(cmd.gsub(":provider", "foo"), 404) + list_send("server/foo", 404) + end + list_deny do + p.each do |p| + list_send(cmd.gsub(":provider", p), 401) + end + list_send(cmd.gsub(":provider", "foo"), 401) + list_send("server/foo", 401) + end + + test_auth "server/foo", {}, 404, "delete" + self.send_delete "server/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "server/foo", nil, h, 415 + h = HEADERS.clone + self.send_delete "server/foo", nil, h, 404 + + all_privileges + server = { + :project => "foo", + :deploy_env => "foo", + :name => "foo", + :without_bootstrap => true, + :force => true, + :groups => [], + :key => "foo" + } + test_headers "server", "post", false + [:project, :deploy_env, :name, :key].each do |k| + s = server.clone + ["", nil, [], {}].each do |v| + next if k == :name and v.nil? + s[k] = v + self.send_post "server", s, HEADERS, 400 + end + end + [:force, :without_bootstrap].each do |k| + s = server.clone + ["", false, [], {}].each do |v| + s[k] = v + self.send_post "server", s, HEADERS, 400 + end + end + s = server.clone + ["", true, [], [true], [{:foo => "foo"}], {}].each do |v| + s[:groups] = v + self.send_post "server", s, HEADERS, 400 + end + + test_auth "server", server + + ["server/foo/pause", "server/foo/unpause"].each do |cmd| + test_auth cmd, nil, 404 + test_headers cmd + end + + bootstrap = { + :instance_id => "foo", + :name => "foo", + :run_list => ["foo"], + :bootstrap_template => "foo" + } + cmd = "server/bootstrap" + test_auth cmd, bootstrap + test_headers cmd, "post", false + + b = bootstrap.clone + ["", [], {}].each do |v| + b[:instance_id] = v + self.send_post cmd, b, HEADERS, 400 + end + + [:name, :bootstrap_template].each do |k| + b = bootstrap.clone + ["", [], {}].each do |v| + b[k] = v + self.send_post cmd, b, HEADERS, 400 + end + end + + b = bootstrap.clone + ["", [nil], [{:foo => "foo"}], [true], {}].each do |v| + b[:run_list] = v + self.send_post cmd, b, HEADERS, 400 + end + + cmd = "server/add" + add = { + :project => "foo", + :deploy_env => "foo", + :key => "foo", + :remote_user => "foo", + :private_ip => "foo", + :public_ip => "foo" + } + test_auth cmd, add + test_headers cmd, "post", false + + [:project, :deploy_env, :key, :remote_user, :private_ip, :public_ip].each do |k| + a = add.clone + [nil, "", [], {}].each do |v| + next if k == :public_ip and v.nil? + a[k] = v + self.send_post cmd, a, HEADERS, 400 + end + end + + end +end diff --git a/devops-service/tests/tag.rb b/devops-service/tests/tag.rb new file mode 100644 index 0000000..a1f61ae --- /dev/null +++ b/devops-service/tests/tag.rb @@ -0,0 +1,33 @@ +require "devops_test" +require "list_command" +require "cud_command" +class Tag < DevopsTest + + include ListCommand + include CudCommand + + def title + "Tag test" + end + + def run + self.username = USERNAME + self.get("tags") + self.check_status 404 + list("tags/foo", nil, 404) + + self.username = USERNAME + + tags = { + :tags => ["tag1"] + } + + test_headers "tags/foo" + test_request "tags/foo", tags + test_auth "tags/foo", tags + + test_headers "tags/foo", "delete" + test_request "tags/foo", tags, "delete", Hash + test_auth "tags/foo", tags, 400, "delete" + end +end diff --git a/devops-service/tests/user.rb b/devops-service/tests/user.rb new file mode 100644 index 0000000..cdbb3f3 --- /dev/null +++ b/devops-service/tests/user.rb @@ -0,0 +1,49 @@ +require "devops_test" +require "list_command" +require "cud_command" +class User < DevopsTest + + include ListCommand + include CudCommand + + def title + "User test" + end + + def run + list("users") + + self.username = USERNAME + + user = { + :username => "foo", + :password => "foo", + } + + test_headers "user" + test_request "user", user + test_auth "user", user + + test_auth "user/foo", user, 404, "delete" + self.send_delete "user/foo", nil, {}, 406 + h = HEADERS.clone + h.delete("Content-Type") + self.send_delete "user/foo", nil, h, 404 + + privileges = { + :privileges => "foo", + :cmd => "foo" + } + test_auth "user/foo", privileges, 404, "put" + test_headers "user/foo", "put" + test_request "user/foo", privileges, "put", Hash + + pass = { + :password => "foo" + } + test_auth "user/foo/password", pass, 400, "put" + test_headers "user/foo/password", "put" + test_request "user/foo/password", pass, "put", Hash + + end +end