fluke/devops-service/lib/executors/server_executor.rb

require "lib/knife/knife_factory"
require "workers/worker"
require "workers/delete_server_worker"
require "hooks"

module Devops
  module Executor
    class ServerExecutor
      include Hooks

      RESULT_CODES = {
        server_bootstrap_fail: 2,
        server_not_in_chef_nodes: 5,
        server_bootstrap_unknown_error: 7,
        deploy_unknown_error: 6,
        deploy_failed: 8
      }

      MAX_SSH_RETRIES_AMOUNT = 20

      #params:
      #  out - container for output data
      #  deploy_info - hash with deploy data
      define_hook :before_deploy
      define_hook :after_deploy

      define_hook :before_create
      define_hook :after_create

      #params:
      #  out - container for output data
      define_hook :before_bootstrap
      define_hook :after_bootstrap

      before_deploy :create_run_list

      def initialize server, out
        if server
          @project = Devops::Db.connector.project(server.project)
          @deploy_env = @project.deploy_env(server.deploy_env)
        end
        @knife_instance = KnifeFactory.instance
        @server = server
        @out = out
        @out.class.send(:define_method, :flush) { } unless @out.respond_to?(:flush)
      end

      def self.result_code(symbolic_code)
        RESULT_CODES.fetch(symbolic_code)
      end

      def self.symbolic_result_code(integer_code)
        RESULT_CODES.key(integer_code) || :unknown_error
      end

      def result_code(symbolic_code)
        self.class.result_code(symbolic_code)
      end

      def report= r
        @report = r
      end

      def project= p
        @project = p
      end

      def deploy_env= e
        @deploy_env = e
      end

      def server
        @server
      end

      def create_server options
        @server = Devops::Model::Server.new({"project" => @project.id, "deploy_env" => @deploy_env.identifier, "created_by" => options["created_by"], "provider" => @deploy_env.provider})
        provider = @server.provider_instance
        mongo = ::Devops::Db.connector
        begin
          @out << "Create server...\n"
          @out.flush

          @server.run_list = options["run_list"] || []
          @server.chef_node_name = options["name"]
          @server.key = options["key"] || provider.ssh_key

          i = mongo.image(@deploy_env.image)
          @server.remote_user = i.remote_user

          res = {}
          @out << "\nBefore create hooks...\n"
          res[:before] = self.run_hook :before_create
          @out << "Done\n"

          return false unless provider.create_server(@server, @deploy_env.image, @deploy_env.flavor, @deploy_env.subnets, @deploy_env.groups, @out)
          mongo.server_insert @server

          @out << "\nAfter create hooks...\n"
          res[:after] = self.run_hook :after_create
          @out << "Done\n"
          @out.flush
          DevopsLogger.logger.info "Server with parameters: #{@server.to_hash.inspect} is running"

          schedule_expiration(@server)

          unless options["without_bootstrap"]
            bootstrap_options = {
              bootstrap_template: i.bootstrap_template
            }
            return two_phase_bootstrap(bootstrap_options)
          else
            return 0
          end
        rescue => e
          @out << e.message
          DevopsLogger.logger.error e.message
          roll_back
          mongo.server_delete @server.id
          # return 5
          return result_code(:server_not_in_chef_nodes)
        end
      end

      def bootstrap options
        @out << "\n\nBootstrap...\n"
        @out.flush
        k = Devops::Db.connector.key(@server.key)
        cert_path = k.path
        options[:cert_path] = k.path
        @out << "\nBefore bootstrap hooks...\n"
        res = self.run_hook(:before_bootstrap, @out)
        @out << "Done\n"
        if @server.private_ip.nil?
          @out << "Error: Private IP is null"
          return false
        end
        ja = {
          :provider => @server.provider,
          :devops_host => `hostname`.strip
        }
        ip = @server.private_ip
        unless @server.public_ip.nil?
          ip = @server.public_ip
          @out << "\nPublic IP is present\n"
        end

        address = "#{@server.remote_user}@#{ip}"

        cmd = 'ssh '
        cmd << "-i #{cert_path} "
        cmd << '-q '
        cmd << '-o StrictHostKeyChecking=no '
        cmd << '-o ConnectTimeout=2 -o ConnectionAttempts=1 '
        cmd << "#{address} 'exit'"
        cmd << " 2>&1"

        @out << "\nWaiting for SSH..."
        @out << "\nTest command: '#{cmd}'\n"
        @out.flush

        retries_amount = 0
        begin
          sleep(5)
          res = `#{cmd}`
          retries_amount += 1
          if retries_amount > MAX_SSH_RETRIES_AMOUNT
            @out.puts "Can not connect to #{address}"
            @out.puts res
            @out.flush
            DevopsLogger.logger.error "Can not connect with command '#{cmd}':\n#{res}"
            return result_code(:server_bootstrap_fail)
          end
          raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
        rescue ArgumentError => e
          @out.puts "SSH command failed, retry (#{retries_amount}/#{MAX_SSH_RETRIES_AMOUNT})"
          @out.flush
          retry
        end

        provider = @server.provider_instance
        @server.chef_node_name = provider.create_default_chef_node_name(@server) if @server.chef_node_name.nil?

        r = @knife_instance.knife_bootstrap(@out, ip, self.bootstrap_options(ja, options))

        if r == 0
          @out << "Chef node name: #{@server.chef_node_name}\n"
          ::Devops::Db.connector.server_set_chef_node_name @server
          @out << "Chef node name has been updated\n"
          @out << "After bootstrap hooks...\n"
          res = self.run_hook(:after_bootstrap, @out)
          @out << "Done\n"
          @out.flush
          DevopsLogger.logger.info "Server with id '#{@server.id}' is bootstraped"
          r
        else
          @out << "Can not bootstrap node '#{@server.id}', error code: #{r}"
          @out.flush
          result_code(:server_bootstrap_fail)
        end
      end

      def bootstrap_options attributes, options
        bootstrap_options = [
          "-x #{@server.remote_user}",
          "-i #{options[:cert_path]}",
          "--json-attributes '#{attributes.to_json}'"
        ]
        bootstrap_options.push "-N #{@server.chef_node_name}" if @server.chef_node_name
        bootstrap_options.push "--sudo" unless @server.remote_user == "root"
        bootstrap_options.push "-t #{options[:bootstrap_template]}" if options[:bootstrap_template]
        rl = options[:run_list]
        bootstrap_options.push "-r #{rl.join(",")}" unless rl.nil?# rl.empty?
        bootstrap_options.push "-c #{options[:config]}" if options[:config]
        bootstrap_options
      end

      def prepare_two_phase_bootstrap options
        @out << "Prepare bootstrap...\n"
        @out << "Done\n"
      end

      def two_phase_bootstrap options
        prepare_two_phase_bootstrap(options)
        # bootstrap phase
        begin
          provider = @server.provider_instance
          mongo = ::Devops::Db.connector
          options[:run_list] = provider.run_list
          bootstrap_status = bootstrap(options)

          if bootstrap_status == 0
            if check_server
              @out << "Server #{@server.chef_node_name} is created"
            else
              @out << roll_back
              @out.flush
              mongo.server_delete @server.id
              return result_code(:server_not_in_chef_nodes)
            end
          else
            # @out << roll_back
            # mongo.server_delete @server.id
            msg = "Failed while bootstraping server with id '#{@server.id}'\n"
            msg << "Bootstraping operation result was #{bootstrap_status}"
            DevopsLogger.logger.error msg
            @out.puts msg
            @out.flush
            return result_code(:server_bootstrap_fail)
          end
        rescue => e
          @out << "\nError: #{e.message}\n"
          @out.flush
          return result_code(:server_bootstrap_unknown_error)
        end

        # deploy phase. Assume that all servers are bootstraped successfully here.
        begin
          #raise "hello"
          @out << "\n"
          run_list = compute_run_list
          @out << "\nComputed run list: #{run_list.join(", ")}"
          @out.flush
          @knife_instance.set_run_list(@server.chef_node_name, run_list)
          deploy_info = @project.deploy_info(@deploy_env)
          deploy_status = deploy_server(deploy_info)
          if deploy_status == 0
            0
          else
            msg = "Failed on chef-client with run list, server with id '#{@server.id}'"
            msg << "\nDeploing server operation status was #{deploy_status}"
            DevopsLogger.logger.error msg
            @out << "\n" + msg + "\n"
            result_code(:deploy_failed)
          end
        rescue => e
          @out << "\nError: #{e.message}\n"
          result_code(:deploy_unknown_error)
        end
      end

      def check_server
        @knife_instance.chef_node_list.include?(@server.chef_node_name) and @knife_instance.chef_client_list.include?(@server.chef_node_name)
      end

      def unbootstrap
        k = Devops::Db.connector.key(@server.key)
        cert_path = k.path
        i = 0
        res = delete_from_chef_server(@server.chef_node_name)
        begin
          new_name = "/etc/chef.backup_#{Time.now.strftime("%d-%m-%Y_%H.%M.%S")}"
#          r = `ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} rm -Rf /etc/chef`
          cmd = "ssh -i #{cert_path} -q #{@server.remote_user}@#{@server.private_ip} \"/bin/sh -c 'if [[ -d /etc/chef ]]; then mv /etc/chef #{new_name}; else echo not found; fi'\""
          DevopsLogger.logger.info("Trying to run command '#{cmd}'")
          r = `#{cmd}`.strip
          if r == 'not found'
            res[:server] = "Directory '/etc/chef' does not exists"
          else
            raise(r) unless $?.success?
            res[:server] = "'/etc/chef' renamed to '#{new_name}'"
          end
        rescue => e
          DevopsLogger.logger.error "Unbootstrap error: " + e.message
          i += 1
          sleep(1)
          retry unless i == 5
          return {error: e.message}
        end
        res
      end

      def deploy_server_with_tags tags, deploy_info
        old_tags_str = nil
        new_tags_str = nil
        unless tags.empty?
          old_tags_str = @knife_instance.tags_list(@server.chef_node_name).join(" ")
          @out << "Server tags: #{old_tags_str}\n"
          @knife_instance.tags_delete(@server.chef_node_name, old_tags_str)

          new_tags_str = tags.join(" ")
          @out << "Server new tags: #{new_tags_str}\n"
          cmd = @knife_instance.tags_create(@server.chef_node_name, new_tags_str)
          unless cmd[1]
            m = "Error: Cannot add tags '#{new_tags_str}' to server '#{@server.chef_node_name}'"
            DevopsLogger.logger.error(m)
            @out << m + "\n"
            return 3
          end
          DevopsLogger.logger.info("Set tags for '#{@server.chef_node_name}': #{new_tags_str}")
        end

        r = deploy_server deploy_info

        unless tags.empty?
          @out << "Restore tags\n"
          cmd = @knife_instance.tags_delete(@server.chef_node_name, new_tags_str)
          DevopsLogger.logger.info("Deleted tags for #{@server.chef_node_name}: #{new_tags_str}")
          cmd = @knife_instance.tags_create(@server.chef_node_name, old_tags_str)
          DevopsLogger.logger.info("Set tags for #{@server.chef_node_name}: #{old_tags_str}")
        end
        return r
      end

      def deploy_server deploy_info
        @out << "\nBefore deploy hooks...\n"
        res = self.run_hook(:before_deploy, @out, deploy_info)
        @out << "Done\n"
        @out << "\nRun chef-client on '#{@server.chef_node_name}'\n"
        cmd = "chef-client --no-color"
        if deploy_info["use_json_file"]
          deploy_info.delete("use_json_file")
          @out << "Deploy Input Parameters:\n"
          json = JSON.pretty_generate(deploy_info)
          @out << json
          @out << "\n"
          file = "#{@server.project}_#{@server.deploy_env}_#{Time.new.to_i}"
          dir = DevopsConfig.config[:project_info_dir]
          File.open(File.join(dir, file), "w") do |f|
            f.write json
          end
          @out.flush
          cmd << " -j http://#{DevopsConfig.config[:address]}:#{DevopsConfig.config[:port]}/#{DevopsConfig.config[:url_prefix]}/v2.0/deploy/data/#{file}"
        else
          cmd << " -r #{deploy_info["run_list"].join(",")}" unless @server.stack.nil?
        end
        ip = if @server.public_ip.nil?
          @server.private_ip
        else
          @out << "Public IP detected\n"
          @server.public_ip
        end
        @out.flush
        k = Devops::Db.connector.key(@server.key)
        lline = @knife_instance.ssh_stream(@out, cmd, ip, @server.remote_user, k.path)
        r = /Chef\sClient\sfinished/i
        if lline[r].nil?
          1
        else
          @out << "\nAfter deploy hooks...\n"
          res = self.run_hook(:after_deploy, @out, deploy_info)
          @out << "Done\n"
          0
        end
      end

      def delete_from_chef_server node_name
        {
          :chef_node => @knife_instance.chef_node_delete(node_name),
          :chef_client => @knife_instance.chef_client_delete(node_name)
        }
      end

=begin
      def delete_etc_chef s, cert_path
        cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
        cmd += " sudo " unless s.remote_user == "root"
        cmd += "rm -Rf /etc/chef"
        r = `#{cmd}`
        raise(r) unless $?.success?
      end
=end

      def delete_server
        mongo = ::Devops::Db.connector
        if @server.static?
          if !@server.chef_node_name.nil?
            unbootstrap
          end
          mongo.server_delete @server.id
          msg = "Static server '#{@server.id}' is removed"
          DevopsLogger.logger.info msg
          return msg, nil
        end
        r = delete_from_chef_server(@server.chef_node_name)
        provider = @server.provider_instance
        begin
          r[:server] = provider.delete_server @server
        rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
          r[:server] = "Server with id '#{@server.id}' not found in '#{provider.name}' servers"
          DevopsLogger.logger.warn r[:server]
        end
        mongo.server_delete @server.id
        info = "Server '#{@server.id}' with name '#{@server.chef_node_name}' for project '#{@server.project}-#{@server.deploy_env}' is removed"
        DevopsLogger.logger.info info
        r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
        return info, r
      end

      def roll_back
        unless @server.id.nil?
          @out << "Server '#{@server.chef_node_name}' with id '#{@server.id}' is not created\n"
          @out << delete_from_chef_server(@server.chef_node_name).values.join("\n")
          begin
            @out << @server.provider_instance.delete_server(@server)
          rescue => e
            @out << e.message
          end
          @out << "\nRolled back\n"
        end
      end

      def create_run_list out, deploy_info
        out << "\nGenerate run list hook...\n"
        if deploy_info["run_list"]
          out << "Deploy info already contains 'run_list': #{deploy_info["run_list"].join(", ")}\n"
          return
        end
        out << "Project run list: #{@project.run_list.join(", ")}\n"
        out << "Deploy environment run list: #{@deploy_env.run_list.join(", ")}\n"
        out << "Server run list: #{@server.run_list.join(", ")}\n"
=begin
        rlist = Set.new.merge(@deploy_env.provider_instance.run_list).merge(@project.run_list).merge(@deploy_env.run_list).merge(@server.run_list)
        if @server.stack
          stack = Devops::Db.connector.stack(@server.stack)
          out << "Stack run list: #{stack.run_list.join(", ")}\n"
          rlist.merge(stack.run_list)
        end
=end
        deploy_info["run_list"] = compute_run_list
        out << "New deploy run list: #{deploy_info["run_list"].join(", ")}\nRun list has been generated\n\n"
      end

      def compute_run_list
        rlist = []
        [@deploy_env.provider_instance.run_list, @project.run_list, @deploy_env.run_list, @server.run_list].each do |sub_run_list|
          rlist += sub_run_list if sub_run_list.is_a?(Array)
        end
        rlist = Set.new(rlist)
        if @server.stack
          stack = Devops::Db.connector.stack(@server.stack)
#          out << "Stack run list: #{stack.run_list.join(", ")}\n"
          srl = stack.run_list
          rlist.merge(srl) if srl.is_a?(Array)
        end
        rlist.to_a
      end

      private

      def schedule_expiration(server)
        expires = @deploy_env.expires
        return unless expires
        interval = interval_in_seconds(expires)
        @out <<  "Planning expiration in #{expires}"
        DeleteServerWorker.perform_in(interval, server_chef_node_name: server.chef_node_name)
      end

      def interval_in_seconds(interval_as_string)
        interval = interval_as_string.to_i
        measure_unit = interval_as_string.chars.last
        case measure_unit
        when 's'
          interval
        when 'm'
          interval * 60
        when 'h'
          interval * 60 * 60
        when 'd'
          interval * 60 * 60 * 24
        when 'w'
          interval * 60 * 60 * 24 * 7
        else
          raise 'Wrong interval format'
        end
      end

    end
  end
end