fluke/devops-service/commands/server.rb

require 'set'
require "commands/knife_commands"
require "commands/deploy"
require "exceptions/record_not_found"

module ServerCommands

  include DeployCommands

  def create_server_proc
    lambda do |out, s, provider|
      mongo = ::Devops::Db.connector
      begin
        out << "Create server...\n"
        out.flush if out.respond_to?(:flush)
        unless provider.create_server(s, out)
          return 3
        end
        s.create
        out.flush if out.respond_to?(:flush)
        DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
        key = mongo.key(s.key)
        return two_phase_bootstrap(s, out, provider, key.path)
      rescue => e
        DevopsLogger.logger.error e.message
        DevopsLogger.logger.warn roll_back(s, provider)
        mongo.server_delete s.id
        return 5
      end
    end
  end

  def create_server project, env, params, user, out
    provider = ::Provider::ProviderFactory.get(env.provider)
    mongo = ::Devops::Db.connector
    begin
      out << "Create server...\n"
      out.flush if out.respond_to?(:flush)

      s = Devops::Model::Server.new
      s.provider = provider.name
      s.project = project.id
      s.deploy_env = env.identifier
      s.run_list = params["run_list"] || []
      s.chef_node_name = params["name"]
      s.key = params["key"] || provider.ssh_key

      i = mongo.image env.image
      s.remote_user = i.remote_user
      s.created_by = user

      return 3 unless s.create(provider, env.image, env.flavor, env.subnets, env.groups, out)
      out.flush if out.respond_to?(:flush)
      DevopsLogger.logger.info "Server with parameters: #{s.to_hash.inspect} is running"
      unless params["without_bootstrap"]
        s.run_list = Set.new.merge(project.run_list).merge(env.run_list).merge(s.run_list)
        key = mongo.key(s.key)
        s.chef_node_name = provider.create_default_chef_node_name(s) if s.chef_node_name.nil?
        return two_phase_bootstrap(s, provider.run_list, i.bootstrap_template, key.path, out)
      else
        return 0
      end
    rescue => e
      DevopsLogger.logger.error e.message
      DevopsLogger.logger.warn roll_back(s, provider)
      mongo.server_delete s.id
      return 5
    end
  end

  def two_phase_bootstrap s, provider_run_list, bootstrap_template, cert_path, out
    mongo = ::Devops::Db.connector
    out << "\n\nBootstrap...\n"
    out.flush if out.respond_to?(:flush)
    status = bootstrap(s, out, bootstrap_template, provider_run_list, cert_path)
    out.flush if out.respond_to?(:flush)
    if status == 0
      DevopsLogger.logger.info "Server with id '#{s.id}' is bootstraped"
      if check_server(s)
        out << "Server #{s.chef_node_name} is created"
      else
        out << roll_back(s, provider)
        mongo.server_delete s.id
        return 5
      end
      out << "\n"
      out.flush if out.respond_to?(:flush)

      run_list = s.run_list + provider_run_list
      out << "\nRun list: #{run_list.inspect}"
#      s.options[:run_list] += run_list
      KnifeCommands.set_run_list(s.chef_node_name, run_list)
      status = deploy_server(out, s, cert_path)
      if status != 0
        msg = "Failed on chef-client with project run list, server with id '#{s.id}'"
        DevopsLogger.logger.error msg
        out << "\n" + msg + "\n"
      end
    else
      msg = "Failed while bootstraping server with id '#{s.id}'"
      DevopsLogger.logger.error msg
      out << "\n" + msg + "\n"
      out << roll_back(s, provider)
      mongo.server_delete s.id
    end
    return status
  end

  def delete_from_chef_server node_name
    {
      :chef_node => KnifeCommands.chef_node_delete(node_name),
      :chef_client => KnifeCommands.chef_client_delete(node_name)
    }
  end

  def self.delete_etc_chef s, cert_path
    cmd = "ssh -i #{cert_path} -t -q #{s.remote_user}@#{s.private_ip}"
    cmd += " sudo " unless s.remote_user == "root"
    cmd += "rm -Rf /etc/chef"
    r = `#{cmd}`
    raise(r) unless $?.success?
  end

  def check_server s
    KnifeCommands.chef_node_list.include?(s.chef_node_name) and KnifeCommands.chef_client_list.include?(s.chef_node_name)
  end

  def bootstrap s, out, bootstrap_template, run_list, cert_path
    out << "Before bootstrap hooks...\n"
    res = s.run_hook(:before_bootstrap, out)
    out << "Done\n"
    if s.private_ip.nil?
      out << "Error: Private IP is null"
      return false
    end
    out << "\nBootstrap with run list: #{run_list.inspect}\n"
    ja = {
      :provider => s.provider,
      :devops_host => `hostname`.strip
    }
    bootstrap_options = [
      "-x #{s.remote_user}",
      "-i #{cert_path}",
      "--json-attributes '#{ja.to_json}'",
      "-N #{s.chef_node_name}"
    ]
    bootstrap_options.push "--sudo" unless s.remote_user == "root"
    bootstrap_options.push "-d #{bootstrap_template}" if bootstrap_template
    bootstrap_options.push "-r #{run_list.join(",")}" unless run_list.empty?
    ip = s.private_ip
    unless s.public_ip.nil? || s.public_ip.strip.empty?
      ip = s.public_ip
      out << "\nPublic IP is present\n"
    end
    out << "\nWaiting for SSH..."
    out.flush if out.respond_to?(:flush)
    i = 0
    cmd = "ssh -i #{cert_path} -q #{s.remote_user}@#{ip} 'exit' 2>&1"
    begin
      sleep(5)
      res = `#{cmd}`
      i += 1
      if i == 120
        out << "\nCan not connect to #{s.remote_user}@#{ip}"
        out << "\n" + res
        DevopsLogger.logger.error "Can not connect with command 'ssh -i #{cert_path} #{s.remote_user}@#{ip}':\n#{res}"
        return false
      end
      raise ArgumentError.new("Can not connect with command '#{cmd}' ") unless $?.success?
    rescue ArgumentError => e
      retry
    end

    r = KnifeCommands.knife_bootstrap(out, ip, bootstrap_options)
    if r == 0
      out << "Chef node name: #{s.chef_node_name}\n"
      ::Devops::Db.connector.server_set_chef_node_name s
      out << "Chef node name has been updated\n"
      out << "After bootstrap hooks...\n"
      res = s.run_hook(:after_bootstrap, out)
      out << "Done\n"
    else
    end
    r
  end

  def self.unbootstrap s, cert_path
    i = 0
    begin
      r = `ssh -i #{cert_path} -q #{s.remote_user}@#{s.private_ip} rm -Rf /etc/chef`
      raise(r) unless $?.success?
    rescue => e
      DevopsLogger.logger.error "Unbootstrap error: " + e.message
      i += 1
      sleep(1)
      retry unless i == 5
      return e.message
    end
    nil
  end

  def delete_server s
    mongo = ::Devops::Db.connector
    if s.static?
      if !s.chef_node_name.nil?
        cert = mongo.key s.key
        ServerCommands.unbootstrap(s, cert.path)
      end
      mongo.server_delete s.id
      msg = "Static server '#{s.id}' is removed"
      DevopsLogger.logger.info msg
      return msg, nil
    end
    r = delete_from_chef_server(s.chef_node_name)
    provider = ::Provider::ProviderFactory.get(s.provider)
    begin
      r[:server] = provider.delete_server s
    rescue Fog::Compute::OpenStack::NotFound, Fog::Compute::AWS::NotFound
      r[:server] = "Server with id '#{s.id}' not found in '#{provider.name}' servers"
      DevopsLogger.logger.warn r[:server]
    end
    mongo.server_delete s.id
    info = "Server '#{s.id}' with name '#{s.chef_node_name}' for project '#{s.project}-#{s.deploy_env}' is removed"
    DevopsLogger.logger.info info
    r.each{|key, log| DevopsLogger.logger.info("#{key} - #{log}")}
    return info, r
  end

  def roll_back s, provider
    str = ""
    unless s.id.nil?
      str << "Server '#{s.chef_node_name}' with id '#{s.id}' is not created\n"
      str << delete_from_chef_server(s.chef_node_name).values.join("\n")
      begin
        str << provider.delete_server(s)
      rescue => e
        str << e.message
      end
      str << "\nRolled back\n"
    end
    return str
  end

end