fluke/devops-service/workers/stack_bootstrap_worker.rb

require "commands/stack"
require "db/mongo/models/stack/stack_factory"
require "db/mongo/models/project"
require "db/mongo/models/report"

class StackCreatingError < StandardError; end
class BootstrapingStackServerError < StandardError; end
class DeployingStackServerError < StandardError; end

class StackBootstrapWorker < Worker
  include StackCommands

  def perform(options)
    stack_attrs = options.fetch('stack_attributes')

    call() do |out, file|
      @out = out
      without_bootstrap = stack_attrs.delete('without_bootstrap')
      @out.puts "Received 'without_bootstrap' option" if without_bootstrap

      report = save_report(file, stack_attrs)

      begin
        stack = create_stack(stack_attrs)
      rescue StackCreatingError
        return 1
      end

      #TODO: errors
      begin
        servers_with_priority = persist_stack_servers!(stack)
        unless without_bootstrap
          sorted_keys = servers_with_priority.keys.sort{|x,y| y <=> x}
          sorted_keys.each do |key|
            bootstrap_servers!(servers_with_priority[key], report)
          end
        end
        @out.puts "Done."
        0
      rescue BootstrapingStackServerError
        @out.puts "\nAn error occured during bootstraping stack servers. Initiating stack rollback."
        rollback_stack!(stack)
        1
      rescue DeployingStackServerError => e
        @out.puts "\nStack was launched, but an error occured during deploying stack servers."
        @out.puts "You can redeploy stack after fixing the error."
        1
      rescue StandardError => e
        @out.puts "\nAn error occured. Initiating stack rollback."
        rollback_stack!(stack)
        raise e
      end
    end
  end

  private

  def rollback_stack!(stack)
    @out.puts "\nStart rollback of a stack"
    stack.delete_stack_in_cloud!
    Devops::Db.connector.stack_servers_delete(stack.name)
    Devops::Db.connector.stack_delete(stack.id)
    @out.puts "Rollback has been completed"
  end

  def create_stack(stack_attrs)
    stack = Devops::Model::StackFactory.create(stack_attrs["provider"], stack_attrs, @out)
    mongo.stack_insert(stack)
    operation_result = sync_stack_proc.call(@out, stack, mongo)

    if operation_result == 0
      @out.puts "\nStack '#{stack.name}' has been created"
      @out.flush
      stack
    else
      human_readable_code = StackCommands.result_codes.key(operation_result)
      @out.puts "An error ocurred during stack creating"
      @out.puts "Stack creating operation result was #{human_readable_code}"
      raise StackCreatingError
    end
  end

  def bootstrap_servers!(servers, report)
    @out << "\nStart bootstraping stack servers\n"

    subreports = []
    data = {}
    servers.each do |server|
      sjid = Worker.start_async(BootstrapWorker,
        server_attrs: server.to_mongo_hash,
        bootstrap_template: 'omnibus',
        owner: server.created_by
      )
      subreports << sjid
      @out.puts "Bootstraping server '#{server.id}'... job id: #{sjid}"
      data[server.id] = sjid
    end
    @out.puts
    @out.flush
    mongo.add_report_subreports(jid, subreports)
    results = []
    data.each do |server_id, subreport_id|
      begin
        sleep(5)
        subreport = mongo.report(subreport_id)
        status = subreport.status
        if status == Worker::STATUS::COMPLETED
          @out.puts "Server '#{server_id}' has been bootstraped with job #{subreport_id}"
          break
        elsif status == Worker::STATUS::FAILED
          results << subreport.job_result_code
          @out.puts "Server '#{server_id}' hasn't been bootstraped with job #{subreport_id}. Job result code is '#{subreport.job_result_code}'"
          break
        end
      end while(true)
    end
    @out.flush
    results.empty? ? 0 : -5
  end

  def check_bootstrap_results!(results)
    if results.values.all?(&:zero?)
      # everything is OK
      @out.puts "Stack servers have been bootstraped"
      @out.flush
      return 0
    end

    @out.puts
    results.each do |chef_node_name, code|
      human_readable_code = Devops::Executor::ServerExecutor.symbolic_result_code(code)
      @out.puts "Operation result for #{chef_node_name}: #{human_readable_code}"
    end

    if errors_in_bootstrapping_present?(results.values)
      raise BootstrapingStackServerError # will cause rollback of a stack
    else
      raise DeployingStackServerError #will not cause rollback of a stack
    end
  end

  def errors_in_bootstrapping_present?(result_codes)
    bootstrap_error_codes = []
    [:server_bootstrap_fail, :server_not_in_chef_nodes, :server_bootstrap_unknown_error].each do |symbolic_code|
      bootstrap_error_codes << Devops::Executor::ServerExecutor.result_code(symbolic_code)
    end

    (bootstrap_error_codes & result_codes).size > 0
  end

  def save_report(file, stack_attrs)
    report = ::Devops::Model::Report.new(
      "file" => file,
      "_id" => jid,
      "created_by" => stack_attrs['owner'],
      "project" => stack_attrs["project"],
      "deploy_env" => stack_attrs["deploy_env"],
      "type" => ::Devops::Model::Report::STACK_TYPE,
      "subreports" => [],
      "stack" => stack_attrs['name']
    )
    mongo.save_report(report)
    report
  end

  # returns
  # {
  #   "priority" => [Servers]
  # }
  def persist_stack_servers!(stack)
    @out.puts "Start syncing stack servers with CID"
    @out.flush
    project = mongo.project(stack.project)
    deploy_env = project.deploy_env(stack.deploy_env)
    provider = stack.provider_instance

    stack_servers_info = provider.stack_servers(stack).group_by{|info| info["tags"]["cid:priority"] || 0}
    stack_servers_with_priority = {}
    stack_servers_info.each do |priority, info_array|
      stack_servers_with_priority[priority] = info_array.map do |extended_info|
        server_attrs = {
          'provider' => provider.name,
          'project' => project.id,
          'deploy_env' => deploy_env.identifier,
          'remote_user' => mongo.image(deploy_env.image).remote_user,
          'key' => extended_info["key_name"] || provider.ssh_key,
          '_id' => extended_info["id"],
          'chef_node_name' => extended_info["name"],
          'private_ip' => extended_info["private_ip"],
          'public_ip' => extended_info["public_ip"],
          'created_by' => stack.owner,
          'run_list' => stack.run_list || [],
          'stack' => stack.name
        }

        server = ::Devops::Model::Server.new(server_attrs)
        mongo.server_insert(server)
        # server.chef_node_name =  provider.create_default_chef_node_name(server)
        server
      end
    end
    @out.puts "Stack servers have been synced with CID"
    stack_servers_with_priority.each do |priority, servers|
      @out.puts "Priority '#{priority}': #{servers.map(&:id).join(", ")}"
    end
    @out.flush
    stack_servers_with_priority
  end

end