Run QC on new 526 backup status polling

[SamStuckey]

Scott recently deployed fixes to 526 state. I want to confirm the backup path polling defined here is working. this is critical to the success of our new state paradigm, and will unblock remediation 'tagging'

The following code, run in Production on 7/10/2024 should return true for both tests

class BenefitsIntakeStatusPoll
  attr_reader :guids, :export_file_name, :export_file_path, :verbose, :tracking_level, :track_submission_id, :only_non_orphaned
  BATCH_SIZE = 1000
  IGNORABLE_STATUSES = %w[success vbms]
  def initialize(guids: nil,
                 verbose: true,
                 guid_import_path: nil,
                 export_file_name: nil,
                 ignore_statuses: IGNORABLE_STATUSES,
                 tracking_level: :all,
                 only_non_orphaned: :false,
                 track_submission_id: false) # this is very slow
    @guids = guids || load_import_guids(guid_import_path)
    @export_file_name = export_file_name
    @verbose = verbose
    @ignorable_statuses = ignore_statuses
    @track_submission_id = track_submission_id
    @only_non_orphaned = only_non_orphaned
    @tracking_level = tracking_level
  end
  def total_failure_count
    @total_failure_count ||= generate_total_failure_count
  end
  def generate_total_failure_count
    total = 0
    report[:error].each do |_err_msg, details|
      total += details.count
    end
    total
  end
  # preloaders -------
  def submissions
    @submissions ||= load_submissions
  end
  def load_submissions
    # we only care about backup because primary path uses EVSS
    Form526Submission.where(backup_submitted_claim_id: guids)
  end
  def error_guids
    @error_guids ||= [].tap do |output|
                       report[:error].each do |_err_type, instances|
                         output << instances&.map { |inst| inst[:lh_guid] } || []
                       end
                     end.flatten
  end
  def error_submissions
    @error_submissions ||= submissions.where(backup_submitted_claim_id: error_guids)
  end
  def vbms_guids
    @vbms_guids ||= report[:vbms].pluck(:lh_guid)
  end
  def vbms_submissions
    @vbms_submissions ||= submissions.where(backup_submitted_claim_id: vbms_guids)
  end
  def success_guids
    @success_guids ||= report[:success].pluck(:lh_guid)
  end
  def success_submissions
    @success_submissions ||= submissions.where(backup_submitted_claim_id: success_guids)
  end
  def processing_guids
    @processing_guids ||= report[:processing].pluck(:lh_guid)
  end
  def processing_submissions
    @processing_submissions ||= submissions.where(backup_submitted_claim_id: processing_guids)
  end
  # ---------------------
  def export_file_path
    "/tmp/#{export_file_name}.csv"
  end
  def run
    if export_file_name.blank?
      write_to_stdout
    else
      write_to_local
    end
  end
  def load_import_guids(path)
    raise 'tried to load guids from nil path' if path.nil?
    csv = CSV.read(path, headers: false)
    csv.to_a.flatten.map(&:strip)
  end
  def write_to_stdout
    puts csv
  end
  def write_to_local
    puts "\n writting to local \n"
    file = File.new(export_file_path, 'w')
    file.puts(csv)
    file.close
    puts export_file_path
  end
  def report
    @report ||= compile_report
  end
  def track_all?
    tracking_level == :all
  end
  def track_selective?
    tracking_level == :selective
  end
  def track_error_only?
    tracking_level == :error_only
  end
  def track_error?
    track_all? || track_error_only?
  end
  def track_only_records_with_submission_id?
    track_submission_id && only_non_orphaned
  end
  def compile_report
    initial_report = { error: {}, success: [], vbms: [], processing: [] }
    lower = 0
    upper = BATCH_SIZE
    number_of_batches = (guids.count / BATCH_SIZE) + 1
    puts "processing #{guids.count} guids in #{number_of_batches} batches of #{BATCH_SIZE}"
    count = 1
    number_of_batches.times do
      puts "starting batch #{count}"
      count += 1
      id_batch = guids[lower...upper]
      batch_report = generate_batch_report({ ids: id_batch })
      batch_report['data'].compact.each do |upload|
        status = upload['attributes']['status']
        lh_id = upload['id']
        submission_id = nil
        sub_data = { lh_guid: lh_id }
        if track_submission_id
          submission_id = submissions.where(backup_submitted_claim_id: lh_id).pluck(:id).first
          next if track_only_records_with_submission_id? && submission_id.blank?
          sub_data.merge!({ va_gov_sub_id: submission_id })
        end
        if status == 'error' && track_error?
          error_key = upload['attributes']['detail']
          # group by error
          initial_report[:error][error_key] ||= []
          initial_report[:error][error_key] << sub_data
        elsif track_all? || (track_selective? && !@ignorable_statuses.include?(status))
          new_status_key = status.to_sym
          initial_report[new_status_key] ||= []
          initial_report[new_status_key] << sub_data
        end
      end
      lower += BATCH_SIZE
      upper += BATCH_SIZE
    end
    initial_report
  end
  def generate_batch_report(data)
    url = URI.parse('https://api.va.gov/services/vba_documents/v1/uploads/report')
    http = Net::HTTP.new(url.host, url.port)
    http.use_ssl = (url.scheme == 'https')
    headers = {
      'Content-Type' => 'application/json',
      'apikey' => Settings.benefits_intake_service.api_key,
      'accept' => 'application/json'
    }
    request = Net::HTTP::Post.new(url.path, headers)
    request.body = data.to_json
    JSON.parse(http.request(request).body)
  end
  def csv
    @csv ||= generate_csv
  end
  def generate_csv
    CSV.generate(headers: false) do |csv|
      csv << ['status', 'count', 'details', 'affected']
      report[:error].each do |error_key, data|
        row = ['error', data[:count], error_key]
        id_cell = if verbose
                    data[:affected_guids]
                  else
                    '(output suppressed)'
                  end
        row << id_cell
        csv << row
      end
      csv << [nil, nil, nil]
      successes = if verbose
                    report[:success].join(',')
                  else
                    '(output suppressed)'
                  end
      csv << ['success', report[:success].count, '', successes]
      csv << [nil, nil, nil]
      vbms = if verbose
               report[:vbms].join(',')
             else
               '(output suppressed)'
             end
      csv << ['vbms', report[:vbms].count, '', vbms]
    end
  end
end

subs = Form526Submission.where.not(backup_submitted_claim_id: nil)
poll = BenefitsIntakeStatusPoll.new(guids: subs.pluck(:backup_submitted_claim_id));

# test 1
polled_successes = poll.success_submissions.pluck(:id)
db_queried_successes = Form526Submission.where(backup_submitted_claim_status: Form526Submission.backup_submitted_claim_statuses[:accepted]).pluck(:id)
# should be true, then we know we are good to go
polled_successes == db_queried_successes

# test 2
polled_rejections = poll.error_submissions.pluck(:id)
db_queried_rejections = Form526Submission.where(backup_submitted_claim_status: Form526Submission.backup_submitted_claim_statuses[:rejected]).pluck(:id)

[SamStuckey]

Determined that the job failed last night, but for a different reaons than it's been failing. Prior to scotts changes (up to Jul 9th) it was blowing up because of a syntax error in the status type else block. @freeheeling fixed that when he added this line.

Last night's job failed due to a simple database timeout (in our database, not the API) because the number of submissions that need updating is so large.

[SamStuckey]

Determined that the job failed last night, but for a different reaons than it's been failing. Prior to scotts changes (up to Jul 9th) it was blowing up because of a syntax error in the status type else block. @freeheeling fixed that when he added this line.

Last night's job failed due to a simple database timeout (in our database, not the API) because the number of submissions that need updating is so large.

[SamStuckey]

Next step is to manually run the backfill in chunks. that should allow the job to keep up moving forward

[SamStuckey]

BLOCKED:

• manual backfill is complete, leaving ~100 submissions in pending
• waiting on deployment of PR to refactor heavy DB query

NEXT STEPS:

• once this PR is out (refactor of query) and this has had a chance to run overnight, we can check datadog for pass / fail logging