#!/usr/bin/env bash
#
# queue-probe.sh — prove that a dispatched job actually completes on THIS host.
#
# ADR-043 Decision 4 requires, before any queued work may be built:
#   "a process-manager unit committed to the repository or to docs/, plus
#    evidence that a dispatched job completed on the target host — not the
#    presence of queue:restart in a deploy script."
#
# This script is that evidence. It dispatches a real job through the real queue
# connection and waits for a real worker to execute it. It writes no secrets and
# prints no configuration values other than the queue driver name.
#
# Usage, from anywhere on the target host:
#     bash /home/<user>/public_html/my.itcarrot.com/deploy/bin/queue-probe.sh
#     bash .../queue-probe.sh status     # liveness only, dispatches nothing
#     PROBE_TIMEOUT=180 bash .../queue-probe.sh
#
# Exit codes: 0 = PASS, 1 = FAIL (a real defect), 2 = could not run the probe.
#
# This script is shipped by deploy.yml and lands at <app_root>/deploy/bin/ on
# the host, which is where the APP_ROOT resolution below expects it. Before
# 2026-08-28 it was silently excluded from the upload and had to be copied up
# by hand; see docs/architecture/RUNBOOK-queue-and-deploy.md.

# Deliberately no `set -e`: this script's whole job is to REPORT failure with
# context. Aborting on the first non-zero exit would hide the diagnosis.
set -uo pipefail

MODE="${1:-probe}"
PROBE_TIMEOUT="${PROBE_TIMEOUT:-90}"

# ---------------------------------------------------------------------------
# Locate the application root (this file lives at <root>/deploy/bin/).
# ---------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

if [ ! -f "$APP_ROOT/artisan" ]; then
  echo "FAIL(setup): no artisan at $APP_ROOT — run this from inside the deployed app tree." >&2
  exit 2
fi
cd "$APP_ROOT" || exit 2

# ---------------------------------------------------------------------------
# Resolve a real CLI php. Bare `php` on this cPanel account can be the CGI SAPI,
# which cannot run artisan. Same resolution order as deploy.yml.
# ---------------------------------------------------------------------------
PHP=""
for CANDIDATE in \
  /opt/cpanel/ea-php83/root/usr/bin/php \
  /opt/cpanel/ea-php84/root/usr/bin/php \
  "$(command -v php 2>/dev/null || true)"
do
  [ -n "$CANDIDATE" ] || continue
  [ -x "$CANDIDATE" ] || continue
  if [ "$("$CANDIDATE" -r 'echo PHP_SAPI;' 2>/dev/null)" = "cli" ]; then
    PHP="$CANDIDATE"
    break
  fi
done

if [ -z "$PHP" ]; then
  echo "FAIL(setup): found no PHP binary reporting SAPI 'cli'." >&2
  echo "  Bare \`php\` here is probably the CGI/FastCGI SAPI. Check WHM -> MultiPHP Manager" >&2
  echo "  and update the path in deploy/supervisor/my-itcarrot-worker.ini to match." >&2
  exit 2
fi

echo "app root : $APP_ROOT"
echo "php      : $PHP"

# ---------------------------------------------------------------------------
# Liveness: is anything actually consuming the queue?
# ---------------------------------------------------------------------------
report_liveness() {
  echo
  echo "--- worker liveness ---"

  # Match only a php BINARY running artisan queue:work — i.e. `php` at the start
  # of the command or after a path separator. A looser pattern also matches the
  # shell that invoked this script (whose own command line contains the words
  # "artisan queue:work"), which inflates the count and reports a worker that is
  # not there. `[p]hp` keeps grep from matching its own process.
  local WORKER_RE='(^|/)[p]hp[0-9.]*[[:space:]].*artisan[[:space:]]+queue:work'
  local worker_count
  worker_count="$(ps -u "$(id -un)" -o args= 2>/dev/null | grep -cE "$WORKER_RE" || true)"
  echo "queue:work processes owned by $(id -un): ${worker_count:-0}"
  ps -u "$(id -un)" -o pid=,etime=,args= 2>/dev/null | grep -E "$WORKER_RE" || true

  if command -v supervisorctl >/dev/null 2>&1; then
    echo "supervisorctl status my-itcarrot-worker:"
    supervisorctl status my-itcarrot-worker 2>&1 | sed 's/^/  /' || true
  else
    echo "supervisorctl: not on PATH for this user (try: sudo supervisorctl status)"
  fi

  if crontab -l 2>/dev/null | grep -q 'queue:work'; then
    echo "cron fallback: PRESENT in this user's crontab"
    crontab -l 2>/dev/null | grep 'queue:work' | sed 's/^/  /'
  else
    echo "cron fallback: absent from this user's crontab"
  fi
}

# ---------------------------------------------------------------------------
# Read the queue driver. A `sync` driver runs jobs inline in the dispatching
# process, so the probe would PASS while proving nothing about a worker. That
# is exactly the false-green this script exists to prevent, so it is a FAIL.
# ---------------------------------------------------------------------------
QUEUE_DRIVER="$("$PHP" artisan tinker --execute='echo config("queue.default");' 2>/dev/null | tr -d '\r\n[:space:]')"

echo "queue    : ${QUEUE_DRIVER:-<unreadable>}"

if [ -z "$QUEUE_DRIVER" ]; then
  echo "FAIL(setup): could not read config('queue.default')." >&2
  echo "  laravel/tinker is in composer.json 'require', so it should survive --no-dev." >&2
  echo "  Check: $PHP artisan tinker --execute='echo 1;'" >&2
  exit 2
fi

if [ "$MODE" = "status" ]; then
  report_liveness
  exit 0
fi

case "$QUEUE_DRIVER" in
  database) ;;
  sync)
    echo
    echo "FAIL: QUEUE_CONNECTION resolves to 'sync'."
    echo "  Jobs run inline in the web request. Nothing is queued, so no worker is"
    echo "  exercised and this probe cannot produce meaningful evidence."
    echo "  Set QUEUE_CONNECTION=database in .env, then: $PHP artisan config:cache"
    exit 1
    ;;
  *)
    echo
    echo "FAIL(setup): queue driver '$QUEUE_DRIVER' is not one this probe understands."
    echo "  It counts rows in the database 'jobs' table; adapt it before trusting it."
    exit 2
    ;;
esac

# ---------------------------------------------------------------------------
# Probe.
# ---------------------------------------------------------------------------
PROBE_TOKEN="probe-$(date -u +%Y%m%dT%H%M%SZ)-$$-${RANDOM}"
PROBE_FILE="$APP_ROOT/storage/app/queue-probe/$PROBE_TOKEN"

count_jobs() {
  "$PHP" artisan tinker --execute='echo \Illuminate\Support\Facades\DB::table(config("queue.connections.database.table", "jobs"))->count();' 2>/dev/null | tr -d '\r\n[:space:]'
}
count_failed() {
  "$PHP" artisan tinker --execute='echo \Illuminate\Support\Facades\DB::table("failed_jobs")->count();' 2>/dev/null | tr -d '\r\n[:space:]'
}

BEFORE="$(count_jobs)"
FAILED_BEFORE="$(count_failed)"

if ! [[ "$BEFORE" =~ ^[0-9]+$ ]]; then
  echo "FAIL(setup): could not count the jobs table (got: '${BEFORE}')." >&2
  echo "  Has 0001_01_01_000002_create_jobs_table.php been migrated on this host?" >&2
  exit 2
fi

echo "jobs table before: $BEFORE (failed_jobs: ${FAILED_BEFORE:-?})"
echo "token            : $PROBE_TOKEN"
echo

# The dispatch code MUST live in a real file on disk, not in `tinker --execute`.
# Laravel's SerializableClosure reconstructs a queued closure by re-reading the
# source file it was declared in; a closure declared in eval'd code has no file,
# so the worker throws "Call to a member function bindTo() on null" and the job
# lands in failed_jobs. That failure looks exactly like a broken worker, which
# would make this probe report a false FAIL on a perfectly healthy host.
# (Verified against laravel/serializable-closure Serializers/Native.php:200.)
#
# The file must survive until the worker has executed the job — it is read at
# unserialize time, not at dispatch time. It is removed in cleanup() below.
#
# $t and $p are captured by value with `use` because the closure runs in the
# WORKER process, which does not inherit this shell's environment; a getenv()
# call inside the closure body would return false there.
DISPATCH_FILE="$APP_ROOT/storage/app/queue-probe/dispatch_${PROBE_TOKEN}.php"
mkdir -p "$APP_ROOT/storage/app/queue-probe" || {
  echo "FAIL(setup): cannot create storage/app/queue-probe — check permissions." >&2
  exit 2
}

cleanup() {
  rm -f "$DISPATCH_FILE"
}
trap cleanup EXIT

cat > "$DISPATCH_FILE" <<'PHPPROBE'
<?php
$t = getenv("PROBE_TOKEN");
$p = getenv("PROBE_FILE");
dispatch(function () use ($t, $p) {
    @mkdir(dirname($p), 0775, true);
    file_put_contents($p, $t . PHP_EOL . date("c") . PHP_EOL . "pid=" . getmypid() . PHP_EOL);
});
echo "dispatched";
PHPPROBE

export PROBE_TOKEN PROBE_FILE
export PROBE_DISPATCH_FILE="$DISPATCH_FILE"
DISPATCH_OUT="$("$PHP" artisan tinker --execute='require getenv("PROBE_DISPATCH_FILE");' 2>&1)"

if ! printf '%s' "$DISPATCH_OUT" | grep -q dispatched; then
  echo "FAIL(setup): dispatch call did not complete."
  printf '%s\n' "$DISPATCH_OUT" | tail -20
  exit 2
fi

AFTER="$(count_jobs)"
echo "jobs table after dispatch: $AFTER"

if [[ "$AFTER" =~ ^[0-9]+$ ]] && [ "$AFTER" -le "$BEFORE" ]; then
  echo
  echo "FAIL: the job never reached the jobs table (before=$BEFORE after=$AFTER)."
  echo "  Either the dispatch was swallowed, or the queue connection in the CLI"
  echo "  environment differs from the one you just read. Nothing to wait for."
  report_liveness
  exit 1
fi

echo "waiting up to ${PROBE_TIMEOUT}s for a worker to execute it..."

START="$(date +%s)"
ELAPSED=0
while [ "$ELAPSED" -lt "$PROBE_TIMEOUT" ]; do
  if [ -f "$PROBE_FILE" ]; then
    echo
    echo "=============================================="
    echo "PASS: job dispatched AND executed by a worker."
    echo "  latency        : ${ELAPSED}s"
    echo "  marker written : $PROBE_FILE"
    sed 's/^/  /' "$PROBE_FILE"
    echo "=============================================="
    echo
    echo "This output is the ADR-043 Decision 4 evidence. Record the date, the host,"
    echo "and the latency. Re-run it after any change to .env, PHP version, or the"
    echo "supervisor unit — it is cheap and it is the only thing that actually proves"
    echo "the queue works."
    rm -f "$PROBE_FILE"
    report_liveness
    exit 0
  fi
  sleep 2
  ELAPSED=$(( $(date +%s) - START ))
done

# ---------------------------------------------------------------------------
# Timed out — diagnose rather than just report.
# ---------------------------------------------------------------------------
NOW_JOBS="$(count_jobs)"
FAILED_AFTER="$(count_failed)"

echo
echo "=============================================="
echo "FAIL: the job was queued but never executed within ${PROBE_TIMEOUT}s."
echo "  jobs table: before=$BEFORE afterDispatch=$AFTER now=$NOW_JOBS"
echo "  failed_jobs: before=${FAILED_BEFORE:-?} now=${FAILED_AFTER:-?}"
echo "=============================================="

if [[ "$FAILED_AFTER" =~ ^[0-9]+$ ]] && [[ "$FAILED_BEFORE" =~ ^[0-9]+$ ]] && [ "$FAILED_AFTER" -gt "$FAILED_BEFORE" ]; then
  echo
  echo "DIAGNOSIS: a worker DID pick the job up and it threw. The worker is alive;"
  echo "the job failed. Inspect it:  $PHP artisan queue:failed"
else
  echo
  echo "DIAGNOSIS: nothing consumed the job. No worker is running, or it is running"
  echo "against a different queue name, a different database, or a different app path."
fi

report_liveness

echo
echo "--- last 30 lines of storage/logs/worker.log ---"
tail -30 "$APP_ROOT/storage/logs/worker.log" 2>/dev/null || echo "(no worker.log — supervisor has probably never started this program)"

echo
echo "The probe job is still queued. Remove it with:"
echo "  $PHP artisan tinker --execute='\\Illuminate\\Support\\Facades\\DB::table(\"jobs\")->truncate();'"
echo "  (only safe while the queue is genuinely idle — it deletes every pending job)"
exit 1