name: Monitor Self-Hosted Runners

on:
  schedule:
    - cron: "0 4 * * 1"   # weekly on Monday at 04:00 UTC (3 hours after heartbeat)
  workflow_dispatch:
    inputs:
      stale_hours:
        description: "Alert if newest heartbeat for a runner is older than this many hours (default tolerates one missed weekly heartbeat)"
        required: false
        default: "192"
      lookback_days:
        description: "Scan heartbeat workflow runs created within this many days (any event type)"
        required: false
        default: "30"

permissions:
  actions: read    # Required to read workflow runs / artifacts / runners
  contents: read   # Required to checkout runners.json

env:
  HEARTBEAT_WORKFLOW: runner_heartbeat.yml
  STALE_HOURS: ${{ github.event.inputs.stale_hours || '192' }}
  LOOKBACK_DAYS: ${{ github.event.inputs.lookback_days || '30' }}
  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  check:
    runs-on: ubuntu-latest
    env:
      # Defined at job scope so the step-level `if:` below can read it via
      # `env.TEAMS_WEBHOOK_URL`. Step-level `env:` is not visible to that
      # same step's `if:` expression.
      TEAMS_WEBHOOK_URL: ${{ secrets.TEAMS_WEBHOOK_URL }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            .github/runners.json
          sparse-checkout-cone-mode: false

      - name: Validate numeric inputs
        # STALE_HOURS and LOOKBACK_DAYS flow from workflow_dispatch inputs
        # into env as strings. Downstream steps use them in shell arithmetic
        # ($(( STALE_HOURS * 3600 ))) and in a date computation, both of
        # which require positive integers. Fail fast with a clear message
        # if a user supplies something non-numeric.
        run: |
          set -euo pipefail
          fail=0
          if ! [[ "$STALE_HOURS" =~ ^[1-9][0-9]*$ ]]; then
            echo "ERROR: stale_hours must be a positive integer, got: '$STALE_HOURS'" >&2
            fail=1
          fi
          if ! [[ "$LOOKBACK_DAYS" =~ ^[1-9][0-9]*$ ]]; then
            echo "ERROR: lookback_days must be a positive integer, got: '$LOOKBACK_DAYS'" >&2
            fail=1
          fi
          # Cap lookback at 1 year to keep the API scan bounded.
          if [ "$fail" = "0" ] && [ "$LOOKBACK_DAYS" -gt 365 ]; then
            echo "ERROR: lookback_days must be <= 365, got: '$LOOKBACK_DAYS'" >&2
            fail=1
          fi
          exit "$fail"

      - name: Load expected runners (source of truth)
        id: expected
        run: |
          set -euo pipefail
          jq -r '.runners[]' .github/runners.json > expected_runners.txt
          echo "Expected runners:"
          cat expected_runners.txt

      - name: Fetch recent heartbeat run artifacts
        id: heartbeats
        run: |
          set -euo pipefail
          # Scan every completed run of the heartbeat workflow in the last
          # LOOKBACK_DAYS, regardless of event type. We intentionally do NOT
          # filter on event=schedule:
          #   * a manual `workflow_dispatch` heartbeat (e.g. after adding a
          #     runner or doing a recovery test) must count toward freshness,
          #     otherwise the monitor keeps alerting until the next scheduled
          #     run completes.
          #   * a manual re-run of *this* monitor must not be able to evict
          #     scheduled runs from the scan window -- so we scan by time
          #     window, not by count, and there is no fixed slot to fall out
          #     of.
          # Per-runner we still take the newest artifact by updated_at across
          # everything found.
          since=$(date -u -d "-${LOOKBACK_DAYS} days" +%Y-%m-%d)
          echo "Scanning heartbeat runs created on or after: $since"

          gh api --paginate \
            "repos/${{ github.repository }}/actions/workflows/${HEARTBEAT_WORKFLOW}/runs?status=completed&created=>=${since}&per_page=100" \
            --jq '.workflow_runs[] | {id, run_started_at, event}' \
            > heartbeat_runs.ndjson

          echo "Recent heartbeat runs scanned:"
          cat heartbeat_runs.ndjson

          if [ ! -s heartbeat_runs.ndjson ]; then
            echo "ERROR: No completed runs of ${HEARTBEAT_WORKFLOW} found in the last ${LOOKBACK_DAYS} days." >&2
            exit 1
          fi

          latest_run_id=$(head -n1 heartbeat_runs.ndjson | jq -r '.id')
          latest_run_started=$(head -n1 heartbeat_runs.ndjson | jq -r '.run_started_at')
          echo "latest_run_id=$latest_run_id" >> "$GITHUB_OUTPUT"
          echo "latest_run_started=$latest_run_started" >> "$GITHUB_OUTPUT"

          # Collect artifacts from each recent run into a single ndjson stream.
          # Do NOT swallow errors from `gh api` here: a transient API failure
          # (rate limit, auth, 5xx, network) would otherwise look indistinguishable
          # from "runner produced no artifact" and the evaluation step below
          # would generate false MISSING alerts. A legitimate run with zero
          # artifacts returns HTTP 200 with an empty array, which jq turns into
          # no output -- that case is fine; only real errors will fail the step.
          : > heartbeat_artifacts.ndjson
          while IFS= read -r line; do
            run_id=$(echo "$line" | jq -r '.id')
            gh api --paginate \
              "repos/${{ github.repository }}/actions/runs/$run_id/artifacts" \
              --jq '.artifacts[] | {name, updated_at, expired}' \
              >> heartbeat_artifacts.ndjson
          done < heartbeat_runs.ndjson

          echo "Total heartbeat artifacts collected:"
          wc -l heartbeat_artifacts.ndjson

      - name: Fetch runner status from GitHub
        id: status
        run: |
          set -euo pipefail
          # Repo-level runners (org-level requires a PAT; skipped intentionally).
          # Two "can't see runners" cases are expected and must be tolerated:
          #   * HTTP 404 -- no repo-scoped runners registered.
          #   * HTTP 403 -- this workflow's GITHUB_TOKEN does not have
          #     `administration: read`, which is required to list repo runners.
          # Both mean the same thing operationally: we can't get this signal,
          # so fall back to heartbeat-only evaluation. Any *other* API failure
          # must propagate so we don't silently downgrade an API outage into
          # "runner status unknown".
          : > runner_status.ndjson
          if gh api --paginate \
              "repos/${{ github.repository }}/actions/runners" \
              --jq '.runners[] | {name, status, busy}' \
              > runner_status.ndjson 2> runner_status_err.log; then
            :
          else
            if grep -qE 'HTTP (403|404)' runner_status_err.log; then
              echo "Cannot list repo-scoped runners (expected for org-level runners or when token lacks administration:read):"
              cat runner_status_err.log
              : > runner_status.ndjson
            else
              echo "ERROR: gh api runners call failed:" >&2
              cat runner_status_err.log >&2
              exit 1
            fi
          fi

          echo "Runner status from API:"
          cat runner_status.ndjson

      - name: Evaluate heartbeats and runner status
        id: evaluate
        run: |
          set -euo pipefail

          now=$(date -u +%s)
          stale_seconds=$(( STALE_HOURS * 3600 ))

          missing_artifact=()
          stale_artifact=()
          api_offline=()
          ok=()

          while IFS= read -r runner; do
            [ -z "$runner" ] && continue

            # --- Signal 1: newest heartbeat artifact across recent runs ---
            # Pick the freshest non-expired artifact (max updated_at) for this
            # runner across all heartbeat runs we scanned. Filtering on
            # expired==false is defensive: today retention-days (30) >=
            # LOOKBACK_DAYS (30) so expired artifacts are already outside the
            # scan window, but if LOOKBACK_DAYS is ever raised past the
            # artifact retention period we must not treat an expired record's
            # updated_at as a fresh heartbeat.
            artifact_line=$(jq -sc --arg name "heartbeat-$runner" \
              '[.[] | select(.name == $name and .expired == false)] | sort_by(.updated_at) | last // empty' \
              heartbeat_artifacts.ndjson)

            if [ -z "$artifact_line" ] || [ "$artifact_line" = "null" ]; then
              missing_artifact+=("$runner")
              hb_status="MISSING (no artifact in last ${LOOKBACK_DAYS} days)"
              age=999999
            else
              updated_at=$(echo "$artifact_line" | jq -r '.updated_at')
              hb_epoch=$(date -u -d "$updated_at" +%s)
              age=$(( now - hb_epoch ))
              age_h=$(( age / 3600 ))
              if [ "$age" -gt "$stale_seconds" ]; then
                stale_artifact+=("$runner (last ${age_h}h ago)")
                hb_status="STALE (${age_h}h, threshold ${STALE_HOURS}h)"
              else
                hb_status="OK (${age_h}h ago)"
              fi
            fi

            # --- Signal 2: GitHub runner registration status ---
            status_line=$(jq -c --arg name "$runner" \
              'select(.name == $name)' runner_status.ndjson | head -n1 || true)

            if [ -z "$status_line" ]; then
              api_status="UNKNOWN (not registered at repo scope)"
            else
              api_status_value=$(echo "$status_line" | jq -r '.status')
              api_busy=$(echo "$status_line" | jq -r '.busy')
              api_status="${api_status_value} (busy=${api_busy})"
              if [ "$api_status_value" != "online" ]; then
                api_offline+=("$runner ($api_status_value)")
              fi
            fi

            echo "Runner: $runner | heartbeat: $hb_status | api: $api_status"

            # A runner is "OK" only if heartbeat is fresh AND
            # (api is online OR api status is unknown -- org runners aren't
            # visible to GITHUB_TOKEN at repo scope, so don't penalize them).
            if [ -n "$artifact_line" ] && [ "$age" -le "$stale_seconds" ]; then
              ok+=("$runner")
            fi
          done < expected_runners.txt

          echo ""
          echo "=========================================="
          echo "Summary"
          echo "=========================================="
          echo "OK:               ${#ok[@]}"
          echo "Missing heartbeat: ${#missing_artifact[@]}"
          echo "Stale heartbeat:   ${#stale_artifact[@]}"
          echo "API offline:       ${#api_offline[@]}"

          # Build a human-readable report block for Teams.
          {
            printf 'Latest heartbeat run: %s (started %s)\n' \
              "${{ steps.heartbeats.outputs.latest_run_id }}" \
              "${{ steps.heartbeats.outputs.latest_run_started }}"
            printf 'Scanned heartbeat runs from the last %s days.\n' "$LOOKBACK_DAYS"
            printf 'Alert threshold: heartbeat older than %sh.\n\n' "$STALE_HOURS"
            if [ "${#missing_artifact[@]}" -gt 0 ]; then
              printf 'Missing heartbeat artifact:\n'
              for r in "${missing_artifact[@]}"; do printf '  - %s\n' "$r"; done
              printf '\n'
            fi
            if [ "${#stale_artifact[@]}" -gt 0 ]; then
              printf 'Stale heartbeat artifact:\n'
              for r in "${stale_artifact[@]}"; do printf '  - %s\n' "$r"; done
              printf '\n'
            fi
            if [ "${#api_offline[@]}" -gt 0 ]; then
              printf 'GitHub API reports offline:\n'
              for r in "${api_offline[@]}"; do printf '  - %s\n' "$r"; done
              printf '\n'
            fi
          } > report.txt

          # Job fails only if at least one runner has a bad heartbeat signal.
          # API-offline alone is reported but does not fail the job, because
          # GITHUB_TOKEN cannot see org-level runners at repo scope.
          bad_count=$(( ${#missing_artifact[@]} + ${#stale_artifact[@]} ))
          echo "bad_count=$bad_count" >> "$GITHUB_OUTPUT"
          {
            echo 'report<<EOF'
            cat report.txt
            echo 'EOF'
          } >> "$GITHUB_OUTPUT"

      - name: Publish summary
        if: always()
        run: |
          {
            echo "## Self-Hosted Runner Monitor"
            echo ""
            echo '```'
            cat report.txt 2>/dev/null || echo "(no report)"
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Fail if any runner has a bad heartbeat
        if: steps.evaluate.outputs.bad_count != '0'
        run: |
          echo "One or more runners failed heartbeat checks."
          exit 1

      - name: Send Teams alert via Power Automate
        if: failure() && env.TEAMS_WEBHOOK_URL != ''
        env:
          # `if: failure()` fires for ANY prior step failure in this job, not
          # just a runner heartbeat failure. If the failure happened in input
          # validation or in the heartbeat/runner-status fetch steps, the
          # evaluate step never ran and `report` is empty. Fall back to a
          # clear pointer at workflow logs so the Teams card is never a blank
          # monospace block.
          REPORT: ${{ steps.evaluate.outputs.report || '(evaluation did not run — see workflow logs)' }}
        run: |
          set -euo pipefail
          echo "Sending Teams alert via Power Automate..."

          payload=$(jq -n --arg report "$REPORT" '
            {
              type: "AdaptiveCard",
              version: "1.3",
              body: [
                { type: "TextBlock", text: "⚠️ Self-hosted Runner Alert",
                  weight: "bolder", size: "medium", color: "attention" },
                { type: "TextBlock",
                  text: "Heartbeat / status check failed for one or more runners.",
                  wrap: true },
                { type: "TextBlock", text: $report,
                  wrap: true, fontType: "monospace" }
              ]
            }')

          response=$(curl -s -w "%{http_code}" -X POST \
            -H "Content-Type: application/json" \
            -d "$payload" \
            "$TEAMS_WEBHOOK_URL")

          http_code="${response: -3}"
          response_body="${response%???}"

          echo "HTTP Status: $http_code"
          echo "Response: $response_body"

          if [ "$http_code" != "200" ] && [ "$http_code" != "202" ]; then
            echo "Power Automate webhook failed with status $http_code"
          else
            echo "Teams alert sent successfully via Power Automate"
          fi
