monthly-llmwiki-refresh.yml

name: Monthly LLMWiki Refresh

on:
  schedule:
    # Sundays only; schedule-gate below limits automatic runs to the first Sunday.
    - cron: "23 7 * * 0"
  workflow_dispatch:
    inputs:
      llmwiki_ref:
        description: "LLMWiki git ref to run"
        required: false
        default: "main"
      max_pages:
        description: "Max developers.telnyx.com pages to scrape"
        required: false
        default: "5000"
      max_workers:
        description: "Concurrent LLM synthesis workers"
        required: false
        default: "4"

permissions:
  contents: write
  pull-requests: write

concurrency:
  group: monthly-llmwiki-refresh
  cancel-in-progress: false

env:
  LLMWIKI_REPO: team-telnyx/LLMWiki
  LLMWIKI_REF: ${{ inputs.llmwiki_ref || 'main' }}
  MAX_PAGES: ${{ inputs.max_pages || '5000' }}
  MAX_WORKERS: ${{ inputs.max_workers || '4' }}
  DEV_DOCS_SOURCES: .llmwiki-cache/dev-docs-sources

jobs:
  schedule-gate:
    name: Check monthly schedule window
    runs-on: telnyx-xsmall
    outputs:
      should_run: ${{ steps.gate.outputs.should_run }}
    steps:
      - name: Allow first Sunday or manual dispatch
        id: gate
        shell: bash
        run: |
          set -euo pipefail
          if [[ "${{ github.event_name }}" != "schedule" ]]; then
            echo "should_run=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          day="$(date -u +%d)"
          if (( 10#$day <= 7 )); then
            echo "should_run=true" >> "$GITHUB_OUTPUT"
          else
            echo "should_run=false" >> "$GITHUB_OUTPUT"
            echo "::notice::Skipping monthly refresh because UTC day ${day} is outside the first-Sunday window."
          fi

  refresh:
    name: Scrape/compile dev-docs and full-compile support-docs
    needs: schedule-gate
    if: needs.schedule-gate.outputs.should_run == 'true'
    runs-on: telnyx-medium
    timeout-minutes: 360
    steps:
      - name: Checkout knowledge-base
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          fetch-depth: 0

      - name: Install uv
        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990  # v8.3.2
        with:
          enable-cache: true
          python-version: '3.13'

      - name: Checkout latest LLMWiki compiler
        shell: bash
        env:
          LLMWIKI_TOKEN: ${{ secrets.LLMWIKI_TOKEN || github.token }}
        run: |
          set -euo pipefail
          git clone --depth 1 --branch "$LLMWIKI_REF" \
            "https://x-access-token:${LLMWIKI_TOKEN}@github.com/${LLMWIKI_REPO}.git" \
            ../LLMWiki
          git -C ../LLMWiki rev-parse HEAD | tee llmwiki-compiler-sha.txt

      - name: Install LLMWiki dependencies
        working-directory: ../LLMWiki
        run: uv sync

      - name: Restore dev-docs scrape cache
        id: devdocs-scrape-cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .llmwiki-cache/dev-docs-sources
          key: devdocs-scrape-${{ runner.os }}-${{ env.LLMWIKI_REF }}-${{ env.MAX_PAGES }}-${{ github.run_id }}

      - name: Scrape developers.telnyx.com source snapshot
        working-directory: ../LLMWiki
        run: |
          set -euo pipefail
          SNAPSHOT="../knowledge-base/${DEV_DOCS_SOURCES}"
          if [[ -f "${SNAPSHOT}/_manifest.json" ]]; then
            echo "Using cached dev-docs scrape snapshot at ${SNAPSHOT}"
            exit 0
          fi

          uv run wiki scrape \
            --source dev-docs \
            --output-dir "${SNAPSHOT}" \
            --max-pages "$MAX_PAGES" \
            2>&1 | tee ../knowledge-base/dev-scrape.log

      - name: Save dev-docs scrape cache
        if: steps.devdocs-scrape-cache.outputs.cache-hit != 'true' && hashFiles('.llmwiki-cache/dev-docs-sources/_manifest.json') != ''
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .llmwiki-cache/dev-docs-sources
          key: devdocs-scrape-${{ runner.os }}-${{ env.LLMWIKI_REF }}-${{ env.MAX_PAGES }}-${{ github.run_id }}

      - name: Full compile support-docs with LLMWiki
        working-directory: ../LLMWiki
        env:
          LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }}
          LITELLM_PROXY_API_BASE: ${{ secrets.LITELLM_PROXY_API_BASE }}
          LLM_PROXY_MODEL_DEFAULT: ${{ secrets.LLM_PROXY_MODEL_DEFAULT }}
        run: |
          set -euo pipefail
          rm -rf ../dist-support wikis/support-docs/.work
          uv run wiki compile \
            --wiki-name support-docs \
            --sources-root ../knowledge-base/support-docs \
            --out-dir ../dist-support \
            --grouping llm \
            --max-workers "$MAX_WORKERS" \
            2>&1 | tee ../knowledge-base/support-compile.log

      - name: Full compile dev-docs with LLMWiki
        working-directory: ../LLMWiki
        env:
          LITELLM_PROXY_API_KEY: ${{ secrets.LITELLM_PROXY_API_KEY }}
          LITELLM_PROXY_API_BASE: ${{ secrets.LITELLM_PROXY_API_BASE }}
          LLM_PROXY_MODEL_DEFAULT: ${{ secrets.LLM_PROXY_MODEL_DEFAULT }}
        run: |
          set -euo pipefail
          rm -rf ../dist-dev wikis/dev-docs/.work
          uv run wiki compile \
            --wiki-name dev-docs \
            --sources-root "../knowledge-base/${DEV_DOCS_SOURCES}" \
            --out-dir ../dist-dev \
            --grouping area \
            --max-workers "$MAX_WORKERS" \
            2>&1 | tee ../knowledge-base/dev-compile.log

      - name: Strip volatile content hashlines
        run: |
          python scripts/monthly_llmwiki_refresh.py strip-content-hashlines \
            --path ../dist-support/support-docs | tee support-strip-hashlines.json
          python scripts/monthly_llmwiki_refresh.py strip-content-hashlines \
            --path ../dist-dev/dev-docs | tee dev-strip-hashlines.json

      - name: Install compiled workspaces and flatten the only committed index
        run: |
          python scripts/monthly_llmwiki_refresh.py install-workspace \
            --workspace support-docs \
            --compiled-wiki-root ../dist-support \
            --wiki-root wiki | tee support-install.json
          python scripts/monthly_llmwiki_refresh.py install-workspace \
            --workspace dev-docs \
            --compiled-wiki-root ../dist-dev \
            --wiki-root wiki | tee dev-install.json
          python scripts/monthly_llmwiki_refresh.py rebuild-root-index \
            --wiki-root wiki | tee root-index.json
          python scripts/monthly_llmwiki_refresh.py remove-workspace-indexes \
            --wiki-root wiki | tee remove-workspace-indexes.json

      - name: Build PR summary
        id: summary
        shell: bash
        run: |
          set -euo pipefail
          SCRAPED_COUNT=$(find "${DEV_DOCS_SOURCES}" -name '*.md' | wc -l | tr -d ' ')
          {
            echo "## Summary"
            echo "- Single monthly LLMWiki refresh for both corpora."
            echo "- Scraped developers.telnyx.com and compiled checked-in support-docs source artifacts, then compiled each corpus with its appropriate grouping strategy."
            echo "- Pulled compiler from \`${LLMWIKI_REPO}@${LLMWIKI_REF}\`."
            echo "- Used a dev-docs scrape snapshot containing \`${SCRAPED_COUNT}\` Markdown source pages."
            echo "- Stripped generated \`content_hash:\` hashlines before committing, per David O'Connell's recommendation."
            echo "- Committed one flattened \`wiki/index.md\`; generated per-workspace indexes are removed before commit."
            echo
            echo "## Compiler SHA"
            echo '```'
            cat llmwiki-compiler-sha.txt
            echo '```'
            echo
            echo "## Diff stat"
            echo '```'
            git diff --stat -- wiki || true
            echo '```'
          } > pr-body.md
          if git diff --quiet -- wiki; then
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Open monthly LLMWiki refresh PR
        if: steps.summary.outputs.has_changes == 'true'
        shell: bash
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          BRANCH="bot/monthly-llmwiki-refresh-$(date -u +%Y-%m)-${GITHUB_RUN_ID}"
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git checkout -b "$BRANCH"
          git add wiki
          git commit -m "docs: monthly LLMWiki refresh"
          git push -u origin "$BRANCH"
          gh pr create \
            --base main \
            --head "$BRANCH" \
            --title "docs: monthly LLMWiki refresh" \
            --body-file pr-body.md

      - name: No corpus changes
        if: steps.summary.outputs.has_changes != 'true'
        run: echo "No wiki corpus changes to publish."