feat: prepare validated nightly catalog refresh automation

Author: KSemenenko

.github/upstream-watch.official-skills.json

@@ -9,78 +9,104 @@
       "issue_name": "Review official dotnet/skills upstream update",
       "notes": "Review external-sources/vendir.lock.yml, sync external-sources/upstreams/dotnet-skills via vendir, rerun scripts/import_external_catalog_sources.py, then regenerate catalog outputs and verify the imported official skills and agents.",
       "skills": [
+        "analyzing-dotnet-performance",
+        "android-tombstone-symbolication",
+        "apple-crash-symbolication",
+        "assertion-quality",
+        "author-component",
+        "binlog-failure-analysis",
+        "binlog-generation",
+        "build-parallelism",
+        "build-perf-baseline",
+        "build-perf-diagnostics",
+        "check-bin-obj-clash",
+        "clr-activation-debugging",
+        "code-testing-agent",
+        "code-testing-extensions",
+        "collect-user-input",
+        "configure-auth",
         "configuring-opentelemetry-dotnet",
-        "minimal-api-file-upload",
+        "convert-blazor-server-to-webapp",
+        "convert-to-cpm",
+        "coordinate-components",
+        "copy-to-output-directory",
+        "coverage-analysis",
+        "crap-score",
+        "create-blazor-project",
+        "create-datadriven-aspnetcore",
+        "csharp-scripts",
+        "detect-static-dependencies",
+        "directory-build-organization",
+        "dotnet-aot-compat",
         "dotnet-maui-doctor",
+        "dotnet-pinvoke",
+        "dotnet-trace-collect",
+        "dotnet-webapi",
+        "dump-collect",
+        "eval-performance",
+        "exp-mock-usage-analysis",
+        "exp-test-maintainability",
+        "extension-points",
+        "fetch-and-send-data",
+        "filter-syntax",
+        "find-untested-sources",
+        "generate-testability-wrappers",
+        "grade-tests",
+        "including-generated-files",
+        "incremental-build",
+        "item-management",
         "maui-app-lifecycle",
         "maui-collectionview",
         "maui-data-binding",
         "maui-dependency-injection",
         "maui-safe-area",
         "maui-shell-navigation",
         "maui-theming",
-        "optimizing-ef-core-queries",
-        "csharp-scripts",
-        "dotnet-pinvoke",
-        "nuget-trusted-publishing",
-        "mcp-csharp-create",
-        "mcp-csharp-debug",
-        "mcp-csharp-publish",
-        "mcp-csharp-test",
-        "technology-selection",
-        "exp-assertion-quality",
-        "exp-dotnet-test-frameworks",
-        "exp-mock-usage-analysis",
-        "exp-simd-vectorization",
-        "exp-test-gap-analysis",
-        "exp-test-maintainability",
-        "exp-test-smell-detection",
-        "exp-test-tagging",
-        "dotnet-aot-compat",
+        "microbenchmarking",
         "migrate-dotnet10-to-dotnet11",
         "migrate-dotnet8-to-dotnet9",
         "migrate-dotnet9-to-dotnet10",
-        "migrate-nullable-references",
-        "thread-abort-migration",
-        "code-testing-agent",
-        "coverage-analysis",
-        "crap-score",
-        "dotnet-test-frameworks",
-        "filter-syntax",
         "migrate-mstest-v1v2-to-v3",
         "migrate-mstest-v3-to-v4",
+        "migrate-nullable-references",
+        "migrate-static-to-wrapper",
         "migrate-vstest-to-mtp",
+        "migrate-xunit-to-mstest",
         "migrate-xunit-to-xunit-v3",
-        "mtp-hot-reload",
-        "platform-detection",
-        "run-tests",
-        "test-anti-patterns",
-        "writing-mstest-tests",
-        "analyzing-dotnet-performance",
-        "android-tombstone-symbolication",
-        "clr-activation-debugging",
-        "dotnet-trace-collect",
-        "dump-collect",
-        "microbenchmarking",
-        "binlog-failure-analysis",
-        "binlog-generation",
-        "build-parallelism",
-        "build-perf-baseline",
-        "build-perf-diagnostics",
-        "check-bin-obj-clash",
-        "directory-build-organization",
-        "eval-performance",
-        "including-generated-files",
-        "incremental-build",
+        "minimal-api-file-upload",
         "msbuild-antipatterns",
         "msbuild-modernization",
         "msbuild-server",
+        "mtp-hot-reload",
+        "nuget-trusted-publishing",
+        "optimizing-ef-core-queries",
+        "plan-ui-change",
+        "platform-detection",
+        "property-patterns",
         "resolve-project-references",
-        "convert-to-cpm",
+        "run-tests",
+        "scaffold-dotnet-test-project",
+        "setup-local-sdk",
+        "support-prerendering",
+        "system-text-json-net11",
+        "target-authoring",
+        "technology-selection",
         "template-authoring",
+        "template-comparison",
         "template-discovery",
         "template-instantiation",
-        "template-validation"
+        "template-smart-defaults",
+        "template-validation",
+        "test-analysis-extensions",
+        "test-anti-patterns",
+        "test-gap-analysis",
+        "test-smell-detection",
+        "test-tagging",
+        "testability-obstacle",
+        "thread-abort-migration",
+        "use-js-interop",
+        "vectorization",
+        "writing-mstest-tests"
       ]
     }
   ]

.github/workflows/catalog-check.yml

@@ -1,6 +1,12 @@
 name: PR Checks
 
 on:
+  workflow_call:
+    inputs:
+      ref:
+        description: Immutable catalog refresh commit to validate
+        required: true
+        type: string
   workflow_dispatch:
   pull_request:
 
@@ -18,6 +24,9 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -50,7 +59,7 @@ jobs:
         run: python3 scripts/import_external_catalog_sources.py --validate-config
 
       - name: Sync and normalize external sources
-        run: bash scripts/sync_external_catalog_sources.sh
+        run: bash scripts/sync_external_catalog_sources.sh --locked
 
       - name: Verify imported external sources are committed
         run: git diff --exit-code -- catalog external-sources
@@ -95,6 +104,9 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
 
       - name: Set up .NET
         uses: actions/setup-dotnet@v5

.github/workflows/publish-catalog.yml

@@ -48,6 +48,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v6
         with:
+          ref: main
           fetch-depth: 0
 
       - name: Check for new commits on main since latest catalog release
@@ -57,16 +58,19 @@ jobs:
         shell: bash
         run: |
           git fetch --force --tags origin main
+          git checkout --detach origin/main
 
           latest_tag=$(gh api "repos/${{ github.repository }}/releases?per_page=100" --jq 'map(select(.draft == false and (.tag_name | startswith("'"${{ env.CATALOG_TAG_PREFIX }}"'")))) | sort_by(.published_at) | reverse | (.[0].tag_name // "")')
           main_ref="origin/main"
           force_release='${{ inputs.force_release }}'
 
           if [[ "${{ github.event_name }}" == "workflow_dispatch" && "$force_release" == "true" ]]; then
-            echo "has_new_commits=true" >> "$GITHUB_OUTPUT"
-            echo "latest_tag=$latest_tag" >> "$GITHUB_OUTPUT"
-            echo "override=true" >> "$GITHUB_OUTPUT"
-            echo "reason=manual_force_release" >> "$GITHUB_OUTPUT"
+            {
+              echo "has_new_commits=true"
+              echo "latest_tag=$latest_tag"
+              echo "override=true"
+              echo "reason=manual_force_release"
+            } >> "$GITHUB_OUTPUT"
             exit 0
           fi
 
@@ -76,9 +80,11 @@ jobs:
             else
               echo "has_new_commits=false" >> "$GITHUB_OUTPUT"
             fi
-            echo "latest_tag=" >> "$GITHUB_OUTPUT"
-            echo "override=false" >> "$GITHUB_OUTPUT"
-            echo "reason=initial_catalog_release" >> "$GITHUB_OUTPUT"
+            {
+              echo "latest_tag="
+              echo "override=false"
+              echo "reason=initial_catalog_release"
+            } >> "$GITHUB_OUTPUT"
             exit 0
           fi
 
@@ -90,9 +96,11 @@ jobs:
             echo "has_new_commits=false" >> "$GITHUB_OUTPUT"
           fi
 
-          echo "latest_tag=$latest_tag" >> "$GITHUB_OUTPUT"
-          echo "override=false" >> "$GITHUB_OUTPUT"
-          echo "reason=$([ "$commits_since_tag" -gt 0 ] && echo commits_since_last_release || echo no_new_commits)" >> "$GITHUB_OUTPUT"
+          {
+            echo "latest_tag=$latest_tag"
+            echo "override=false"
+            echo "reason=$([ "$commits_since_tag" -gt 0 ] && echo commits_since_last_release || echo no_new_commits)"
+          } >> "$GITHUB_OUTPUT"
 
       - name: Skip release when no new commits exist
         if: steps.changes.outputs.has_new_commits != 'true'

.github/workflows/upstream-watch.yml

@@ -1,13 +1,14 @@
-name: Upstream Watch
+name: Nightly Catalog Refresh
 
 on:
   workflow_dispatch:
   schedule:
-    - cron: "17 9 * * *"
+    - cron: "17 0 * * *"
 
 permissions:
   contents: write
   issues: write
+  pull-requests: write
 
 concurrency:
   group: upstream-watch
@@ -19,10 +20,19 @@ env:
 jobs:
   watch:
     runs-on: ubuntu-latest
+    timeout-minutes: 180
+    outputs:
+      changed: ${{ steps.publish.outputs.changed }}
+      head: ${{ steps.publish.outputs.head }}
+      base: ${{ steps.publish.outputs.base }}
+      pr: ${{ steps.publish.outputs.pr }}
 
     steps:
       - name: Check out repository
         uses: actions/checkout@v6
+        with:
+          ref: main
+          fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -57,8 +67,89 @@ jobs:
         run: python3 scripts/upstream_watch.py --state "${{ env.UPSTREAM_WATCH_STATE_FILE }}"
 
       - name: Save upstream watch state cache
-        if: always()
         uses: actions/cache/save@v4
         with:
           path: ${{ env.UPSTREAM_WATCH_STATE_FILE }}
           key: upstream-watch-state-${{ github.repository_id }}-${{ github.ref_name }}-${{ github.run_id }}
+
+      - name: Install vendir
+        run: |
+          mkdir -p .tools
+          curl -fsSL https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR="$PWD/.tools" bash
+          echo "$PWD/.tools" >> "$GITHUB_PATH"
+
+      - name: Refresh external catalog sources
+        run: bash scripts/sync_external_catalog_sources.sh
+
+      - name: Refresh pending watched skills
+        env:
+          GH_TOKEN: ${{ github.token }}
+          NIGHTLY_SKILL_REFRESH_COMMAND: ${{ vars.NIGHTLY_SKILL_REFRESH_COMMAND }}
+        run: python3 scripts/nightly_skill_refresh.py --state "${{ env.UPSTREAM_WATCH_STATE_FILE }}"
+
+      - name: Validate refreshed catalog
+        run: |
+          python3 scripts/generate_catalog.py --validate-only
+          python3 scripts/generate_agent_catalog.py --validate-only
+          python3 scripts/generate_pages.py
+
+      - name: Create or update nightly pull request
+        id: publish
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 scripts/nightly_catalog_pr.py publish
+
+      - name: Upload refresh report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-refresh
+          path: artifacts/nightly-refresh/
+          retention-days: 7
+
+  validate:
+    needs: watch
+    if: needs.watch.outputs.changed == 'true'
+    uses: ./.github/workflows/catalog-check.yml
+    with:
+      ref: ${{ needs.watch.outputs.head }}
+
+  merge:
+    needs: [watch, validate]
+    if: needs.validate.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Merge the verified catalog commit
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ needs.watch.outputs.pr }}
+          VALIDATED_HEAD: ${{ needs.watch.outputs.head }}
+          VALIDATED_BASE: ${{ needs.watch.outputs.base }}
+        run: python3 scripts/nightly_catalog_pr.py merge --pr "$PR_NUMBER" --head "$VALIDATED_HEAD" --base "$VALIDATED_BASE"
+
+  report:
+    needs: [watch, validate, merge]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Report failure or close recovered failure issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+          WATCH_RESULT: ${{ needs.watch.result }}
+          VALIDATION_RESULT: ${{ needs.validate.result }}
+          MERGE_RESULT: ${{ needs.merge.result }}
+          CHANGED: ${{ needs.watch.outputs.changed }}
+        run: |
+          args=()
+          if [[ "$WATCH_RESULT" != success ]] || [[ "$CHANGED" == true && ( "$VALIDATION_RESULT" != success || "$MERGE_RESULT" != success ) ]]; then
+            args+=(--failed)
+          fi
+          python3 scripts/nightly_catalog_pr.py report "${args[@]}" --details "Refresh: $WATCH_RESULT; validation: $VALIDATION_RESULT; merge: $MERGE_RESULT."