Update llm classifier guardrail experiment config paths #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI (full) | |
| on: | |
| push: | |
| branches: [main, master] | |
| workflow_dispatch: | |
| jobs: | |
| quality-checks: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=4096 # increase default to 4GB for all steps in this job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.12'] # Can add '3.13' when dependencies support it | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Ensure empty .env (for safety) | |
| run: | | |
| rm -f .env | |
| touch .env | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock', 'pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv- | |
| - name: Install dependencies | |
| run: make install | |
| - name: Check notebooks have no outputs | |
| run: uv run make check-notebooks | |
| - name: Run linting | |
| run: uv run make lint | |
| - name: Run full test suite with coverage | |
| run: uv run make coverage | |
| - name: Coverage comment (update baseline) | |
| if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' | |
| continue-on-error: true | |
| uses: py-cov-action/python-coverage-comment-action@v3 | |
| with: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| MERGE_COVERAGE_FILES: false | |
| - name: Upload coverage artifacts | |
| if: always() && hashFiles('logs/coverage/**') != '' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-${{ matrix.python-version }} | |
| path: logs/coverage/ | |
| open-issue-on-failure: | |
| name: Open issue on full CI failure | |
| runs-on: ubuntu-latest | |
| needs: quality-checks | |
| if: always() && github.event_name == 'push' && needs.quality-checks.result == 'failure' | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - name: Open or update issue on failure | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const branchName = context.ref.replace("refs/heads/", ""); | |
| const shortSha = context.sha.substring(0, 7); | |
| const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; | |
| const actorLogin = context.actor; | |
| const issueLabel = "ci-failure"; | |
| const marker = `<!-- ci-full-failure:${branchName} -->`; | |
| let labelExists = true; | |
| try { | |
| await github.rest.issues.getLabel({ | |
| owner, | |
| repo, | |
| name: issueLabel, | |
| }); | |
| } catch (error) { | |
| if (error.status === 404) { | |
| labelExists = false; | |
| try { | |
| await github.rest.issues.createLabel({ | |
| owner, | |
| repo, | |
| name: issueLabel, | |
| color: "B60205", | |
| description: "Automated full CI failure tracker", | |
| }); | |
| labelExists = true; | |
| } catch (labelCreateError) { | |
| if (labelCreateError.status === 422) { | |
| labelExists = true; | |
| } else { | |
| core.warning(`Could not create label "${issueLabel}": ${labelCreateError.message}`); | |
| } | |
| } | |
| } else { | |
| throw error; | |
| } | |
| } | |
| const openIssues = await github.paginate(github.rest.issues.listForRepo, { | |
| owner, | |
| repo, | |
| state: "open", | |
| per_page: 100, | |
| }); | |
| const existingIssue = openIssues.find( | |
| (issue) => !issue.pull_request && issue.body && issue.body.includes(marker), | |
| ); | |
| const failureSummaryLine = `- ${new Date().toISOString()}: \`${shortSha}\` by @${actorLogin} ([run link](${runUrl}))`; | |
| if (existingIssue) { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: existingIssue.number, | |
| body: [ | |
| "Another `CI (full)` failure was detected on this branch.", | |
| "", | |
| failureSummaryLine, | |
| ].join("\n"), | |
| }); | |
| } else { | |
| const labels = labelExists ? [issueLabel] : []; | |
| await github.rest.issues.create({ | |
| owner, | |
| repo, | |
| title: `CI (full) failing on ${branchName}`, | |
| body: [ | |
| `The full CI workflow failed after a push to \`${branchName}\`.`, | |
| "", | |
| `- **Latest commit:** ${context.sha}`, | |
| `- **Actor:** @${actorLogin}`, | |
| `- **Failed run:** ${runUrl}`, | |
| "", | |
| "This issue is automatically updated while failures continue on this branch.", | |
| "", | |
| "Recent failures:", | |
| failureSummaryLine, | |
| "", | |
| marker, | |
| ].join("\n"), | |
| labels, | |
| }); | |
| } | |
| close-issue-on-success: | |
| name: Close issue when full CI recovers | |
| runs-on: ubuntu-latest | |
| needs: quality-checks | |
| if: always() && github.event_name == 'push' && needs.quality-checks.result == 'success' | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - name: Close matching failure issue on success | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const branchName = context.ref.replace("refs/heads/", ""); | |
| const shortSha = context.sha.substring(0, 7); | |
| const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; | |
| const marker = `<!-- ci-full-failure:${branchName} -->`; | |
| const openIssues = await github.paginate(github.rest.issues.listForRepo, { | |
| owner, | |
| repo, | |
| state: "open", | |
| per_page: 100, | |
| }); | |
| const matchingIssue = openIssues.find( | |
| (issue) => !issue.pull_request && issue.body && issue.body.includes(marker), | |
| ); | |
| if (!matchingIssue) { | |
| return; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: matchingIssue.number, | |
| body: [ | |
| "Full CI is passing again on this branch.", | |
| "", | |
| `- **Recovered commit:** ${context.sha}`, | |
| `- **Successful run:** ${runUrl}`, | |
| "", | |
| `Closing this issue after recovery at \`${shortSha}\`.`, | |
| ].join("\n"), | |
| }); | |
| await github.rest.issues.update({ | |
| owner, | |
| repo, | |
| issue_number: matchingIssue.number, | |
| state: "closed", | |
| }); |