|
| 1 | +# Optional live-LLM conversation E2E (manual / scheduled — does not block PR merges). |
| 2 | +name: TestQL Conversations (Live LLM) |
| 3 | + |
| 4 | +on: |
| 5 | + workflow_dispatch: |
| 6 | + inputs: |
| 7 | + scenario: |
| 8 | + description: Scenario file relative to repo root (empty = all) |
| 9 | + required: false |
| 10 | + default: "" |
| 11 | + schedule: |
| 12 | + - cron: "0 6 * * 1" |
| 13 | + |
| 14 | +permissions: |
| 15 | + contents: read |
| 16 | + |
| 17 | +concurrency: |
| 18 | + group: testql-live-llm-${{ github.ref }} |
| 19 | + cancel-in-progress: false |
| 20 | + |
| 21 | +jobs: |
| 22 | + live-llm-e2e: |
| 23 | + runs-on: ubuntu-latest |
| 24 | + continue-on-error: true |
| 25 | + steps: |
| 26 | + - name: Checkout koru |
| 27 | + uses: actions/checkout@v4 |
| 28 | + |
| 29 | + - name: Checkout testql (editable) |
| 30 | + uses: actions/checkout@v4 |
| 31 | + with: |
| 32 | + repository: oqlos/testql |
| 33 | + path: vendor/testql |
| 34 | + ref: main |
| 35 | + continue-on-error: true |
| 36 | + |
| 37 | + - name: Set up Python 3.12 |
| 38 | + uses: actions/setup-python@v5 |
| 39 | + with: |
| 40 | + python-version: "3.12" |
| 41 | + |
| 42 | + - name: Install dependencies |
| 43 | + run: | |
| 44 | + python -m pip install --upgrade pip |
| 45 | + if [ -d vendor/testql ]; then pip install -e vendor/testql; else pip install testql; fi |
| 46 | + pip install -e packages/nlpshim |
| 47 | + pip install httpx pyyaml |
| 48 | +
|
| 49 | + - name: Live LLM conversation run |
| 50 | + env: |
| 51 | + TESTQL_LIVE_LLM: "1" |
| 52 | + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} |
| 53 | + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} |
| 54 | + NLP2DSL_URL: ${{ secrets.NLP2DSL_URL }} |
| 55 | + TESTQL_MOCK_LLM_FILE: testql-scenarios/artifacts/mock-llm-replies.yaml |
| 56 | + run: | |
| 57 | + if [ -z "${OPENROUTER_API_KEY}${LLM_API_KEY}" ]; then |
| 58 | + echo "Skipping: no LLM API key configured in repository secrets" |
| 59 | + exit 0 |
| 60 | + fi |
| 61 | + args=(scripts/run_testql_conversations.py --live-llm) |
| 62 | + if [ -n "${{ github.event.inputs.scenario }}" ]; then |
| 63 | + args+=("${{ github.event.inputs.scenario }}") |
| 64 | + fi |
| 65 | + python "${args[@]}" |
| 66 | +
|
| 67 | + - name: Live LLM pytest (when testql source present) |
| 68 | + env: |
| 69 | + TESTQL_LIVE_LLM: "1" |
| 70 | + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} |
| 71 | + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} |
| 72 | + run: | |
| 73 | + if [ -z "${OPENROUTER_API_KEY}${LLM_API_KEY}" ]; then |
| 74 | + echo "Skipping pytest: no LLM API key" |
| 75 | + exit 0 |
| 76 | + fi |
| 77 | + if [ -d vendor/testql ]; then |
| 78 | + cd vendor/testql && python -m pytest tests/test_conversation_live_llm.py -q -m live_llm |
| 79 | + else |
| 80 | + echo "testql source not available; skipping pytest live_llm" |
| 81 | + fi |
0 commit comments