daily-dataset #85
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: daily-dataset | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: routesentinel-daily-dataset-${{ github.ref }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install system MRT parser | |
| run: sudo apt-get update && sudo apt-get install -y bgpdump | |
| - name: Install RouteSentinel | |
| run: python -m pip install ".[dev]" | |
| - name: Test | |
| run: pytest | |
| - name: Set date | |
| id: date | |
| run: | | |
| today="$(date -u +%F)" | |
| previous="$(date -u -d "${today} -1 day" +%F)" | |
| generated_at="$(date -u '+%Y-%m-%d %H:%M UTC')" | |
| { | |
| echo "today=${today}" | |
| echo "previous=${previous}" | |
| echo "generated_at=${generated_at}" | |
| echo "release_url=https://github.com/${GITHUB_REPOSITORY}/releases/tag/${today}" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Restore source cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: data/raw | |
| key: routesentinel-raw-${{ steps.date.outputs.today }} | |
| - name: Fetch BGP RIB and VRPs | |
| run: | | |
| mkdir -p data/raw data/normalized out | |
| routesentinel fetch https://data.ris.ripe.net/rrc00/latest-bview.gz data/raw/rrc00-latest-bview.gz | |
| routesentinel fetch https://data.ris.ripe.net/rrc01/latest-bview.gz data/raw/rrc01-latest-bview.gz | |
| routesentinel fetch https://data.ris.ripe.net/rrc10/latest-bview.gz data/raw/rrc10-latest-bview.gz | |
| routesentinel fetch https://console.rpki-client.org/vrps.json data/raw/vrps.json | |
| - name: Parse MRT | |
| run: | | |
| routesentinel parse-mrt data/raw/rrc00-latest-bview.gz data/normalized/rrc00.csv --collector rrc00 | |
| routesentinel parse-mrt data/raw/rrc01-latest-bview.gz data/normalized/rrc01.csv --collector rrc01 | |
| routesentinel parse-mrt data/raw/rrc10-latest-bview.gz data/normalized/rrc10.csv --collector rrc10 | |
| - name: Build snapshot | |
| run: | | |
| routesentinel snapshot \ | |
| --announcements data/normalized/rrc00.csv \ | |
| --announcements data/normalized/rrc01.csv \ | |
| --announcements data/normalized/rrc10.csv \ | |
| --vrps data/raw/vrps.json \ | |
| --out out | |
| - name: Download previous release state | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p previous | |
| gh release download "${{ steps.date.outputs.previous }}" \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --pattern route-origin-status.csv \ | |
| --dir previous || true | |
| - name: Build changelog | |
| run: | | |
| python scripts/build_changelog.py \ | |
| --current-status out/route-origin-status.csv \ | |
| --current-summary out/rpki-summary.json \ | |
| --previous-status previous/route-origin-status.csv \ | |
| --date "${{ steps.date.outputs.today }}" \ | |
| --updated-at "${{ steps.date.outputs.generated_at }}" \ | |
| --out-md out/changelog.md \ | |
| --out-json out/daily-diff.json | |
| - name: Sync branch before README update | |
| run: git pull --rebase origin "${GITHUB_REF_NAME}" | |
| - name: Update README stats | |
| run: | | |
| python scripts/update_readme_stats.py \ | |
| --summary out/rpki-summary.json \ | |
| --date "${{ steps.date.outputs.today }}" \ | |
| --release-url "${{ steps.date.outputs.release_url }}" \ | |
| --updated-at "${{ steps.date.outputs.generated_at }}" | |
| - name: Commit README stats | |
| run: | | |
| if git diff --quiet README.md; then | |
| echo "README stats unchanged" | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add README.md | |
| git commit -m "Update RouteSentinel snapshot stats" | |
| git push || { | |
| git pull --rebase origin "${GITHUB_REF_NAME}" | |
| git push | |
| } | |
| - name: Recreate release with fresh timestamp | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| target="$(git rev-parse HEAD)" | |
| gh release delete "${{ steps.date.outputs.today }}" \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --cleanup-tag \ | |
| --yes || true | |
| gh release create "${{ steps.date.outputs.today }}" \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --target "${target}" \ | |
| --title "RouteSentinel ${{ steps.date.outputs.today }}" \ | |
| --notes-file out/changelog.md \ | |
| --latest \ | |
| out/rpki-summary.json \ | |
| out/rpki-invalids.csv \ | |
| out/route-origin-status.csv \ | |
| out/rpki-covered-prefixes.csv \ | |
| out/top-invalid-asns.csv \ | |
| out/suspected-events.jsonl \ | |
| out/daily-diff.json \ | |
| out/changelog.md |