03_fetch_readmes.py

"""
Step 03 - fetch the README of every repository in the corpus.

Calls the contents API once per repository and stores the decoded text
alongside the metadata from step 02. This is the last step that touches
GitHub; everything after it works from local files.

INPUT   data/processed/github/github_repos_dedup.csv
OUTPUT  data/raw/github/raw_github_readmes.jsonl      one record per fetch
        data/raw/github/raw_github_readmes.csv        same, flattened
        data/processed/github/github_repos_with_readme.csv  metadata + text
        outputs/github/github_readme_fetch_summary.csv      status counts

RESUMABLE
Each record is appended to the JSONL as it arrives, and a new run reads that
file first and fetches only what is missing. A run interrupted at hour three
resumes rather than restarting - which matters, since this is the slowest step.

LARGE READMES
GitHub omits inline content above roughly 1 MB and returns encoding="none" with
a download_url instead. Those are fetched from raw.githubusercontent.com, a
separate CDN that does not draw on the API rate limit. Without this fallback
the longest READMEs would come back empty, exactly the ones most worth having.

FAILURE MODES ARE RECORDED, NOT DROPPED
Every outcome gets an explicit readme_status: ok, ok_via_download_url,
no_readme_or_not_found, empty_readme_content, api_error, base64_decode_error,
download_url_fetch_failed, unknown_encoding, script_exception. A repository
without a README stays in the corpus with empty text, so later steps can decide
what to do with it rather than finding it silently absent.

RATE LIMITS
5000 requests/hour authenticated. Requests are spaced accordingly; the delay can
be overridden with GITHUB_README_REQUEST_DELAY_SECONDS. Secondary limits back
off with growing sleeps up to five retries, server errors (500-504) up to three.

RUN OF AUGUST 2026
24 105 repositories: 23 440 ok, 79 via download_url, 557 without a README, 28
empty, 1 script exception. Median README is 6 632 characters.
"""

import os
import time
import json
import base64
from pathlib import Path
from datetime import datetime, timezone

import pandas as pd
import requests
from dotenv import load_dotenv
from tqdm import tqdm


SCRIPT_VERSION = "2026-06-02-github-readme-fetch-only-v1"

load_dotenv(dotenv_path=Path(".env"), override=False)

GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")

INPUT_CSV = Path("data/processed/github/github_repos_dedup.csv")

RAW_DIR = Path("data/raw/github")
PROCESSED_DIR = Path("data/processed/github")
OUTPUTS_DIR = Path("outputs/github")

RAW_DIR.mkdir(parents=True, exist_ok=True)
PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)

README_JSONL = RAW_DIR / "raw_github_readmes.jsonl"
README_CSV = RAW_DIR / "raw_github_readmes.csv"
ENRICHED_CSV = PROCESSED_DIR / "github_repos_with_readme.csv"
SUMMARY_CSV = OUTPUTS_DIR / "github_readme_fetch_summary.csv"

API_BASE = "https://api.github.com"

HEADERS = {
    "Accept": "application/vnd.github+json",
    "X-GitHub-Api-Version": "2022-11-28",
    "User-Agent": "world-models-signals-research-readme-fetch",
}

if GITHUB_TOKEN:
    HEADERS["Authorization"] = f"Bearer {GITHUB_TOKEN}"


# One request per repository, against the core REST API (not Search API):
# 5000 req/hour authenticated, but only 60 req/hour unauthenticated.
# 1.5 sec ≈ 2400 requests/hour, safely under the authenticated limit.
# Without a token this would be wildly too fast for the 60/hour cap, so the
# default falls back to a pace that actually matches that limit.
# Override explicitly if needed:
# export GITHUB_README_REQUEST_DELAY_SECONDS="3"
_default_delay = 1.5 if GITHUB_TOKEN else 65
REQUEST_DELAY_SECONDS = float(os.getenv("GITHUB_README_REQUEST_DELAY_SECONDS", str(_default_delay)))

SECONDARY_LIMIT_BASE_SLEEP_SECONDS = 60
MAX_SECONDARY_RETRIES = 5

SERVER_ERROR_SLEEP_SECONDS = 60
MAX_SERVER_ERROR_RETRIES = 3

_last_request_time = 0


def now_utc_iso():
    return datetime.now(timezone.utc).isoformat()


def polite_pause():
    global _last_request_time

    now = time.time()
    elapsed = now - _last_request_time

    if elapsed < REQUEST_DELAY_SECONDS:
        time.sleep(REQUEST_DELAY_SECONDS - elapsed)

    _last_request_time = time.time()


def github_get_readme(owner, repo):
    url = f"{API_BASE}/repos/{owner}/{repo}/readme"

    secondary_retries = 0
    server_retries = 0

    while True:
        polite_pause()

        response = requests.get(
            url,
            headers=HEADERS,
            timeout=60,
            allow_redirects=True,
        )

        if response.status_code in (200, 404):
            return response

        if response.status_code == 401:
            print("\nGitHub returned 401 Unauthorized.")
            print("The token is missing, expired, or invalid.")
            print(response.text[:800])
            response.raise_for_status()

        if response.status_code in (403, 429):
            text = response.text or ""
            text_lower = text.lower()

            retry_after = response.headers.get("Retry-After")
            reset = response.headers.get("X-RateLimit-Reset")
            remaining = response.headers.get("X-RateLimit-Remaining")

            if retry_after:
                sleep_for = int(retry_after) + 30
                print(f"\nRetry-After received. Sleeping for {sleep_for} seconds.")
                time.sleep(sleep_for)
                continue

            if reset and remaining == "0":
                sleep_for = max(int(reset) - int(time.time()) + 30, 60)
                print(f"\nPrimary GitHub rate limit reached. Sleeping for {sleep_for} seconds.")
                time.sleep(sleep_for)
                continue

            if "secondary rate limit" in text_lower or "abuse" in text_lower:
                secondary_retries += 1

                if secondary_retries > MAX_SECONDARY_RETRIES:
                    return response

                sleep_for = SECONDARY_LIMIT_BASE_SLEEP_SECONDS * (2 ** (secondary_retries - 1))
                print("\nGitHub returned secondary rate limit.")
                print(text[:500])
                print(f"Sleeping for {sleep_for} seconds before retry {secondary_retries}/{MAX_SECONDARY_RETRIES}.")
                time.sleep(sleep_for)
                continue

            return response

        if response.status_code in (500, 502, 503, 504):
            server_retries += 1

            if server_retries > MAX_SERVER_ERROR_RETRIES:
                return response

            sleep_for = SERVER_ERROR_SLEEP_SECONDS * server_retries
            print(f"\nGitHub server error {response.status_code}. Sleeping for {sleep_for} seconds.")
            time.sleep(sleep_for)
            continue

        return response


def decode_readme_response(response):
    if response.status_code == 404:
        return {
            "readme_status": "no_readme_or_not_found",
            "readme_text": "",
            "readme_error": "",
            "readme_name": "",
            "readme_path": "",
            "readme_sha": "",
            "readme_size": None,
            "readme_encoding": "",
            "readme_html_url": "",
            "readme_download_url": "",
            "readme_api_url": "",
        }

    if response.status_code != 200:
        return {
            "readme_status": "api_error",
            "readme_text": "",
            "readme_error": response.text[:1000] if response.text else "",
            "readme_name": "",
            "readme_path": "",
            "readme_sha": "",
            "readme_size": None,
            "readme_encoding": "",
            "readme_html_url": "",
            "readme_download_url": "",
            "readme_api_url": "",
        }

    try:
        data = response.json()
    except Exception as e:
        return {
            "readme_status": "json_decode_error",
            "readme_text": "",
            "readme_error": str(e),
            "readme_name": "",
            "readme_path": "",
            "readme_sha": "",
            "readme_size": None,
            "readme_encoding": "",
            "readme_html_url": "",
            "readme_download_url": "",
            "readme_api_url": "",
        }

    encoding = data.get("encoding") or ""
    content = data.get("content") or ""
    download_url = data.get("download_url") or ""

    text = ""
    error = ""
    status = "ok"

    if encoding == "base64" and content:
        try:
            raw_bytes = base64.b64decode(content)
            text = raw_bytes.decode("utf-8", errors="replace")
        except Exception as e:
            status = "base64_decode_error"
            error = str(e)

    elif encoding == "none":
        # GitHub omits inline content for files over ~1MB and provides
        # download_url instead. Without this fallback, large/thorough
        # READMEs (e.g. big awesome-lists) would silently come back empty.
        if download_url:
            try:
                # download_url points at raw.githubusercontent.com, a separate
                # CDN that does not draw from the api.github.com rate limit
                # (verified directly), so no polite_pause() needed here.
                dl_response = requests.get(download_url, headers=HEADERS, timeout=60)
                if dl_response.status_code == 200:
                    text = dl_response.text
                    status = "ok_via_download_url"
                else:
                    status = "download_url_fetch_failed"
                    error = f"download_url returned status {dl_response.status_code}"
            except Exception as e:
                status = "download_url_fetch_error"
                error = str(e)
        else:
            status = "content_not_returned_by_api"
            error = "GitHub returned encoding='none' and no download_url was provided."

    elif not content:
        status = "empty_readme_content"
        error = f"encoding={encoding}; content is empty."

    else:
        status = "unknown_encoding"
        error = f"encoding={encoding}; content_present={bool(content)}"

    return {
        "readme_status": status,
        "readme_text": text,
        "readme_error": error,
        "readme_name": data.get("name") or "",
        "readme_path": data.get("path") or "",
        "readme_sha": data.get("sha") or "",
        "readme_size": data.get("size"),
        "readme_encoding": encoding,
        "readme_html_url": data.get("html_url") or "",
        "readme_download_url": data.get("download_url") or "",
        "readme_api_url": data.get("url") or "",
    }


def build_readme_record(repo_row):
    full_name = str(repo_row["full_name"])

    if "/" not in full_name:
        return {
            "full_name": full_name,
            "html_url": repo_row.get("html_url", ""),
            "readme_fetched_at": now_utc_iso(),
            "readme_http_status": "",
            "readme_status": "invalid_full_name",
            "readme_error": "full_name does not contain owner/repo",
            "readme_text": "",
            "readme_length": 0,
        }

    owner, repo = full_name.split("/", 1)

    response = github_get_readme(owner, repo)
    decoded = decode_readme_response(response)
    text = decoded.get("readme_text", "")

    record = {
        "full_name": full_name,
        "html_url": repo_row.get("html_url", ""),
        "readme_fetched_at": now_utc_iso(),
        "readme_http_status": response.status_code,
        "readme_rate_limit_remaining": response.headers.get("X-RateLimit-Remaining", ""),
        "readme_rate_limit_reset": response.headers.get("X-RateLimit-Reset", ""),
        **decoded,
        "readme_length": len(text) if text else 0,
    }

    return record


def load_existing_readme_records():
    if not README_JSONL.exists():
        return {}

    records_by_full_name = {}

    with README_JSONL.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()

            if not line:
                continue

            try:
                record = json.loads(line)
            except json.JSONDecodeError:
                continue

            full_name = record.get("full_name")
            if full_name:
                records_by_full_name[full_name] = record

    return records_by_full_name


def append_record_jsonl(record):
    with README_JSONL.open("a", encoding="utf-8") as f:
        f.write(json.dumps(record, ensure_ascii=False) + "\n")


def save_outputs(repos, records_by_full_name):
    readmes = pd.DataFrame(list(records_by_full_name.values()))

    if readmes.empty:
        readmes = pd.DataFrame(columns=["full_name"])

    readmes.to_csv(README_CSV, index=False)

    enriched = repos.merge(readmes, on="full_name", how="left")
    enriched.to_csv(ENRICHED_CSV, index=False)

    if "readme_status" in readmes.columns:
        status_summary = (
            readmes.groupby("readme_status", dropna=False)
            .size()
            .reset_index(name="n_repositories")
            .sort_values("n_repositories", ascending=False)
        )
    else:
        status_summary = pd.DataFrame(columns=["readme_status", "n_repositories"])

    summary_rows = [
        {"metric": "script_version", "value": SCRIPT_VERSION},
        {"metric": "input_repositories", "value": len(repos)},
        {"metric": "readme_records_saved", "value": len(readmes)},
        {"metric": "remaining_without_readme_record", "value": len(repos) - len(readmes)},
        {"metric": "request_delay_seconds", "value": REQUEST_DELAY_SECONDS},
    ]

    summary = pd.DataFrame(summary_rows)

    with SUMMARY_CSV.open("w", encoding="utf-8") as f:
        summary.to_csv(f, index=False)
        f.write("\n")
        status_summary.to_csv(f, index=False)

    print("\nSaved:")
    print(README_JSONL)
    print(README_CSV)
    print(ENRICHED_CSV)
    print(SUMMARY_CSV)

    print("\nREADME status summary:")
    if not status_summary.empty:
        print(status_summary.to_string(index=False))
    else:
        print("No README records yet.")

    print("\nRecords saved:", len(readmes))
    print("Input repos:", len(repos))
    print("Remaining:", len(repos) - len(readmes))


def main():
    print(f"SCRIPT_VERSION: {SCRIPT_VERSION}")
    print("GITHUB_TOKEN found:", bool(GITHUB_TOKEN))

    if GITHUB_TOKEN:
        print("GITHUB_TOKEN prefix:", GITHUB_TOKEN[:4])
        print("GITHUB_TOKEN length:", len(GITHUB_TOKEN))
    else:
        print("Warning: GITHUB_TOKEN is not set. Unauthenticated limit is much lower.")

    print(f"Input file: {INPUT_CSV}")
    print(f"Request delay: {REQUEST_DELAY_SECONDS} seconds")

    if not INPUT_CSV.exists():
        raise FileNotFoundError(
            f"Input file not found: {INPUT_CSV}\n"
            "Run scripts/github/02_dedup_github_repos.py first."
        )

    repos = pd.read_csv(INPUT_CSV)

    if repos.empty:
        print("Input file is empty.")
        save_outputs(repos, {})
        return

    if "full_name" not in repos.columns:
        raise ValueError("Input CSV must contain a 'full_name' column.")

    # Hard methodological boundary: no pre-2018 repositories.
    if "created_at" in repos.columns:
        repos["created_at_dt"] = pd.to_datetime(repos["created_at"], errors="coerce", utc=True)
        repos = repos[repos["created_at_dt"] >= pd.Timestamp("2018-01-01", tz="UTC")]
        repos = repos.drop(columns=["created_at_dt"])

    repos = repos.drop_duplicates(subset=["full_name"]).reset_index(drop=True)

    existing = load_existing_readme_records()
    print("Existing README records:", len(existing))

    remaining = repos[~repos["full_name"].isin(existing.keys())].copy()
    print("Remaining to fetch:", len(remaining))

    records_by_full_name = dict(existing)

    try:
        for _, repo_row in tqdm(remaining.iterrows(), total=len(remaining)):
            full_name = str(repo_row["full_name"])

            try:
                record = build_readme_record(repo_row)
            except Exception as e:
                record = {
                    "full_name": full_name,
                    "html_url": repo_row.get("html_url", ""),
                    "readme_fetched_at": now_utc_iso(),
                    "readme_http_status": "",
                    "readme_status": "script_exception",
                    "readme_error": str(e),
                    "readme_text": "",
                    "readme_length": 0,
                }

            records_by_full_name[full_name] = record
            append_record_jsonl(record)

    except KeyboardInterrupt:
        print("\nStopped by user. Saving partial results...")

    finally:
        save_outputs(repos, records_by_full_name)


if __name__ == "__main__":
    main()