BED-9508: support GHES REST and GraphQL endpoints

README.md

@@ -40,6 +40,19 @@ Enterprise GitHub App credentials accept either `client_id` or `app_id` as the
 JWT issuer. When both are configured, `client_id` is preferred. At least one
 identifier must be supplied together with `key_path` and `enterprise_name`.
 
+### GitHub Enterprise Server endpoints
+
+GitHub.com is the default deployment and does not require endpoint configuration.
+To collect from GitHub Enterprise Server, set both API endpoints in `config.toml`:
+
+```toml
+[sources.github]
+rest_api_url = "https://ghe.example/api/v3"
+graphql_url = "https://ghe.example/api/graphql"
+```
+
+Both values must be provided together when overriding the GitHub.com defaults.
+
 ### Enterprise SCIM and hybrid correlations
 
 A token with enterprise SCIM access is used to collect both `/scim/v2/enterprises/{enterprise}/Users` and `/scim/v2/enterprises/{enterprise}/Groups`. The collector emits normalized `SCIM_Organization`, `SCIM_User`, and `SCIM_Group` nodes plus `SCIM_Contains`, `SCIM_MemberOf`, and `SCIM_Provisioned` relationships. Install the BloodHound SCIM extension alongside this extension to register the shared SCIM kinds.

src/openhound_github/resources/enterprise.py

@@ -53,14 +53,28 @@ class SourceContext:
     """Shared context for GitHub API access."""
 
     client: RESTClient
+    graphql_client: RESTClient | None = None
     sso_client: RESTClient | None = None
+    sso_graphql_client: RESTClient | None = None
     org_name: str | None = None
     enterprise_name: str | None = None
     emit_legacy_scim_correlations: bool = False
     github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID
     github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN
 
 
+def _graphql_client(ctx: SourceContext) -> tuple[RESTClient, str]:
+    if ctx.graphql_client:
+        return ctx.graphql_client, ""
+    return ctx.client, "/graphql"
+
+
+def _sso_graphql_client(ctx: SourceContext) -> tuple[RESTClient | None, str]:
+    if ctx.sso_graphql_client:
+        return ctx.sso_graphql_client, ""
+    return ctx.sso_client, "/graphql"
+
+
 def iter_enterprise_scim_resources(
     client: RESTClient,
     enterprise_slug: str,
@@ -111,8 +125,9 @@ def enterprise(ctx: SourceContext):
         "variables": {"slug": ctx.enterprise_name, "after": None},
     }
 
+    client, graphql_path = _graphql_client(ctx)
     try:
-        response = ctx.client.post("/graphql", json=data).json()
+        response = client.post(graphql_path, json=data).json()
         page_enterprise = (response.get("data") or {}).get("enterprise")
         if page_enterprise:
             yield page_enterprise
@@ -139,9 +154,10 @@ def enterprise_organizations(enterprise_data: Enterprise, ctx: SourceContext):
         "variables": {"slug": ctx.enterprise_name, "after": None},
     }
 
+    client, graphql_path = _graphql_client(ctx)
     try:
-        for page_data in ctx.client.paginate(
-            "/graphql",
+        for page_data in client.paginate(
+            graphql_path,
             method="POST",
             json=data,
             paginator=paginator,
@@ -263,8 +279,9 @@ def enterprise_members(enterprise_data: Enterprise, ctx: SourceContext):
         "query": ENTERPRISE_MEMBERS_QUERY,
         "variables": {"slug": ctx.enterprise_name, "count": 100, "after": None},
     }
-    for page_data in ctx.client.paginate(
-        "/graphql",
+    client, graphql_path = _graphql_client(ctx)
+    for page_data in client.paginate(
+        graphql_path,
         method="POST",
         json=data,
         paginator=paginator,
@@ -635,8 +652,9 @@ def enterprise_admins(enterprise_data: Enterprise, ctx: SourceContext):
         "query": ENTERPRISE_ADMINS_QUERY,
         "variables": {"slug": ctx.enterprise_name, "count": 100, "after": None},
     }
-    for page_data in ctx.client.paginate(
-        "/graphql",
+    client, graphql_path = _graphql_client(ctx)
+    for page_data in client.paginate(
+        graphql_path,
         method="POST",
         json=data,
         paginator=paginator,
@@ -665,7 +683,7 @@ def enterprise_admins(enterprise_data: Enterprise, ctx: SourceContext):
     parallelized=True
 )
 def enterprise_saml_provider(enterprise_data: Enterprise, ctx: SourceContext):
-    client = ctx.sso_client
+    client, graphql_path = _sso_graphql_client(ctx)
     if not client:
         logger.info(
             "Skipping enterprise_saml_provider for enterprise '%s': no SSO client configured",
@@ -679,7 +697,7 @@ def enterprise_saml_provider(enterprise_data: Enterprise, ctx: SourceContext):
     }
 
     try:
-        response = client.post("/graphql", json=data).json()
+        response = client.post(graphql_path, json=data).json()
     except Exception as e:
         logger.error(
             f"Error in resource 'enterprise_saml_provider' processing enterprise '{ctx.enterprise_name}': {e}",
@@ -779,7 +797,7 @@ def enterprise_saml_issuer(saml_provider: SamlProvider, ctx: SourceContext):
 def enterprise_external_identity(
     saml_provider: SamlProvider, ctx: SourceContext
 ):
-    client = ctx.sso_client
+    client, graphql_path = _sso_graphql_client(ctx)
     if not client:
         logger.info(
             "Skipping enterprise_external_identity for enterprise '%s': no SSO client configured",
@@ -801,7 +819,7 @@ def enterprise_external_identity(
 
     try:
         for page_data in client.paginate(
-            "/graphql",
+            graphql_path,
             method="POST",
             json=data,
             paginator=paginator,

src/openhound_github/resources/organization.py

@@ -83,11 +83,14 @@
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_GITHUB_REST_API_URL = "https://api.github.com"
+
 
 @dataclass
 class OrgContext:
     client: RESTClient
     org_name: str
+    graphql_client: RESTClient | None = None
     enterprise_name: str | None = None
     github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID
     github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN
@@ -97,6 +100,7 @@ class OrgContext:
 class SourceContext:
     client: RESTClient
     organizations: list[OrgContext] = field(default_factory=list)
+    graphql_client: RESTClient | None = None
     enterprise_name: str | None = None
     github_deployment_id: str = DEFAULT_GITHUB_DEPLOYMENT_ID
     github_web_origin: str = DEFAULT_GITHUB_WEB_ORIGIN
@@ -132,6 +136,24 @@ def _client_for_org(ctx: SourceContext, org_login: str) -> RESTClient:
     return ctx.client
 
 
+def _graphql_client_for_org(
+    ctx: SourceContext, org_login: str
+) -> tuple[RESTClient, str]:
+    for org in ctx.organizations:
+        if org.org_name == org_login:
+            if org.graphql_client:
+                return org.graphql_client, ""
+            return org.client, "/graphql"
+    if ctx.graphql_client:
+        return ctx.graphql_client, ""
+    return ctx.client, "/graphql"
+
+
+def _rest_api_url(client: RESTClient) -> str:
+    base_url = getattr(client, "base_url", DEFAULT_GITHUB_REST_API_URL)
+    return str(base_url).strip().rstrip("/") or DEFAULT_GITHUB_REST_API_URL
+
+
 def _encode_path_segment(value: str) -> str:
     return quote(value, safe="")
 
@@ -452,7 +474,7 @@ def users(ctx: SourceContext) -> Iterator[dict[str, Any]]:
 
     for org in ctx.organizations:
         org_name = org.org_name
-        client = org.client
+        client, graphql_path = _graphql_client_for_org(ctx, org_name)
         try:
             paginator = GraphQLCursorPaginator(
                 page_info_path="data.organization.membersWithRole.pageInfo",
@@ -466,7 +488,7 @@ def users(ctx: SourceContext) -> Iterator[dict[str, Any]]:
             }
 
             for page_data in client.paginate(
-                "/graphql",
+                graphql_path,
                 method="POST",
                 json=data,
                 paginator=paginator,
@@ -503,7 +525,7 @@ def teams(ctx: SourceContext):
 
     for org in ctx.organizations:
         org_name = org.org_name
-        client = org.client
+        client, graphql_path = _graphql_client_for_org(ctx, org_name)
         try:
             paginator = GraphQLCursorPaginator(
                 page_info_path="data.organization.teams.pageInfo",
@@ -516,7 +538,7 @@ def teams(ctx: SourceContext):
                 "variables": {"login": org_name, "count": 100, "after": None},
             }
             for page_data in client.paginate(
-                "/graphql",
+                graphql_path,
                 method="POST",
                 json=data,
                 paginator=paginator,
@@ -614,7 +636,7 @@ def team_members(team: Team, ctx: SourceContext):
             raise RuntimeError(
                 f"GitHub team {team.org_login}/{team.slug} has more members but no endCursor"
             )
-        client = _client_for_org(ctx, team.org_login)
+        client, graphql_path = _graphql_client_for_org(ctx, team.org_login)
         data = {
             "query": TEAM_MEMBERS_OVERFLOW_QUERY,
             "variables": {
@@ -625,7 +647,7 @@ def team_members(team: Team, ctx: SourceContext):
             },
         }
         for page_data in client.paginate(
-            "/graphql",
+            graphql_path,
             method="POST",
             json=data,
             paginator=paginator,
@@ -915,7 +937,7 @@ def repositories_graphql(ctx: SourceContext):
     """
     for org in ctx.organizations:
         org_name = org.org_name
-        client = org.client
+        client, graphql_path = _graphql_client_for_org(ctx, org_name)
         repository_cursor: str | None = None
         emitted_repositories = 0
         try:
@@ -931,7 +953,7 @@ def repositories_graphql(ctx: SourceContext):
             }
 
             for page_data in client.paginate(
-                "/graphql",
+                graphql_path,
                 method="POST",
                 json=data,
                 paginator=paginator,
@@ -1003,7 +1025,7 @@ def branches(repository: RepositoryQL, ctx: SourceContext):
         }
 
     if repository.refs.page_info.has_next_page:
-        client = _client_for_org(ctx, repository.org_login)
+        client, graphql_path = _graphql_client_for_org(ctx, repository.org_login)
         data = {
             "query": REF_OVERFLOW_QUERY,
             "variables": {
@@ -1015,7 +1037,7 @@ def branches(repository: RepositoryQL, ctx: SourceContext):
         }
 
         for page_data in client.paginate(
-            "/graphql",
+            graphql_path,
             method="POST",
             json=data,
             paginator=paginator,
@@ -1057,12 +1079,12 @@ def branch_protection_rules(repository: RepositoryQL, ctx: SourceContext):
                 rule_ids_seen.add(rule_id)
 
     rule_ids_list = list(rule_ids_seen)
-    client = _client_for_org(ctx, repository.org_login)
+    client, graphql_path = _graphql_client_for_org(ctx, repository.org_login)
     for i in range(0, len(rule_ids_list), 100):
         rules_chunk = rule_ids_list[i : i + 100]
         if rules_chunk:
             data = {"query": PROTECTION_RULES_QUERY, "variables": {"ids": rules_chunk}}
-            response = client.post("/graphql", json=data).json()
+            response = client.post(graphql_path, json=data).json()
             for rule in response["data"].get("nodes", []):
                 # GitHub can return null actors for deleted or inaccessible allowance actors.
                 for allowance_key in ("bypassPullRequestAllowances", "pushAllowances"):
@@ -1646,7 +1668,7 @@ def secret_scanning_alerts(ctx: SourceContext):
                     ):
                         try:
                             resp = requests.get(
-                                "https://api.github.com/user",
+                                f"{_rest_api_url(client)}/user",
                                 headers={"Authorization": f"Bearer {secret}"},
                                 timeout=10,
                             )
@@ -1782,14 +1804,14 @@ def saml_provider(ctx: SourceContext):
     """
     for org in ctx.organizations:
         org_name = org.org_name
-        client = org.client
+        client, graphql_path = _graphql_client_for_org(ctx, org_name)
         try:
             data = {
                 "query": SAML_QUERY,
                 "variables": {"login": org_name, "count": 100, "after": None},
             }
 
-            response = client.post("/graphql", json=data).json()
+            response = client.post(graphql_path, json=data).json()
             response_data = response.get("data", {})
             org_data = response_data.get("organization", {})
             if response_data and org_data:
@@ -1826,7 +1848,7 @@ def external_identities(ctx: SourceContext):
     """
     for org in ctx.organizations:
         org_name = org.org_name
-        client = org.client
+        client, graphql_path = _graphql_client_for_org(ctx, org_name)
         github_deployment_id = org.github_deployment_id
         try:
             paginator = GraphQLCursorPaginator(
@@ -1842,7 +1864,7 @@ def external_identities(ctx: SourceContext):
             }
 
             for page_data in client.paginate(
-                "/graphql",
+                graphql_path,
                 method="POST",
                 json=data,
                 paginator=paginator,