Skip to content

Commit ab56a54

Browse files
maxachismaxachis
authored
Merge pull request #598 from Police-Data-Accessibility-Project/issue-341-url-health
Add URL health view and expose health in data source sync/API
2 parents ee09ea1 + 3c36514 commit ab56a54

15 files changed

Lines changed: 277 additions & 42 deletions

File tree

‎alembic/versions/2026_02_27_1200-7a6c2e1b9d44_add_url_health_materialized_view.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
"""Add url health view
2+
3+
Revision ID: 7a6c2e1b9d44
4+
Revises: 1fb2286a016c
5+
Create Date: 2026-02-27 12:00:00.000000
6+
7+
"""
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision: str = "7a6c2e1b9d44"
15+
down_revision: Union[str, None] = "1fb2286a016c"
16+
branch_labels: Union[str, Sequence[str], None] = None
17+
depends_on: Union[str, Sequence[str], None] = None
18+
19+
20+
def _create_url_health_view() -> None:
21+
op.execute(
22+
"""
23+
CREATE VIEW url_health_view AS
24+
WITH latest_redirect AS (
25+
SELECT DISTINCT ON (lur.source_url_id)
26+
lur.source_url_id,
27+
lur.destination_url_id
28+
FROM link_urls_redirect_url lur
29+
ORDER BY lur.source_url_id, lur.updated_at DESC, lur.created_at DESC
30+
)
31+
SELECT
32+
u.id AS url_id,
33+
CASE
34+
WHEN uwm.status_code = 200 THEN 'OK'
35+
WHEN lr.destination_url_id IS NOT NULL AND redirect_uwm.status_code = 200 THEN 'OK'
36+
WHEN uiapm.archive_url IS NOT NULL THEN 'Archived'
37+
ELSE 'Broken'
38+
END AS health,
39+
CASE
40+
WHEN uwm.status_code = 200 THEN 100
41+
WHEN lr.destination_url_id IS NOT NULL AND redirect_uwm.status_code = 200 THEN 150
42+
WHEN uiapm.archive_url IS NOT NULL THEN 200
43+
ELSE 300
44+
END AS code,
45+
uwm.status_code,
46+
lr.destination_url_id AS redirect_url_id,
47+
CASE
48+
WHEN redirect_u.scheme IS NOT NULL AND redirect_u.trailing_slash = TRUE
49+
THEN redirect_u.scheme || '://' || redirect_u.url || '/'
50+
WHEN redirect_u.scheme IS NOT NULL AND redirect_u.trailing_slash = FALSE
51+
THEN redirect_u.scheme || '://' || redirect_u.url
52+
ELSE redirect_u.url
53+
END AS redirect_url,
54+
redirect_uwm.status_code AS redirect_status_code,
55+
(lr.destination_url_id IS NOT NULL) AS has_redirect,
56+
(lr.destination_url_id IS NOT NULL AND redirect_uwm.status_code = 200) AS redirect_is_healthy,
57+
(uiapm.archive_url IS NOT NULL) AS has_archive,
58+
uiapm.archive_url
59+
FROM urls u
60+
LEFT JOIN url_web_metadata uwm
61+
ON uwm.url_id = u.id
62+
LEFT JOIN latest_redirect lr
63+
ON lr.source_url_id = u.id
64+
LEFT JOIN urls redirect_u
65+
ON redirect_u.id = lr.destination_url_id
66+
LEFT JOIN url_web_metadata redirect_uwm
67+
ON redirect_uwm.url_id = lr.destination_url_id
68+
LEFT JOIN url_internet_archives_probe_metadata uiapm
69+
ON uiapm.url_id = u.id
70+
"""
71+
)
72+
73+
74+
def upgrade() -> None:
75+
_create_url_health_view()
76+
77+
78+
def downgrade() -> None:
79+
op.execute("DROP VIEW IF EXISTS url_health_view")

‎alembic/versions/2026_03_18_1400-a1b2c3d4e5f6_merge_url_health_head.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
"""Merge url_health head with dev head
2+
3+
Revision ID: a1b2c3d4e5f6
4+
Revises: f831e447b1cb, 7a6c2e1b9d44
5+
Create Date: 2026-03-18 14:00:00.000000
6+
7+
"""
8+
from typing import Optional, Sequence
9+
10+
11+
# revision identifiers, used by Alembic.
12+
revision: str = 'a1b2c3d4e5f6'
13+
down_revision: Optional[tuple[str, ...]] = ('f831e447b1cb', '7a6c2e1b9d44')
14+
branch_labels: Optional[str | Sequence[str]] = None
15+
depends_on: Optional[str | Sequence[str]] = None
16+
17+
18+
def upgrade() -> None:
19+
"""Merge multiple heads."""
20+
pass
21+
22+
23+
def downgrade() -> None:
24+
"""Downgrade merge."""
25+
pass

‎src/api/endpoints/data_source/_shared/build.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from src.db.models.impl.url.core.sqlalchemy import URL
88
from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata
99
from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType
10+
from src.db.models.materialized_views.url_health.sqlalchemy import URLHealthMaterializedView
1011

1112

1213
def build_data_source_get_query() -> Select:
@@ -38,7 +39,18 @@ def build_data_source_get_query() -> Select:
3839
URLOptionalDataSourceMetadata.scraper_url,
3940
URLOptionalDataSourceMetadata.submission_notes,
4041
URLOptionalDataSourceMetadata.access_notes,
41-
URLOptionalDataSourceMetadata.access_types
42+
URLOptionalDataSourceMetadata.access_types,
43+
44+
URLHealthMaterializedView.health,
45+
URLHealthMaterializedView.code,
46+
URLHealthMaterializedView.status_code,
47+
URLHealthMaterializedView.redirect_url_id,
48+
URLHealthMaterializedView.redirect_url,
49+
URLHealthMaterializedView.redirect_status_code,
50+
URLHealthMaterializedView.has_redirect,
51+
URLHealthMaterializedView.redirect_is_healthy,
52+
URLHealthMaterializedView.has_archive,
53+
URLHealthMaterializedView.archive_url,
4254
)
4355
.join(
4456
URLRecordType,
@@ -59,6 +71,10 @@ def build_data_source_get_query() -> Select:
5971
URLOptionalDataSourceMetadata,
6072
URLOptionalDataSourceMetadata.url_id == URL.id
6173
)
74+
.outerjoin(
75+
URLHealthMaterializedView,
76+
URLHealthMaterializedView.url_id == URL.id,
77+
)
6278
.options(
6379
selectinload(URL.confirmed_agencies),
6480
)

‎src/api/endpoints/data_source/_shared/process.py‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
from sqlalchemy import RowMapping
22

3-
from src.api.endpoints.data_source.get.response import DataSourceGetResponse
3+
from src.api.endpoints.data_source.get.response import DataSourceGetResponse, DataSourceURLHealthResponse
44
from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL
55
from src.db.models.impl.url.core.sqlalchemy import URL
66
from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata
77
from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType
8+
from src.db.models.materialized_views.url_health.sqlalchemy import URLHealthMaterializedView
89

910

1011
def process_data_source_get_mapping(
@@ -16,6 +17,22 @@ def process_data_source_get_mapping(
1617
for agency in url.confirmed_agencies:
1718
url_agency_ids.append(agency.id)
1819

20+
url_health: DataSourceURLHealthResponse | None = None
21+
health_value = mapping[URLHealthMaterializedView.health]
22+
if health_value is not None:
23+
url_health = DataSourceURLHealthResponse(
24+
value=health_value,
25+
code=mapping[URLHealthMaterializedView.code],
26+
http_status_code=mapping[URLHealthMaterializedView.status_code],
27+
redirect_url_id=mapping[URLHealthMaterializedView.redirect_url_id],
28+
redirect_url=mapping[URLHealthMaterializedView.redirect_url],
29+
redirect_http_status_code=mapping[URLHealthMaterializedView.redirect_status_code],
30+
has_redirect=mapping[URLHealthMaterializedView.has_redirect] or False,
31+
redirect_is_healthy=mapping[URLHealthMaterializedView.redirect_is_healthy] or False,
32+
has_archive=mapping[URLHealthMaterializedView.has_archive] or False,
33+
archive_url=mapping[URLHealthMaterializedView.archive_url] or None,
34+
)
35+
1936
return DataSourceGetResponse(
2037
url_id=mapping[URL.id],
2138
url=mapping[URL.url],
@@ -39,5 +56,6 @@ def process_data_source_get_mapping(
3956
scraper_url=mapping[URLOptionalDataSourceMetadata.scraper_url],
4057
submission_notes=mapping[URLOptionalDataSourceMetadata.submission_notes],
4158
access_notes=mapping[URLOptionalDataSourceMetadata.access_notes],
42-
access_types=mapping[URLOptionalDataSourceMetadata.access_types] or []
59+
access_types=mapping[URLOptionalDataSourceMetadata.access_types] or [],
60+
url_health=url_health,
4361
)

‎src/api/endpoints/data_source/get/response.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,20 @@
55
from src.core.enums import RecordType
66
from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \
77
RetentionScheduleEnum, AccessTypeEnum
8+
from src.db.models.materialized_views.url_health.enums import URLHealthViewEnum
9+
10+
11+
class DataSourceURLHealthResponse(BaseModel):
12+
value: URLHealthViewEnum
13+
code: int
14+
http_status_code: int | None = None
15+
redirect_url_id: int | None = None
16+
redirect_url: str | None = None
17+
redirect_http_status_code: int | None = None
18+
has_redirect: bool = False
19+
redirect_is_healthy: bool = False
20+
has_archive: bool = False
21+
archive_url: str | None = None
822

923

1024
class DataSourceGetResponse(BaseModel):
@@ -37,6 +51,7 @@ class DataSourceGetResponse(BaseModel):
3751
submission_notes: str | None = None
3852
access_notes: str | None = None
3953
access_types: list[AccessTypeEnum]
54+
url_health: DataSourceURLHealthResponse | None = None
4055

4156
class DataSourceGetOuterResponse(BaseModel):
4257
results: list[DataSourceGetResponse]

‎src/core/tasks/scheduled/impl/sync_to_ds/impl/data_sources/add/queries/get.py‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,13 @@
66
from src.core.tasks.scheduled.impl.sync_to_ds.constants import PER_REQUEST_ENTITY_LIMIT
77
from src.core.tasks.scheduled.impl.sync_to_ds.impl.data_sources.add.queries.cte import \
88
DSAppLinkSyncDataSourceAddPrerequisitesCTEContainer
9-
from src.core.tasks.scheduled.impl.sync_to_ds.shared.convert import convert_sm_url_status_to_ds_url_status
9+
from src.core.tasks.scheduled.impl.sync_to_ds.shared.convert import convert_sm_url_health_to_ds_url_status
1010
from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency
1111
from src.db.models.impl.url.core.sqlalchemy import URL
12-
from src.db.models.impl.url.internet_archives.probe.sqlalchemy import URLInternetArchivesProbeMetadata
1312
from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata
1413
from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType
15-
from src.db.models.impl.url.web_metadata.sqlalchemy import URLWebMetadata
16-
from src.db.models.materialized_views.url_status.sqlalchemy import URLStatusMaterializedView
14+
from src.db.models.materialized_views.url_health.sqlalchemy import URLHealthMaterializedView
1715
from src.db.queries.base.builder import QueryBuilderBase
18-
from src.external.pdap.enums import DataSourcesURLStatus
1916
from src.external.pdap.impl.sync.data_sources._shared.content import DataSourceSyncContentModel
2017
from src.external.pdap.impl.sync.data_sources.add.request import AddDataSourcesOuterRequest, AddDataSourcesInnerRequest
2118

@@ -42,7 +39,6 @@ async def run(self, session: AsyncSession) -> AddDataSourcesOuterRequest:
4239
# Required
4340
URL.full_url,
4441
URL.name,
45-
URLWebMetadata.status_code,
4642
URLRecordType.record_type,
4743
agency_id_cte.c.agency_ids,
4844
# Optional
@@ -61,7 +57,9 @@ async def run(self, session: AsyncSession) -> AddDataSourcesOuterRequest:
6157
URLOptionalDataSourceMetadata.scraper_url,
6258
URLOptionalDataSourceMetadata.access_notes,
6359
URLOptionalDataSourceMetadata.access_types,
64-
URLInternetArchivesProbeMetadata.archive_url,
60+
URLHealthMaterializedView.health,
61+
URLHealthMaterializedView.has_archive,
62+
URLHealthMaterializedView.archive_url,
6563
)
6664
.select_from(
6765
cte.cte
@@ -75,12 +73,8 @@ async def run(self, session: AsyncSession) -> AddDataSourcesOuterRequest:
7573
URL.id == URLOptionalDataSourceMetadata.url_id,
7674
)
7775
.outerjoin(
78-
URLWebMetadata,
79-
URL.id == URLWebMetadata.url_id
80-
)
81-
.outerjoin(
82-
URLInternetArchivesProbeMetadata,
83-
URL.id == URLInternetArchivesProbeMetadata.url_id,
76+
URLHealthMaterializedView,
77+
URL.id == URLHealthMaterializedView.url_id,
8478
)
8579
.join(
8680
URLRecordType,
@@ -124,15 +118,15 @@ async def run(self, session: AsyncSession) -> AddDataSourcesOuterRequest:
124118
scraper_url=mapping[URLOptionalDataSourceMetadata.scraper_url],
125119
access_notes=mapping[URLOptionalDataSourceMetadata.access_notes],
126120
access_types=mapping[URLOptionalDataSourceMetadata.access_types] or [],
127-
# TODO: Change to convert web metadata result to URL Status
128-
url_status=convert_sm_url_status_to_ds_url_status(
129-
mapping[URLWebMetadata.status_code],
121+
url_status=convert_sm_url_health_to_ds_url_status(
122+
health=mapping[URLHealthMaterializedView.health],
123+
has_archive=mapping[URLHealthMaterializedView.has_archive] or False,
130124
),
131-
internet_archives_url=mapping[URLInternetArchivesProbeMetadata.archive_url] or None,
125+
internet_archives_url=mapping[URLHealthMaterializedView.archive_url] or None,
132126
)
133127
)
134128
)
135129

136130
return AddDataSourcesOuterRequest(
137131
data_sources=inner_requests,
138-
)
132+
)

‎src/core/tasks/scheduled/impl/sync_to_ds/impl/data_sources/update/queries/cte.py‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1-
from sqlalchemy import select, or_, Column, CTE
1+
from sqlalchemy import select, or_, Column, CTE, and_, exists
22

3+
from src.db.models.impl.link.url_redirect_url.sqlalchemy import LinkURLRedirectURL
34
from src.db.models.impl.url.core.sqlalchemy import URL
45
from src.db.models.impl.url.data_source.sqlalchemy import DSAppLinkDataSource
56
from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata
67
from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType
8+
from src.db.models.impl.url.web_metadata.sqlalchemy import URLWebMetadata
79

810

911
class DSAppLinkSyncDataSourceUpdatePrerequisitesCTEContainer:
@@ -32,6 +34,35 @@ def __init__(self):
3234
URLOptionalDataSourceMetadata.updated_at > DSAppLinkDataSource.last_synced_at,
3335
URLRecordType.created_at > DSAppLinkDataSource.last_synced_at,
3436
URLRecordType.updated_at > DSAppLinkDataSource.last_synced_at,
37+
exists(
38+
select(URLWebMetadata.url_id).where(
39+
and_(
40+
URLWebMetadata.url_id == DSAppLinkDataSource.url_id,
41+
URLWebMetadata.updated_at > DSAppLinkDataSource.last_synced_at,
42+
)
43+
)
44+
),
45+
exists(
46+
select(LinkURLRedirectURL.source_url_id).where(
47+
and_(
48+
LinkURLRedirectURL.source_url_id == DSAppLinkDataSource.url_id,
49+
LinkURLRedirectURL.updated_at > DSAppLinkDataSource.last_synced_at,
50+
)
51+
)
52+
),
53+
exists(
54+
select(LinkURLRedirectURL.source_url_id)
55+
.join(
56+
URLWebMetadata,
57+
URLWebMetadata.url_id == LinkURLRedirectURL.destination_url_id,
58+
)
59+
.where(
60+
and_(
61+
LinkURLRedirectURL.source_url_id == DSAppLinkDataSource.url_id,
62+
URLWebMetadata.updated_at > DSAppLinkDataSource.last_synced_at,
63+
)
64+
)
65+
),
3566
)
3667
).cte("ds_app_link_sync_data_source_update_prerequisites")
3768
)
@@ -46,4 +77,4 @@ def ds_data_source_id(self) -> Column[int]:
4677

4778
@property
4879
def cte(self) -> CTE:
49-
return self._cte
80+
return self._cte

0 commit comments

Comments
 (0)