feat: add analysis pages and raster risk map
Ship a new app version with broader analytics, restructured dashboards, and a server-rendered risk map. Frontend: - Add Overview, Demographic, Disease, and Environmental Health analysis pages - Add AnomalyMarkers, CalendarHeatmap, and MetricHeatmapTable components - Rebuild Alerts map onto server-rendered raster risk tiles; expand Monitoring, Trend, and District Comparison views - Extend API client, stores, and TypeScript types Backend: - Add environment router (pollutants, lag correlations) - Add risk_raster util serving XYZ 100m risk tiles - Expand cases endpoints (demographics, seasonality, diagnoses) and insights; harden auth and file-based loaders Data & tooling: - Add processed outpatient/inpatient/combined case parquet (LFS) - Add nested CLAUDE.md guides, pyrightconfig, and test updates
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
import asyncio
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
from datetime import datetime, timedelta
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
@@ -6,6 +7,7 @@ from typing import Optional
|
||||
import logging
|
||||
import sys
|
||||
import math
|
||||
import pandas as pd
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
@@ -22,47 +24,33 @@ from models import (
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["grid"])
|
||||
|
||||
logger = logging.getLogger("cbpoa.grid")
|
||||
|
||||
_parquet_cache: dict[str, "pd.DataFrame"] = {}
|
||||
_parquet_cache: dict[str, pd.DataFrame] = {}
|
||||
|
||||
def _load_parquet(path: Path) -> "pd.DataFrame":
|
||||
import pandas as pd
|
||||
def _load_parquet(path: Path) -> pd.DataFrame:
|
||||
key = str(path)
|
||||
if key not in _parquet_cache:
|
||||
_parquet_cache[key] = pd.read_parquet(path)
|
||||
return _parquet_cache[key]
|
||||
|
||||
|
||||
@router.get("/history/aggregated", response_model=HistoricalAggregationResponse)
|
||||
async def get_historical_aggregated(
|
||||
start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
|
||||
end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
|
||||
aggregation: str = Query("daily", description="Aggregation level: daily, weekly, monthly"),
|
||||
district: Optional[str] = Query(None, description="Filter by district name"),
|
||||
):
|
||||
"""
|
||||
Historical data aggregation API.
|
||||
|
||||
Returns aggregated case and weather data by district and date.
|
||||
"""
|
||||
try:
|
||||
start = datetime.strptime(start_date, "%Y-%m-%d")
|
||||
end = datetime.strptime(end_date, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
|
||||
|
||||
if (end - start).days > 365:
|
||||
raise HTTPException(status_code=400, detail="Date range exceeds 365 days")
|
||||
|
||||
import pandas as pd
|
||||
|
||||
def _compute_historical_aggregation(
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
aggregation: str,
|
||||
district: Optional[str],
|
||||
) -> HistoricalAggregationResponse:
|
||||
"""Run the full pandas aggregation pipeline (called in thread pool)."""
|
||||
try:
|
||||
cases_df = _load_parquet(PROJECT_ROOT / "processed" / "cases_by_district_daily.parquet")
|
||||
except FileNotFoundError:
|
||||
return HistoricalAggregationResponse(
|
||||
aggregations=[], total_records=0,
|
||||
date_range=(start_date, end_date), timestamp=datetime.now().isoformat(),
|
||||
date_range=(start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")),
|
||||
timestamp=datetime.now().isoformat(),
|
||||
)
|
||||
cases_df = cases_df.copy()
|
||||
cases_df['date'] = pd.to_datetime(cases_df['date'])
|
||||
|
||||
filtered_cases = cases_df[
|
||||
@@ -76,6 +64,7 @@ async def get_historical_aggregated(
|
||||
]
|
||||
|
||||
if aggregation == "weekly":
|
||||
filtered_cases = filtered_cases.copy()
|
||||
filtered_cases['period'] = filtered_cases['date'].dt.to_period('W').astype(str)
|
||||
grouped = filtered_cases.groupby(['period', 'district']).agg({
|
||||
'total_cases': 'sum',
|
||||
@@ -84,6 +73,7 @@ async def get_historical_aggregated(
|
||||
}).reset_index()
|
||||
grouped['date'] = grouped['period']
|
||||
elif aggregation == "monthly":
|
||||
filtered_cases = filtered_cases.copy()
|
||||
filtered_cases['period'] = filtered_cases['date'].dt.to_period('M').astype(str)
|
||||
grouped = filtered_cases.groupby(['period', 'district']).agg({
|
||||
'total_cases': 'sum',
|
||||
@@ -98,7 +88,8 @@ async def get_historical_aggregated(
|
||||
try:
|
||||
weather_df = _load_parquet(PROJECT_ROOT / "processed" / "weather" / "station_daily_2022.parquet")
|
||||
except FileNotFoundError:
|
||||
weather_df = pd.DataFrame(columns=['date', 'AQI', 'PM25', 'PM10'])
|
||||
weather_df = pd.DataFrame({'date': pd.Series(dtype='str'), 'AQI': pd.Series(dtype='float64'), 'PM25': pd.Series(dtype='float64'), 'PM10': pd.Series(dtype='float64')})
|
||||
weather_df = weather_df.copy()
|
||||
weather_df['date'] = pd.to_datetime(weather_df['date']).dt.strftime('%Y-%m-%d')
|
||||
|
||||
# Weather data doesn't have district - aggregate by date only
|
||||
@@ -114,135 +105,173 @@ async def get_historical_aggregated(
|
||||
aggregations = []
|
||||
for _, row in merged.iterrows():
|
||||
aggregations.append(DistrictAggregation(
|
||||
district=row['district'],
|
||||
district=str(row['district']),
|
||||
date=str(row['date']),
|
||||
total_cases=int(row['total_cases']),
|
||||
outpatient_count=int(row['outpatient_count']),
|
||||
inpatient_count=int(row['inpatient_count']),
|
||||
avg_AQI=float(row['AQI']) if pd.notna(row['AQI']) else 0.0,
|
||||
avg_PM25=float(row['PM25']) if pd.notna(row['PM25']) else 0.0,
|
||||
avg_PM10=float(row['PM10']) if pd.notna(row['PM10']) else 0.0,
|
||||
avg_AQI=float(row['AQI']) if bool(pd.notna(row['AQI'])) else 0.0,
|
||||
avg_PM25=float(row['PM25']) if bool(pd.notna(row['PM25'])) else 0.0,
|
||||
avg_PM10=float(row['PM10']) if bool(pd.notna(row['PM10'])) else 0.0,
|
||||
))
|
||||
|
||||
return HistoricalAggregationResponse(
|
||||
aggregations=aggregations,
|
||||
total_records=len(aggregations),
|
||||
date_range=(start_date, end_date),
|
||||
date_range=(start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")),
|
||||
timestamp=datetime.now().isoformat(),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/history/aggregated", response_model=HistoricalAggregationResponse)
|
||||
async def get_historical_aggregated(
|
||||
start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
|
||||
end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
|
||||
aggregation: str = Query("daily", description="Aggregation level: daily, weekly, monthly"),
|
||||
district: Optional[str] = Query(None, description="Filter by district name"),
|
||||
):
|
||||
"""
|
||||
Historical data aggregation API.
|
||||
|
||||
Returns aggregated case and weather data by district and date.
|
||||
Pandas processing runs in a thread pool to avoid blocking the async event loop.
|
||||
"""
|
||||
try:
|
||||
start = datetime.strptime(start_date, "%Y-%m-%d")
|
||||
end = datetime.strptime(end_date, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
|
||||
|
||||
if (end - start).days > 365:
|
||||
raise HTTPException(status_code=400, detail="Date range exceeds 365 days")
|
||||
|
||||
# Offload all pandas I/O and processing to a thread pool
|
||||
# to prevent blocking the async event loop
|
||||
return await asyncio.to_thread(
|
||||
_compute_historical_aggregation, start, end, aggregation, district
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _grid_geojson_base():
|
||||
"""Date-independent base merge: grid centroid + district + real population.
|
||||
|
||||
Merged once and cached (the source frames are ~1M rows each, so the join
|
||||
must not run per request). Raises FileNotFoundError if the core grid files
|
||||
are missing (caller handles it).
|
||||
"""
|
||||
import pandas as pd
|
||||
grid_df = _load_parquet(PROJECT_ROOT / "processed" / "grid_100m_index.parquet")
|
||||
district_map = _load_parquet(PROJECT_ROOT / "processed" / "grid_district_mapping.parquet")
|
||||
base = grid_df.merge(district_map, on='grid_id', how='left')
|
||||
try:
|
||||
pop_df = _load_parquet(PROJECT_ROOT / "processed" / "grid_100m_with_dem_pop.parquet")
|
||||
base = base.merge(pop_df[['grid_id', 'population_density']], on='grid_id', how='left')
|
||||
except FileNotFoundError:
|
||||
base['population_density'] = 0.0
|
||||
base['population_density'] = base['population_density'].fillna(0.0)
|
||||
return base
|
||||
|
||||
|
||||
def _risk_level_of(v: float) -> str:
|
||||
if v >= 0.7:
|
||||
return "high"
|
||||
if v >= 0.5:
|
||||
return "medium"
|
||||
if v >= 0.3:
|
||||
return "medium_low"
|
||||
return "low"
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _grids_geojson_body(date: str, district: Optional[str], risk_level: Optional[str]) -> str:
|
||||
"""Build + serialize the grid GeoJSON once per (date, district, risk_level).
|
||||
|
||||
Risk is computed vectorised over the full grid (no per-row Python loop) and
|
||||
the highest-risk grids are returned as hotspots, so the map shows real
|
||||
high→low variation. Cached, so warm calls are near-instant. Raises
|
||||
FileNotFoundError if the core grid files are missing.
|
||||
"""
|
||||
merged = _grid_geojson_base()
|
||||
if district:
|
||||
merged = merged[merged['district_name'].str.contains(district.replace('区', ''), na=False, regex=False)]
|
||||
|
||||
cases_df = _load_parquet(PROJECT_ROOT / "processed" / "cases_by_district_daily.parquet").copy()
|
||||
cases_df['date'] = pd.to_datetime(cases_df['date']).dt.strftime('%Y-%m-%d')
|
||||
cases_df = cases_df[cases_df['date'] == date]
|
||||
|
||||
# Normalise district case load to 0..1 across districts for this date.
|
||||
max_district_cases = float(cases_df['total_cases'].max()) if len(cases_df) else 0.0
|
||||
if max_district_cases <= 0:
|
||||
max_district_cases = 1.0
|
||||
|
||||
merged = merged.merge(cases_df[['district', 'total_cases']], left_on='district_name', right_on='district', how='left')
|
||||
merged = merged.copy()
|
||||
merged['total_cases'] = merged['total_cases'].fillna(0).astype(int)
|
||||
merged['center_lon'] = pd.to_numeric(merged['center_lon'], errors='coerce').fillna(0.0)
|
||||
merged['center_lat'] = pd.to_numeric(merged['center_lat'], errors='coerce').fillna(0.0)
|
||||
merged['population_density'] = merged['population_density'].fillna(0.0).clip(lower=0.0)
|
||||
|
||||
# Drop grids without coordinates.
|
||||
merged = merged[(merged['center_lon'] != 0.0) | (merged['center_lat'] != 0.0)]
|
||||
|
||||
# Demo risk model (vectorised): a district's relative case load × each grid's
|
||||
# own population exposure. Sparse cells stay low; densely-populated cells in
|
||||
# high-case districts rise toward 1.0.
|
||||
district_load = (merged['total_cases'] / max_district_cases).clip(upper=1.0)
|
||||
pop_factor = (merged['population_density'] / 50.0).clip(upper=1.0)
|
||||
merged['risk_value'] = (0.1 + 0.85 * district_load * pop_factor).clip(upper=1.0).round(3)
|
||||
|
||||
# Show the highest-risk grids (hotspots), not arbitrary cells.
|
||||
merged = merged.nlargest(10000, 'risk_value')
|
||||
|
||||
features = []
|
||||
for rec in merged.to_dict('records'):
|
||||
rv = float(rec['risk_value'])
|
||||
lvl = _risk_level_of(rv)
|
||||
if risk_level and lvl != risk_level:
|
||||
continue
|
||||
name = rec.get('district_name')
|
||||
if not isinstance(name, str):
|
||||
name = "未知"
|
||||
lon = round(float(rec['center_lon']), 6)
|
||||
lat = round(float(rec['center_lat']), 6)
|
||||
features.append({
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [lon, lat]},
|
||||
"properties": {
|
||||
"grid_id": str(rec.get('grid_id', '')),
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"district": name,
|
||||
"total_cases": int(rec.get('total_cases', 0)),
|
||||
"population_density": round(float(rec.get('population_density', 0.0)), 2),
|
||||
"risk_value": rv,
|
||||
"risk_level": lvl,
|
||||
}
|
||||
})
|
||||
|
||||
return GridGeoJSONResponse(
|
||||
type="FeatureCollection",
|
||||
features=features,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
).model_dump_json()
|
||||
|
||||
|
||||
@router.get("/grids/geojson", response_model=GridGeoJSONResponse)
|
||||
async def get_grids_geojson(
|
||||
date: str = Query(..., description="Date (YYYY-MM-DD)"),
|
||||
district: Optional[str] = Query(None, description="Filter by district"),
|
||||
risk_level: Optional[str] = Query(None, description="Filter by risk level"),
|
||||
):
|
||||
"""
|
||||
Get grid data as GeoJSON for map visualization.
|
||||
"""
|
||||
import pandas as pd
|
||||
|
||||
"""Get grid data as GeoJSON for map visualization (cached per query)."""
|
||||
try:
|
||||
grid_df = _load_parquet(PROJECT_ROOT / "processed" / "grid_100m_index.parquet")
|
||||
# Offload the parquet merges + vectorised compute to a thread so the
|
||||
# cold-cache build doesn't block the event loop.
|
||||
body = await asyncio.to_thread(_grids_geojson_body, date, district, risk_level)
|
||||
except FileNotFoundError:
|
||||
return GridGeoJSONResponse(type="FeatureCollection", features=[], timestamp=datetime.now().isoformat())
|
||||
|
||||
try:
|
||||
district_map = _load_parquet(PROJECT_ROOT / "processed" / "grid_district_mapping.parquet")
|
||||
except FileNotFoundError:
|
||||
return GridGeoJSONResponse(type="FeatureCollection", features=[], timestamp=datetime.now().isoformat())
|
||||
|
||||
merged = grid_df.merge(district_map, on='grid_id', how='left')
|
||||
|
||||
if district:
|
||||
merged = merged[merged['district_name'].str.contains(district.replace('区', ''), na=False, regex=False)]
|
||||
|
||||
try:
|
||||
cases_df = _load_parquet(PROJECT_ROOT / "processed" / "cases_by_district_daily.parquet")
|
||||
except FileNotFoundError:
|
||||
return GridGeoJSONResponse(type="FeatureCollection", features=[], timestamp=datetime.now().isoformat())
|
||||
cases_df['date'] = pd.to_datetime(cases_df['date']).dt.strftime('%Y-%m-%d')
|
||||
|
||||
cases_df = cases_df[cases_df['date'] == date]
|
||||
|
||||
merged = merged.merge(cases_df, left_on='district_name', right_on='district', how='left')
|
||||
merged['total_cases'] = merged['total_cases'].fillna(0).astype(int)
|
||||
|
||||
def safe_float(val, default=0.0):
|
||||
try:
|
||||
v = float(val)
|
||||
return default if math.isnan(v) or math.isinf(v) else v
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def sanitize(obj):
|
||||
"""Replace NaN/Inf with None for JSON serialization."""
|
||||
if isinstance(obj, float):
|
||||
if math.isnan(obj) or math.isinf(obj):
|
||||
return None
|
||||
return obj
|
||||
if isinstance(obj, dict):
|
||||
return {k: sanitize(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [sanitize(v) for v in obj]
|
||||
return obj
|
||||
|
||||
features = []
|
||||
for _, row in merged.iterrows():
|
||||
lon = safe_float(row.get('center_lon'))
|
||||
lat = safe_float(row.get('center_lat'))
|
||||
if lon == 0.0 and lat == 0.0:
|
||||
continue
|
||||
|
||||
# MVP: Simple risk calculation based on cases and population density
|
||||
total_cases = safe_float(row.get('total_cases', 0), 0)
|
||||
total_cases = int(total_cases)
|
||||
pop_density = safe_float(row.get('population_density', 0))
|
||||
|
||||
# Risk formula: cases per 10k population + baseline
|
||||
risk_value = min(1.0, (total_cases / max(pop_density, 1)) * 10 + 0.1)
|
||||
|
||||
if risk_value >= 0.7:
|
||||
risk_level = "high"
|
||||
elif risk_value >= 0.5:
|
||||
risk_level = "medium"
|
||||
elif risk_value >= 0.3:
|
||||
risk_level = "medium_low"
|
||||
else:
|
||||
risk_level = "low"
|
||||
|
||||
district = row.get('district_name')
|
||||
if isinstance(district, float) and (math.isnan(district) or math.isinf(district)):
|
||||
district = "未知"
|
||||
|
||||
feature = {
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [lon, lat]
|
||||
},
|
||||
"properties": {
|
||||
"grid_id": str(row.get('grid_id', '')),
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"district": district,
|
||||
"total_cases": total_cases,
|
||||
"population_density": pop_density,
|
||||
"risk_value": round(risk_value, 3),
|
||||
"risk_level": risk_level,
|
||||
}
|
||||
}
|
||||
features.append(feature)
|
||||
|
||||
if len(features) >= 10000:
|
||||
break
|
||||
|
||||
return GridGeoJSONResponse(
|
||||
type="FeatureCollection",
|
||||
features=features,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
)
|
||||
return Response(content=body, media_type="application/json")
|
||||
|
||||
|
||||
@router.post("/predict/multi-day", response_model=MultiDayPredictionResponse)
|
||||
@@ -278,9 +307,9 @@ async def predict_multi_day(request: MultiDayPredictionRequest):
|
||||
]
|
||||
|
||||
for _, row in features_df.iterrows():
|
||||
risk_1d = float(row.get('risk_1day', 0.5))
|
||||
risk_3d = float(row.get('risk_3day', 0.5))
|
||||
risk_7d = float(row.get('risk_7day', 0.5))
|
||||
risk_1d = float(row.get('risk_1day', 0.5)) # type: ignore[arg-type]
|
||||
risk_3d = float(row.get('risk_3day', 0.5)) # type: ignore[arg-type]
|
||||
risk_7d = float(row.get('risk_7day', 0.5)) # type: ignore[arg-type]
|
||||
|
||||
if risk_1d >= 0.8:
|
||||
risk_level = "high"
|
||||
@@ -323,14 +352,8 @@ async def predict_multi_day(request: MultiDayPredictionRequest):
|
||||
)
|
||||
|
||||
|
||||
@router.get("/grids/{grid_id}/history")
|
||||
async def get_grid_history(
|
||||
grid_id: str,
|
||||
days: int = Query(30, ge=1, le=365, description="Number of days of history"),
|
||||
):
|
||||
"""
|
||||
Get historical data for a specific grid cell.
|
||||
"""
|
||||
def _compute_grid_history(grid_id: str, days: int) -> dict:
|
||||
"""Heavy synchronous parquet reads + per-row loop (called in thread pool)."""
|
||||
import pandas as pd
|
||||
|
||||
district_map = _load_parquet(PROJECT_ROOT / "processed" / "grid_district_mapping.parquet")
|
||||
@@ -367,4 +390,18 @@ async def get_grid_history(
|
||||
"district": district,
|
||||
"history": history,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@router.get("/grids/{grid_id}/history")
|
||||
async def get_grid_history(
|
||||
grid_id: str,
|
||||
days: int = Query(30, ge=1, le=365, description="Number of days of history"),
|
||||
):
|
||||
"""
|
||||
Get historical data for a specific grid cell.
|
||||
|
||||
Parquet reads + aggregation run in a thread pool to avoid blocking the
|
||||
async event loop.
|
||||
"""
|
||||
return await asyncio.to_thread(_compute_grid_history, grid_id, days)
|
||||
|
||||
Reference in New Issue
Block a user