Admin/fquiz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat:[FL-211][高程管理扁平化为文件记录]

Browse Source 
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
chengkai3
2026-06-20 18:03:27 +08:00
parent fac00c0536
commit 899d5316cf
11 changed files with 732 additions and 1882 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/app/api/v1/elevation.py
+48
View File
@@ -30,7 +30,9 @@ from ...schemas.elevation import (
ElevationFileRecordListResponse,
ElevationFileRecordPreviewResponse,
ElevationFileRecordSummary,
ElevationFileRecordTaskStatusResponse,
ElevationFileRecordTerrainBuildResponse,
ElevationFileRecordTerrainTaskStatusResponse,
ElevationFileRecordUpdateRequest,
ElevationFileRecordUploadResponse,
)
@@ -40,6 +42,10 @@ from ...services.elevation_service import (
delete_dataset,
get_data_import_job_by_id,
get_dataset_analysis_task_status,
get_file_record_analysis_task_status,
get_file_record_terrain_layer,
get_file_record_terrain_task_status,
get_file_record_terrain_tile,
get_dataset_terrain_layer,
get_dataset_terrain_task_status,
get_dataset_terrain_tile,
@@ -180,6 +186,46 @@ def preview_elevation_file_record(
)
@router.get("/records/{record_id}/analysis-status", response_model=ElevationFileRecordTaskStatusResponse)
def get_elevation_file_record_analysis_status(
record_id: str,
_: CurrentUser = Depends(require_any_permission("elevation.read", "elevation.manage")),
db: Session = Depends(get_db),
) -> ElevationFileRecordTaskStatusResponse:
return get_file_record_analysis_task_status(db, record_id=record_id)
@router.get("/records/{record_id}/terrain-status", response_model=ElevationFileRecordTerrainTaskStatusResponse)
def get_elevation_file_record_terrain_status(
record_id: str,
_: CurrentUser = Depends(require_any_permission("elevation.read", "elevation.manage")),
db: Session = Depends(get_db),
) -> ElevationFileRecordTerrainTaskStatusResponse:
return get_file_record_terrain_task_status(db, record_id=record_id)
@router.get("/records/{record_id}/terrain/layer.json", response_model=ElevationTerrainLayerResponse)
def get_elevation_file_record_terrain_layer(
record_id: str,
_: CurrentUser = Depends(require_any_permission("elevation.read", "elevation.manage")),
db: Session = Depends(get_db),
) -> ElevationTerrainLayerResponse:
return get_file_record_terrain_layer(db, record_id=record_id)
@router.get("/records/{record_id}/terrain/{z}/{x}/{y}.terrain")
def get_elevation_file_record_terrain_tile_endpoint(
record_id: str,
z: int,
x: int,
y: int,
_: CurrentUser = Depends(require_any_permission("elevation.read", "elevation.manage")),
db: Session = Depends(get_db),
) -> Response:
content = get_file_record_terrain_tile(db, record_id=record_id, z=z, x=x, y=y)
return Response(content=content, media_type="application/octet-stream")
# ============================================================================
# Legacy Dataset API (向后兼容，逐步废弃)
# ============================================================================
@@ -378,6 +424,7 @@ def get_elevation_dataset_terrain_tile_endpoint(
def get_elevation_jobs(
line_id: str | None = Query(default=None),
dataset_id: str | None = Query(default=None),
file_record_id: str | None = Query(default=None),
status_filter: str | None = Query(default=None, alias="status"),
limit: int = Query(default=50, ge=1, le=200),
_: CurrentUser = Depends(require_any_permission("elevation.read", "elevation.manage")),
@@ -387,6 +434,7 @@ def get_elevation_jobs(
db,
line_id=line_id,
dataset_id=dataset_id,
file_record_id=file_record_id,
status_filter=status_filter,
limit=limit,
)
api/app/schemas/elevation.py
+28 -2
View File
@@ -293,6 +293,29 @@ class ElevationDatasetTerrainTaskStatusResponse(BaseModel):
update_date: datetime | None = None
class ElevationFileRecordTaskStatusResponse(BaseModel):
record_id: str
file_name: str
task_id: str | None = None
status: Literal["queued", "running", "success", "failed", "unknown", "not_found"] = "not_found"
detail: str | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
update_date: datetime | None = None
class ElevationFileRecordTerrainTaskStatusResponse(BaseModel):
record_id: str
file_name: str
task_id: str | None = None
status: Literal["queued", "running", "success", "failed", "unknown", "not_found"] = "not_found"
detail: str | None = None
terrain_url_template: str | None = None
terrain_min_zoom: int | None = None
terrain_max_zoom: int | None = None
update_date: datetime | None = None
class ElevationTerrainLayerResponse(BaseModel):
tilejson: str = "2.1.0"
format: str = "heightmap-1.0"
@@ -313,7 +336,7 @@ class ElevationApplyJobSummary(BaseModel):
line_id: str
line_code: str | None = None
line_name: str | None = None
file_record_id: str
file_record_id: str | None = None
file_record_name: str | None = None
dataset_id: str | None = None
dataset_code: str | None = None
@@ -354,7 +377,9 @@ class ElevationApplyJobCreateResponse(BaseModel):
class ElevationDataImportJobSummary(BaseModel):
id: str
dataset_id: str
dataset_id: str | None = None
file_record_id: str | None = None
file_record_name: str | None = None
dataset_code: str | None = None
dataset_name: str | None = None
status: ElevationDataImportJobStatus
@@ -384,4 +409,5 @@ class ElevationDataImportJobListResponse(BaseModel):
total: int
ElevationFileRecordPreviewResponse.model_rebuild()
ElevationDatasetDataImportResponse.model_rebuild()
api/app/services/elevation_file_record_service.py
-11
View File
@@ -27,32 +27,21 @@ from ..schemas.elevation import (
)
from .elevation_service import (
ELEVATION_FILE_EXT_FORMAT_MAP,
ELEVATION_TOPIC,
IMPORTABLE_ELEVATION_EXTENSIONS,
RASTER_FILE_FORMATS,
TERRAIN_SUPPORTED_DATASET_FORMATS,
_analyze_dataset_content,
_build_raster_preview,
_decode_csv_bytes,
_default_terrain_status_for_format,
_detect_file_format,
_fire_and_forget,
_load_dataset_points,
_normalize_str,
_publish_elevation_change,
_queue_dataset_terrain_build_after_analysis,
_require_mount,
_require_rasterio_available,
_resolve_dataset_dir,
_resolve_dataset_file_path,
_resolve_dataset_mount_code,
_sample_preview_points_from_csv,
_supports_terrain_build,
_sync_dataset_terrain_support,
ElevationDatasetPreviewDiagnostics,
ElevationDatasetPreviewPoint,
join_virtual_path,
publish_topic,
)
from .file_service import _build_driver_or_400
api/app/services/elevation_service.py
+376 -101
View File
@@ -19,7 +19,7 @@ from sqlalchemy.orm import Session
from ..core.database import SessionLocal
from ..models.base import utcnow
from ..models.elevation import ElevationApplyJob, ElevationDataImportJob, ElevationDataset, ElevationDatasetFileMeta
from ..models.elevation import ElevationApplyJob, ElevationDataImportJob, ElevationDataset, ElevationDatasetFileMeta, ElevationFileRecord
from ..models.line import Line
from ..models.line_tower import LineTower
from ..models.user import User
@@ -36,6 +36,8 @@ from ..schemas.elevation import (
ElevationDatasetDataImportResponse,
ElevationDatasetTerrainBuildResponse,
ElevationDatasetTerrainTaskStatusResponse,
ElevationFileRecordTaskStatusResponse,
ElevationFileRecordTerrainTaskStatusResponse,
ElevationTerrainLayerResponse,
ElevationDatasetPreviewCell,
ElevationDatasetPreviewDiagnostics,
@@ -181,11 +183,14 @@ def serialize_dataset(item: ElevationDataset) -> ElevationDatasetSummary:
def serialize_job(item: ElevationApplyJob) -> ElevationApplyJobSummary:
line = item.line
dataset = item.dataset
file_record = item.file_record
return ElevationApplyJobSummary(
id=item.id,
line_id=item.line_id,
line_code=line.code if line else None,
line_name=line.name if line else None,
file_record_id=item.file_record_id,
file_record_name=file_record.file_name if file_record else None,
dataset_id=item.dataset_id,
dataset_code=dataset.code if dataset else None,
dataset_name=dataset.name if dataset else None,
@@ -209,9 +214,12 @@ def serialize_job(item: ElevationApplyJob) -> ElevationApplyJobSummary:
def serialize_data_import_job(item: ElevationDataImportJob) -> ElevationDataImportJobSummary:
dataset = item.dataset
file_record = item.file_record
return ElevationDataImportJobSummary(
id=item.id,
dataset_id=item.dataset_id,
file_record_id=item.file_record_id,
file_record_name=file_record.file_name if file_record else None,
dataset_code=dataset.code if dataset else None,
dataset_name=dataset.name if dataset else None,
status=item.status, # type: ignore[arg-type]
@@ -277,6 +285,12 @@ def get_dataset_by_id(db: Session, dataset_id: str) -> ElevationDataset | None:
).scalar_one_or_none()
def get_file_record_by_id(db: Session, record_id: str) -> ElevationFileRecord | None:
return db.execute(
select(ElevationFileRecord).where(ElevationFileRecord.id == record_id)
).scalar_one_or_none()
def get_dataset_by_code(db: Session, code: str) -> ElevationDataset | None:
normalized = code.strip()
if not normalized:
@@ -309,6 +323,10 @@ def _supports_terrain_build(dataset: ElevationDataset) -> bool:
return _resolve_dataset_file_format(dataset) in TERRAIN_SUPPORTED_DATASET_FORMATS
def _supports_file_record_terrain_build(record: ElevationFileRecord) -> bool:
return _resolve_file_record_format(record) in TERRAIN_SUPPORTED_DATASET_FORMATS
def _default_terrain_status_for_format(file_format: str) -> str:
return "pending" if file_format in TERRAIN_SUPPORTED_DATASET_FORMATS else "not_supported"
@@ -339,10 +357,26 @@ def _resolve_dataset_terrain_tile_path(*, dataset_code: str, z: int, x: int, y:
return join_virtual_path(join_virtual_path(join_virtual_path(_resolve_dataset_terrain_dir(dataset_code), str(z)), str(x)), f"{y}.terrain")
def _resolve_file_record_terrain_dir(record_id: str) -> str:
return f"/elevation/terrain/records/{record_id[:2]}/{record_id[2:4]}/{record_id}"
def _resolve_file_record_terrain_layer_path(record_id: str) -> str:
return join_virtual_path(_resolve_file_record_terrain_dir(record_id), "layer.json")
def _resolve_file_record_terrain_tile_path(*, record_id: str, z: int, x: int, y: int) -> str:
return join_virtual_path(join_virtual_path(join_virtual_path(_resolve_file_record_terrain_dir(record_id), str(z)), str(x)), f"{y}.terrain")
def _build_dataset_terrain_url_template(dataset_id: str) -> str:
return f"/api/v1/elevation/datasets/{dataset_id}/terrain/{{z}}/{{x}}/{{y}}.terrain?v={TERRAIN_TILE_VERSION}"
def _build_file_record_terrain_url_template(record_id: str) -> str:
return f"/api/v1/elevation/records/{record_id}/terrain/{{z}}/{{x}}/{{y}}.terrain?v={TERRAIN_TILE_VERSION}"
def _map_dataset_task_status(status_value: str | None) -> str:
status_map = {
"queued": "queued",
@@ -1276,6 +1310,7 @@ def get_dataset_terrain_layer(
data = json.loads(payload.content.decode("utf-8"))
except Exception as exc:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="地形 layer.json 解析失败") from exc
data.setdefault("minzoom", 0)
return ElevationTerrainLayerResponse.model_validate(data)
@@ -1307,6 +1342,106 @@ def get_dataset_terrain_tile(
return payload.content
def get_file_record_analysis_task_status(
db: Session,
*,
record_id: str,
) -> ElevationFileRecordTaskStatusResponse:
record = get_file_record_by_id(db, record_id)
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程文件记录不存在")
return ElevationFileRecordTaskStatusResponse(
record_id=record.id,
file_name=record.file_name,
task_id=record.analysis_task_id,
status=_map_dataset_task_status(record.analysis_status), # type: ignore[arg-type]
detail=record.analysis_error_message,
started_at=record.analysis_started_at,
finished_at=record.analysis_finished_at,
update_date=record.update_date,
)
def get_file_record_terrain_task_status(
db: Session,
*,
record_id: str,
) -> ElevationFileRecordTerrainTaskStatusResponse:
record = get_file_record_by_id(db, record_id)
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程文件记录不存在")
return ElevationFileRecordTerrainTaskStatusResponse(
record_id=record.id,
file_name=record.file_name,
task_id=record.terrain_task_id,
status=_map_dataset_task_status(record.terrain_status), # type: ignore[arg-type]
detail=record.terrain_error_message,
terrain_url_template=record.terrain_url_template,
terrain_min_zoom=record.terrain_min_zoom,
terrain_max_zoom=record.terrain_max_zoom,
update_date=record.update_date,
)
def get_file_record_terrain_layer(
db: Session,
*,
record_id: str,
) -> ElevationTerrainLayerResponse:
record = get_file_record_by_id(db, record_id)
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程文件记录不存在")
if record.terrain_status != "ready" or not record.terrain_root_path:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="地形瓦片尚未就绪")
mount = _require_mount(db, record.mount_code)
driver = _build_driver_or_400(mount)
layer_path = _resolve_file_record_terrain_layer_path(record.id)
try:
payload = driver.read_file(layer_path)
except StoragePathNotFoundError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="地形 layer.json 不存在") from exc
except StorageInvalidPathError as exc:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
except StorageDriverError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
try:
data = json.loads(payload.content.decode("utf-8"))
except Exception as exc:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="地形 layer.json 解析失败") from exc
data.setdefault("minzoom", 0)
return ElevationTerrainLayerResponse.model_validate(data)
def get_file_record_terrain_tile(
db: Session,
*,
record_id: str,
z: int,
x: int,
y: int,
) -> bytes:
record = get_file_record_by_id(db, record_id)
if not record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程文件记录不存在")
if record.terrain_status != "ready" or not record.terrain_root_path:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="地形瓦片尚未就绪")
mount = _require_mount(db, record.mount_code)
driver = _build_driver_or_400(mount)
tile_path = _resolve_file_record_terrain_tile_path(record_id=record.id, z=z, x=x, y=y)
try:
payload = driver.read_file(tile_path)
except StoragePathNotFoundError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="地形瓦片不存在") from exc
except StorageInvalidPathError as exc:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
except StorageDriverError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
return payload.content
def update_dataset(
db: Session,
dataset_id: str,
@@ -1563,6 +1698,7 @@ def list_jobs(
dataset_id: str | None,
status_filter: str | None,
limit: int,
file_record_id: str | None = None,
) -> ElevationApplyJobListResponse:
stmt = select(ElevationApplyJob)
total_stmt = select(func.count()).select_from(ElevationApplyJob)
@@ -1573,6 +1709,9 @@ def list_jobs(
if dataset_id:
stmt = stmt.where(ElevationApplyJob.dataset_id == dataset_id)
total_stmt = total_stmt.where(ElevationApplyJob.dataset_id == dataset_id)
if file_record_id:
stmt = stmt.where(ElevationApplyJob.file_record_id == file_record_id)
total_stmt = total_stmt.where(ElevationApplyJob.file_record_id == file_record_id)
if status_filter in {"pending", "running", "success", "failed"}:
stmt = stmt.where(ElevationApplyJob.status == status_filter)
total_stmt = total_stmt.where(ElevationApplyJob.status == status_filter)
@@ -1603,11 +1742,22 @@ def create_apply_job(
if not line:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="线路不存在")
dataset = get_dataset_by_id(db, payload.dataset_id)
if not dataset:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程数据集不存在")
if dataset.status != "active":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="高程数据集未启用")
dataset: ElevationDataset | None = None
file_record: ElevationFileRecord | None = None
if payload.file_record_id:
file_record = get_file_record_by_id(db, payload.file_record_id)
if not file_record:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程文件记录不存在")
if file_record.status != "active":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="高程文件记录未启用")
elif payload.dataset_id:
dataset = get_dataset_by_id(db, payload.dataset_id)
if not dataset:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="高程数据集不存在")
if dataset.status != "active":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="高程数据集未启用")
else:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="必须提供 file_record_id 或 dataset_id")
allowed_modes = {"fill_null_only", "overwrite_all"}
if payload.mode not in allowed_modes:
@@ -1627,7 +1777,8 @@ def create_apply_job(
now = utcnow()
job = ElevationApplyJob(
line_id=line.id,
dataset_id=dataset.id,
dataset_id=dataset.id if dataset is not None else None,
file_record_id=file_record.id if file_record is not None else None,
mode=payload.mode,
status="pending",
total_tower_count=total_tower_count,
@@ -1652,11 +1803,18 @@ def create_apply_job(
if not latest:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="任务派发失败")
_refresh_dataset_usage_status(db, dataset_id=latest.dataset_id)
if latest.dataset_id:
_refresh_dataset_usage_status(db, dataset_id=latest.dataset_id)
_publish_elevation_change(
"elevation.job.created",
{"action": "job_created", "job_id": latest.id, "line_id": latest.line_id},
{
"action": "job_created",
"job_id": latest.id,
"line_id": latest.line_id,
"dataset_id": latest.dataset_id,
"file_record_id": latest.file_record_id,
},
)
return ElevationApplyJobCreateResponse(job=serialize_job(latest), queued=True)
@@ -1699,34 +1857,50 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
job.started_at = utcnow()
job.update_date = utcnow()
db.commit()
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
if job.dataset_id:
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
_publish_elevation_change(
"elevation.job.running",
{"action": "job_running", "job_id": job.id, "line_id": job.line_id},
{
"action": "job_running",
"job_id": job.id,
"line_id": job.line_id,
"dataset_id": job.dataset_id,
"file_record_id": job.file_record_id,
},
)
line = db.execute(select(Line).where(Line.id == job.line_id)).scalar_one_or_none()
dataset = get_dataset_by_id(db, job.dataset_id)
if not line or not dataset:
dataset = get_dataset_by_id(db, job.dataset_id) if job.dataset_id else None
file_record = get_file_record_by_id(db, job.file_record_id) if job.file_record_id else None
elevation_source = file_record or dataset
if not line or elevation_source is None:
job.status = "failed"
job.error_message = "线路或高程数据集不存在"
job.error_message = "线路或高程文件记录不存在"
job.finished_at = utcnow()
job.update_date = utcnow()
db.commit()
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
if job.dataset_id:
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
_publish_elevation_change(
"elevation.job.failed",
{"action": "job_failed", "job_id": job.id, "line_id": job.line_id},
{
"action": "job_failed",
"job_id": job.id,
"line_id": job.line_id,
"dataset_id": job.dataset_id,
"file_record_id": job.file_record_id,
},
)
return
file_format = _resolve_dataset_file_format(dataset)
file_format = _resolve_elevation_file_format(elevation_source)
if file_format == "csv":
points, warnings = _load_dataset_points(db, dataset)
points, warnings = _load_dataset_points(db, elevation_source)
stats = _apply_points_to_line_towers(
db,
line_id=line.id,
dataset=dataset,
elevation_source=elevation_source,
mode=job.mode,
points=points,
)
@@ -1734,7 +1908,7 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
stats, warnings = _apply_raster_to_line_towers(
db,
line_id=line.id,
dataset=dataset,
elevation_source=elevation_source,
mode=job.mode,
)
else:
@@ -1743,7 +1917,8 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
detail=f"不支持的高程文件格式: {file_format}",
)
_refresh_dataset_usage_status(db, dataset_id=dataset.id)
if dataset is not None:
_refresh_dataset_usage_status(db, dataset_id=dataset.id)
warning_note = "; ".join(warnings[:5]) if warnings else None
job.updated_tower_count = stats["updated_tower_count"]
@@ -1762,8 +1937,10 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
payload={
"prepared_at": utcnow().isoformat(),
"prepared_by_user_id": resolved_actor_user_id,
"dataset_id": dataset.id,
"dataset_code": dataset.code,
"dataset_id": dataset.id if dataset is not None else None,
"dataset_code": dataset.code if dataset is not None else None,
"file_record_id": file_record.id if file_record is not None else None,
"file_record_name": file_record.file_name if file_record is not None else None,
"job_id": job.id,
"mode": job.mode,
"updated_tower_count": job.updated_tower_count,
@@ -1772,7 +1949,8 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
},
)
db.commit()
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
if job.dataset_id:
_refresh_dataset_usage_status(db, dataset_id=job.dataset_id)
_publish_elevation_change(
"elevation.job.success",
@@ -1780,6 +1958,8 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
"action": "job_success",
"job_id": job.id,
"line_id": line.id,
"dataset_id": job.dataset_id,
"file_record_id": job.file_record_id,
"updated_tower_count": job.updated_tower_count,
"skipped_tower_count": job.skipped_tower_count,
},
@@ -1797,10 +1977,17 @@ def execute_apply_job(job_id: str, actor_user_id: str | None = None) -> None:
failed.finished_at = utcnow()
failed.update_date = utcnow()
db.commit()
_refresh_dataset_usage_status(db, dataset_id=failed.dataset_id)
if failed.dataset_id:
_refresh_dataset_usage_status(db, dataset_id=failed.dataset_id)
_publish_elevation_change(
"elevation.job.failed",
{"action": "job_failed", "job_id": failed.id, "line_id": failed.line_id},
{
"action": "job_failed",
"job_id": failed.id,
"line_id": failed.line_id,
"dataset_id": failed.dataset_id,
"file_record_id": failed.file_record_id,
},
)
raise
finally:
@@ -2120,6 +2307,35 @@ def _queue_dataset_terrain_build_after_analysis(db: Session, *, dataset: Elevati
db.commit()
def _queue_file_record_terrain_build_after_analysis(db: Session, *, record: ElevationFileRecord, actor_user_id: str | None) -> None:
if not _supports_file_record_terrain_build(record):
return
if record.terrain_status in {"processing", "ready"}:
return
if record.terrain_status == "pending" and record.terrain_task_id:
return
record.terrain_status = "pending"
record.terrain_error_message = None
record.update_date = utcnow()
db.commit()
try:
from ..tasks.elevation_tasks import build_elevation_file_record_terrain_job
task = build_elevation_file_record_terrain_job.delay(record.id, actor_user_id)
record.terrain_task_id = str(task.id)
record.update_date = utcnow()
db.commit()
_publish_elevation_change(
"elevation.file_record.terrain.queued",
{"action": "file_record_terrain_queued", "file_record_id": record.id, "task_id": record.terrain_task_id},
)
except Exception as exc:
record.terrain_status = "failed"
record.terrain_error_message = str(exc)
record.update_date = utcnow()
db.commit()
def _ensure_dataset_file_exists(db: Session, *, mount_code: str, file_path: str) -> None:
mount = _require_mount(db, mount_code.strip())
driver = _build_driver_or_400(mount)
@@ -2342,7 +2558,7 @@ def _refresh_dataset_usage_status(db: Session, *, dataset_id: str) -> None:
def _load_dataset_points(
db: Session,
dataset: ElevationDataset,
dataset: ElevationDataset | ElevationFileRecord,
) -> tuple[list[ElevationSamplePoint], list[str]]:
mount = _require_mount(db, dataset.mount_code)
driver = _build_driver_or_400(mount)
@@ -2461,7 +2677,7 @@ def _apply_points_to_line_towers(
db: Session,
*,
line_id: str,
dataset: ElevationDataset,
elevation_source: ElevationDataset | ElevationFileRecord,
mode: str,
points: list[ElevationSamplePoint],
) -> dict[str, int]:
@@ -2521,8 +2737,7 @@ def _apply_points_to_line_towers(
raw_extra = dict(tower.raw_extra_json or {})
raw_extra["elevation"] = {
"dataset_id": dataset.id,
"dataset_code": dataset.code,
**_elevation_source_metadata(elevation_source),
"sample_method": "nearest",
"sample_distance_m": round(distance_m, 3),
"sample_distance_source": "computed",
@@ -2755,6 +2970,39 @@ def _resolve_dataset_file_format(dataset: ElevationDataset) -> str:
return detected
def _resolve_file_record_format(record: ElevationFileRecord) -> str:
declared = (record.file_format or "").strip().lower()
detected = _detect_file_format(record.file_path)
if declared and declared in ELEVATION_FILE_EXT_FORMAT_MAP.values():
if declared == detected:
return declared
if declared in RASTER_FILE_FORMATS and detected in RASTER_FILE_FORMATS:
return detected
return detected
def _resolve_elevation_file_format(elevation_source: ElevationDataset | ElevationFileRecord) -> str:
if isinstance(elevation_source, ElevationFileRecord):
return _resolve_file_record_format(elevation_source)
return _resolve_dataset_file_format(elevation_source)
def _elevation_source_metadata(elevation_source: Any) -> dict[str, str | None]:
if isinstance(elevation_source, ElevationFileRecord) or hasattr(elevation_source, "file_name"):
return {
"dataset_id": None,
"dataset_code": None,
"file_record_id": getattr(elevation_source, "id", None),
"file_record_name": getattr(elevation_source, "file_name", None),
}
return {
"dataset_id": getattr(elevation_source, "id", None),
"dataset_code": getattr(elevation_source, "code", None),
"file_record_id": None,
"file_record_name": None,
}
def _require_rasterio_available() -> Any:
try:
import rasterio
@@ -2772,9 +3020,9 @@ def _require_rasterio_available() -> Any:
def _open_raster_dataset(
db: Session,
dataset: ElevationDataset,
dataset: ElevationDataset | ElevationFileRecord,
) -> _OpenedRasterDataset:
file_format = _resolve_dataset_file_format(dataset)
file_format = _resolve_elevation_file_format(dataset)
if file_format not in RASTER_FILE_FORMATS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@@ -3164,6 +3412,84 @@ def _build_dataset_terrain_tiles(db: Session, dataset: ElevationDataset) -> _Ter
)
def _build_file_record_terrain_tiles(db: Session, record: ElevationFileRecord) -> _TerrainBuildArtifacts:
mount = _require_mount(db, record.mount_code)
driver = _build_driver_or_400(mount)
terrain_dir = _resolve_file_record_terrain_dir(record.id)
_delete_virtual_directory_if_exists(driver, terrain_dir)
_ensure_virtual_directory(driver, terrain_dir)
with _open_raster_dataset(db, record) as opened:
rasterio = opened.rasterio
src = opened.dataset
bounds = _compute_raster_wgs84_bounds(rasterio=rasterio, src=src)
min_zoom, max_zoom = _resolve_terrain_zoom_limits(bounds=bounds, resolution_m=record.resolution_m)
availability = _build_terrain_available_ranges(bounds=bounds, max_zoom=max_zoom)
tile_count = 0
for level in range(min_zoom, max_zoom + 1):
ranges = availability.get(level, [])
for tile_range in ranges:
for tile_x in range(tile_range["startX"], tile_range["endX"] + 1):
for tile_y in range(tile_range["startY"], tile_range["endY"] + 1):
tile_count += 1
tile_bounds = _tile_bounds_from_tms(level, tile_x, tile_y)
child_mask = _build_terrain_child_mask(
availability=availability,
level=level,
x=tile_x,
y=tile_y,
max_zoom=max_zoom,
)
tile_bytes = _generate_heightmap_tile_bytes(
rasterio=rasterio,
src=src,
tile_bounds=tile_bounds,
child_mask=child_mask,
is_blank_root=level == 0 and not _tile_intersects_bounds(tile_bounds, bounds),
)
tile_path = _resolve_file_record_terrain_tile_path(record_id=record.id, z=level, x=tile_x, y=tile_y)
_write_virtual_file(driver, path=tile_path, content=tile_bytes, content_type=TERRAIN_CONTENT_TYPE)
layer_url = f"/api/v1/elevation/records/{record.id}/terrain/layer.json"
terrain_url_template = _build_file_record_terrain_url_template(record.id)
layer_payload = ElevationTerrainLayerResponse(
tiles=[f"{{z}}/{{x}}/{{y}}.terrain?v={TERRAIN_TILE_VERSION}"],
minzoom=0,
maxzoom=max_zoom,
attribution=record.file_name,
bounds=[bounds["west"], bounds["south"], bounds["east"], bounds["north"]],
available=[availability.get(level, []) for level in range(0, max_zoom + 1)],
).model_dump(mode="json")
_write_virtual_file(
driver,
path=_resolve_file_record_terrain_layer_path(record.id),
content=json.dumps(layer_payload, ensure_ascii=True, separators=(",", ":")).encode("utf-8"),
content_type="application/json",
)
metadata = {
"tool": TERRAIN_BUILD_TOOL,
"format": TERRAIN_TILE_FORMAT,
"projection": TERRAIN_TILE_PROJECTION,
"tile_size": TERRAIN_TILE_SIZE,
"generated_at": utcnow().isoformat(),
"source_crs": str(src.crs) if src.crs is not None else None,
"source_nodata": src.nodatavals[0] if src.nodatavals else None,
"resolution_m": record.resolution_m,
"tile_count": tile_count,
"layer_url": layer_url,
"terrain_url_template": terrain_url_template,
}
return _TerrainBuildArtifacts(
min_zoom=min_zoom,
max_zoom=max_zoom,
bounds=bounds,
metadata=metadata,
)
def _generate_heightmap_tile_bytes(
*,
rasterio: Any,
@@ -3414,7 +3740,7 @@ def _apply_raster_to_line_towers(
db: Session,
*,
line_id: str,
dataset: ElevationDataset,
elevation_source: ElevationDataset | ElevationFileRecord,
mode: str,
) -> tuple[dict[str, int], list[str]]:
towers = db.execute(
@@ -3429,7 +3755,7 @@ def _apply_raster_to_line_towers(
unmatched_count = 0
warnings: list[str] = []
with _open_raster_dataset(db, dataset) as opened:
with _open_raster_dataset(db, elevation_source) as opened:
rasterio = opened.rasterio
src = opened.dataset
warning_text = _append_non_wgs84_bounds_warning(rasterio=rasterio, src=src)
@@ -3476,8 +3802,7 @@ def _apply_raster_to_line_towers(
raw_extra = dict(tower.raw_extra_json or {})
raw_extra["elevation"] = {
"dataset_id": dataset.id,
"dataset_code": dataset.code,
**_elevation_source_metadata(elevation_source),
"sample_method": "raster_pixel",
"sample_distance_m": 0.0,
"sample_distance_source": "pixel_lookup",
@@ -3558,10 +3883,15 @@ def _publish_elevation_change(event_name: str, payload: dict[str, Any]) -> None:
ELEVATION_TOPIC,
name=event_name,
payload=payload,
requires_refetch=["/api/v1/elevation/datasets", "/api/v1/elevation/jobs", "/api/v1/elevation/import-jobs"],
requires_refetch=[
"/api/v1/elevation/records",
"/api/v1/elevation/datasets",
"/api/v1/elevation/jobs",
"/api/v1/elevation/import-jobs",
],
dedupe_key=(
f"{event_name}:"
f"{payload.get('import_job_id') or payload.get('job_id') or payload.get('dataset_id') or 'unknown'}"
f"{payload.get('import_job_id') or payload.get('job_id') or payload.get('file_record_id') or payload.get('dataset_id') or 'unknown'}"
),
)
)
@@ -3841,6 +4171,7 @@ def execute_file_record_analysis_job(*, record_id: str, actor_user_id: str | Non
"elevation.file_record.analysis.success",
{"action": "file_record_analysis_success", "file_record_id": saved.id},
)
_queue_file_record_terrain_build_after_analysis(db, record=saved, actor_user_id=actor.id)
except Exception as exc:
from .elevation_file_record_service import get_file_record_by_id
failed = get_file_record_by_id(db, record_id)
@@ -3899,75 +4230,19 @@ def execute_file_record_terrain_build_job(*, record_id: str, actor_user_id: str
)
return
# Build terrain tiles
mount = _require_mount(db, item.mount_code)
driver = _build_driver_or_400(mount)
# Create terrain output directory
terrain_dir = f"/elevation/terrain/records/{item.id[:2]}/{item.id[2:4]}/{item.id}"
driver.ensure_directory(terrain_dir)
# Read source file
try:
read_result = driver.read_file(item.file_path)
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"文件不存在: {item.file_path}"
) from exc
# Process with ctb-tile (similar to dataset terrain build)
_require_rasterio_available()
import tempfile
import subprocess
import os
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(item.file_path).suffix) as src_tmp:
src_tmp.write(read_result.content)
src_path = src_tmp.name
try:
with tempfile.TemporaryDirectory() as output_tmp:
# Run ctb-tile
cmd = ["ctb-tile", "-f", "Mesh", "-C", "-N", "-o", output_tmp, src_path]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600)
if result.returncode != 0:
raise Exception(f"ctb-tile failed: {result.stderr}")
# Upload generated tiles to storage
for root, dirs, files in os.walk(output_tmp):
for file in files:
local_path = os.path.join(root, file)
rel_path = os.path.relpath(local_path, output_tmp)
remote_path = join_virtual_path(terrain_dir, rel_path)
with open(local_path, "rb") as f:
content = f.read()
driver.write_file(remote_path, content=content, content_type="application/octet-stream")
# Read layer.json if exists
layer_json_path = os.path.join(output_tmp, "layer.json")
if os.path.exists(layer_json_path):
with open(layer_json_path, "r") as f:
import json
layer_data = json.load(f)
item.terrain_min_zoom = layer_data.get("minzoom", 0)
item.terrain_max_zoom = layer_data.get("maxzoom", 18)
item.terrain_bounds = {"bounds": layer_data.get("bounds")}
item.terrain_metadata = layer_data
finally:
os.unlink(src_path)
artifacts = _build_file_record_terrain_tiles(db, item)
saved = get_file_record_by_id(db, record_id)
if saved is None:
return
saved.terrain_status = "ready"
saved.terrain_error_message = None
saved.terrain_root_path = terrain_dir
saved.terrain_url_template = f"/api/v1/elevation/records/{record_id}/terrain/{{z}}/{{x}}/{{y}}.terrain"
saved.terrain_root_path = _resolve_file_record_terrain_dir(saved.id)
saved.terrain_url_template = _build_file_record_terrain_url_template(saved.id)
saved.terrain_min_zoom = artifacts.min_zoom
saved.terrain_max_zoom = artifacts.max_zoom
saved.terrain_bounds = artifacts.bounds
saved.terrain_metadata = artifacts.metadata
saved.update_date = utcnow()
saved.update_user = actor.id
db.commit()
api/tests/test_async_dispatch_services.py
+52 -2
View File
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
from app.core.database import Base
from app.models.atp_model import AtpModel, AtpModelVersion, AtpSimulationRun
from app.models.elevation import ElevationDataImportJob, ElevationDataset
from app.models.elevation import ElevationDataImportJob, ElevationDataset, ElevationFileRecord
from app.models.user import User
from app.models.wine import WineRun
from app.schemas.atp_model import AtpSimulationRunRequest
@@ -233,6 +233,53 @@ def test_queue_dataset_terrain_build_reuses_existing_running_task(monkeypatch) -
session.close()
def test_file_record_terrain_layer_and_tile_read_from_record_storage(monkeypatch) -> None:
testing_session = _build_sessionmaker(ElevationFileRecord.__table__)
session: Session = testing_session()
try:
record = ElevationFileRecord(
id="abcdef1234567890abcdef1234567890",
file_name="terrain.tif",
file_path="/elevation/records/ab/cd/terrain.tif",
file_format="tif",
file_size=128,
mount_code="default",
status="active",
terrain_status="ready",
terrain_root_path="/elevation/terrain/records/ab/cd/abcdef1234567890abcdef1234567890",
terrain_url_template="/api/v1/elevation/records/abcdef1234567890abcdef1234567890/terrain/{z}/{x}/{y}.terrain?v=1.0.0",
terrain_min_zoom=0,
terrain_max_zoom=0,
)
session.add(record)
session.commit()
driver = _MemoryStorageDriver()
layer_payload = b'{"tilejson":"2.1.0","format":"heightmap-1.0","version":"1.0.0","scheme":"tms","projection":"EPSG:4326","tiles":["{z}/{x}/{y}.terrain?v=1.0.0"],"minzoom":0,"maxzoom":0}'
driver.write_file(
"/elevation/terrain/records/ab/cd/abcdef1234567890abcdef1234567890/layer.json",
content=layer_payload,
content_type="application/json",
)
driver.write_file(
"/elevation/terrain/records/ab/cd/abcdef1234567890abcdef1234567890/0/0/0.terrain",
content=b"tile-bytes",
content_type="application/octet-stream",
)
monkeypatch.setattr(elevation_service, "_require_mount", lambda *_args, **_kwargs: SimpleNamespace(code="default"))
monkeypatch.setattr(elevation_service, "_build_driver_or_400", lambda *_args, **_kwargs: driver)
layer = elevation_service.get_file_record_terrain_layer(session, record_id=record.id)
tile = elevation_service.get_file_record_terrain_tile(session, record_id=record.id, z=0, x=0, y=0)
assert layer.maxzoom == 0
assert layer.tiles == ["{z}/{x}/{y}.terrain?v=1.0.0"]
assert tile == b"tile-bytes"
finally:
session.close()
def test_import_dataset_data_files_queue_job_and_worker_keeps_preferred_raster(monkeypatch) -> None:
testing_session = _build_sessionmaker(ElevationDataset.__table__, ElevationDataImportJob.__table__)
session: Session = testing_session()
@@ -296,7 +343,10 @@ def test_import_dataset_data_files_queue_job_and_worker_keeps_preferred_raster(m
assert first.job.uploaded_file_count == 1
assert first.job.analysis_task_queued is False
assert import_calls == [(first.job.id, actor.id)]
assert any(path.endswith(".img") and "/.imports/" in path for path in driver.files)
saved_pending_job = session.get(ElevationDataImportJob, first.job.id)
assert saved_pending_job is not None
assert saved_pending_job.staged_files_json[0]["filename"] == "terrain.img"
assert "content_base64" in saved_pending_job.staged_files_json[0]
second = elevation_service.import_dataset_data_files(
session,
api/tests/test_elevation_apply_service.py
+129 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from app.core.database import Base
from app.models.elevation import ElevationApplyJob, ElevationDataset
from app.models.elevation import ElevationApplyJob, ElevationDataset, ElevationFileRecord
from app.models.line import Line
from app.models.line_tower import LineTower
from app.models.tower_profile import TowerProfile
@@ -28,6 +28,7 @@ def _build_session_factory() -> sessionmaker[Session]:
LineTower.__table__,
TowerProfile.__table__,
ElevationDataset.__table__,
ElevationFileRecord.__table__,
ElevationApplyJob.__table__,
],
)
@@ -81,6 +82,53 @@ def test_create_apply_job_dispatches_actor_user_id(monkeypatch) -> None:
session.close()
def test_create_apply_job_accepts_file_record_id(monkeypatch) -> None:
testing_session = _build_session_factory()
session = testing_session()
try:
line = Line(code="L-APPLY-003", name="文件记录回填线路", voltage_kv=220, lightning_param_json={})
record = ElevationFileRecord(
file_name="record.csv",
file_path="/elevation/records/ab/cd/record.csv",
file_format="csv",
file_size=32,
mount_code="default",
status="active",
)
session.add_all([line, record])
session.flush()
session.add(LineTower(line_id=line.id, seq_no=1, tower_no="T1", longitude=120.0, latitude=30.0))
session.commit()
dispatched: dict[str, str | None] = {}
def _fake_dispatch(*, job_id: str, actor_user_id: str | None) -> SimpleNamespace:
dispatched["job_id"] = job_id
dispatched["actor_user_id"] = actor_user_id
return SimpleNamespace(id="celery-task-file-record")
monkeypatch.setattr(elevation_service, "_dispatch_elevation_apply_task", _fake_dispatch)
monkeypatch.setattr(elevation_service, "_publish_elevation_change", lambda *args, **kwargs: None)
response = elevation_service.create_apply_job(
session,
ElevationApplyJobCreateRequest(line_id=line.id, file_record_id=record.id, mode="overwrite_all"),
actor=SimpleNamespace(id="tester"),
)
saved_job = session.get(ElevationApplyJob, response.job.id)
assert response.queued is True
assert dispatched == {"job_id": response.job.id, "actor_user_id": "tester"}
assert saved_job is not None
assert saved_job.file_record_id == record.id
assert saved_job.dataset_id is None
assert saved_job.task_id == "celery-task-file-record"
assert response.job.file_record_id == record.id
assert response.job.file_record_name == "record.csv"
finally:
session.close()
def test_execute_apply_job_uses_saved_actor_for_preparation_source(monkeypatch) -> None:
testing_session = _build_session_factory()
session = testing_session()
@@ -160,3 +208,83 @@ def test_execute_apply_job_uses_saved_actor_for_preparation_source(monkeypatch)
verification_session.close()
finally:
session.close()
def test_execute_apply_job_uses_file_record_source(monkeypatch) -> None:
testing_session = _build_session_factory()
session = testing_session()
try:
line = Line(code="L-APPLY-004", name="文件记录高程回填线路", voltage_kv=110, lightning_param_json={})
record = ElevationFileRecord(
file_name="record.csv",
file_path="/elevation/records/ab/cd/record.csv",
file_format="csv",
file_size=32,
mount_code="default",
status="active",
)
session.add_all([line, record])
session.flush()
meter_to_lat = 1 / 111_320.0
session.add_all(
[
LineTower(line_id=line.id, seq_no=1, tower_no="P1", longitude=120.0, latitude=30.0 + 300 * meter_to_lat),
LineTower(line_id=line.id, seq_no=2, tower_no="P2", longitude=120.0, latitude=30.0 + 600 * meter_to_lat),
LineTower(line_id=line.id, seq_no=3, tower_no="P3", longitude=120.0, latitude=30.0 + 900 * meter_to_lat),
]
)
session.flush()
job = ElevationApplyJob(
line_id=line.id,
file_record_id=record.id,
mode="overwrite_all",
status="pending",
total_tower_count=3,
create_user="tester",
update_user="tester",
)
session.add(job)
session.commit()
points = [
elevation_service.ElevationSamplePoint(
lon=120.0,
lat=30.0 + distance_m * meter_to_lat,
altitude_m=100.0 + distance_m * 0.12,
)
for distance_m in range(0, 1251, 50)
]
monkeypatch.setattr(elevation_service, "SessionLocal", testing_session)
monkeypatch.setattr(elevation_service, "_load_dataset_points", lambda *_args, **_kwargs: (points, []))
monkeypatch.setattr(elevation_service, "_publish_elevation_change", lambda *args, **kwargs: None)
monkeypatch.setattr(elevation_service, "_publish_line_change", lambda *args, **kwargs: None)
elevation_service.execute_apply_job(job.id)
verification_session = testing_session()
try:
saved_job = verification_session.get(ElevationApplyJob, job.id)
saved_line = verification_session.get(Line, line.id)
towers = verification_session.execute(
select(LineTower).where(LineTower.line_id == line.id).order_by(LineTower.seq_no.asc())
).scalars().all()
assert saved_job is not None
assert saved_job.status == "success"
assert saved_line is not None
assert saved_line.update_user == "tester"
assert all(tower.altitude_m is not None for tower in towers)
source = saved_line.lightning_param_json["preparation_sources"]["ground_slope"]
assert source["prepared_by_user_id"] == "tester"
assert source["file_record_id"] == record.id
assert source["file_record_name"] == "record.csv"
assert source["dataset_id"] is None
assert source["job_id"] == job.id
finally:
verification_session.close()
finally:
session.close()
api/tests/test_line_preparation_flow.py
+1 -1
View File
@@ -158,7 +158,7 @@ def test_apply_points_to_line_towers_computes_ground_slopes() -> None:
stats = elevation_service._apply_points_to_line_towers(
session,
line_id=line.id,
dataset=SimpleNamespace(id="ds-1", code="DEM-001"),
elevation_source=SimpleNamespace(id="ds-1", code="DEM-001"),
mode="overwrite_all",
points=points,
)