From 07735fb23fa84a602ba5bb09a2a51273693b59d6 Mon Sep 17 00:00:00 2001 From: chengkai3 Date: Sat, 13 Jun 2026 07:58:43 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20[FL-104][=E9=AB=98=E7=A8=8B=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=AE=A1=E7=90=86=E4=B8=AD=E6=96=87=E4=BB=B6=E6=98=8E?= =?UTF-8?q?=E7=BB=86=E8=A6=81=E5=B1=95=E7=A4=BA=E5=90=84=E4=B8=AA=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=9A=84=E5=9D=90=E6=A0=87=E8=8C=83=E5=9B=B4]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加 ElevationDatasetFileMeta 数据库模型存储文件级别坐标范围 - 更新 API schema 和 service，返回每个文件的 bbox 信息 - 修改高程数据分析任务，遍历目录所有文件并提取坐标范围 - 前端文件明细表格新增坐标范围列 - 创建数据库迁移脚本 Co-Authored-By: Claude Sonnet 4.6 Co-authored-by: multica-agent --- api/app/models/elevation.py | 35 +++++ api/app/schemas/elevation.py | 4 + api/app/services/elevation_service.py | 185 +++++++++++++++++++++++-- migrations/add_elevation_file_meta.sql | 25 ++++ web/src/app/admin/elevation/page.tsx | 18 ++- web/src/types/auth.ts | 4 + 6 files changed, 262 insertions(+), 9 deletions(-) create mode 100644 migrations/add_elevation_file_meta.sql diff --git a/api/app/models/elevation.py b/api/app/models/elevation.py index faad18f..9c2c2ff 100644 --- a/api/app/models/elevation.py +++ b/api/app/models/elevation.py @@ -163,3 +163,38 @@ class ElevationDataImportJob(Base): update_user: Mapped[str | None] = mapped_column(String(64), index=True) dataset: Mapped[ElevationDataset] = relationship("ElevationDataset", lazy="selectin") + + +class ElevationDatasetFileMeta(Base): + __tablename__ = "elevation_dataset_file_meta" + __table_args__ = ( + Index("idx_elevation_file_meta_dataset", "dataset_id"), + Index("idx_elevation_file_meta_path", "dataset_id", "file_path"), + ) + + id: Mapped[str] = mapped_column( + String(32), + primary_key=True, + default=lambda: uuid4().hex, + ) + dataset_id: Mapped[str] = mapped_column( + String(32), + ForeignKey("elevation_dataset.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + file_path: Mapped[str] = mapped_column(String(2048), nullable=False) + file_name: Mapped[str] = mapped_column(String(512), nullable=False) + bbox_min_lon: Mapped[float | None] = mapped_column(Float) + bbox_max_lon: Mapped[float | None] = mapped_column(Float) + bbox_min_lat: Mapped[float | None] = mapped_column(Float) + bbox_max_lat: Mapped[float | None] = mapped_column(Float) + sample_count: Mapped[int] = mapped_column(Integer, default=0) + create_date: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, index=True) + update_date: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utcnow, + onupdate=utcnow, + ) + + dataset: Mapped[ElevationDataset] = relationship("ElevationDataset", lazy="selectin") diff --git a/api/app/schemas/elevation.py b/api/app/schemas/elevation.py index c3cac4c..257ea15 100644 --- a/api/app/schemas/elevation.py +++ b/api/app/schemas/elevation.py @@ -165,6 +165,10 @@ class ElevationDatasetFileItem(BaseModel): size: int modified_at: datetime | None = None mime_type: str | None = None + bbox_min_lon: float | None = None + bbox_max_lon: float | None = None + bbox_min_lat: float | None = None + bbox_max_lat: float | None = None class ElevationDatasetFileListResponse(BaseModel): diff --git a/api/app/services/elevation_service.py b/api/app/services/elevation_service.py index bf9c43d..44b5dc9 100644 --- a/api/app/services/elevation_service.py +++ b/api/app/services/elevation_service.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import Session from ..core.database import SessionLocal from ..models.base import utcnow -from ..models.elevation import ElevationApplyJob, ElevationDataImportJob, ElevationDataset +from ..models.elevation import ElevationApplyJob, ElevationDataImportJob, ElevationDataset, ElevationDatasetFileMeta from ..models.line import Line from ..models.line_tower import LineTower from ..models.user import User @@ -380,6 +380,12 @@ def list_dataset_files( except StorageDriverError as exc: raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc + # Query file metadata with bbox information + stmt = select(ElevationDatasetFileMeta).where( + ElevationDatasetFileMeta.dataset_id == dataset_id + ) + file_metas = {meta.file_path: meta for meta in db.execute(stmt).scalars().all()} + files = [ ElevationDatasetFileItem( path=item.path, @@ -387,6 +393,10 @@ def list_dataset_files( size=max(0, int(item.size)), modified_at=item.modified_at, mime_type=item.mime_type, + bbox_min_lon=file_metas[item.path].bbox_min_lon if item.path in file_metas else None, + bbox_max_lon=file_metas[item.path].bbox_max_lon if item.path in file_metas else None, + bbox_min_lat=file_metas[item.path].bbox_min_lat if item.path in file_metas else None, + bbox_max_lat=file_metas[item.path].bbox_max_lat if item.path in file_metas else None, ) for item in entries if not item.is_dir @@ -2398,16 +2408,53 @@ def _analyze_dataset_content( db: Session, dataset: ElevationDataset, ) -> tuple[dict[str, float | int], list[str]]: + # First, analyze the main dataset file (for overall stats) file_format = _resolve_dataset_file_format(dataset) if file_format == "csv": points, warnings = _load_dataset_points(db, dataset) - return _compute_dataset_stats(points), warnings - if file_format in RASTER_FILE_FORMATS: - return _compute_raster_stats(db, dataset) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"不支持的高程文件格式: {file_format}", - ) + overall_stats = _compute_dataset_stats(points) + elif file_format in RASTER_FILE_FORMATS: + overall_stats, warnings = _compute_raster_stats(db, dataset) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"不支持的高程文件格式: {file_format}", + ) + + # Then, analyze each file in the dataset directory for file-level metadata + try: + mount = _require_mount(db, dataset.mount_code) + driver = _build_driver_or_400(mount) + dataset_dir = _resolve_dataset_dir(dataset.code) + + try: + entries = driver.list_dir(dataset_dir) + except (StoragePathNotFoundError, StorageInvalidPathError, StorageDriverError): + entries = [] + + for entry in entries: + if entry.is_dir: + continue + + file_ext = Path(entry.path).suffix.lower() + if file_ext not in ELEVATION_FILE_EXT_FORMAT_MAP: + continue + + file_stats_result = _analyze_single_file(db, dataset, entry.path) + if file_stats_result is not None: + file_stats, _ = file_stats_result + _store_file_metadata( + db, + dataset_id=dataset.id, + file_path=entry.path, + file_name=entry.name, + stats=file_stats, + ) + except Exception: + # If file-level analysis fails, don't fail the whole dataset analysis + pass + + return overall_stats, warnings def _apply_points_to_line_towers( @@ -3541,3 +3588,125 @@ def _fire_and_forget(coro: object) -> None: close() return loop.create_task(coro) + + +def _analyze_single_file( + db: Session, + dataset: ElevationDataset, + file_path: str, +) -> tuple[dict[str, float | int], list[str]] | None: + """Analyze a single elevation file and return its bbox stats.""" + mount = _require_mount(db, dataset.mount_code) + driver = _build_driver_or_400(mount) + + file_ext = Path(file_path).suffix.lower() + if file_ext not in ELEVATION_FILE_EXT_FORMAT_MAP: + return None + + file_format = ELEVATION_FILE_EXT_FORMAT_MAP[file_ext] + + try: + if file_format == "csv": + read_result = driver.read_file(file_path) + text = _decode_csv_bytes(read_result.content) + rows = list(csv.DictReader(io.StringIO(text))) + if not rows: + return None + + points: list[ElevationSamplePoint] = [] + for row in rows: + lon = _pick_float(row, ["longitude", "lon", "lng", "经度"]) + lat = _pick_float(row, ["latitude", "lat", "纬度"]) + altitude = _pick_float(row, ["altitude_m", "altitude", "elevation", "dem", "海拔m", "高程"]) + if lon is None or lat is None or altitude is None: + continue + if lon < -180 or lon > 180 or lat < -90 or lat > 90: + continue + points.append(ElevationSamplePoint(lon=lon, lat=lat, altitude_m=altitude)) + + if not points: + return None + + lon_values = [p.lon for p in points] + lat_values = [p.lat for p in points] + return { + "sample_count": len(points), + "bbox_min_lon": min(lon_values), + "bbox_max_lon": max(lon_values), + "bbox_min_lat": min(lat_values), + "bbox_max_lat": max(lat_values), + }, [] + + elif file_format in RASTER_FILE_FORMATS: + # For raster files, open and extract bounds + try: + import rasterio + except ImportError: + return None + + read_result = driver.read_file(file_path) + with NamedTemporaryFile(delete=False, suffix=file_ext) as tmp_file: + tmp_file.write(read_result.content) + tmp_path = tmp_file.name + + try: + with rasterio.open(tmp_path) as src: + bounds = src.bounds + wgs84_bounds = _compute_raster_wgs84_bounds(rasterio=rasterio, src=src) + width = int(src.width or 0) + height = int(src.height or 0) + sample_count = width * height + if sample_count > MAX_SAMPLE_COUNT_INT: + sample_count = MAX_SAMPLE_COUNT_INT + + return { + "sample_count": sample_count, + "bbox_min_lon": float(wgs84_bounds["west"]), + "bbox_max_lon": float(wgs84_bounds["east"]), + "bbox_min_lat": float(wgs84_bounds["south"]), + "bbox_max_lat": float(wgs84_bounds["north"]), + }, [] + finally: + import os + os.unlink(tmp_path) + + return None + except Exception: + return None + + +def _store_file_metadata( + db: Session, + dataset_id: str, + file_path: str, + file_name: str, + stats: dict[str, float | int], +) -> None: + """Store or update file metadata in database.""" + stmt = select(ElevationDatasetFileMeta).where( + ElevationDatasetFileMeta.dataset_id == dataset_id, + ElevationDatasetFileMeta.file_path == file_path, + ) + existing = db.execute(stmt).scalar_one_or_none() + + if existing: + existing.bbox_min_lon = stats.get("bbox_min_lon") + existing.bbox_max_lon = stats.get("bbox_max_lon") + existing.bbox_min_lat = stats.get("bbox_min_lat") + existing.bbox_max_lat = stats.get("bbox_max_lat") + existing.sample_count = int(stats.get("sample_count", 0)) + existing.update_date = utcnow() + else: + meta = ElevationDatasetFileMeta( + dataset_id=dataset_id, + file_path=file_path, + file_name=file_name, + bbox_min_lon=stats.get("bbox_min_lon"), + bbox_max_lon=stats.get("bbox_max_lon"), + bbox_min_lat=stats.get("bbox_min_lat"), + bbox_max_lat=stats.get("bbox_max_lat"), + sample_count=int(stats.get("sample_count", 0)), + ) + db.add(meta) + + db.commit() diff --git a/migrations/add_elevation_file_meta.sql b/migrations/add_elevation_file_meta.sql new file mode 100644 index 0000000..91385df --- /dev/null +++ b/migrations/add_elevation_file_meta.sql @@ -0,0 +1,25 @@ +-- Migration: Add elevation_dataset_file_meta table for storing file-level coordinate ranges +-- Date: 2026-06-13 +-- Description: Create new table to store bbox and metadata for each elevation file in a dataset + +CREATE TABLE IF NOT EXISTS elevation_dataset_file_meta ( + id VARCHAR(32) PRIMARY KEY, + dataset_id VARCHAR(32) NOT NULL, + file_path VARCHAR(2048) NOT NULL, + file_name VARCHAR(512) NOT NULL, + bbox_min_lon DOUBLE PRECISION, + bbox_max_lon DOUBLE PRECISION, + bbox_min_lat DOUBLE PRECISION, + bbox_max_lat DOUBLE PRECISION, + sample_count INTEGER DEFAULT 0, + create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + update_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + FOREIGN KEY (dataset_id) REFERENCES elevation_dataset(id) ON DELETE CASCADE +); + +CREATE INDEX idx_elevation_file_meta_dataset ON elevation_dataset_file_meta(dataset_id); +CREATE INDEX idx_elevation_file_meta_path ON elevation_dataset_file_meta(dataset_id, file_path); + +-- Notes: +-- After running this migration, run the elevation dataset analysis task for each dataset +-- to populate the file metadata with coordinate ranges. diff --git a/web/src/app/admin/elevation/page.tsx b/web/src/app/admin/elevation/page.tsx index 9753735..4b4a8ce 100644 --- a/web/src/app/admin/elevation/page.tsx +++ b/web/src/app/admin/elevation/page.tsx @@ -618,6 +618,22 @@ export default function AdminElevationPage() { width: 160, render: (value: string | null) => value || "-", }, + { + title: "坐标范围", + key: "bbox", + width: 320, + render: (_, row) => { + if (row.bbox_min_lon === null || row.bbox_max_lon === null || + row.bbox_min_lat === null || row.bbox_max_lat === null) { + return -; + } + return ( + + {formatNumber(row.bbox_min_lon, 6)}, {formatNumber(row.bbox_min_lat, 6)} ~ {formatNumber(row.bbox_max_lon, 6)}, {formatNumber(row.bbox_max_lat, 6)} + + ); + }, + }, ], [], ); @@ -1489,7 +1505,7 @@ export default function AdminElevationPage() { columns={fileColumns} dataSource={datasetFiles} pagination={false} - scroll={{ x: 1000 }} + scroll={{ x: 1320 }} /> )} diff --git a/web/src/types/auth.ts b/web/src/types/auth.ts index 08cdee1..b8b3084 100644 --- a/web/src/types/auth.ts +++ b/web/src/types/auth.ts @@ -343,6 +343,10 @@ export type ElevationDatasetFileItem = { name: string; size: number; modified_at: string | null; + bbox_min_lon: number | null; + bbox_max_lon: number | null; + bbox_min_lat: number | null; + bbox_max_lat: number | null; mime_type: string | null; };