From c3ebe5cd2fad9f7f7a0bbb3f198a324604024b0c Mon Sep 17 00:00:00 2001 From: Anastasiia Ivanchenko Date: Sat, 25 Jul 2026 12:19:33 +0300 Subject: [PATCH] Derive grid extent and resolution from the store on fetch (#30) Spatial extent, resolution, dimension sizes, and native chunk footprint are no longer hardcoded. They are read from the remote Zarr store's metadata at runtime, with the previous constants kept as a fallback for first paint and error states. - Add GridSpec type and deriveGridSpec() that reads shape, chunks and lon/lat coordinate arrays from the store - Memoize derivation on ZarrChunkReader via getGridSpec() - Thread GridSpec through geogrid helpers and map components, defaulting to DEFAULT_GRID_SPEC - Add unit tests covering derivation and the fallback path --- src/components/map/EarthMap.tsx | 55 +++++++--- src/components/map/GlobeSelectionOverlay.tsx | 18 +++- src/components/map/MapReadout.tsx | 12 ++- src/lib/constants/store.ts | 14 +++ src/lib/map/geogrid.ts | 36 +++++-- src/lib/zarr/ZarrChunkReader.ts | 16 ++- src/lib/zarr/gridSpec.test.ts | 81 +++++++++++++++ src/lib/zarr/gridSpec.ts | 104 +++++++++++++++++++ src/types/map.ts | 29 ++++++ 9 files changed, 330 insertions(+), 35 deletions(-) create mode 100644 src/lib/zarr/gridSpec.test.ts create mode 100644 src/lib/zarr/gridSpec.ts diff --git a/src/components/map/EarthMap.tsx b/src/components/map/EarthMap.tsx index 52cfbda..553984e 100644 --- a/src/components/map/EarthMap.tsx +++ b/src/components/map/EarthMap.tsx @@ -20,7 +20,13 @@ import { import { openZarrStore } from "@/lib/zarr/store"; import { ZarrChunkReader } from "@/lib/zarr/ZarrChunkReader"; import { DEFAULT_HISTORY_YEARS } from "@/lib/zarr/timeRange"; -import type { MapSelection, MapViewMode, MapViewState } from "@/types/map"; +import { DEFAULT_GRID_SPEC } from "@/lib/constants/store"; +import type { + GridSpec, + MapSelection, + MapViewMode, + MapViewState, +} from "@/types/map"; import { useTheme } from "@/providers/ThemeProvider"; import { EditorShell, @@ -73,10 +79,23 @@ export function EarthMap() { const [seriesError, setSeriesError] = useState(null); const [seriesValues, setSeriesValues] = useState(null); const [seriesUnits, setSeriesUnits] = useState(null); + const [gridSpec, setGridSpec] = useState(DEFAULT_GRID_SPEC); const isSphere = viewMode === "sphere"; const mapStyle = isLight ? MAP_BASE_STYLES.light : MAP_BASE_STYLES.dark; + const ensureReader = useCallback(() => { + if (!readerPromiseRef.current) { + readerPromiseRef.current = openZarrStore() + .then((ds) => new ZarrChunkReader(ds)) + .catch((error) => { + readerPromiseRef.current = null; + throw error; + }); + } + return readerPromiseRef.current; + }, []); + useEffect(() => { const node = mapStageRef.current; if (!node) return; @@ -91,6 +110,21 @@ export function EarthMap() { return () => observer.disconnect(); }, []); + useEffect(() => { + let cancelled = false; + ensureReader() + .then((reader) => reader.getGridSpec()) + .then((spec) => { + if (!cancelled) setGridSpec(spec); + }) + .catch(() => { + // Keep DEFAULT_GRID_SPEC on failure; time-series loads surface errors. + }); + return () => { + cancelled = true; + }; + }, [ensureReader]); + const loadTimeSeries = useCallback( async (nextSelection: MapSelection, years: number) => { const requestId = ++requestIdRef.current; @@ -101,16 +135,7 @@ export function EarthMap() { setSeriesUnits(null); try { - if (!readerPromiseRef.current) { - readerPromiseRef.current = openZarrStore() - .then((ds) => new ZarrChunkReader(ds)) - .catch((error) => { - readerPromiseRef.current = null; - throw error; - }); - } - - const reader = await readerPromiseRef.current; + const reader = await ensureReader(); const { values, units } = await reader.getTimeSeries( nextSelection.grid, @@ -140,7 +165,7 @@ export function EarthMap() { } } }, - [], + [ensureReader], ); const flyToView = useCallback((next: MapViewState, duration = 0) => { @@ -168,7 +193,7 @@ export function EarthMap() { (lon: number, lat: number) => { const nextSelection: MapSelection = { click: { lon, lat }, - grid: geoPointToZarrGrid({ lon, lat }), + grid: geoPointToZarrGrid({ lon, lat }, gridSpec), }; setSelection(nextSelection); @@ -180,7 +205,7 @@ export function EarthMap() { flyToView(focused, SELECTION_FOCUS_TRANSITION_MS); void loadTimeSeries(nextSelection, historyYears); }, - [flyToView, historyYears, loadTimeSeries, viewMode, viewState], + [flyToView, gridSpec, historyYears, loadTimeSeries, viewMode, viewState], ); const handleMapClick = useCallback( @@ -260,6 +285,7 @@ export function EarthMap() { sidebar={ 0 && mapSize.height > 0 ? ( { const guidePaths = gridCellToGuidePaths( - gridCellToBounds(cell), + gridCellToBounds(cell, gridSpec), viewportToGeoBounds(viewState, mapSize.width, mapSize.height), ); return selectionGuideGeoJson(cell, guidePaths, { densifyGuides: isSphere, + spec: gridSpec, patchBounds: showPatch - ? chunkPatchBounds(cell, ZARR_STORE.nativeChunks.lat, ZARR_STORE.nativeChunks.lon) + ? chunkPatchBounds( + cell, + gridSpec.nativeChunks.lat, + gridSpec.nativeChunks.lon, + gridSpec, + ) : null, }); - }, [cell, isSphere, mapSize.height, mapSize.width, showPatch, viewState]); + }, [cell, gridSpec, isSphere, mapSize.height, mapSize.width, showPatch, viewState]); const guideColor = rgba( isLight ? SELECTION_GUIDE_COLOR.light : SELECTION_GUIDE_COLOR.dark, diff --git a/src/components/map/MapReadout.tsx b/src/components/map/MapReadout.tsx index ba938c3..e9d97cf 100644 --- a/src/components/map/MapReadout.tsx +++ b/src/components/map/MapReadout.tsx @@ -1,8 +1,8 @@ "use client"; -import type { MapSelection } from "@/types/map"; +import type { GridSpec, MapSelection } from "@/types/map"; import { formatCoordinate, formatGeoPoint } from "@/lib/map/geogrid"; -import { ZARR_STORE } from "@/lib/constants/store"; +import { DEFAULT_GRID_SPEC, ZARR_STORE } from "@/lib/constants/store"; import { ZARR_TIME } from "@/lib/zarr/timeRange"; import { TimeSeriesPlot } from "@/components/map/TimeSeriesPlot"; import { TimeSeriesPlotLoading } from "@/components/map/TimeSeriesPlotLoading"; @@ -12,6 +12,7 @@ type SeriesProgress = { loaded: number; total: number }; type MapReadoutProps = { selection: MapSelection | null; + gridSpec?: GridSpec; historyYears: number; onHistoryYearsChange: (years: number) => void; loadingSeries: boolean; @@ -26,6 +27,7 @@ const META = "font-mono text-[11px] text-editor-fg-tertiary"; export function MapReadout({ selection, + gridSpec = DEFAULT_GRID_SPEC, historyYears, onHistoryYearsChange, loadingSeries, @@ -56,8 +58,8 @@ export function MapReadout({ ); } - const patchCells = ZARR_STORE.nativeChunks.lon; - const patchDeg = patchCells * ZARR_STORE.spatialResolutionDeg; + const patchCells = gridSpec.nativeChunks.lon; + const patchDeg = patchCells * gridSpec.spatialResolutionDeg; const historyControl = (
@@ -122,7 +124,7 @@ export function MapReadout({

{ZARR_STORE.kicker} · snapped to the nearest{" "} - {ZARR_STORE.spatialResolutionDeg}° cell + {gridSpec.spatialResolutionDeg}° cell

Each click downloads a {patchCells}×{patchCells} patch ({patchDeg}° ×{" "} diff --git a/src/lib/constants/store.ts b/src/lib/constants/store.ts index 954f049..fdd3043 100644 --- a/src/lib/constants/store.ts +++ b/src/lib/constants/store.ts @@ -1,3 +1,5 @@ +import type { GridSpec } from "@/types/map"; + /** ESDC v3 low-resolution test cube (2.5° × 16-day). */ export const ZARR_STORE = { id: "nee-d-0.05deg", @@ -32,3 +34,15 @@ export const ZARR_STORE = { } as const; export type ZarrDataset = typeof ZARR_STORE; + +/** + * Fallback grid used for first paint and whenever deriving the spec from the + * remote store fails. Mirrors the hardcoded `ZARR_STORE` values; the live spec + * is fetched at runtime by `@/lib/zarr/gridSpec`. + */ +export const DEFAULT_GRID_SPEC: GridSpec = { + grid: { ...ZARR_STORE.grid }, + dimensions: { ...ZARR_STORE.dimensions }, + nativeChunks: { ...ZARR_STORE.nativeChunks }, + spatialResolutionDeg: ZARR_STORE.spatialResolutionDeg, +}; diff --git a/src/lib/map/geogrid.ts b/src/lib/map/geogrid.ts index e58772f..194f4e2 100644 --- a/src/lib/map/geogrid.ts +++ b/src/lib/map/geogrid.ts @@ -1,7 +1,5 @@ -import { ZARR_STORE } from "@/lib/constants/store"; -import type { GeoPoint, GridCell } from "@/types/map"; - -const { grid, dimensions } = ZARR_STORE; +import { DEFAULT_GRID_SPEC } from "@/lib/constants/store"; +import type { GeoPoint, GridCell, GridSpec } from "@/types/map"; function clamp(value: number, min: number, max: number): number { return Math.min(max, Math.max(min, value)); @@ -19,7 +17,11 @@ function snapToAxis(value: number, start: number, step: number, count: number): } /** Snap a WGS-84 point to the nearest Zarr grid cell center. */ -export function geoPointToZarrGrid(point: GeoPoint): GridCell { +export function geoPointToZarrGrid( + point: GeoPoint, + spec: GridSpec = DEFAULT_GRID_SPEC, +): GridCell { + const { grid, dimensions } = spec; const lon = snapToAxis(point.lon, grid.lonStart, grid.lonStep, dimensions.lon); const lat = snapToAxis(point.lat, grid.latStart, grid.latStep, dimensions.lat); @@ -39,7 +41,11 @@ export type GridCellBounds = { }; /** Half-open cell edges around a snapped cell center. */ -export function gridCellToBounds(cell: GridCell): GridCellBounds { +export function gridCellToBounds( + cell: GridCell, + spec: GridSpec = DEFAULT_GRID_SPEC, +): GridCellBounds { + const { grid } = spec; const halfLon = Math.abs(grid.lonStep) / 2; const halfLat = Math.abs(grid.latStep) / 2; @@ -64,8 +70,11 @@ export function boundsToPolygon(bounds: GridCellBounds): GeoPoint[] { } /** Closed lon/lat ring for the cell footprint (counter-clockwise). */ -export function gridCellToPolygon(cell: GridCell): GeoPoint[] { - return boundsToPolygon(gridCellToBounds(cell)); +export function gridCellToPolygon( + cell: GridCell, + spec: GridSpec = DEFAULT_GRID_SPEC, +): GeoPoint[] { + return boundsToPolygon(gridCellToBounds(cell, spec)); } /** @@ -77,7 +86,9 @@ export function chunkPatchBounds( cell: GridCell, chunkLat: number, chunkLon: number, + spec: GridSpec = DEFAULT_GRID_SPEC, ): GridCellBounds { + const { grid, dimensions } = spec; const firstLatIdx = Math.floor(cell.latIndex / chunkLat) * chunkLat; const lastLatIdx = Math.min(firstLatIdx + chunkLat, dimensions.lat) - 1; const firstLonIdx = Math.floor(cell.lonIndex / chunkLon) * chunkLon; @@ -179,10 +190,15 @@ function closedRing(corners: GeoPoint[]): LonLatPath { export function selectionGuideGeoJson( cell: GridCell, guidePaths: LonLatPath[], - options?: { densifyGuides?: boolean; patchBounds?: GridCellBounds | null }, + options?: { + densifyGuides?: boolean; + patchBounds?: GridCellBounds | null; + spec?: GridSpec; + }, ): SelectionGuideGeoJson { const densifyGuides = options?.densifyGuides ?? true; const patchBounds = options?.patchBounds ?? null; + const spec = options?.spec ?? DEFAULT_GRID_SPEC; return { type: "FeatureCollection", @@ -210,7 +226,7 @@ export function selectionGuideGeoJson( { type: "Feature", properties: { kind: "cell" }, - geometry: { type: "Polygon", coordinates: [closedRing(gridCellToPolygon(cell))] }, + geometry: { type: "Polygon", coordinates: [closedRing(gridCellToPolygon(cell, spec))] }, }, ], }; diff --git a/src/lib/zarr/ZarrChunkReader.ts b/src/lib/zarr/ZarrChunkReader.ts index d75332d..e651ea1 100644 --- a/src/lib/zarr/ZarrChunkReader.ts +++ b/src/lib/zarr/ZarrChunkReader.ts @@ -24,7 +24,8 @@ import { type ZarrArrayHandle, type ZarrStore, } from "@/lib/zarr/store"; -import type { GridCell } from "@/types/map"; +import { deriveGridSpec } from "@/lib/zarr/gridSpec"; +import type { GridCell, GridSpec } from "@/types/map"; type CachedNativeChunk = { data: Float32Array; @@ -46,6 +47,7 @@ export class ZarrChunkReader { private arrayPromises = new Map>(); private chunkLoadsInFlight = new Map>(); private prefetchInFlight = new Map>(); + private gridSpecPromise?: Promise; /** Default holds ~8 pixels of full history (6 native chunks per pixel). */ constructor(ds: ZarrStore, maxCacheSize = 48) { @@ -53,6 +55,18 @@ export class ZarrChunkReader { this.cache = new LRUCache(maxCacheSize); } + /** + * Derive the dataset's spatial grid from store metadata, memoized so the + * coordinate arrays are only read once. Falls back to DEFAULT_GRID_SPEC + * inside deriveGridSpec if the store cannot be read. + */ + getGridSpec(variable = ZARR_STORE.defaultVariable): Promise { + if (!this.gridSpecPromise) { + this.gridSpecPromise = deriveGridSpec(this.ds, variable); + } + return this.gridSpecPromise; + } + private getArray(variable: string): Promise { const existing = this.arrayPromises.get(variable); if (existing) return existing; diff --git a/src/lib/zarr/gridSpec.test.ts b/src/lib/zarr/gridSpec.test.ts new file mode 100644 index 0000000..a6f881e --- /dev/null +++ b/src/lib/zarr/gridSpec.test.ts @@ -0,0 +1,81 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import * as zarr from "zarrita"; +import { deriveGridSpec } from "@/lib/zarr/gridSpec"; +import { DEFAULT_GRID_SPEC } from "@/lib/constants/store"; +import type { ZarrStore } from "@/lib/zarr/store"; + +vi.mock("zarrita", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + open: vi.fn(), + get: vi.fn(), + }; +}); + +const mockOpen = vi.mocked(zarr.open); +const mockGet = vi.mocked(zarr.get); + +const ds = { + store: {}, + root: { resolve: vi.fn((name: string) => name) }, +} as unknown as ZarrStore; + +// First two cell centers per coordinate axis, keyed by resolved array path. +const AXIS_HEADS: Record = { + lon: [-179.975, -179.925], + lat: [89.975, 89.925], +}; + +describe("deriveGridSpec", () => { + beforeEach(() => { + mockOpen.mockReset(); + mockGet.mockReset(); + + mockOpen.mockImplementation(async (loc: unknown) => { + const path = String(loc); + if (path === "NEE") { + return { + shape: [7670, 24, 3600, 7200], + chunks: [1461, 24, 40, 40], + attrs: { _ARRAY_DIMENSIONS: ["time", "hour", "lat", "lon"] }, + } as never; + } + return { shape: [path === "lon" ? 7200 : 3600] } as never; + }); + + mockGet.mockImplementation(async (array: unknown) => { + // The array handle is whatever `open` returned; recover its axis by shape. + const shape = (array as { shape: number[] }).shape; + const head = shape[0] === 7200 ? AXIS_HEADS.lon : AXIS_HEADS.lat; + return { data: head } as never; + }); + }); + + it("reads extent, resolution, dimensions and native chunks from the store", async () => { + const spec = await deriveGridSpec(ds, "NEE"); + + expect(spec.grid).toEqual({ + lonStart: -179.975, + lonStep: expect.closeTo(0.05, 10), + latStart: 89.975, + latStep: expect.closeTo(-0.05, 10), + }); + expect(spec.dimensions).toEqual({ + time: 7670, + hour: 24, + lat: 3600, + lon: 7200, + }); + expect(spec.nativeChunks).toEqual({ lat: 40, lon: 40 }); + expect(spec.spatialResolutionDeg).toBeCloseTo(0.05, 10); + }); + + it("falls back to the default spec when the variable cannot be opened", async () => { + mockOpen.mockRejectedValue(new Error("network")); + + const spec = await deriveGridSpec(ds, "NEE"); + + expect(spec).toBe(DEFAULT_GRID_SPEC); + }); +}); diff --git a/src/lib/zarr/gridSpec.ts b/src/lib/zarr/gridSpec.ts new file mode 100644 index 0000000..86e4dfc --- /dev/null +++ b/src/lib/zarr/gridSpec.ts @@ -0,0 +1,104 @@ +import * as zarr from "zarrita"; +import { DEFAULT_GRID_SPEC, ZARR_STORE } from "@/lib/constants/store"; +import type { GridSpec } from "@/types/map"; +import type { ZarrStore } from "@/lib/zarr/store"; + +/** Axis names we recognize, in the order the data variable is expected to use. */ +const FALLBACK_DIM_ORDER = ["time", "hour", "lat", "lon"] as const; + +type OpenedArray = { + shape: number[]; + chunks: number[]; + attrs: Record; +}; + +async function openArray(store: ZarrStore, name: string): Promise { + const array = await zarr.open(store.root.resolve(name), { kind: "array" }); + return array as unknown as OpenedArray; +} + +/** Dimension names of a variable, from `_ARRAY_DIMENSIONS`, if present. */ +function arrayDimensions(attrs: Record): string[] | null { + const dims = attrs["_ARRAY_DIMENSIONS"]; + if (Array.isArray(dims) && dims.every((d) => typeof d === "string")) { + return dims as string[]; + } + return null; +} + +/** First two cell centers of a 1-D coordinate array → start and step. */ +async function readAxisStartStep( + store: ZarrStore, + name: string, +): Promise<{ start: number; step: number }> { + const array = await openArray(store, name); + const head = await zarr.get(array as Parameters[0], [ + zarr.slice(0, 2), + ]); + const data = head.data as { [index: number]: number; length: number }; + const start = Number(data[0]); + const step = Number(data[1]) - Number(data[0]); + return { start, step }; +} + +/** + * Read the dataset's spatial grid from the remote store's metadata instead of + * trusting the hardcoded constants. Opens the data variable for its shape, + * chunks and dimension order, then reads the first two entries of the `lon` + * and `lat` coordinate arrays to recover each axis start and step. + * + * Any failure (missing array, unexpected layout, network error) falls back to + * `DEFAULT_GRID_SPEC` so the map keeps working against the known store. + */ +export async function deriveGridSpec( + store: ZarrStore, + variable: string = ZARR_STORE.defaultVariable, +): Promise { + try { + const data = await openArray(store, variable); + const dims = arrayDimensions(data.attrs) ?? [...FALLBACK_DIM_ORDER]; + + const fallbackOrder: readonly string[] = FALLBACK_DIM_ORDER; + const axisIndex = (name: string) => { + const idx = dims.indexOf(name); + return idx === -1 ? fallbackOrder.indexOf(name) : idx; + }; + + const timeAxis = axisIndex("time"); + const hourAxis = axisIndex("hour"); + const latAxis = axisIndex("lat"); + const lonAxis = axisIndex("lon"); + + if (latAxis < 0 || lonAxis < 0) return DEFAULT_GRID_SPEC; + + const [lon, lat] = await Promise.all([ + readAxisStartStep(store, dims[lonAxis] ?? "lon"), + readAxisStartStep(store, dims[latAxis] ?? "lat"), + ]); + + const dimensionSize = (axis: number, fallback: number) => + Number.isInteger(data.shape[axis]) ? data.shape[axis] : fallback; + + return { + grid: { + lonStart: lon.start, + lonStep: lon.step, + latStart: lat.start, + latStep: lat.step, + }, + dimensions: { + time: dimensionSize(timeAxis, DEFAULT_GRID_SPEC.dimensions.time), + hour: dimensionSize(hourAxis, DEFAULT_GRID_SPEC.dimensions.hour), + lat: data.shape[latAxis], + lon: data.shape[lonAxis], + }, + nativeChunks: { + lat: data.chunks[latAxis] ?? DEFAULT_GRID_SPEC.nativeChunks.lat, + lon: data.chunks[lonAxis] ?? DEFAULT_GRID_SPEC.nativeChunks.lon, + }, + spatialResolutionDeg: Math.abs(lon.step), + }; + } catch { + return DEFAULT_GRID_SPEC; + } +} diff --git a/src/types/map.ts b/src/types/map.ts index 939b62c..660fd7a 100644 --- a/src/types/map.ts +++ b/src/types/map.ts @@ -10,6 +10,35 @@ export type GridCell = GeoPoint & { latIndex: number; }; +/** + * Everything the map geometry needs to know about the dataset's spatial grid. + * Hardcoded as a fallback in `@/lib/constants/store` and derived from the + * remote store's metadata at runtime by `@/lib/zarr/gridSpec`. + */ +export type GridSpec = { + /** Cell-center coordinates of the first cell and per-cell step on each axis. */ + grid: { + lonStart: number; + lonStep: number; + latStart: number; + latStep: number; + }; + /** Sizes of each array dimension. */ + dimensions: { + time: number; + hour: number; + lon: number; + lat: number; + }; + /** On-disk chunk footprint along lat/lon (cells per patch side). */ + nativeChunks: { + lat: number; + lon: number; + }; + /** Cell size in degrees (absolute lon step). */ + spatialResolutionDeg: number; +}; + export type MapSelection = { click: GeoPoint; grid: GridCell;