From 0f7790004962c6120b7dad27669361438a72ba37 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Thu, 24 Sep 2026 08:01:34 +0200 Subject: [PATCH] Generate regular GRIB coordinates lazily --- .../grib/collection/GribCoordinateReader.java | 40 ++++++++ .../nc2/grib/collection/GribIospBuilder.java | 12 +-- .../collection/TestGribCoordinateReader.java | 91 +++++++++++++++++++ 3 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 grib/src/main/java/ucar/nc2/grib/collection/GribCoordinateReader.java create mode 100644 grib/src/test/java/ucar/nc2/grib/collection/TestGribCoordinateReader.java diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCoordinateReader.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCoordinateReader.java new file mode 100644 index 0000000000..b15cf366d7 --- /dev/null +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCoordinateReader.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.collection; + +import ucar.ma2.Array; +import ucar.ma2.IndexIterator; +import ucar.ma2.Section; +import ucar.nc2.ProxyReader; +import ucar.nc2.Variable; +import ucar.nc2.util.CancelTask; + +/** Generates a regular horizontal coordinate only when it is read. */ +final class GribCoordinateReader implements ProxyReader { + private final double start; + private final double increment; + + GribCoordinateReader(double start, double increment) { + this.start = start; + this.increment = increment; + } + + @Override + public Array reallyRead(Variable client, CancelTask cancelTask) { + return Array.makeArray(client.getDataType(), (int) client.getSize(), start, increment); + } + + @Override + public Array reallyRead(Variable client, Section section, CancelTask cancelTask) { + Array result = Array.factory(client.getDataType(), section.getShape()); + IndexIterator iterator = result.getIndexIterator(); + for (int index : section.getRange(0)) { + // Preserve the full coordinate's arithmetic, including rounding, for sections and strides. + iterator.setDoubleNext(start + index * increment); + } + return result; + } +} diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribIospBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/GribIospBuilder.java index b61a3173c6..f042d6f3ff 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribIospBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribIospBuilder.java @@ -100,13 +100,13 @@ private void makeGroup(Group.Builder g, GribCollectionImmutable.GroupGC group, G g.addVariable(rlat); rlat.addAttribute(new Attribute(CF.STANDARD_NAME, CF.GRID_LATITUDE)); rlat.addAttribute(new Attribute(CDM.UNITS, CDM.RLATLON_UNITS)); - rlat.setCachedData(Array.makeArray(DataType.FLOAT, hcs.ny, hcs.starty, hcs.dy), false); + rlat.setProxyReader(new GribCoordinateReader(hcs.starty, hcs.dy)); Variable.Builder rlon = Variable.builder().setName("rlon").setDataType(DataType.FLOAT).setParentGroupBuilder(g) .setDimensionsByName("rlon"); g.addVariable(rlon); rlon.addAttribute(new Attribute(CF.STANDARD_NAME, CF.GRID_LONGITUDE)); rlon.addAttribute(new Attribute(CDM.UNITS, CDM.RLATLON_UNITS)); - rlon.setCachedData(Array.makeArray(DataType.FLOAT, hcs.nx, hcs.startx, hcs.dx), false); + rlon.setProxyReader(new GribCoordinateReader(hcs.startx, hcs.dx)); } else if (isLatLon2D) { // CurvilinearOrthogonal - lat and lon fields must be present in the file horizDims = "lat lon"; @@ -135,14 +135,14 @@ private void makeGroup(Group.Builder g, GribCollectionImmutable.GroupGC group, G lat.setCachedData(hcs.getGaussianLats(), false); lat.addAttribute(new Attribute(CDM.GAUSSIAN, "true")); } else { - lat.setCachedData(Array.makeArray(DataType.FLOAT, hcs.ny, hcs.starty, hcs.dy), false); + lat.setProxyReader(new GribCoordinateReader(hcs.starty, hcs.dy)); } Variable.Builder lon = Variable.builder().setName("lon").setDataType(DataType.FLOAT).setParentGroupBuilder(g) .setDimensionsByName("lon"); g.addVariable(lon); lon.addAttribute(new Attribute(CDM.UNITS, CDM.LON_UNITS)); - lon.setCachedData(Array.makeArray(DataType.FLOAT, hcs.nx, hcs.startx, hcs.dx), false); + lon.setProxyReader(new GribCoordinateReader(hcs.startx, hcs.dx)); } else { // make horiz coordsys coordinate variable @@ -162,14 +162,14 @@ private void makeGroup(Group.Builder g, GribCollectionImmutable.GroupGC group, G g.addVariable(xcv); xcv.addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_X_COORDINATE)); xcv.addAttribute(new Attribute(CDM.UNITS, "km")); - xcv.setCachedData(Array.makeArray(DataType.FLOAT, hcs.nx, hcs.startx, hcs.dx), false); + xcv.setProxyReader(new GribCoordinateReader(hcs.startx, hcs.dx)); Variable.Builder ycv = Variable.builder().setName("y").setDataType(DataType.FLOAT).setParentGroupBuilder(g).setDimensionsByName("y"); g.addVariable(ycv); ycv.addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_Y_COORDINATE)); ycv.addAttribute(new Attribute(CDM.UNITS, "km")); - ycv.setCachedData(Array.makeArray(DataType.FLOAT, hcs.ny, hcs.starty, hcs.dy), false); + ycv.setProxyReader(new GribCoordinateReader(hcs.starty, hcs.dy)); } for (Coordinate coord : group.coords) { diff --git a/grib/src/test/java/ucar/nc2/grib/collection/TestGribCoordinateReader.java b/grib/src/test/java/ucar/nc2/grib/collection/TestGribCoordinateReader.java new file mode 100644 index 0000000000..53a4cf96d3 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/collection/TestGribCoordinateReader.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.collection; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.ma2.Section; +import ucar.nc2.Group; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; +import ucar.unidata.util.test.TestDir; + +public class TestGribCoordinateReader { + @Test + public void fullAndSectionReadsMatchEagerCoordinates() throws IOException, InvalidRangeException { + for (double[] axis : new double[][] {{0, 1}, {-180, 0.1}, {90, -0.01}, {103.829457843, 0.0000011234567}, + {-0.1, 0.1}, {100000, -123.456789}, {1, 0}}) { + Variable coordinate = coordinate(8193, axis[0], axis[1]); + Array eager = Array.makeArray(DataType.FLOAT, 8193, axis[0], axis[1]); + assertThat(coordinate.hasCachedData()).isFalse(); + Section section = new Section("129:8100:31"); + assertThat(coordinate.read(section).copyTo1DJavaArray()) + .isEqualTo(eager.sectionNoReduce(section.getRanges()).copyTo1DJavaArray()); + assertThat(coordinate.hasCachedData()).isFalse(); + assertThat(coordinate.read().copyTo1DJavaArray()).isEqualTo(eager.copyTo1DJavaArray()); + } + } + + @Test + public void returnedDataDoesNotChangeSubsequentReads() throws IOException, InvalidRangeException { + // Exercise both automatically cached small axes and uncached large axes. + for (int length : new int[] {8, 8193}) { + Variable coordinate = coordinate(length, -180, 0.1); + Array first = coordinate.read(); + first.setFloat(0, 999); + assertThat(coordinate.read().getFloat(0)).isEqualTo(-180); + Array section = coordinate.read(new Section("0:3")); + section.setFloat(0, 999); + assertThat(coordinate.read(new Section("0:3")).getFloat(0)).isEqualTo(-180); + } + } + + @Test + public void sectionSliceAndCopiedVariablePreserveCoordinates() throws IOException, InvalidRangeException { + Variable coordinate = coordinate(8193, 103.829457843, 0.0000011234567); + Array eager = Array.makeArray(DataType.FLOAT, 8193, 103.829457843, 0.0000011234567); + Section first = new Section("7:8000:3"); + Section second = new Section("11:37:2"); + Array expected = eager.sectionNoReduce(first.getRanges()).sectionNoReduce(second.getRanges()); + assertThat(coordinate.section(first).read(second).copyTo1DJavaArray()).isEqualTo(expected.copyTo1DJavaArray()); + assertThat(coordinate.slice(0, 77).readScalarFloat()).isEqualTo(eager.getFloat(77)); + assertThat(coordinate.toBuilder().build(Group.builder().build()).read().copyTo1DJavaArray()) + .isEqualTo(eager.copyTo1DJavaArray()); + } + + @Test + public void openingGribDoesNotMaterializeRegularCoordinates() throws IOException { + for (String file : new String[] {"cosmo-eu.grib2", "sref.pds2.grib2", "HLYA10.grib2"}) { + try (NetcdfFile nc = NetcdfFiles.open(TestDir.localTestDataDir + file)) { + int coordinates = 0; + for (Variable variable : nc.getVariables()) { + if (variable.isCoordinateVariable() && variable.getDataType() == DataType.FLOAT) { + String name = variable.getShortName(); + if (name.equals("x") || name.equals("y") || name.equals("lat") || name.equals("lon") || name.equals("rlat") + || name.equals("rlon")) { + assertThat(variable.hasCachedData()).isFalse(); + Array data = variable.read(); + assertThat(data.getShape()).isEqualTo(variable.getShape()); + coordinates++; + } + } + } + assertThat(coordinates).isEqualTo(2); + } + } + } + + private static Variable coordinate(int length, double start, double increment) { + return Variable.builder().setName("x").setDataType(DataType.FLOAT).setDimensionsAnonymous(new int[] {length}) + .setProxyReader(new GribCoordinateReader(start, increment)).build(Group.builder().build()); + } +}