Merge pull request #194 from JohnLCaron/all_btree

Use readChunksConcurrent for all btree1 reading.

Author: JohnLCaron

Readme.md

@@ -1,5 +1,5 @@
 # netchdf
-_last updated: 7/22/2025_
+_last updated: 7/25/2025_
 
 This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. 
 
@@ -133,7 +133,7 @@ For HDF5 files using deflate filters, the deflate library dominates the read tim
 are about 2X slower than native code. Unless the deflate libraries get better, there's not much gain in trying to make
 other parts of the code faster.
 
-We will investigate using Kotlin coroutines to speed up performance bottlenecks.
+We are seeing 10x speedup on data reading. see https://github.com/JohnLCaron/netchdf/issues/189.
 
 
 ### Goals and scope

core/src/commonMain/kotlin/com/sunya/cdm/api/Netchdf.kt

@@ -20,8 +20,9 @@ interface Netchdf : AutoCloseable {
     // iterate over all the chunks in section, order is arbitrary. TODO where is intersection with wantSection done ??
     fun <T> chunkIterator(v2: Variable<T>, wantSection: SectionPartial? = null, maxElements : Int? = null) : Iterator<ArraySection<T>>
 
+    // iterate over all the chunks in section, order is arbitrary, callbacks are in multiple threads.
     fun <T> readChunksConcurrent(v2: Variable<T>,
-                                 lamda : (ArraySection<*>) -> Unit,
+                                 lamda : (ArraySection<T>) -> Unit,
                                  done : () -> Unit,
                                  wantSection: SectionPartial? = null,
                                  nthreads: Int? = null) {
@@ -30,9 +31,9 @@ interface Netchdf : AutoCloseable {
 }
 
 // the section describes the array chunk reletive to the variable's shape.
-data class ArraySection<T>(val array : ArrayTyped<T>, val section : Section) {
+data class ArraySection<T>(val array : ArrayTyped<T>, val chunkSection : Section) {
     fun intersect(wantSection: SectionPartial) : ArrayTyped<T> {
-        // TODO
+        // TODO ??
         return array
     }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayByte.kt

@@ -37,4 +37,10 @@ class ArrayByte(shape : IntArray, val values: ByteArray) : ArrayTyped<Byte>(Data
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as ByteArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayDouble.kt

@@ -36,4 +36,10 @@ class ArrayDouble(shape : IntArray, val values: DoubleArray) : ArrayTyped<Double
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as DoubleArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayFloat.kt

@@ -37,4 +37,10 @@ class ArrayFloat(shape : IntArray, val values: FloatArray) : ArrayTyped<Float>(D
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as FloatArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayInt.kt

@@ -36,4 +36,10 @@ class ArrayInt(shape : IntArray, val values: IntArray) : ArrayTyped<Int>(Datatyp
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as IntArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayLong.kt

@@ -36,4 +36,10 @@ class ArrayLong(shape : IntArray, val values: LongArray) : ArrayTyped<Long>(Data
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as LongArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayOpaque.kt

@@ -75,5 +75,10 @@ class ArrayOpaque(shape : IntArray, val values : List<ByteArray>, val size : Int
         }
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as MutableList<ByteArray>
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayShort.kt

@@ -36,4 +36,10 @@ class ArrayShort(shape : IntArray, val values: ShortArray) : ArrayTyped<Short>(D
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as ShortArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayString.kt

@@ -8,6 +8,7 @@ import com.fleeksoft.charset.decodeToString
 import com.sunya.cdm.api.*
 import com.sunya.cdm.layout.IndexND
 import com.sunya.cdm.layout.IndexSpace
+import com.sunya.cdm.layout.TransferChunk
 
 // fake ByteBuffer
 class ArrayString(shape : IntArray, val values : List<String>) : ArrayTyped<String>(Datatype.STRING, shape) {
@@ -37,6 +38,12 @@ class ArrayString(shape : IntArray, val values : List<String>) : ArrayTyped<Stri
         }
         return ArrayString(section.shape.toIntArray(), sectionList)
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as MutableList<String>
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }
 
 /**