From eb17a1cd00f6f73a202e4c74098573b30f937ca0 Mon Sep 17 00:00:00 2001 From: RecursivePineapple Date: Fri, 28 Nov 2025 13:07:12 -0500 Subject: [PATCH 1/4] Add batched reads (cherry picked from commit 454d8545411b393b0ebe07299970fd2029c325a9) --- .../regionlib/api/region/BatchReadResult.java | 18 +++++++ .../regionlib/api/region/IRegion.java | 26 ++++++++++ .../regionlib/api/storage/SaveSection.java | 47 +++++++++++++++++++ .../regionlib/impl/SaveCubeColumns.java | 32 +++++++++++++ .../cubicchunks/regionlib/lib/Region.java | 7 ++- 5 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java diff --git a/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java b/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java new file mode 100644 index 0000000..24fdcac --- /dev/null +++ b/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java @@ -0,0 +1,18 @@ +package cubicchunks.regionlib.api.region; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + +public class BatchReadResult { + + public final ImmutableMap read; + public final ImmutableMap errored; + + public BatchReadResult(Map read, Map errored) { + this.read = ImmutableMap.copyOf(read); + this.errored = ImmutableMap.copyOf(errored); + } +} diff --git a/src/main/java/cubicchunks/regionlib/api/region/IRegion.java b/src/main/java/cubicchunks/regionlib/api/region/IRegion.java index b232d69..26907c1 100644 --- a/src/main/java/cubicchunks/regionlib/api/region/IRegion.java +++ b/src/main/java/cubicchunks/regionlib/api/region/IRegion.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -101,6 +103,30 @@ default void writeValues(Map entries) throws IOException { */ Optional readValue(K key) throws IOException; + /** + * Reads multiple values at their corresponding keys within this region + * + * @param keys The locations to read + * @return An object containing all locations that were successfully read and all exceptions from locations that failed + */ + default BatchReadResult readValues(Collection keys) { + HashMap out = new HashMap<>(keys.size()); + HashMap errors = new HashMap<>(); + + for (K key : keys) { + try { + //attempt to read one value at a time + Optional result = readValue(key); + + if (result.isPresent()) out.put(key, result.get()); + } catch (IOException e) { + errors.put(key, e); + } + } + + return new BatchReadResult<>(out, errors); + } + /** * Returns true if something was stored there before within this region. */ diff --git a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java index 1f0f8db..e89a1dd 100644 --- a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java +++ b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java @@ -23,8 +23,15 @@ */ package cubicchunks.regionlib.api.storage; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.Multimaps; import cubicchunks.regionlib.MultiUnsupportedDataException; import cubicchunks.regionlib.UnsupportedDataException; +import cubicchunks.regionlib.api.region.BatchReadResult; import cubicchunks.regionlib.api.region.IRegion; import cubicchunks.regionlib.api.region.IRegionProvider; import cubicchunks.regionlib.api.region.key.IKey; @@ -33,6 +40,7 @@ import cubicchunks.regionlib.util.CheckedFunction; import cubicchunks.regionlib.util.SaveSectionException; import cubicchunks.regionlib.util.Utils; +import io.netty.buffer.ByteBuf; import java.io.Closeable; import java.io.Flushable; @@ -40,6 +48,7 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.*; +import java.util.Map.Entry; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -193,6 +202,44 @@ public Optional load(K key, boolean createRegion) throws IOException return Optional.empty(); } + public BatchReadResult load(Collection positions, boolean createRegion) throws IOException { + ListMultimap exceptions = MultimapBuilder.hashKeys().arrayListValues().build(); + + //group positions into batches based on their containing region + Map> positionsByRegion = positions.stream().collect(Collectors.groupingBy(IKey::getRegionKey, Collectors.toList())); + + HashMap loaded = new HashMap<>(positions.size()); + + for (List inRegion : positionsByRegion.values()) { + CheckedConsumer, IOException> reader = region -> { + BatchReadResult result = region.readValues(inRegion); + + loaded.putAll(result.read); + result.errored.forEach(exceptions::put); + }; + + //for each position group (corresponding to a single region), emulate behavior of save(key, value) + for (IRegionProvider prov : regionProviders) { + + //we can safely retrieve element 0 as an arbitrary member of the positions in this region, as the list is guaranteed to be non-empty + if (createRegion) { + prov.forRegion(inRegion.get(0), reader); + } else { + prov.forExistingRegion(inRegion.get(0), reader); + } + } + } + + Map errored = Multimaps.asMap(exceptions) + .entrySet() + .stream() + .collect(Collectors.toMap( + Entry::getKey, + e -> new SaveSectionException("Could not load object: " + e.getKey(), e.getValue()))); + + return new BatchReadResult<>(loaded, errored); + } + /** * Gets a {@link Stream} over all the already saved keys. *

diff --git a/src/main/java/cubicchunks/regionlib/impl/SaveCubeColumns.java b/src/main/java/cubicchunks/regionlib/impl/SaveCubeColumns.java index c14d046..e504581 100644 --- a/src/main/java/cubicchunks/regionlib/impl/SaveCubeColumns.java +++ b/src/main/java/cubicchunks/regionlib/impl/SaveCubeColumns.java @@ -28,9 +28,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; +import java.util.Collection; import java.util.Map; import java.util.Optional; +import cubicchunks.regionlib.api.region.BatchReadResult; import cubicchunks.regionlib.impl.save.SaveSection2D; import cubicchunks.regionlib.impl.save.SaveSection3D; import cubicchunks.regionlib.util.Utils; @@ -121,6 +123,21 @@ public Optional load(EntryLocation3D location, boolean createRegion) return saveSection3D.load(location, createRegion); } + /** + * Reads entry at given location. + *

+ * This can be accessed from multiple threads. (thread safe) + * + * @param positions the locations of the entry data to load + * @param createRegion if true, a new region file will be created and cached. This is the preferred option. + * @throws IOException when an unexpected IO error occurs + * + * @return An Optional containing the value if it exists + */ + public BatchReadResult load3D(Collection positions, boolean createRegion) throws IOException { + return saveSection3D.load(positions, createRegion); + } + /** * Reads entry at given location. *

@@ -136,6 +153,21 @@ public Optional load(EntryLocation2D location, boolean createRegion) return saveSection2D.load(location, createRegion); } + /** + * Reads entry at given location. + *

+ * This can be accessed from multiple threads. (thread safe) + * + * @param positions the locations of the entry data to load + * @param createRegion if true, a new region file will be created and cached. This is the preferred option. + * @throws IOException when an unexpected IO error occurs + * + * @return An Optional containing the value if it exists + */ + public BatchReadResult load2D(Collection positions, boolean createRegion) throws IOException { + return saveSection2D.load(positions, createRegion); + } + /** * @param directory directory for the save * @throws IOException when an unexpected IO error occurs diff --git a/src/main/java/cubicchunks/regionlib/lib/Region.java b/src/main/java/cubicchunks/regionlib/lib/Region.java index e4736c4..ea6ca7f 100644 --- a/src/main/java/cubicchunks/regionlib/lib/Region.java +++ b/src/main/java/cubicchunks/regionlib/lib/Region.java @@ -27,6 +27,7 @@ import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; +import cubicchunks.regionlib.api.region.BatchReadResult; import cubicchunks.regionlib.api.region.IRegion; import cubicchunks.regionlib.api.region.IRegionProvider; import cubicchunks.regionlib.api.region.header.IHeaderDataEntryProvider; @@ -47,6 +48,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.function.BiConsumer; @@ -123,7 +125,8 @@ private void updateHeaders(K key) throws IOException { } } - @Override public synchronized Optional readValue(K key) throws IOException { + @Override + public synchronized Optional readValue(K key) throws IOException { // a hack because Optional can't throw checked exceptions try { return sectorMap.trySpecialValue(key) @@ -134,7 +137,7 @@ private void updateHeaders(K key) throws IOException { } } - private Optional doReadKey(K key) { + private Optional doReadKey(K key) { return sectorMap.getEntryLocation(key).flatMap(loc -> { try { int sectorOffset = loc.getOffset(); From b04a97f8cbcb2d762cfb885ae962e5df7e0af333 Mon Sep 17 00:00:00 2001 From: RecursivePineapple Date: Thu, 28 May 2026 15:08:53 -0400 Subject: [PATCH 2/4] Remove google common collections --- .../regionlib/api/region/BatchReadResult.java | 10 ++++------ .../regionlib/api/storage/SaveSection.java | 15 +++++---------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java b/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java index 24fdcac..122a1dd 100644 --- a/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java +++ b/src/main/java/cubicchunks/regionlib/api/region/BatchReadResult.java @@ -4,15 +4,13 @@ import java.nio.ByteBuffer; import java.util.Map; -import com.google.common.collect.ImmutableMap; - public class BatchReadResult { - public final ImmutableMap read; - public final ImmutableMap errored; + public final Map read; + public final Map errored; public BatchReadResult(Map read, Map errored) { - this.read = ImmutableMap.copyOf(read); - this.errored = ImmutableMap.copyOf(errored); + this.read = read; + this.errored = errored; } } diff --git a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java index e89a1dd..02eafbc 100644 --- a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java +++ b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java @@ -23,12 +23,6 @@ */ package cubicchunks.regionlib.api.storage; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.MultimapBuilder; -import com.google.common.collect.Multimaps; import cubicchunks.regionlib.MultiUnsupportedDataException; import cubicchunks.regionlib.UnsupportedDataException; import cubicchunks.regionlib.api.region.BatchReadResult; @@ -40,7 +34,6 @@ import cubicchunks.regionlib.util.CheckedFunction; import cubicchunks.regionlib.util.SaveSectionException; import cubicchunks.regionlib.util.Utils; -import io.netty.buffer.ByteBuf; import java.io.Closeable; import java.io.Flushable; @@ -203,7 +196,7 @@ public Optional load(K key, boolean createRegion) throws IOException } public BatchReadResult load(Collection positions, boolean createRegion) throws IOException { - ListMultimap exceptions = MultimapBuilder.hashKeys().arrayListValues().build(); + Map> exceptions = new HashMap<>(); //group positions into batches based on their containing region Map> positionsByRegion = positions.stream().collect(Collectors.groupingBy(IKey::getRegionKey, Collectors.toList())); @@ -215,7 +208,9 @@ public BatchReadResult load(Collection positions, boolean createRegion) th BatchReadResult result = region.readValues(inRegion); loaded.putAll(result.read); - result.errored.forEach(exceptions::put); + result.errored.forEach((k, v) -> { + exceptions.computeIfAbsent(k, $ -> new ArrayList<>()).add(v); + }); }; //for each position group (corresponding to a single region), emulate behavior of save(key, value) @@ -230,7 +225,7 @@ public BatchReadResult load(Collection positions, boolean createRegion) th } } - Map errored = Multimaps.asMap(exceptions) + Map errored = exceptions .entrySet() .stream() .collect(Collectors.toMap( From 6d12f61ab8028cd68bafd4293c1aa60b9bfa76b8 Mon Sep 17 00:00:00 2001 From: RecursivePineapple Date: Thu, 28 May 2026 15:34:20 -0400 Subject: [PATCH 3/4] Cleanup + documentation --- .../java/cubicchunks/regionlib/api/storage/SaveSection.java | 5 +++++ src/main/java/cubicchunks/regionlib/lib/Region.java | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java index 02eafbc..34aaeae 100644 --- a/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java +++ b/src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java @@ -195,6 +195,11 @@ public Optional load(K key, boolean createRegion) throws IOException return Optional.empty(); } + /// Groups position keys based on their region, so that we can call + /// [IRegionProvider#forRegion(IKey, CheckedConsumer)] as little as possible. + /// Most positions will be in a few regions, so loading a new region for each is pointless. + /// Regions could also parallelize [IRegion#readValues(Collection)], but currently the default implementation is + /// just a simple for-each loop. public BatchReadResult load(Collection positions, boolean createRegion) throws IOException { Map> exceptions = new HashMap<>(); diff --git a/src/main/java/cubicchunks/regionlib/lib/Region.java b/src/main/java/cubicchunks/regionlib/lib/Region.java index ea6ca7f..83b1f61 100644 --- a/src/main/java/cubicchunks/regionlib/lib/Region.java +++ b/src/main/java/cubicchunks/regionlib/lib/Region.java @@ -27,7 +27,6 @@ import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; -import cubicchunks.regionlib.api.region.BatchReadResult; import cubicchunks.regionlib.api.region.IRegion; import cubicchunks.regionlib.api.region.IRegionProvider; import cubicchunks.regionlib.api.region.header.IHeaderDataEntryProvider; From 48e99965861da962af737182675417384b327584 Mon Sep 17 00:00:00 2001 From: RecursivePineapple Date: Thu, 28 May 2026 15:41:20 -0400 Subject: [PATCH 4/4] missed one --- src/main/java/cubicchunks/regionlib/lib/Region.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/cubicchunks/regionlib/lib/Region.java b/src/main/java/cubicchunks/regionlib/lib/Region.java index 83b1f61..940b388 100644 --- a/src/main/java/cubicchunks/regionlib/lib/Region.java +++ b/src/main/java/cubicchunks/regionlib/lib/Region.java @@ -47,7 +47,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.function.BiConsumer;