Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package cubicchunks.regionlib.api.region;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;

public class BatchReadResult<K> {

public final Map<K, ByteBuffer> read;
public final Map<K, IOException> errored;

public BatchReadResult(Map<K, ByteBuffer> read, Map<K, IOException> errored) {
this.read = read;
this.errored = errored;
}
}
26 changes: 26 additions & 0 deletions src/main/java/cubicchunks/regionlib/api/region/IRegion.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -101,6 +103,30 @@ default void writeValues(Map<K, ByteBuffer> entries) throws IOException {
*/
Optional<ByteBuffer> readValue(K key) throws IOException;

/**
* Reads multiple values at their corresponding keys within this region
*
* @param keys The locations to read
* @return An object containing all locations that were successfully read and all exceptions from locations that failed
*/
default BatchReadResult<K> readValues(Collection<K> keys) {
HashMap<K, ByteBuffer> out = new HashMap<>(keys.size());
HashMap<K, IOException> errors = new HashMap<>();

for (K key : keys) {
try {
//attempt to read one value at a time
Optional<ByteBuffer> result = readValue(key);

if (result.isPresent()) out.put(key, result.get());
} catch (IOException e) {
errors.put(key, e);
}
}

return new BatchReadResult<>(out, errors);
}

/**
* Returns true if something was stored there before within this region.
*/
Expand Down
47 changes: 47 additions & 0 deletions src/main/java/cubicchunks/regionlib/api/storage/SaveSection.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import cubicchunks.regionlib.MultiUnsupportedDataException;
import cubicchunks.regionlib.UnsupportedDataException;
import cubicchunks.regionlib.api.region.BatchReadResult;
import cubicchunks.regionlib.api.region.IRegion;
import cubicchunks.regionlib.api.region.IRegionProvider;
import cubicchunks.regionlib.api.region.key.IKey;
Expand All @@ -40,6 +41,7 @@
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.Map.Entry;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -193,6 +195,51 @@ public Optional<ByteBuffer> load(K key, boolean createRegion) throws IOException
return Optional.empty();
}

/// Groups position keys based on their region, so that we can call
/// [IRegionProvider#forRegion(IKey, CheckedConsumer)] as little as possible.
/// Most positions will be in a few regions, so loading a new region for each is pointless.
/// Regions could also parallelize [IRegion#readValues(Collection)], but currently the default implementation is
/// just a simple for-each loop.
public BatchReadResult<K> load(Collection<K> positions, boolean createRegion) throws IOException {
Map<K, List<IOException>> exceptions = new HashMap<>();

//group positions into batches based on their containing region
Map<RegionKey, List<K>> positionsByRegion = positions.stream().collect(Collectors.groupingBy(IKey::getRegionKey, Collectors.toList()));

HashMap<K, ByteBuffer> loaded = new HashMap<>(positions.size());

for (List<K> inRegion : positionsByRegion.values()) {
CheckedConsumer<IRegion<K>, IOException> reader = region -> {
BatchReadResult<K> result = region.readValues(inRegion);

loaded.putAll(result.read);
result.errored.forEach((k, v) -> {
exceptions.computeIfAbsent(k, $ -> new ArrayList<>()).add(v);
});
};

//for each position group (corresponding to a single region), emulate behavior of save(key, value)
for (IRegionProvider<K> prov : regionProviders) {

//we can safely retrieve element 0 as an arbitrary member of the positions in this region, as the list is guaranteed to be non-empty
if (createRegion) {
prov.forRegion(inRegion.get(0), reader);
} else {
prov.forExistingRegion(inRegion.get(0), reader);
}
}
}

Map<K, IOException> errored = exceptions
.entrySet()
.stream()
.collect(Collectors.toMap(
Entry::getKey,
e -> new SaveSectionException("Could not load object: " + e.getKey(), e.getValue())));

return new BatchReadResult<>(loaded, errored);
}

/**
* Gets a {@link Stream} over all the already saved keys.
* <p>
Expand Down
32 changes: 32 additions & 0 deletions src/main/java/cubicchunks/regionlib/impl/SaveCubeColumns.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;

import cubicchunks.regionlib.api.region.BatchReadResult;
import cubicchunks.regionlib.impl.save.SaveSection2D;
import cubicchunks.regionlib.impl.save.SaveSection3D;
import cubicchunks.regionlib.util.Utils;
Expand Down Expand Up @@ -121,6 +123,21 @@ public Optional<ByteBuffer> load(EntryLocation3D location, boolean createRegion)
return saveSection3D.load(location, createRegion);
}

/**
* Reads entry at given location.
* <p>
* This can be accessed from multiple threads. (thread safe)
*
* @param positions the locations of the entry data to load
* @param createRegion if true, a new region file will be created and cached. This is the preferred option.
* @throws IOException when an unexpected IO error occurs
*
* @return An Optional containing the value if it exists
*/
public BatchReadResult<EntryLocation3D> load3D(Collection<EntryLocation3D> positions, boolean createRegion) throws IOException {
return saveSection3D.load(positions, createRegion);
}

/**
* Reads entry at given location.
* <p>
Expand All @@ -136,6 +153,21 @@ public Optional<ByteBuffer> load(EntryLocation2D location, boolean createRegion)
return saveSection2D.load(location, createRegion);
}

/**
* Reads entry at given location.
* <p>
* This can be accessed from multiple threads. (thread safe)
*
* @param positions the locations of the entry data to load
* @param createRegion if true, a new region file will be created and cached. This is the preferred option.
* @throws IOException when an unexpected IO error occurs
*
* @return An Optional containing the value if it exists
*/
public BatchReadResult<EntryLocation2D> load2D(Collection<EntryLocation2D> positions, boolean createRegion) throws IOException {
return saveSection2D.load(positions, createRegion);
}

/**
* @param directory directory for the save
* @throws IOException when an unexpected IO error occurs
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/cubicchunks/regionlib/lib/Region.java
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ private void updateHeaders(K key) throws IOException {
}
}

@Override public synchronized Optional<ByteBuffer> readValue(K key) throws IOException {
@Override
public synchronized Optional<ByteBuffer> readValue(K key) throws IOException {
// a hack because Optional can't throw checked exceptions
try {
return sectorMap.trySpecialValue(key)
Expand All @@ -134,7 +135,7 @@ private void updateHeaders(K key) throws IOException {
}
}

private Optional<ByteBuffer> doReadKey(K key) {
private Optional<ByteBuffer> doReadKey(K key) {
return sectorMap.getEntryLocation(key).flatMap(loc -> {
try {
int sectorOffset = loc.getOffset();
Expand Down