Skip to content

Commit

Permalink
Move chunk serialization off main thread
Browse files Browse the repository at this point in the history 
And use a shallow copy instead of a full serialize-deserialize copy to create
the fake chunks for unloaded real ones.
  • Loading branch information
@Johni0702
Johni0702 committed Nov 19, 2022
1 parent 48932ee commit 6c477c3
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 29 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
- Sort fake chunks before loading so nearby chunks load first
- Fix render distance resetting after restart if above 32 (#109)
- Fix errors about Starlight in log when Starlight is not installed
- Fix lag spikes caused by chunk serialization on main thread when crossing chunk borders (#95)

### 4.0.0
- Update to Minecraft 1.19
Expand Down
12 changes: 12 additions & 0 deletions src/main/java/de/johni0702/minecraft/bobby/FakeChunk.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@
import de.johni0702.minecraft.bobby.ext.ChunkLightProviderExt;
import net.minecraft.client.MinecraftClient;
import net.minecraft.client.render.WorldRenderer;
import net.minecraft.nbt.NbtList;
import net.minecraft.util.math.ChunkPos;
import net.minecraft.util.math.ChunkSectionPos;
import net.minecraft.world.Heightmap;
import net.minecraft.world.LightType;
import net.minecraft.world.World;
import net.minecraft.world.chunk.ChunkNibbleArray;
import net.minecraft.world.chunk.ChunkSection;
import net.minecraft.world.chunk.UpgradeData;
import net.minecraft.world.chunk.WorldChunk;
Expand All @@ -18,6 +21,11 @@ public class FakeChunk extends WorldChunk {

private boolean isTainted;

// Keeping these around, so we can safely serialize the chunk from any thread
public ChunkNibbleArray[] blockLight;
public ChunkNibbleArray[] skyLight;
public NbtList serializedBlockEntities;

public FakeChunk(World world, ChunkPos pos, ChunkSection[] sections) {
super(world, pos, UpgradeData.NO_UPGRADE_DATA, new ChunkTickScheduler<>(), new ChunkTickScheduler<>(), 0L, sections, null, null);
}
Expand Down Expand Up @@ -54,4 +62,8 @@ private void updateTaintedState(ChunkLightProviderExt lightProvider, int x, int
}
lightProvider.bobby_setTainted(ChunkSectionPos.asLong(x, y, z), delta);
}

public void setHeightmap(Heightmap.Type type, Heightmap heightmap) {
this.heightmaps.put(type, heightmap);
}
}
29 changes: 17 additions & 12 deletions src/main/java/de/johni0702/minecraft/bobby/FakeChunkManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import net.minecraft.nbt.NbtCompound;
import net.minecraft.server.integrated.IntegratedServer;
import net.minecraft.util.Identifier;
import net.minecraft.util.Pair;
import net.minecraft.util.Util;
import net.minecraft.util.math.ChunkPos;
import net.minecraft.util.math.ChunkSectionPos;
Expand All @@ -30,6 +29,7 @@
import net.minecraft.world.chunk.WorldChunk;
import net.minecraft.world.chunk.light.LightingProvider;
import net.minecraft.world.level.storage.LevelStorage;
import org.apache.commons.lang3.tuple.Pair;

import java.nio.file.Path;
import java.util.ArrayDeque;
Expand Down Expand Up @@ -73,6 +73,9 @@ public class FakeChunkManager {
private static final ExecutorService loadExecutor = Executors.newFixedThreadPool(8, new DefaultThreadFactory("bobby-loading", true));
private final Long2ObjectMap<LoadingJob> loadingJobs = new Long2ObjectLinkedOpenHashMap<>();

// Executor for serialization and saving. Single-threaded so we do not have to worry about races between multiple saves for the same chunk.
private static final ExecutorService saveExecutor = Executors.newSingleThreadExecutor(new DefaultThreadFactory("bobby-saving", true));

public FakeChunkManager(ClientWorld world, ClientChunkManager clientChunkManager) {
this.world = world;
this.clientChunkManager = clientChunkManager;
Expand Down Expand Up @@ -147,7 +150,7 @@ private void update(boolean blocking, BooleanSupplier shouldKeepTicking, int new
// Chunk is now outside view distance, can be unloaded / cancelled
cancelLoad(chunkPos);
toBeUnloaded.put(chunkPos, time);
unloadQueue.add(new Pair<>(chunkPos, time));
unloadQueue.add(Pair.of(chunkPos, time));
}, chunkPos -> {
// Chunk is now inside view distance, load it
int x = ChunkPos.getPackedX(chunkPos);
Expand Down Expand Up @@ -271,7 +274,7 @@ private CompletableFuture<Optional<Pair<NbtCompound, FakeChunkStorage>>> loadTag
FakeChunkStorage storage = storages.get(storageIndex);
return storage.loadTag(chunkPos).thenCompose(maybeTag -> {
if (maybeTag.isPresent()) {
return CompletableFuture.completedFuture(Optional.of(new Pair<>(maybeTag.get(), storage)));
return CompletableFuture.completedFuture(Optional.of(Pair.of(maybeTag.get(), storage)));
}
if (storageIndex + 1 < storages.size()) {
return loadTag(chunkPos, storageIndex + 1);
Expand All @@ -280,15 +283,7 @@ private CompletableFuture<Optional<Pair<NbtCompound, FakeChunkStorage>>> loadTag
});
}

public void load(int x, int z, NbtCompound tag, FakeChunkStorage storage) {
Supplier<WorldChunk> chunkSupplier = storage.deserialize(new ChunkPos(x, z), tag, world);
if (chunkSupplier == null) {
return;
}
load(x, z, chunkSupplier.get());
}

protected void load(int x, int z, WorldChunk chunk) {
public void load(int x, int z, WorldChunk chunk) {
fakeChunks.put(ChunkPos.toLong(x, z), chunk);

world.resetChunkColor(new ChunkPos(x, z));
Expand Down Expand Up @@ -332,6 +327,16 @@ private void cancelLoad(long chunkPos) {
}
}

public Supplier<WorldChunk> save(WorldChunk chunk) {
Pair<WorldChunk, Supplier<WorldChunk>> copy = storage.shallowCopy(chunk);
LightingProvider lightingProvider = chunk.getWorld().getLightingProvider();
saveExecutor.execute(() -> {
NbtCompound nbt = storage.serialize(copy.getLeft(), lightingProvider);
storage.save(chunk.getPos(), nbt);
});
return copy.getRight();
}

private static String getCurrentWorldOrServerName() {
IntegratedServer integratedServer = client.getServer();
if (integratedServer != null) {
Expand Down
80 changes: 73 additions & 7 deletions src/main/java/de/johni0702/minecraft/bobby/FakeChunkStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import net.minecraft.world.gen.chunk.FlatChunkGenerator;
import net.minecraft.world.storage.StorageIoWorker;
import net.minecraft.world.storage.VersionedChunkStorage;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
Expand Down Expand Up @@ -210,13 +211,17 @@ public NbtCompound serialize(WorldChunk chunk, LightingProvider lightingProvider
empty = false;
}

ChunkNibbleArray blockLight = lightingProvider.get(LightType.BLOCK).getLightSection(ChunkSectionPos.from(chunkPos, y));
ChunkNibbleArray blockLight = chunk instanceof FakeChunk fakeChunk
? fakeChunk.blockLight[i + 1]
: lightingProvider.get(LightType.BLOCK).getLightSection(ChunkSectionPos.from(chunkPos, y));
if (blockLight != null && !blockLight.isUninitialized()) {
sectionTag.putByteArray("BlockLight", blockLight.asByteArray());
empty = false;
}

ChunkNibbleArray skyLight = lightingProvider.get(LightType.SKY).getLightSection(ChunkSectionPos.from(chunkPos, y));
ChunkNibbleArray skyLight = chunk instanceof FakeChunk fakeChunk
? fakeChunk.skyLight[i + 1]
: lightingProvider.get(LightType.SKY).getLightSection(ChunkSectionPos.from(chunkPos, y));
if (skyLight != null && !skyLight.isUninitialized()) {
sectionTag.putByteArray("SkyLight", skyLight.asByteArray());
empty = false;
Expand All @@ -229,11 +234,16 @@ public NbtCompound serialize(WorldChunk chunk, LightingProvider lightingProvider

level.put("sections", sectionsTag);

NbtList blockEntitiesTag = new NbtList();
for (BlockPos pos : chunk.getBlockEntityPositions()) {
NbtCompound blockEntityTag = chunk.getPackedBlockEntityNbt(pos);
if (blockEntityTag != null) {
blockEntitiesTag.add(blockEntityTag);
NbtList blockEntitiesTag;
if (chunk instanceof FakeChunk fakeChunk) {
blockEntitiesTag = fakeChunk.serializedBlockEntities;
} else {
blockEntitiesTag = new NbtList();
for (BlockPos pos : chunk.getBlockEntityPositions()) {
NbtCompound blockEntityTag = chunk.getPackedBlockEntityNbt(pos);
if (blockEntityTag != null) {
blockEntitiesTag.add(blockEntityTag);
}
}
}
level.put("block_entities", blockEntitiesTag);
Expand Down Expand Up @@ -379,7 +389,20 @@ public NbtCompound serialize(WorldChunk chunk, LightingProvider lightingProvider
}
}

return loadChunk(chunk, blockLight, skyLight, config);
}

private Supplier<WorldChunk> loadChunk(
FakeChunk chunk,
ChunkNibbleArray[] blockLight,
ChunkNibbleArray[] skyLight,
BobbyConfig config
) {
return () -> {
ChunkPos pos = chunk.getPos();
World world = chunk.getWorld();
ChunkSection[] chunkSections = chunk.getSectionArray();

boolean hasSkyLight = world.getDimension().hasSkyLight();
ChunkManager chunkManager = world.getChunkManager();
LightingProvider lightingProvider = chunkManager.getLightingProvider();
Expand Down Expand Up @@ -414,6 +437,49 @@ public NbtCompound serialize(WorldChunk chunk, LightingProvider lightingProvider
};
}

// This method is called before the original chunk is unloaded and needs to return a supplier
// that can be called after the chunk has been unloaded to load a fake chunk in its place.
// It also returns a fake chunk immediately that isn't loaded into the game (yet) but can safely
// be serialized on another thread.
public Pair<WorldChunk, Supplier<WorldChunk>> shallowCopy(WorldChunk original) {
BobbyConfig config = Bobby.getInstance().getConfig();

World world = original.getWorld();
ChunkPos chunkPos = original.getPos();

ChunkSection[] chunkSections = original.getSectionArray();

ChunkNibbleArray[] blockLight = new ChunkNibbleArray[chunkSections.length + 2];
ChunkNibbleArray[] skyLight = new ChunkNibbleArray[chunkSections.length + 2];
LightingProvider lightingProvider = world.getChunkManager().getLightingProvider();
for (int y = lightingProvider.getBottomY(), i = 0; y < lightingProvider.getTopY(); y++, i++) {
blockLight[i] = lightingProvider.get(LightType.BLOCK).getLightSection(ChunkSectionPos.from(chunkPos, y));
skyLight[i] = lightingProvider.get(LightType.SKY).getLightSection(ChunkSectionPos.from(chunkPos, y));
}

FakeChunk fake = new FakeChunk(world, chunkPos, chunkSections);
fake.blockLight = blockLight;
fake.skyLight = skyLight;

for (Map.Entry<Heightmap.Type, Heightmap> entry : original.getHeightmaps()) {
fake.setHeightmap(entry.getKey(), entry.getValue());
}

NbtList blockEntitiesTag = new NbtList();
for (BlockPos pos : original.getBlockEntityPositions()) {
NbtCompound blockEntityTag = original.getPackedBlockEntityNbt(pos);
if (blockEntityTag != null) {
blockEntitiesTag.add(blockEntityTag);
if (!config.isNoBlockEntities()) {
fake.addPendingBlockEntityNbt(blockEntityTag);
}
}
}
fake.serializedBlockEntities = blockEntitiesTag;

return Pair.of(fake, loadChunk(fake, blockLight, skyLight, config));
}

private static ChunkNibbleArray floodSkylightFromAbove(ChunkNibbleArray above) {
if (above.isUninitialized()) {
return new ChunkNibbleArray();
Expand Down
16 changes: 6 additions & 10 deletions src/main/java/de/johni0702/minecraft/bobby/mixin/ClientChunkManagerMixin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import de.johni0702.minecraft.bobby.Bobby;
import de.johni0702.minecraft.bobby.FakeChunk;
import de.johni0702.minecraft.bobby.FakeChunkManager;
import de.johni0702.minecraft.bobby.FakeChunkStorage;
import de.johni0702.minecraft.bobby.VisibleChunksTracker;
import de.johni0702.minecraft.bobby.ext.ClientChunkManagerExt;
import net.minecraft.client.world.ClientChunkManager;
Expand All @@ -14,7 +13,6 @@
import net.minecraft.util.math.ChunkPos;
import net.minecraft.world.chunk.ChunkStatus;
import net.minecraft.world.chunk.WorldChunk;
import net.minecraft.world.chunk.light.LightingProvider;
import org.apache.commons.lang3.tuple.Pair;
import org.jetbrains.annotations.Nullable;
import org.spongepowered.asm.mixin.Final;
Expand All @@ -29,13 +27,13 @@
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Supplier;

@Mixin(ClientChunkManager.class)
public abstract class ClientChunkManagerMixin implements ClientChunkManagerExt {
@Shadow @Final private WorldChunk emptyChunk;

@Shadow @Nullable public abstract WorldChunk getChunk(int i, int j, ChunkStatus chunkStatus, boolean bl);
@Shadow public abstract LightingProvider getLightingProvider();
@Shadow private static int getChunkMapRadius(int loadDistance) { throw new AssertionError(); }

protected FakeChunkManager bobbyChunkManager;
Expand All @@ -45,7 +43,7 @@ public abstract class ClientChunkManagerMixin implements ClientChunkManagerExt {
private final VisibleChunksTracker realChunksTracker = new VisibleChunksTracker();

// List of real chunks saved just before they are unloaded, so we can restore fake ones in their place afterwards
private final List<Pair<Long, NbtCompound>> bobbyChunkReplacements = new ArrayList<>();
private final List<Pair<Long, Supplier<WorldChunk>>> bobbyChunkReplacements = new ArrayList<>();

@Inject(method = "<init>", at = @At("RETURN"))
private void bobbyInit(ClientWorld world, int loadDistance, CallbackInfo ci) {
Expand Down Expand Up @@ -111,24 +109,22 @@ private void saveRealChunk(long chunkPos) {
return;
}

FakeChunkStorage storage = bobbyChunkManager.getStorage();
NbtCompound tag = storage.serialize(chunk, getLightingProvider());
storage.save(chunk.getPos(), tag);
Supplier<WorldChunk> copy = bobbyChunkManager.save(chunk);

if (bobbyChunkManager.shouldBeLoaded(chunkX, chunkZ)) {
bobbyChunkReplacements.add(Pair.of(chunkPos, tag));
bobbyChunkReplacements.add(Pair.of(chunkPos, copy));

bobby_pauseChunkStatusListener();
}
}

@Unique
private void substituteFakeChunksForUnloadedRealOnes() {
for (Pair<Long, NbtCompound> entry : bobbyChunkReplacements) {
for (Pair<Long, Supplier<WorldChunk>> entry : bobbyChunkReplacements) {
long chunkPos = entry.getKey();
int chunkX = ChunkPos.getPackedX(chunkPos);
int chunkZ = ChunkPos.getPackedZ(chunkPos);
bobbyChunkManager.load(chunkX, chunkZ, entry.getValue(), bobbyChunkManager.getStorage());
bobbyChunkManager.load(chunkX, chunkZ, entry.getValue().get());
}
bobbyChunkReplacements.clear();

Expand Down

0 comments on commit 6c477c3

Please sign in to comment.