Pre-parse book data into SNBT Minecraft books

LibraryFloor.py

@@ -267,7 +267,7 @@ def bookCapacity(self) -> int:
             capacity += building.bookCapacity
         return capacity
 
-    def placeBooks(self, books: List[Dict]):
+    def placeBooks(self, books: List[str]):
         tilesFilledWithBooks: List[ivec3] = []
         for index, building in self.placedTiles.items():
             if building.bookCapacity > 0:

StructureBase.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import functools
-from typing import TYPE_CHECKING, Any, Dict, List
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
 import bookTools
 import vectorTools
@@ -16,7 +16,7 @@
 import globals
 import worldTools
 from Adjacency import StructureAdjacency
-from gdpc.src.gdpc.interface import placeStructure
+from gdpc.src.gdpc.interface import placeStructure, placeBlocks
 from gdpc.src.gdpc.vector_tools import Box, Rect
 
 
@@ -29,7 +29,7 @@ class Structure:
     customProperties: Dict[str, Any]
 
     preProcessingSteps: Dict[ivec3, Block]
-    postProcessingSteps: Dict[ivec3, Block]
+    postProcessingSteps: List[Tuple[ivec3, Block]]
 
     adjecencies: StructureAdjacency
 
@@ -55,7 +55,7 @@ def __init__(
         self.customProperties = dict()
 
         self.preProcessingSteps = dict()
-        self.postProcessingSteps = dict()
+        self.postProcessingSteps = []
 
         self.rotation = withRotation
         self.tile = tile
@@ -140,22 +140,19 @@ def place(self):
         )
 
     def doPostProcessingSteps(self):
-        for index, block in self.postProcessingSteps.items():
-            print(f'Run post processing step {index} {block}')
-            pos = vectorTools.rotatePointAroundOrigin3D(self.box.center, index, self.rotation)
-            globals.editor.placeBlockGlobal(
-                position=pos + self.boxInWorldSpace.offset,
-                block=block,
-            )
+        placeBlocks(
+            blocks=self.postProcessingSteps,
+            doBlockUpdates=False,
+        )
 
     @property
     def bookCapacity(self) -> int:
         # The minecraft:chiseled_bookshelf block has a capacity of 6 books.
         return len(self.bookShelves) * 6
 
-    def addBooks(self, booksData: List[Dict]):
-        # TODO this function could be ran in seperate threads if needed.
-        bookShelfEntries: List[Dict] = []
+    def addBooks(self, booksData: List[str]):
+        # TODO for books consider modifying NBT data itself.
+        bookShelfEntries: List[str] = []
         bookShelfBlockPositions = self.bookShelves.copy()
         for book in booksData:
             bookShelfEntries.append(book)
@@ -168,7 +165,14 @@ def addBooks(self, booksData: List[Dict]):
                     block_state_tools.rotateFacing(bookShelfRotation, self.rotation)
                 )
                 bookShelfBlock = bookTools.fillBookShelf(bookShelfEntries, bookShelfBlock)
-                self.postProcessingSteps[bookShelfPosition] = bookShelfBlock
+
+                bookShelfPosition = vectorTools.rotatePointAroundOrigin3D(
+                    self.box.center, bookShelfPosition, self.rotation
+                ) + self.boxInWorldSpace.offset
+
+                self.postProcessingSteps.append(
+                    (bookShelfPosition, bookShelfBlock)
+                )
                 bookShelfEntries.clear()
 
     def __eq__(self, other):

bookTools.py

@@ -5,10 +5,8 @@
 from glob import glob
 from typing import Dict, List
 
-from glm import ivec3
 from pylatexenc.latex2text import latex2text
 
-import globals
 from gdpc.src.gdpc import Block, minecraft_tools
 
 
@@ -98,8 +96,8 @@ def lastVersionYear(data: Dict) -> str:
     return re.sub(r'^\w\w\w,\s\d+\s\w\w\w\s|\s.+$', '', data['versions'][-1]['created'])
 
 
-def primaryAuthor(data: Dict) -> str:
-    return data['authors_parsed'][0][0]
+def primaryAuthor(snbt: str) -> str:
+    return re.sub(r'^.+?title:\'|(?: et al\.)? \(§7.+', '', snbt)
 
 
 def truncatedBookTitle(data: Dict) -> str:
@@ -184,49 +182,23 @@ def createBookShelfBlock(facing: str) -> Block:
     )
 
 
-def fillBookShelf(bookSources: List[Dict], block: Block) -> Block:
+def fillBookShelf(bookSources: List[str], block: Block) -> Block:
     if len(bookSources) > 6:
-        print('Too many book sources provided at once!')
-        return block
+        raise Exception('Too many book sources provided at once!')
     shelfSNBT = '{Items: ['
     for bookIndex in range(6):
-        book = writeBookData(bookSources[bookIndex])
-        shelfSNBT += f'{{Slot: {bookIndex}b, Count: 1b, id: "minecraft:written_book", tag: {book}}},'
+        shelfSNBT += f'{{Slot: {bookIndex}b, Count: 1b, id: "minecraft:written_book", tag: {bookSources[bookIndex]}}},'
         block.states[f'slot_{bookIndex}_occupied'] = 'true'
     shelfSNBT = shelfSNBT[:-1]
     shelfSNBT += ']}'
     block.data = shelfSNBT
     return block
 
 
-def createMegaBookWall():
-    globals.initialize()
-    with open('dataset/arxiv-metadata-oai-snapshot.json') as f:
-        bookShelfEntries: List[Dict] = []
-        pos = ivec3(globals.buildarea.begin)
-        bookCount = 0
-        facing = 'north'
-
-        for line in f:
-            entry = json.loads(line)
-            # print(entry)
-            bookShelfEntries.append(entry)
-            if len(bookShelfEntries) == 6:
-                pos.x = pos.x + 1
-                if pos.x > globals.buildarea.end.x:
-                    pos.y = pos.y + 1
-                    pos.x = globals.buildarea.begin.x
-
-                bookShelfBlock = createBookShelfBlock(facing)
-                bookshelfBlock = fillBookShelf(bookShelfEntries, bookShelfBlock)
-                globals.editor.placeBlockGlobal(block=bookshelfBlock, position=pos)
-                bookShelfEntries = []
-            bookCount += 1
-
-
 def writeCategoryFile(line: str, category: str, year: str):
     with open(f'dataset/{category}-{year}.json', 'a') as f:
         f.write(line)
+        f.write('\n')
 
 
 def splitByCategory():
@@ -235,20 +207,27 @@ def splitByCategory():
             entry = json.loads(line)
             primaryCategory = entry['categories'].split(' ')[0]
             year = lastVersionYear(entry)
-            writeCategoryFile(line, primaryCategory, year)
+            bookData = writeBookData(entry)
+            writeCategoryFile(bookData, primaryCategory, year)
 
 
-def gatherBooksOfCategory(category: str = 'cs.') -> List[Dict]:
+def gatherBooksOfCategory(category: str = 'cs.') -> List[str]:
     groupedBooks = []
     startYear = 1980
     endYear = datetime.today().year
     for year in range(startYear, endYear + 1):
-        bookGroup: List[Dict] = []
+        bookGroup: List[str] = []
         inputFiles = glob(f'dataset/{category}*-{year}.json')
         for inputFile in inputFiles:
             with open(inputFile) as f:
                 for line in f:
-                    bookGroup.append(json.loads(line))
+                    # Somewhere in the data is a bomb of 0.2 megabyte (221257 bytes).
+                    # To prevent Minecraft chunk size from growing such
+                    # that is crashes the game, skip all books that are
+                    # over roughly 10k bytes
+                    # TODO re-run book parser to trim the abstracts, notes and author lists down
+                    if len(line) < 10000:
+                        bookGroup.append(line)
         bookGroup.sort(key=lambda x: primaryAuthor(x))
         groupedBooks.extend(bookGroup)
     return groupedBooks

globals.py

@@ -49,7 +49,7 @@ def initialize():
     global buildarea
     buildarea = interface.getBuildArea()
     global editor
-    editor = Editor()
+    editor = Editor(buffering=True)
     # editor.loadWorldSlice(rect=buildarea.toRect(), cache=True)
 
     # TODO implement algorithm to find and define an suitable build volume

main.py

@@ -1,4 +1,5 @@
 import os
+from typing import List
 
 from glm import ivec3
 from termcolor import cprint
@@ -12,8 +13,8 @@
 
 globals.initialize()
 
-computerScienceBooks = bookTools.gatherBooksOfCategory('cs.')
-volumeRotatiom = 0
+computerScienceBooks: List[str] = bookTools.gatherBooksOfCategory('cs.')
+volumeRotation = 0
 volumeY = 100
 VOLUME_Y_SIZE = 10
 floorNumber = 0
@@ -32,7 +33,7 @@
     )
     libraryFloor = LibraryFloor(
         volume=floorVolume,
-        volumeRotation=volumeRotatiom,
+        volumeRotation=volumeRotation,
     )
 
     bookCapacity = libraryFloor.bookCapacity
@@ -43,5 +44,5 @@
     del computerScienceBooks[-bookCapacity:]
 
     volumeY -= VOLUME_Y_SIZE
-    volumeRotatiom += 1
+    volumeRotation += 1
     floorNumber -= 1