Added data indexing & indexed queries, improved locking mechanism, ad…

…ded tests, fixed bugs, much more

.vscode/launch.json

@@ -7,6 +7,7 @@
         {
             "type": "node",
             "request": "launch",
+            "protocol": "inspector",
             "name": "Launch Program",
             "program": "${workspaceFolder}/acebase-test\\src\\index.js"
         }

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 appy-one
+Copyright (c) 2018 Ewout Stortenbeker (me@appy.one)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

acebase-client/src/api-web.js

@@ -235,6 +235,15 @@ class WebApi extends Api {
                 return results.list;
             });
     }
+
+    createIndex(path, key) {
+        const data = JSON.stringify({ action: "create", path, key });
+        return _request("POST", `${this.url}/index/${this.dbname}`, data);
+    }
+
+    getIndexes() {
+        return _request("GET", `${this.url}/index/${this.dbname}`);
+    }
 }
 
 module.exports = { WebApi };

acebase-server/src/acebase-server.js

@@ -182,8 +182,8 @@ class AceBaseServer extends EventEmitter {
                 // Execute query
                 const path = req.path.substr(dbname.length + 8);
                 const data = transport.deserialize(req.body);
-                const ref = db.ref(path);
-                const query = ref.query();
+                //const ref = db.ref(path);
+                const query = db.query(path);
                 data.query.filters.forEach(filter => {
                     query.where(filter.key, filter.op, filter.compare);
                 });
@@ -213,6 +213,30 @@ class AceBaseServer extends EventEmitter {
                 });
             });
 
+            app.get(`/index/${dbname}`, (req, res) => {
+                // Get all indexes
+                db.indexes.list()
+                .then(indexes => {
+                    res.send(indexes);
+                });
+            });
+
+            app.post(`/index/${dbname}`, (req, res) => {
+                // create index
+                const data = req.body;
+                if (data.action === "create") {
+                    db.indexes.create(data.path, data.key)
+                    .then(() => {
+                        res.send({ success: true });
+                    })
+                    .catch(err => {
+                        console.error(err);
+                        res.statusCode = 500;
+                        res.send(err);         
+                    })
+                }
+            });
+
             // Websocket implementation:
             const clients = {
                 list: [],

acebase-test/src/index.js

@@ -1,7 +1,4 @@
-// require("./tests/indexing");
-// return;
-
-const local = true;
+const local = true;    // Set to false to spawn an acebase webserver and run tests over http connection
 const db = local 
     ? require('./local') 
     : require('./client-server');
@@ -11,14 +8,15 @@ db.on("ready", () => {
 
     // Run tests
     let tests = [
+        require("./tests/events")(db),
         require("./tests/movies")(db),
         require("./tests/users")(db),
         require("./tests/exclude")(db),
         require("./tests/regexp")(db),
         require("./tests/binary")(db),
         require("./tests/transaction")(db),
         require("./tests/getset")(db),
-        local ? require("./tests/indexed-query")(db) : null,
+        require("./tests/indexed_data")(db),
         // require("./tests/stats")(db)
     ];
 

acebase-test/src/tests/events.js

@@ -0,0 +1,155 @@
+const { AceBase, DataReference, PathReference } = require('acebase');
+
+/**
+ * 
+ * @param {AceBase} db
+ * @param {string} path 
+ * @param {string} type 
+ * @param {number} nr
+ * @returns {Promise<void>} returns a promise that resolves once the event has been fired on the reference nr of times
+ */
+function createEvent(db, path, type, nr) {
+    let resolve, reject;
+    const p = new Promise((rs, rj) => { resolve = rs; reject = rj; });
+    //return new Promise((resolve, reject) => {
+        let n = 0;
+        const ref = db.ref(path);
+        const subscription = ref.on(type).subscribe(snap => {
+            n++;
+            console.log(`${n}/${nr}: Event "${type}" fired on "/${ref.path}" with source path "/${snap.ref.path}" and data:`, snap.val())
+            if (n === nr) { 
+                subscription.stop(); // Stop subscription
+                resolve(snap.val()); 
+            }
+        });
+        if (nr === 0) {
+            // Set a timeout for events that should not fire at all (nr === 0)
+            // 1s timeout should be sufficient, unless you have breakpoints in your code (the timeout will fire before any potential event callback)
+            setTimeout(() => {
+                if (n === 0) { resolve(); }
+                else {
+                    reject(`This event should not have fired!`);
+                }
+            }, 1000);
+        }
+        else {
+            setTimeout(() => {
+                if (n < nr) {
+                    console.warn(`Still waiting for event "${type}" on path "/${path}", only hit ${n}/${nr}`);
+                }
+            }, 1000);            
+        }
+    //});
+    return p;
+}
+
+/**
+ * 
+ * @param {AceBase} db 
+ */
+const run = (db) => {
+
+    // Initialize data
+    return db.ref("events/welcome_august")
+    .set({
+        name: "Welcome August Party",
+        location: "Amsterdam",
+        start_date: new Date("2018/07/31 20:00"),
+        end_date: new Date("2018/08/01 05:00"),
+        venue: "Museumplein",
+        tickets_total: 5000,
+        tickets_sold: 3587,
+        price: 25,
+        price_currency: "EUR",
+        line_up: {
+            dj1: {
+                name: "The first DJ",
+                start_date: new Date("2018/07/31 20:00"),
+                end_date: new Date("2018/07/31 21:00"),
+                bio: new PathReference("events/djs/the_first_dj")
+            },
+            dj2: {
+                name: "Second to None",
+                start_date: new Date("2018/07/31 21:00"),
+                start_date: new Date("2018/07/31 22:30"),
+                bio: new PathReference("events/djs/second_to_none")
+            },
+            dj3: {
+                name: "3 Strikes' Out",
+                start_date: new Date("2018/07/31 22:30"),
+                start_date: new Date("2018/08/01 00:00"),
+                bio: new PathReference("events/djs/3_strikes_out")
+            },
+            dj4: {
+                name: "The Night Shift",
+                start_date: new Date("2018/07/31 00:00"),
+                start_date: new Date("2018/08/01 03:00"),
+                bio: new PathReference("events/djs/the_night_shift")                
+            }
+        }
+    })
+    .then(ref => {
+        // Setup events
+        const events = [
+            createEvent(db, "", "child_changed", 1),
+            createEvent(db, "events", "value", 1),
+            createEvent(db, "events", "child_changed", 1),
+            createEvent(db, "events/welcome_august", "value", 1),
+            createEvent(db, "events/welcome_august", "child_changed", 1),
+            createEvent(db, "events/welcome_august/name", "value", 1),
+        ];
+
+        // Fiddle with the data
+        db.ref("events/welcome_august/name").update("Welcome August");
+
+        // Wait for all events to have fired nr of times
+        return Promise.all(events);
+    })
+    .then((results) => {
+        // Try some more events
+        const events = [
+            createEvent(db, "", "child_changed", 1),
+            createEvent(db, "events", "child_changed", 1),
+            createEvent(db, "events/welcome_august", "child_changed", 1),
+            createEvent(db, "events/welcome_august/line_up", "value", 1),
+            createEvent(db, "events/welcome_august/line_up", "child_changed", 0),
+            createEvent(db, "events/welcome_august/line_up", "child_added", 1),
+            createEvent(db, "events/welcome_august/line_up", "child_removed", 0)
+        ];
+
+        // Fiddle with the data
+        let dj5Ref = db.ref("events/welcome_august/line_up/dj5");
+        dj5Ref.set({
+            name: "Last BN Least",
+            start_date: new Date("2018/07/31 03:00"),
+            start_date: new Date("2018/08/01 05:00"),
+            bio: new PathReference("events/djs/last_bn_least")
+        })
+
+        // Wait for all events to have fired nr of times
+        return Promise.all(events).then(res => {
+            return dj5Ref;
+        });   
+    })
+    .then(djRef => {
+        const events = [
+            createEvent(db, "", "child_changed", 1),
+            createEvent(db, "events", "child_changed", 1),
+            createEvent(db, "events/welcome_august", "child_changed", 1),
+            createEvent(db, "events/welcome_august/line_up", "value", 1),
+            createEvent(db, "events/welcome_august/line_up", "child_changed", 0),
+            createEvent(db, "events/welcome_august/line_up", "child_added", 0),
+            createEvent(db, "events/welcome_august/line_up", "child_removed", 1)
+        ];
+
+        djRef.remove(); // Delete it again
+
+        // Wait for all events to have fired nr of times
+        return Promise.all(events).then(res => {
+            return true;
+        });
+    });
+
+};
+
+module.exports = run;

acebase-test/src/tests/exclude.js

@@ -1,6 +1,7 @@
 const { AceBase } = require('acebase');
 /**
- * 
+ * This test adds chats to the database, retrieves them while excluding certain properties.
+ * It also tests using a wildcard index to query all chat messages by user
  * @param {AceBase} db 
  */
 const run = (db) => {
@@ -74,8 +75,59 @@ const run = (db) => {
         Object.keys(chats).forEach(id => {
             const chat = chats[id];
             console.assert(!chat.messages, "retrieved chats should NOT include messages!");
+        });
+    })
+    .then(() => {
+        // Add another chat
+        return db.ref("chats/anotherchat").set({
+            members: ["ewout", "friend"],
+            title: "Friend chat",
+            messages: {
+                msg1: {
+                    sent: new Date("2018-06-19T13:02:09Z"),
+                    user: "ewout",
+                    text: "We should grab a beer soon 🍺",
+                    receipts: {
+                        friend: {
+                            received: new Date("2018-06-19T13:02:10Z"),
+                            read: new Date("2018-06-19T13:03:54Z")
+                        }
+                    } 
+                },
+                msg2: {
+                    sent: new Date("2018-06-19T13:05:09Z"),
+                    user: "friend",
+                    text: "Good idea. I'm pretty thirsty now. How about tonight?",
+                    receipts: {
+                        ewout: {
+                            received: new Date("2018-06-19T13:05:09Z"),
+                            read: new Date("2018-06-19T13:05:54Z")
+                        }
+                    }
+                },
+                msg3: {
+                    sent: new Date("2018-06-19T13:06:01Z"),
+                    user: "ewout",
+                    text: "👍 Meet you around 9?",
+                    receipts: {} 
+                }
+            }
         });        
     })
+    .then(() => {
+        // Create a wildcard index so we can query all chat messages by user
+        return db.indexes.create("chats/*/messages", "user");
+    })
+    .then(() => {
+        // Now use the wildcard query to get all messages by ewout, in all chats
+        return db.query("chats/*/messages")
+        .where("user", "==", "ewout")
+        .get();
+    })
+    .then(snapshots => {
+        const messages = snapshots.map(snapshot => snapshot.val());
+        console.log(messages);
+    })
     ;
 };
 

acebase-test/src/tests/indexed-query.js

@@ -5,19 +5,32 @@ const { Storage } = require('acebase/src/storage');
  * @param {AceBase} db 
  */
 const run = (db) => {
-    /**
-     * @type Storage
-     */
-    const storage = db.api.storage; // This manual plumbing is temporary (won't work in client/server mode), will be refactored once indexing is solid
-    const indexes = storage.indexes;
-    return indexes.create("movies", "year")
-    .then(() => {
-        return indexes.query("movies", "year", "between", [1990, 2000]);
-    })
-    .then(results => {
-        console.log(`Got ${results.length} indexed query results`);
-        console.log(results);
-        //console.assert(results.length === 7, "There must be 7 movies matching")
+    // Create an index on "/movies" as soon as it is created, or changed.
+    db.ref("movies")
+    .on("value")
+    .subscribe(snapshot => {
+        if (!snapshot.exists()) { 
+            return; 
+        }
+        /**
+         * @type Storage
+         */
+        const storage = db.api.storage; // This manual plumbing is temporary (won't work in client/server mode), will be refactored once indexing is solid
+        const indexes = storage.indexes;    
+        return Promise.all(
+            indexes.create("movies", "year"),
+            indexes.create("movies", "rating")
+        )
+        .then(indexes => {
+            return Promise.all(
+                indexes[0].query(indexes[0], "between", [1990, 2000]),
+                indexes[1].query(indexes[1], ">=", 9)
+            );
+        })
+        .then(results => {
+            console.log(`Got ${results.length} indexed query results`);
+            console.log(results);
+        });
     });
 }