Skip to content

Commit

Permalink
JSON-based index names (rethinkdb#650)
Browse files Browse the repository at this point in the history 
* fixing eslint errors

* added json-encoded index names

* warn about index parsing failures rather than abort

* filter indexes that start with 'hz_' and more logging on indexes
  • Loading branch information
@Tryneus
Tryneus authored Jul 28, 2016
1 parent 135a8c3 commit 81402f8
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 77 deletions.
140 changes: 86 additions & 54 deletions server/src/metadata/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,63 +1,92 @@
'use strict';

const check = require('../error').check;
const logger = require('../logger');

// Index names are of the format "field1_field2_field3", where the fields
// are given in order of use in a compound index. If the field names contain
// the characters '\' or '_', they will be escaped with a '\'.
// TODO: what about empty field names?

const name_to_fields = (name) => {
let escaped = false;
let field = '';
const fields = [ ];
for (const c of name) {
if (escaped) {
check(c === '\\' || c === '_', `Unexpected index name: "${name}"`);
escaped = false;
field += c;
} else if (c === '\\') {
escaped = true;
} else if (c === '_') {
fields.push(field);
field = '';
// Index names are of the format "hz_[<flags>_]<JSON>" where <flags> may be
// omitted or "multi_<offset>" or "geo" (at the moment). <JSON> is a JSON array
// specifying which fields are indexed in which order. The value at each index
// in the array is either a nested array (for indexing nested fields) or a string
// for a root-level field name.
//
// Example:
// Fields indexed: foo.bar, baz
// Index name: hz_[["foo","bar"],"baz"]
const primary_index_name = 'id';

const name_to_info = (name) => {
if (name === primary_index_name) {
return { geo: false, multi: false, fields: [ 'id' ] };
}

const re = /^hz_(?:(geo)_)?(?:multi_([0-9])+_)?\[/;

const matches = name.match(re);
check(matches !== null, `Unexpected index name (invalid format): "${name}"`);

const json_offset = matches[0].length - 1;

const info = { name, geo: Boolean(matches[1]), multi: isNaN(matches[2]) ? false : Number(matches[2]) };

// Parse remainder as JSON
try {
info.fields = JSON.parse(name.slice(json_offset));
} catch (err) {
check(false, `Unexpected index name (invalid JSON): "${name}"`);
}

// Sanity check fields
const validate_field = (f) => {
if (Array.isArray(f)) {
f.forEach((s) => check(typeof s === 'string',
`Unexpected index name (invalid field): "${name}"`));
} else {
field += c;
check(typeof f === 'string',
`Unexpected index name (field is not a string or array): "${name}"`);
}
}
check(!escaped, `Unexpected index name: "${name}"`);
fields.push(field);
return fields;
};

check(Array.isArray(info.fields),
`Unexpected index name (fields are not an array): "${name}"`);
check((info.multi === null) || info.multi < info.fields.length,
`Unexpected index name (multi index out of bounds): "${name}"`);
info.fields.forEach(validate_field);
return info;
};

const fields_to_name = (fields) => {
let res = '';
for (const field of fields) {
if (res.length > 0) {
res += '_';
}
for (const c of field) {
if (c === '\\' || c === '_') {
res += '\\';
}
res += c;
}
const info_to_name = (info) => {
let res = 'hz_';
if (info.geo) {
res += 'geo_';
}
if (info.multi !== null) {
res += 'multi_' + info.multi + '_';
}
res += JSON.stringify(info.fields);
return res;
};

const primary_index_name = fields_to_name([ 'id' ]);

class Index {
constructor(name, table, conn) {
logger.debug(`${table} index registered: ${name}`);
const info = name_to_info(name);
this.name = name;
this.fields = Index.name_to_fields(name);
this.geo = info.geo; // true or false
this.multi = info.multi; // false or the offset of the multi field
this.fields = info.fields; // array of fields or nested field paths

this._waiters = [ ];
this._result = null;

if (this.geo) {
logger.warn(`Unsupported index (geo): ${this.name}`);
} else if (this.multi !== false) {
logger.warn(`Unsupported index (multi): ${this.name}`);
}

if (name !== primary_index_name) {
table.indexWait(name).run(conn).then(() => {
logger.debug(`${table} index ready: ${name}`);
this._result = true;
this._waiters.forEach((w) => w());
this._waiters = [ ];
Expand All @@ -67,6 +96,7 @@ class Index {
this._waiters = [ ];
});
} else {
logger.debug(`${table} index ready: ${name}`);
this._result = true;
}
}
Expand All @@ -92,29 +122,31 @@ class Index {

// `fuzzy_fields` may be in any order at the beginning of the index.
// These must be immediately followed by `ordered_fields` in the exact
// order given. There may be no other fields present in the index until
// after all of `fuzzy_fields` and `ordered_fields` are present.
// order given. There may be no other fields present in the index
// (because the absence of a field would mean that row is not indexed).
// `fuzzy_fields` may overlap with `ordered_fields`.
is_match(fuzzy_fields, ordered_fields) {
// TODO: multi index matching
if (this.geo || this.multi !== false) {
return false;
}

if (this.fields.length > fuzzy_fields.length + ordered_fields.length) {
return false;
}

for (let i = 0; i < fuzzy_fields.length; ++i) {
const pos = this.fields.indexOf(fuzzy_fields[i]);
if (pos < 0 || pos >= fuzzy_fields.length) { return false; }
}

outer: // eslint-disable-line no-labels
for (let i = 0; i <= fuzzy_fields.length && i + ordered_fields.length <= this.fields.length; ++i) {
for (let j = 0; j < ordered_fields.length; ++j) {
if (this.fields[i + j] !== ordered_fields[j]) {
continue outer; // eslint-disable-line no-labels
}
}
return true;
for (let i = 0; i < ordered_fields.length; ++i) {
const pos = this.fields.length - ordered_fields.length + i;
if (pos < 0 || this.fields[pos] !== ordered_fields[i]) { return false; }
}
return false;

return true;
}
}

Index.name_to_fields = name_to_fields;
Index.fields_to_name = fields_to_name;

module.exports = { Index };
module.exports = { Index, primary_index_name, name_to_info, info_to_name };
9 changes: 7 additions & 2 deletions server/src/metadata/metadata.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,11 @@ class Metadata {
.table('table_config')
.filter((row) => r.and(row('db').eq(this._db),
row('name').match('^hz_').not()))
.pluck('name', 'id', 'indexes')
.map((row) => ({
id: row('id'),
name: row('name'),
indexes: row('indexes').filter((idx) => idx.match('^hz_'))
}))
.changes({ squash: true,
includeInitial: true,
includeStates: true,
Expand Down Expand Up @@ -265,8 +269,9 @@ class Metadata {
logger.debug('redirecting users table');
// Redirect the 'users' table to the one in the internal db
const users_table = new Table('users', table_id, this._db, this._conn);
const users_collection = new Collection('users', table_id);
users_table.update_indexes([ ]);

const users_collection = new Collection({ id: 'users', table: 'users' }, this._internal_db);
users_collection.set_table(users_table);

this._tables.set(table_id, users_table);
Expand Down
50 changes: 29 additions & 21 deletions server/src/metadata/table.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
'use strict';

const error = require('../error');
const Index = require('./index').Index;
const index = require('./index');
const logger = require('../logger');

const r = require('rethinkdb');

Expand All @@ -10,7 +11,6 @@ class Table {
this.collection = null; // This will be set when we are attached to a collection
this.table = r.db(db).table(table);
this.indexes = new Map();
this.update_indexes([ ]);

this._waiters = [ ];
this._result = null;
Expand Down Expand Up @@ -52,35 +52,43 @@ class Table {
}

update_indexes(indexes, conn) {
logger.debug(`${this.table} indexes changed, reevaluating`);

// Initialize the primary index, which won't show up in the changefeed
indexes.push(Index.fields_to_name([ 'id' ]));
indexes.push(index.primary_index_name);

const new_index_map = new Map();
indexes.map((name) => {
const old_index = this.indexes.get(name);
const new_index = new Index(name, this.table, conn);
if (old_index) {
// Steal any waiters from the old index
new_index._waiters = old_index._waiters;
old_index._waiters = [ ];
indexes.forEach((name) => {
try {
const old_index = this.indexes.get(name);
const new_index = new index.Index(name, this.table, conn);
if (old_index) {
// Steal any waiters from the old index
new_index._waiters = old_index._waiters;
old_index._waiters = [ ];
}
new_index_map.set(name, new_index);
} catch (err) {
logger.warn(`${err}`);
}
new_index_map.set(name, new_index);
});

this.indexes.forEach((i) => i.close());
this.indexes = new_index_map;
logger.debug(`${this.table} indexes updated`);
}

// TODO: support geo and multi indexes
create_index(fields, conn, done) {
const index_name = Index.fields_to_name(fields);
const index_name = index.info_to_name({ geo: false, multi: null, fields });
error.check(!this.indexes.get(index_name), 'index already exists');

const success = () => {
// Create the Index object now so we don't try to create it again before the
// feed notifies us of the index creation
const index = new Index(index_name, this.table, conn);
this.indexes.set(index_name, index);
return index.on_ready(done);
const new_index = new index.Index(index_name, this.table, conn);
this.indexes.set(index_name, new_index);
return new_index.on_ready(done);
};

this.table.indexCreate(index_name, (row) => fields.map((key) => row(key)))
Expand All @@ -100,16 +108,16 @@ class Table {
// fuzzy_fields and ordered_fields should both be arrays
get_matching_index(fuzzy_fields, ordered_fields) {
if (fuzzy_fields.length === 0 && ordered_fields.length === 0) {
return this.indexes.get(Index.fields_to_name([ 'id' ]));
return this.indexes.get(index.primary_index_name);
}

let match;
for (const index of this.indexes.values()) {
if (index.is_match(fuzzy_fields, ordered_fields)) {
if (index.ready()) {
return index;
for (const i of this.indexes.values()) {
if (i.is_match(fuzzy_fields, ordered_fields)) {
if (i.ready()) {
return i;
} else if (!match) {
match = index;
match = i;
}
}
}
Expand Down

0 comments on commit 81402f8

Please sign in to comment.