lib ldb key_value: Add get_size method

Add the get_size method to the ldb_key_value layer, this will allow the
reindexing code to get an estimate of the number of records in the
database.

The lmdb backend returns an accurate count of the number of records in
the database withe the mdb_env_stat call.

The tdb backend does not provide a low cost method to determine the
number of records on the database.  It does provide a tdb_summary call
however this this walks the entire database.

So for tdb we use the map size divided by 500, this over estimates the counts
for small domains, but the extra memory allocated for the cache should
not be significant.

Signed-off-by: Gary Lockyer <[email protected]>
Reviewed-by: Andrew Bartlett <[email protected]>

lib/ldb/ldb_key_value/ldb_kv.h

@@ -43,6 +43,7 @@ struct kv_db_ops {
 	const char *(*name)(struct ldb_kv_private *ldb_kv);
 	bool (*has_changed)(struct ldb_kv_private *ldb_kv);
 	bool (*transaction_active)(struct ldb_kv_private *ldb_kv);
+	size_t (*get_size)(struct ldb_kv_private *ldb_kv);
 };
 
 /* this private structure is used by the key value backends in the

lib/ldb/ldb_mdb/ldb_mdb.c

@@ -576,6 +576,27 @@ static bool lmdb_changed(struct ldb_kv_private *ldb_kv)
 	return true;
 }
 
+/*
+ * Get the number of records in the database.
+ *
+ * The mdb_env_stat call returns an accurate count, so we return the actual
+ * number of records in the database rather than an estimate.
+ */
+static size_t lmdb_get_size(struct ldb_kv_private *ldb_kv) {
+
+	struct MDB_stat stats = {0};
+	struct lmdb_private *lmdb = ldb_kv->lmdb_private;
+	int ret = 0;
+
+	ret = mdb_env_stat(lmdb->env, &stats);
+	if (ret != 0) {
+		return 0;
+	}
+	return stats.ms_entries;
+}
+
+
+
 static struct kv_db_ops lmdb_key_value_ops = {
 	.store              = lmdb_store,
 	.delete             = lmdb_delete,
@@ -593,6 +614,7 @@ static struct kv_db_ops lmdb_key_value_ops = {
 	.name               = lmdb_name,
 	.has_changed        = lmdb_changed,
 	.transaction_active = lmdb_transaction_active,
+	.get_size           = lmdb_get_size,
 };
 
 static const char *lmdb_get_path(const char *url)

lib/ldb/ldb_tdb/ldb_tdb.c

@@ -400,6 +400,23 @@ static bool ltdb_transaction_active(struct ldb_kv_private *ldb_kv)
 	return tdb_transaction_active(ldb_kv->tdb);
 }
 
+/*
+ * Get an estimate of the number of records in a tdb database.
+ *
+ * This implementation will overestimate the number of records in a sparsely
+ * populated database. The size estimate is only used for allocating
+ * an in memory tdb to cache index records during a reindex, overestimating
+ * the contents is acceptable, and preferable to underestimating
+ */
+#define RECORD_SIZE 500
+static size_t ltdb_get_size(struct ldb_kv_private *ldb_kv)
+{
+	size_t map_size = tdb_map_size(ldb_kv->tdb);
+	size_t size = map_size / RECORD_SIZE;
+
+	return size;
+}
+
 static const struct kv_db_ops key_value_ops = {
     .store = ltdb_store,
     .delete = ltdb_delete,
@@ -417,6 +434,7 @@ static const struct kv_db_ops key_value_ops = {
     .name = ltdb_name,
     .has_changed = ltdb_changed,
     .transaction_active = ltdb_transaction_active,
+    .get_size = ltdb_get_size,
 };
 
 /*

lib/ldb/tests/ldb_kv_ops_test.c

@@ -43,6 +43,9 @@
  * - supports iteration over all records in the database
  * - supports the update_in_iterate operation allowing entries to be
  *   re-keyed.
+ * - has a get_size implementation that returns an estimate of the number of
+ *   records in the database.  Note that this can be an estimate rather than
+ *   an accurate size.
  */
 #include <stdarg.h>
 #include <stddef.h>
@@ -1523,6 +1526,71 @@ static void test_delete_transaction_isolation(void **state)
 }
 
 
+/*
+ * Test that get_size returns a sensible estimate of the number of records
+ * in the database.
+ */
+static void test_get_size(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ldb_kv_private *ldb_kv = get_ldb_kv(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+	size_t size = 0;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	size = ldb_kv->kv_ops->get_size(ldb_kv);
+#if defined(TEST_LMDB)
+	assert_int_equal(2, size);
+#else
+	/*
+	 * The tdb implementation of get_size over estimates for sparse files
+	 * which is perfectly acceptable for it's intended use.
+	 */
+	assert_true( size > 2500);
+#endif
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ldb_kv->kv_ops->begin_write(ldb_kv);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ldb_kv->kv_ops->store(ldb_kv, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ldb_kv->kv_ops->finish_write(ldb_kv);
+	assert_int_equal(ret, 0);
+
+	size = ldb_kv->kv_ops->get_size(ldb_kv);
+#ifdef TEST_LMDB
+	assert_int_equal(3, size);
+#endif
+	talloc_free(tmp_ctx);
+}
+
 int main(int argc, const char **argv)
 {
 	const struct CMUnitTest tests[] = {
@@ -1570,6 +1638,10 @@ int main(int argc, const char **argv)
 			test_delete_transaction_isolation,
 			setup,
 			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_get_size,
+			setup,
+			teardown),
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);

lib/ldb/wscript

@@ -526,7 +526,7 @@ def build(bld):
 
             bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
                              source='tests/ldb_kv_ops_test.c',
-                             cflags='-DTEST_BE=\"mdb\"',
+                             cflags='-DTEST_BE=\"mdb\" -DTEST_LMDB=1',
                              deps='cmocka ldb',
                              install=False)
         else: