From 404d3fc4c5d1183e5a29433ea7fe52ef88546710 Mon Sep 17 00:00:00 2001 From: Kelsey Schubert Date: Fri, 13 Apr 2018 10:05:52 -0400 Subject: [PATCH] Import wiredtiger: 5fc85c47caba5dbd4fc49ad6fa924fee4e3d5695 from branch mongodb-3.8 ref: 5bfcc92407..5fc85c47ca for: 3.7.4 WT-3959 Recovery timestamp set on restart scenarios need addressing WT-4032 parent pages can be evicted while being split. WT-4034 Re-entering eviction can result in checkpoint corruption --- src/third_party/wiredtiger/dist/api_data.py | 2 - src/third_party/wiredtiger/dist/s_string.ok | 1 + src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/btree/bt_split.c | 18 +- .../wiredtiger/src/config/config_def.c | 26 ++- .../wiredtiger/src/conn/conn_api.c | 5 +- .../wiredtiger/src/conn/conn_dhandle.c | 3 +- .../wiredtiger/src/cursor/cur_backup.c | 10 +- .../wiredtiger/src/evict/evict_page.c | 19 ++- .../wiredtiger/src/include/extern.h | 1 + src/third_party/wiredtiger/src/include/meta.h | 3 + src/third_party/wiredtiger/src/include/txn.h | 1 + .../wiredtiger/src/meta/meta_ckpt.c | 59 +++++-- .../wiredtiger/src/support/generation.c | 6 + src/third_party/wiredtiger/src/txn/txn_ckpt.c | 59 +++++-- .../wiredtiger/src/txn/txn_recover.c | 154 +++++++++++------- .../wiredtiger/src/utilities/util_list.c | 5 +- .../wiredtiger/test/suite/test_backup08.py | 142 ++++++++++++++++ .../wiredtiger/test/suite/test_timestamp10.py | 121 ++++++++------ 19 files changed, 457 insertions(+), 180 deletions(-) create mode 100644 src/third_party/wiredtiger/test/suite/test_backup08.py diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 89072b51df704..9ecd76d76ecd6 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -351,8 +351,6 @@ def __cmp__(self, other): the file checkpoint entries'''), Config('checkpoint_lsn', '', r''' LSN of the last checkpoint'''), - Config('checkpoint_timestamp', '', r''' - stable timestamp of the last checkpoint'''), Config('id', '', r''' the file's ID number'''), Config('version', '(major=0,minor=0)', r''' diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 3b4c5eb88834a..6ceb4c2013ba2 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1201,6 +1201,7 @@ sx sy sys syscall +sysinfo sz t's tV diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 39867b1ec14f3..cac59dc11d325 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "5bfcc924079afdcd6dda22c29b5fa60a14ec3dc9", + "commit": "5fc85c47caba5dbd4fc49ad6fa924fee4e3d5695", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.8" diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 57effcbae7c3a..d58851a2a2302 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -402,8 +402,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) root_decr = root_incr = 0; complete = WT_ERR_RETURN; - /* The root page will be marked dirty, make sure that will succeed. */ + /* Mark the root page dirty. */ WT_RET(__wt_page_modify_init(session, root)); + __wt_page_modify_set(session, root); /* * Our caller is holding the root page locked to single-thread splits, @@ -574,10 +575,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); root_decr += size; - /* Adjust the root's memory footprint and mark it dirty. */ + /* Adjust the root's memory footprint. */ __wt_cache_page_inmem_incr(session, root, root_incr); __wt_cache_page_inmem_decr(session, root, root_decr); - __wt_page_modify_set(session, root); err: switch (complete) { case WT_ERR_RETURN: @@ -629,8 +629,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, empty_parent = false; complete = WT_ERR_RETURN; - /* The parent page will be marked dirty, make sure that will succeed. */ + /* Mark the page dirty. */ WT_RET(__wt_page_modify_init(session, parent)); + __wt_page_modify_set(session, parent); /* * We've locked the parent, which means it cannot split (which is the @@ -862,10 +863,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, WT_TRET(__split_safe_free(session, split_gen, exclusive, pindex, size)); parent_decr += size; - /* Adjust the parent's memory footprint and mark it dirty. */ + /* Adjust the parent's memory footprint. */ __wt_cache_page_inmem_incr(session, parent, parent_incr); __wt_cache_page_inmem_decr(session, parent, parent_decr); - __wt_page_modify_set(session, parent); err: __wt_scr_free(session, &scr); /* @@ -929,8 +929,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_STAT_CONN_INCR(session, cache_eviction_split_internal); WT_STAT_DATA_INCR(session, cache_eviction_split_internal); - /* The page will be marked dirty, make sure that will succeed. */ + /* Mark the page dirty. */ WT_RET(__wt_page_modify_init(session, page)); + __wt_page_modify_set(session, page); btree = S2BT(session); alloc_index = replace_index = NULL; @@ -1136,10 +1137,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); page_decr += size; - /* Adjust the page's memory footprint, and mark it dirty. */ + /* Adjust the page's memory footprint. */ __wt_cache_page_inmem_incr(session, page, page_incr); __wt_cache_page_inmem_decr(session, page, page_decr); - __wt_page_modify_set(session, page); err: switch (complete) { case WT_ERR_RETURN: diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 9c692fd80755a..09f031a182872 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -599,7 +599,6 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, { "checkpoint", "string", NULL, NULL, NULL, 0 }, { "checkpoint_lsn", "string", NULL, NULL, NULL, 0 }, - { "checkpoint_timestamp", "string", NULL, NULL, NULL, 0 }, { "checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0 }, @@ -1435,19 +1434,18 @@ static const WT_CONFIG_ENTRY config_entries[] = { "access_pattern_hint=none,allocation_size=4KB,app_metadata=," "assert=(commit_timestamp=none,read_timestamp=none)," "block_allocation=best,block_compressor=,cache_resident=false," - "checkpoint=,checkpoint_lsn=,checkpoint_timestamp=," - "checksum=uncompressed,collator=,columns=,dictionary=0," - "encryption=(keyid=,name=),format=btree,huffman_key=," - "huffman_value=,id=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB," - "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,value_format=u," - "version=(major=0,minor=0)", - confchk_file_meta, 41 + "checkpoint=,checkpoint_lsn=,checksum=uncompressed,collator=," + "columns=,dictionary=0,encryption=(keyid=,name=),format=btree," + "huffman_key=,huffman_value=,id=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u,version=(major=0,minor=0)", + confchk_file_meta, 40 }, { "index.meta", "app_metadata=,collator=,columns=,extractor=,immutable=false," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index c67ec597f66f5..a8e906a9f1935 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1055,9 +1055,10 @@ __conn_close(WT_CONNECTION *wt_conn, const char *config) if (cval.val != 0) F_SET(conn, WT_CONN_LEAK_MEMORY); WT_TRET(__wt_config_gets(session, cfg, "use_timestamp", &cval)); - if (cval.val != 0 && conn->txn_global.has_stable_timestamp) { + if (cval.val != 0) { ckpt_cfg = "use_timestamp=true"; - F_SET(conn, WT_CONN_CLOSING_TIMESTAMP); + if (conn->txn_global.has_stable_timestamp) + F_SET(conn, WT_CONN_CLOSING_TIMESTAMP); } err: /* diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 799c75d6a0e26..7c24f3c126fdc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -790,7 +790,8 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) restart: TAILQ_FOREACH(dhandle, &conn->dhqh, q) { if (WT_IS_METADATA(dhandle) || - strcmp(dhandle->name, WT_LAS_URI) == 0) + strcmp(dhandle->name, WT_LAS_URI) == 0 || + WT_PREFIX_MATCH(dhandle->name, WT_SYSTEM_PREFIX)) continue; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index f25e3b48db4d5..a9e08cfa4d844 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -470,12 +470,13 @@ __backup_list_uri_append( !WT_PREFIX_MATCH(name, "colgroup:") && !WT_PREFIX_MATCH(name, "index:") && !WT_PREFIX_MATCH(name, "lsm:") && + !WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX) && !WT_PREFIX_MATCH(name, "table:")) WT_RET_MSG(session, ENOTSUP, "hot backup is not supported for objects of type %s", name); - /* Ignore the lookaside table. */ + /* Ignore the lookaside table or system info. */ if (strcmp(name, WT_LAS_URI) == 0) return (0); @@ -485,6 +486,13 @@ __backup_list_uri_append( __wt_free(session, value); WT_RET(ret); + /* + * We want to retain the system information in the backup metadata + * file above, but there is no file object to copy so return now. + */ + if (WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX)) + return (0); + /* Add file type objects to the list of files to be copied. */ if (WT_PREFIX_MATCH(name, "file:")) WT_RET(__backup_list_append(session, cb, name)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index d0203fea19c40..93add0596a4fa 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -121,16 +121,24 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_PAGE *page; - bool clean_page, inmem_split, tree_dead; + bool clean_page, inmem_split, local_gen, tree_dead; conn = S2C(session); page = ref->page; + local_gen = false; __wt_verbose(session, WT_VERB_EVICT, "page %p (%s)", (void *)page, __wt_page_type_string(page->type)); - /* Enter the eviction generation. */ - __wt_session_gen_enter(session, WT_GEN_EVICT); + /* + * Enter the eviction generation. If we re-enter eviction, leave the + * previous eviction generation (which must be as low as the current + * generation), untouched. + */ + if (__wt_session_gen(session, WT_GEN_EVICT) == 0) { + local_gen = true; + __wt_session_gen_enter(session, WT_GEN_EVICT); + } /* * Get exclusive access to the page if our caller doesn't have the tree @@ -221,8 +229,9 @@ err: if (!closing) WT_STAT_DATA_INCR(session, cache_eviction_fail); } -done: /* Leave the eviction generation. */ - __wt_session_gen_leave(session, WT_GEN_EVICT); +done: /* Leave any local eviction generation. */ + if (local_gen) + __wt_session_gen_leave(session, WT_GEN_EVICT); return (ret); } diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 7b932f3ec49e8..caa481808670a 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -509,6 +509,7 @@ extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, W extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep); extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt); +extern int __wt_meta_sysinfo_set(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_remove(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index e64e06d08f8be..97d86b44051d1 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -28,6 +28,9 @@ #define WT_LAS_URI "file:WiredTigerLAS.wt" /* Lookaside table URI*/ +#define WT_SYSTEM_PREFIX "system:" /* System URI prefix */ +#define WT_SYSTEM_CKPT_URI "system:checkpoint" /* Checkpoint URI */ + /* * Optimize comparisons against the metafile URI, flag handles that reference * the metadata file. diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 19e0be2d695a0..32234dca23ef7 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -103,6 +103,7 @@ struct __wt_txn_global { WT_DECL_TIMESTAMP(commit_timestamp) WT_DECL_TIMESTAMP(last_ckpt_timestamp) + WT_DECL_TIMESTAMP(meta_ckpt_timestamp) WT_DECL_TIMESTAMP(oldest_timestamp) WT_DECL_TIMESTAMP(pinned_timestamp) WT_DECL_TIMESTAMP(recovery_timestamp) diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 67604399a2e21..f32a1cbeb199c 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -375,7 +375,6 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, time_t secs; int64_t maxorder; const char *sep; - char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 2]; WT_ERR(__wt_scr_alloc(session, 0, &buf)); maxorder = 0; @@ -453,21 +452,6 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, WT_ERR(__wt_buf_catfmt(session, buf, ",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")", ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset)); - hex_timestamp[0] = '0'; - hex_timestamp[1] = '\0'; -#ifdef HAVE_TIMESTAMPS - /* - * We need to record the timestamp of the checkpoint in the metadata's - * checkpoint record. Although the read_timestamp remains set for the - * duration of the checkpoint, we set and unset the flag based on the - * file's durability. Record the timestamp if the flag is set. - */ - if (F_ISSET(&session->txn, WT_TXN_HAS_TS_READ)) - WT_ERR(__wt_timestamp_to_hex_string(session, hex_timestamp, - &session->txn.read_timestamp)); -#endif - WT_ERR(__wt_buf_catfmt(session, buf, - ",checkpoint_timestamp=\"%s\"", hex_timestamp)); WT_ERR(__ckpt_set(session, fname, buf->mem)); err: __wt_scr_free(session, &buf); @@ -509,6 +493,49 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_CLEAR(*ckpt); /* Clear to prepare for re-use. */ } +/* + * __wt_meta_sysinfo_set -- + * Set the system information in the metadata. + */ +int +__wt_meta_sysinfo_set(WT_SESSION_IMPL *session) +{ + WT_DECL_ITEM(buf); + WT_DECL_RET; + char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 2]; + + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + hex_timestamp[0] = '0'; + hex_timestamp[1] = '\0'; +#ifdef HAVE_TIMESTAMPS + /* + * We need to record the timestamp of the checkpoint in the metadata. + * The timestamp value is set at a higher level, either in checkpoint + * or in recovery. + */ + WT_ERR(__wt_timestamp_to_hex_string(session, hex_timestamp, + &S2C(session)->txn_global.meta_ckpt_timestamp)); +#endif + + /* + * Don't leave a zero entry in the metadata: remove it. This avoids + * downgrade issues if the metadata is opened with an older version of + * WiredTiger that does not understand the new entry. + */ + if (strcmp(hex_timestamp, "0") == 0) + WT_ERR_NOTFOUND_OK( + __wt_metadata_remove(session, WT_SYSTEM_CKPT_URI)); + else { + WT_ERR(__wt_buf_catfmt(session, buf, + "checkpoint_timestamp=\"%s\"", hex_timestamp)); + WT_ERR(__wt_metadata_update( + session, WT_SYSTEM_CKPT_URI, buf->data)); + } + +err: __wt_scr_free(session, &buf); + return (ret); +} + /* * __ckpt_version_chk -- * Check the version major/minor numbers. diff --git a/src/third_party/wiredtiger/src/support/generation.c b/src/third_party/wiredtiger/src/support/generation.c index 34f3ab6afb95d..b962bca739a1b 100644 --- a/src/third_party/wiredtiger/src/support/generation.c +++ b/src/third_party/wiredtiger/src/support/generation.c @@ -190,6 +190,12 @@ __wt_session_gen(WT_SESSION_IMPL *session, int which) void __wt_session_gen_enter(WT_SESSION_IMPL *session, int which) { + /* + * Don't enter a generation we're already in, it will likely result in + * code intended to be protected by a generation running outside one. + */ + WT_ASSERT(session, session->generations[which] == 0); + /* * Assign the thread's resource generation and publish it, ensuring * threads waiting on a resource to drain see the new value. Check we diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index d3f11c5fa69d0..5a71135918a66 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -60,7 +60,10 @@ __checkpoint_name_check(WT_SESSION_IMPL *session, const char *uri) * devices. If a target list is configured for the checkpoint, this * function is called with each target list entry; check the entry to * make sure it's backed by a file. If no target list is configured, - * confirm the metadata file contains no non-file objects. + * confirm the metadata file contains no non-file objects. Skip any + * internal system objects. We don't want spurious error messages, + * other code will skip over them and the user has no control over + * their existence. */ if (uri == NULL) { WT_RET(__wt_metadata_cursor(session, &cursor)); @@ -69,6 +72,7 @@ __checkpoint_name_check(WT_SESSION_IMPL *session, const char *uri) if (!WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "index:") && + !WT_PREFIX_MATCH(uri, WT_SYSTEM_PREFIX) && !WT_PREFIX_MATCH(uri, "table:")) { fail = uri; break; @@ -707,12 +711,31 @@ __checkpoint_prepare( WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_READ | WT_TXN_PUBLIC_TS_COMMIT | WT_TXN_PUBLIC_TS_READ)); - if (use_timestamp && txn_global->has_stable_timestamp) { - __wt_timestamp_set( - &txn->read_timestamp, &txn_global->stable_timestamp); - F_SET(txn, WT_TXN_HAS_TS_READ); - } else + if (use_timestamp) { + /* + * If the user wants timestamps then set the metadata + * checkpoint timestamp based on whether or not a stable + * timestamp is actually in use. Only set it when we're not + * running recovery because recovery doesn't set the recovery + * timestamp until its checkpoint is complete. + */ + if (txn_global->has_stable_timestamp) { + __wt_timestamp_set(&txn->read_timestamp, + &txn_global->stable_timestamp); + F_SET(txn, WT_TXN_HAS_TS_READ); + if (!F_ISSET(conn, WT_CONN_RECOVERING)) + __wt_timestamp_set( + &txn_global->meta_ckpt_timestamp, + &txn->read_timestamp); + } else if (!F_ISSET(conn, WT_CONN_RECOVERING)) + __wt_timestamp_set(&txn_global->meta_ckpt_timestamp, + &txn_global->recovery_timestamp); + } else { __wt_timestamp_set_zero(&txn->read_timestamp); + if (!F_ISSET(conn, WT_CONN_RECOVERING)) + __wt_timestamp_set_zero( + &txn_global->meta_ckpt_timestamp); + } #else WT_UNUSED(use_timestamp); #endif @@ -871,6 +894,21 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) */ session->dhandle = NULL; +#ifdef HAVE_TIMESTAMPS + /* + * Record the timestamp from the transaction if we were successful. + * Store it in a temp variable now because it will be invalidated during + * commit but we don't want to set it until we know the checkpoint + * is successful. We have to set the system information before we + * release the snapshot. + */ + __wt_timestamp_set_zero(&ckpt_tmp_ts); + if (full) { + WT_ERR(__wt_meta_sysinfo_set(session)); + __wt_timestamp_set(&ckpt_tmp_ts, &txn->read_timestamp); + } +#endif + /* Release the snapshot so we aren't pinning updates in cache. */ __wt_txn_release_snapshot(session); @@ -900,15 +938,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * checkpointing the metadata since we know that all files in the * checkpoint are now in a consistent state. */ -#ifdef HAVE_TIMESTAMPS - /* - * Record the timestamp from the transaction if we were successful. - * Store it in a temp variable now because it will be invalidated during - * commit but we don't want to set it until we know the checkpoint - * is successful. - */ - __wt_timestamp_set(&ckpt_tmp_ts, &txn->read_timestamp); -#endif WT_ERR(__wt_txn_commit(session, NULL)); /* diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 422810ce850a8..c45059b425176 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -24,7 +24,6 @@ typedef struct { u_int max_fileid; /* Maximum file ID seen. */ WT_LSN max_lsn; /* Maximum checkpoint LSN seen. */ u_int nfiles; /* Number of files in the metadata. */ - WT_DECL_TIMESTAMP(max_timestamp) WT_LSN ckpt_lsn; /* Start LSN for main recovery loop. */ @@ -339,6 +338,75 @@ __txn_log_recover(WT_SESSION_IMPL *session, return (0); } +/* + * __recovery_set_checkpoint_timestamp -- + * Set the checkpoint timestamp as retrieved from the metadata file. + */ +static int +__recovery_set_checkpoint_timestamp(WT_RECOVERY *r) +{ +#ifdef HAVE_TIMESTAMPS + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_DECL_TIMESTAMP(ckpt_timestamp) + WT_SESSION_IMPL *session; + char *sys_config; + + sys_config = NULL; + + session = r->session; + conn = S2C(session); + /* + * Read the system checkpoint information from the metadata file and + * save the stable timestamp of the last checkpoint for later query. + * This gets saved in the connection. + */ + __wt_timestamp_set_zero(&ckpt_timestamp); + + /* Search in the metadata for the system information. */ + WT_ERR_NOTFOUND_OK( + __wt_metadata_search(session, WT_SYSTEM_CKPT_URI, &sys_config)); + if (sys_config != NULL) { + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_getones( + session, sys_config, "checkpoint_timestamp", &cval)); + if (cval.len != 0) { + __wt_verbose(session, WT_VERB_RECOVERY, + "Recovery timestamp %.*s", + (int)cval.len, cval.str); + WT_ERR(__wt_txn_parse_timestamp_raw(session, + "recovery", &ckpt_timestamp, &cval)); + } + } + + /* + * Set the recovery checkpoint timestamp and the metadata checkpoint + * timestamp so that the checkpoint after recovery writes the correct + * value into the metadata. + */ + __wt_timestamp_set( + &conn->txn_global.meta_ckpt_timestamp, &ckpt_timestamp); + __wt_timestamp_set( + &conn->txn_global.recovery_timestamp, &ckpt_timestamp); + + if (WT_VERBOSE_ISSET(session, + WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS)) { + char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 1]; + WT_TRET(__wt_timestamp_to_hex_string(session, + hex_timestamp, &conn->txn_global.recovery_timestamp)); + __wt_verbose(session, + WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, + "Set global recovery timestamp: %s", hex_timestamp); + } +err: __wt_free(session, sys_config); + return (ret); +#else + WT_UNUSED(r); + return (0); +#endif +} + /* * __recovery_setup_file -- * Set up the recovery slot for a file. @@ -347,7 +415,7 @@ static int __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) { WT_CONFIG_ITEM cval; - WT_DECL_TIMESTAMP(ckpt_timestamp) + WT_DECL_RET; WT_LSN lsn; uint32_t fileid, lsnfile, lsnoffset; @@ -364,31 +432,8 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) r->nfiles = fileid + 1; } -#ifdef HAVE_TIMESTAMPS - /* - * If we're reading the config for the metadata from the turtle file - * save the stable timestamp of the last checkpoint for later query. - * This gets saved in the connection. - */ - WT_CLEAR(cval); - WT_RET_NOTFOUND_OK(__wt_config_getones(r->session, - config, "checkpoint_timestamp", &cval)); - if (cval.len != 0) { - __wt_verbose(r->session, WT_VERB_RECOVERY, - "%s: Recovery timestamp %.*s", - uri, (int)cval.len, cval.str); - WT_RET(__wt_txn_parse_timestamp_raw(r->session, "recovery", - &ckpt_timestamp, &cval)); - /* - * Keep track of the largest checkpoint timestamp seen. - */ - if (__wt_timestamp_cmp(&ckpt_timestamp, &r->max_timestamp) > 0) - __wt_timestamp_set(&r->max_timestamp, &ckpt_timestamp); - } -#endif - - WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri)); - WT_RET( + WT_ERR(__wt_strdup(r->session, uri, &r->files[fileid].uri)); + WT_ERR( __wt_config_getones(r->session, config, "checkpoint_lsn", &cval)); /* If there is checkpoint logged for the file, apply everything. */ if (cval.type != WT_CONFIG_ITEM_STRUCT) @@ -397,7 +442,7 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) "(%" SCNu32 ",%" SCNu32 ")", &lsnfile, &lsnoffset) == 2) WT_SET_LSN(&lsn, lsnfile, lsnoffset); else - WT_RET_MSG(r->session, EINVAL, + WT_ERR_MSG(r->session, EINVAL, "Failed to parse checkpoint LSN '%.*s'", (int)cval.len, cval.str); r->files[fileid].ckpt_lsn = lsn; @@ -410,7 +455,7 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) (WT_IS_MAX_LSN(&r->max_lsn) || __wt_log_cmp(&lsn, &r->max_lsn) > 0)) r->max_lsn = lsn; - return (0); +err: return (ret); } @@ -484,11 +529,13 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_RECOVERY r; WT_RECOVERY_FILE *metafile; char *config; - bool eviction_started, needs_rec, was_backup; + bool do_checkpoint, eviction_started, needs_rec, was_backup; conn = S2C(session); WT_CLEAR(r); WT_INIT_LSN(&r.ckpt_lsn); + config = NULL; + do_checkpoint = true; eviction_started = false; was_backup = F_ISSET(conn, WT_CONN_WAS_BACKUP); @@ -499,7 +546,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_MAX_LSN(&r.max_lsn); #ifdef HAVE_TIMESTAMPS __wt_timestamp_set_zero(&conn->txn_global.recovery_timestamp); - __wt_timestamp_set_zero(&r.max_timestamp); + __wt_timestamp_set_zero(&conn->txn_global.meta_ckpt_timestamp); #endif F_SET(conn, WT_CONN_RECOVERING); @@ -534,11 +581,11 @@ __wt_txn_recover(WT_SESSION_IMPL *session) if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && WT_IS_MAX_LSN(&metafile->ckpt_lsn) && - !WT_IS_MAX_LSN(&r.max_lsn)) { + !WT_IS_MAX_LSN(&r.max_lsn)) WT_ERR(__wt_log_reset(session, r.max_lsn.l.file)); - goto ckpt; - } else - goto done; + else + do_checkpoint = false; + goto done; } /* @@ -617,8 +664,10 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_ERR(WT_RUN_RECOVERY); } - if (F_ISSET(conn, WT_CONN_READONLY)) + if (F_ISSET(conn, WT_CONN_READONLY)) { + do_checkpoint = false; goto done; + } /* * Recovery can touch more data than fits in cache, so it relies on @@ -649,30 +698,15 @@ __wt_txn_recover(WT_SESSION_IMPL *session) conn->next_file_id = r.max_fileid; - /* - * If recovery ran successfully forcibly log a checkpoint so the next - * open is fast and keep the metadata up to date with the checkpoint - * LSN and archiving. - */ -ckpt: WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); -done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); -#ifdef HAVE_TIMESTAMPS - /* - * After recovery, set the recovery timestamp to the largest one we - * recovered. This is done at the end so that it is set whether we - * ran a full recovery or not. In all cases, we've reviewed all the - * files in the metadata. - */ - { - char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 1]; - __wt_timestamp_set( - &conn->txn_global.recovery_timestamp, &r.max_timestamp); - WT_TRET(__wt_timestamp_to_hex_string(session, - hex_timestamp, &conn->txn_global.recovery_timestamp)); - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, - "Set global recovery timestamp: %s", hex_timestamp); - } -#endif +done: WT_ERR(__recovery_set_checkpoint_timestamp(&r)); + if (do_checkpoint) + /* + * Forcibly log a checkpoint so the next open is fast and keep + * the metadata up to date with the checkpoint LSN and + * archiving. + */ + WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); err: WT_TRET(__recovery_free(&r)); __wt_free(session, config); diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index 7ff17394f79e3..54e3cd61f8bec 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -139,8 +139,11 @@ list_print(WT_SESSION *session, const char *uri, bool cflag, bool vflag) * We don't normally say anything about the WiredTiger metadata * and lookaside tables, they're not application/user "objects" * in the database. I'm making an exception for the checkpoint - * and verbose options. + * and verbose options. However, skip over the metadata system + * information for anything except the verbose option. */ + if (!vflag && WT_PREFIX_MATCH(key, WT_SYSTEM_PREFIX)) + continue; if (cflag || vflag || (strcmp(key, WT_METADATA_URI) != 0 && strcmp(key, WT_LAS_URI) != 0)) diff --git a/src/third_party/wiredtiger/test/suite/test_backup08.py b/src/third_party/wiredtiger/test/suite/test_backup08.py new file mode 100644 index 0000000000000..145a36c5a584e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup08.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_backup08.py +# Timestamps: Verify the saved checkpoint timestamp survives a backup. +# + +import os, shutil +import wiredtiger, wttest +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +class test_backup08(wttest.WiredTigerTestCase): + conn_config = 'config_base=false,create,log=(enabled)' + dir = 'backup.dir' + coll1_uri = 'table:collection10.1' + coll2_uri = 'table:collection10.2' + coll3_uri = 'table:collection10.3' + oplog_uri = 'table:oplog10' + + nentries = 10 + table_cnt = 3 + + types = [ + ('all', dict(use_stable='false')), + ('default', dict(use_stable='default')), + ('stable', dict(use_stable='true')), + ] + scenarios = make_scenarios(types) + + def data_and_checkpoint(self): + # + # Create several collection-like tables that are checkpoint durability. + # Add data to each of them separately and checkpoint so that each one + # has a different stable timestamp. + # + self.session.create(self.oplog_uri, 'key_format=i,value_format=i') + self.session.create(self.coll1_uri, 'key_format=i,value_format=i,log=(enabled=false)') + self.session.create(self.coll2_uri, 'key_format=i,value_format=i,log=(enabled=false)') + self.session.create(self.coll3_uri, 'key_format=i,value_format=i,log=(enabled=false)') + c_op = self.session.open_cursor(self.oplog_uri) + c = [] + c.append(self.session.open_cursor(self.coll1_uri)) + c.append(self.session.open_cursor(self.coll2_uri)) + c.append(self.session.open_cursor(self.coll3_uri)) + + # Begin by adding some data. + for table in range(1,self.table_cnt+1): + curs = c[table - 1] + start = self.nentries * table + end = start + self.nentries + ts = (end - 3) + self.pr("table: " + str(table)) + for i in range(start,end): + self.session.begin_transaction() + c_op[i] = i + curs[i] = i + self.pr("i: " + str(i)) + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(i)) + # Set the oldest and stable timestamp a bit earlier than the data + # we inserted. Take a checkpoint to the stable timestamp. + self.pr("stable ts: " + str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + + ',stable_timestamp=' + timestamp_str(ts)) + # This forces a different checkpoint timestamp for each table. + # Default is to use the stable timestamp. + expected = ts + ckpt_config = '' + if self.use_stable == 'false': + expected = 0 + ckpt_config = 'use_timestamp=false' + elif self.use_stable == 'true': + ckpt_config = 'use_timestamp=true' + self.session.checkpoint(ckpt_config) + q = self.conn.query_timestamp('get=last_checkpoint') + self.assertTimestampsEqual(q, timestamp_str(expected)) + return expected + + def backup_and_recover(self, expected_rec_ts): + # + # Perform a live backup. Then open the backup and query the + # recovery timestamp verifying it is the expected value. + # + os.mkdir(self.dir) + cursor = self.session.open_cursor('backup:') + while True: + ret = cursor.next() + if ret != 0: + break + shutil.copy(cursor.get_key(), self.dir) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + cursor.close() + + backup_conn = self.wiredtiger_open(self.dir) + q = backup_conn.query_timestamp('get=recovery') + self.pr("query recovery ts: " + q) + self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + + def test_timestamp_backup(self): + if not wiredtiger.timestamp_build(): + self.skipTest('requires a timestamp build') + + # Add some data and checkpoint using the timestamp or not + # depending on the configuration. Get the expected timestamp + # where the data is checkpointed for the backup. + ckpt_ts = self.data_and_checkpoint() + + # Backup and run recovery checking the recovery timestamp. + # This tests that the stable timestamp information is transferred + # with the backup. It should be part of the backup metadata file. + self.backup_and_recover(ckpt_ts) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 02b22e6afbeff..ca703ec07f402 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -27,12 +27,12 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_timestamp10.py -# Timestamps: Saving and querying the last checkpoint and recovery timestamps +# Timestamps: Saving and querying the last checkpoint and recovery timestamps. # -import fnmatch, os, shutil from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios def timestamp_str(t): return '%x' % t @@ -44,26 +44,23 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): coll3_uri = 'table:collection10.3' oplog_uri = 'table:oplog10' - def copy_dir(self, olddir, newdir): - ''' Simulate a crash from olddir and restart in newdir. ''' - # with the connection still open, copy files to new directory - shutil.rmtree(newdir, ignore_errors=True) - os.mkdir(newdir) - for fname in os.listdir(olddir): - fullname = os.path.join(olddir, fname) - # Skip lock file on Windows since it is locked - if os.path.isfile(fullname) and \ - "WiredTiger.lock" not in fullname and \ - "Tmplog" not in fullname and \ - "Preplog" not in fullname: - shutil.copy(fullname, newdir) - # close the original connection. - self.close_conn() + nentries = 10 + table_cnt = 3 - def test_timestamp_recovery(self): - if not wiredtiger.timestamp_build(): - self.skipTest('requires a timestamp build') + types = [ + ('all', dict(use_stable='false', run_wt=0)), + ('all+wt', dict(use_stable='false', run_wt=1)), + ('all+wt2', dict(use_stable='false', run_wt=2)), + ('default', dict(use_stable='default', run_wt=0)), + ('default+wt', dict(use_stable='default', run_wt=1)), + ('default+wt2', dict(use_stable='default', run_wt=2)), + ('stable', dict(use_stable='true', run_wt=0)), + ('stable+wt', dict(use_stable='true', run_wt=1)), + ('stable+wt2', dict(use_stable='true', run_wt=2)), + ] + scenarios = make_scenarios(types) + def data_and_checkpoint(self): # # Create several collection-like tables that are checkpoint durability. # Add data to each of them separately and checkpoint so that each one @@ -80,13 +77,12 @@ def test_timestamp_recovery(self): c.append(self.session.open_cursor(self.coll3_uri)) # Begin by adding some data. - nentries = 10 - table_cnt = 3 - for table in range(1,table_cnt+1): + for table in range(1,self.table_cnt+1): curs = c[table - 1] - start = nentries * table - end = start + nentries + start = self.nentries * table + end = start + self.nentries ts = (end - 3) + self.pr("table: " + str(table)) for i in range(start,end): self.session.begin_transaction() c_op[i] = i @@ -103,49 +99,68 @@ def test_timestamp_recovery(self): self.session.checkpoint() q = self.conn.query_timestamp('get=last_checkpoint') self.assertTimestampsEqual(q, timestamp_str(ts)) + return ts + + def close_and_recover(self, expected_rec_ts): + # + # Close with the close configuration string and optionally run + # the 'wt' command. Then open the connection again and query the + # recovery timestamp verifying it is the expected value. + # + if self.use_stable == 'true': + close_cfg = 'use_timestamp=true' + elif self.use_stable == 'false': + close_cfg = 'use_timestamp=false' + else: + close_cfg = '' + self.close_conn(close_cfg) - # Copy to a new database and then recover. - self.copy_dir(".", "RESTART") - self.copy_dir(".", "SAVE") - new_conn = self.wiredtiger_open("RESTART", self.conn_config) - # Query the recovery timestamp and verify the data in the new database. - new_session = new_conn.open_session() - q = new_conn.query_timestamp('get=recovery') + # Run the wt command some number of times to get some runs in that do + # not use timestamps. Make sure the recovery checkpoint is maintained. + for i in range(0, self.run_wt): + self.runWt(['-h', '.', '-R', 'list', '-v'], outfilename="list.out") + + self.open_conn() + q = self.conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(ts)) + self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + + def test_timestamp_recovery(self): + if not wiredtiger.timestamp_build(): + self.skipTest('requires a timestamp build') - c_op = new_session.open_cursor(self.oplog_uri) + # Add some data and checkpoint at a stable timestamp. + last_stable = self.data_and_checkpoint() + + expected = 0 + # Note: assumes default is true. + if self.use_stable != 'false': + expected = last_stable + # Close and run recovery checking the stable timestamp. + self.close_and_recover(expected) + + # Verify the data in the recovered database. + c_op = self.session.open_cursor(self.oplog_uri) c = [] - c.append(new_session.open_cursor(self.coll1_uri)) - c.append(new_session.open_cursor(self.coll2_uri)) - c.append(new_session.open_cursor(self.coll3_uri)) - for table in range(1,table_cnt+1): + c.append(self.session.open_cursor(self.coll1_uri)) + c.append(self.session.open_cursor(self.coll2_uri)) + c.append(self.session.open_cursor(self.coll3_uri)) + for table in range(1,self.table_cnt+1): curs = c[table - 1] - start = nentries * table - end = start + nentries + start = self.nentries * table + end = start + self.nentries ts = (end - 3) for i in range(start,end): + # The oplog-like table is logged so it always has all the data. self.assertEquals(c_op[i], i) curs.set_key(i) # Earlier tables have all the data because later checkpoints # will save the last bit of data. Only the last table will # be missing some. - if i <= ts or table != table_cnt: + if self.use_stable == 'false' or i <= ts or table != self.table_cnt: self.assertEquals(curs[i], i) else: self.assertEqual(curs.search(), wiredtiger.WT_NOTFOUND) - new_conn.close() - # - # Run the wt command so that we get a non-logged recovery. - # - self.runWt(['-h', 'RESTART', 'list', '-v'], outfilename="list.out") - new_conn = self.wiredtiger_open("RESTART", self.conn_config) - # Query the recovery timestamp and verify the data in the new database. - new_session = new_conn.open_session() - q = new_conn.query_timestamp('get=recovery') - self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(ts)) - if __name__ == '__main__': wttest.run()