Import wiredtiger: 5fc85c47caba5dbd4fc49ad6fa924fee4e3d5695 from bran…

…ch mongodb-3.8

ref: 5bfcc92407..5fc85c47ca
for: 3.7.4

WT-3959       Recovery timestamp set on restart scenarios need addressing
WT-4032       parent pages can be evicted while being split.
WT-4034       Re-entering eviction can result in checkpoint corruption

src/third_party/wiredtiger/dist/api_data.py

@@ -351,8 +351,6 @@ def __cmp__(self, other):
         the file checkpoint entries'''),
     Config('checkpoint_lsn', '', r'''
         LSN of the last checkpoint'''),
-    Config('checkpoint_timestamp', '', r'''
-        stable timestamp of the last checkpoint'''),
     Config('id', '', r'''
         the file's ID number'''),
     Config('version', '(major=0,minor=0)', r'''

src/third_party/wiredtiger/dist/s_string.ok

@@ -1201,6 +1201,7 @@ sx
 sy
 sys
 syscall
+sysinfo
 sz
 t's
 tV

src/third_party/wiredtiger/import.data

@@ -1,5 +1,5 @@
 {
-    "commit": "5bfcc924079afdcd6dda22c29b5fa60a14ec3dc9", 
+    "commit": "5fc85c47caba5dbd4fc49ad6fa924fee4e3d5695", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-3.8"

src/third_party/wiredtiger/src/btree/bt_split.c

@@ -402,8 +402,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root)
 	root_decr = root_incr = 0;
 	complete = WT_ERR_RETURN;
 
-	/* The root page will be marked dirty, make sure that will succeed. */
+	/* Mark the root page dirty. */
 	WT_RET(__wt_page_modify_init(session, root));
+	__wt_page_modify_set(session, root);
 
 	/*
 	 * Our caller is holding the root page locked to single-thread splits,
@@ -574,10 +575,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root)
 	WT_TRET(__split_safe_free(session, split_gen, false, pindex, size));
 	root_decr += size;
 
-	/* Adjust the root's memory footprint and mark it dirty. */
+	/* Adjust the root's memory footprint. */
 	__wt_cache_page_inmem_incr(session, root, root_incr);
 	__wt_cache_page_inmem_decr(session, root, root_decr);
-	__wt_page_modify_set(session, root);
 
 err:	switch (complete) {
 	case WT_ERR_RETURN:
@@ -629,8 +629,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	empty_parent = false;
 	complete = WT_ERR_RETURN;
 
-	/* The parent page will be marked dirty, make sure that will succeed. */
+	/* Mark the page dirty. */
 	WT_RET(__wt_page_modify_init(session, parent));
+	__wt_page_modify_set(session, parent);
 
 	/*
 	 * We've locked the parent, which means it cannot split (which is the
@@ -862,10 +863,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	WT_TRET(__split_safe_free(session, split_gen, exclusive, pindex, size));
 	parent_decr += size;
 
-	/* Adjust the parent's memory footprint and mark it dirty. */
+	/* Adjust the parent's memory footprint. */
 	__wt_cache_page_inmem_incr(session, parent, parent_incr);
 	__wt_cache_page_inmem_decr(session, parent, parent_decr);
-	__wt_page_modify_set(session, parent);
 
 err:	__wt_scr_free(session, &scr);
 	/*
@@ -929,8 +929,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page)
 	WT_STAT_CONN_INCR(session, cache_eviction_split_internal);
 	WT_STAT_DATA_INCR(session, cache_eviction_split_internal);
 
-	/* The page will be marked dirty, make sure that will succeed. */
+	/* Mark the page dirty. */
 	WT_RET(__wt_page_modify_init(session, page));
+	__wt_page_modify_set(session, page);
 
 	btree = S2BT(session);
 	alloc_index = replace_index = NULL;
@@ -1136,10 +1137,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page)
 	WT_TRET(__split_safe_free(session, split_gen, false, pindex, size));
 	page_decr += size;
 
-	/* Adjust the page's memory footprint, and mark it dirty. */
+	/* Adjust the page's memory footprint. */
 	__wt_cache_page_inmem_incr(session, page, page_incr);
 	__wt_cache_page_inmem_decr(session, page, page_decr);
-	__wt_page_modify_set(session, page);
 
 err:	switch (complete) {
 	case WT_ERR_RETURN:

src/third_party/wiredtiger/src/config/config_def.c

@@ -599,7 +599,6 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = {
 	{ "cache_resident", "boolean", NULL, NULL, NULL, 0 },
 	{ "checkpoint", "string", NULL, NULL, NULL, 0 },
 	{ "checkpoint_lsn", "string", NULL, NULL, NULL, 0 },
-	{ "checkpoint_timestamp", "string", NULL, NULL, NULL, 0 },
 	{ "checksum", "string",
 	    NULL, "choices=[\"on\",\"off\",\"uncompressed\"]",
 	    NULL, 0 },
@@ -1435,19 +1434,18 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "access_pattern_hint=none,allocation_size=4KB,app_metadata=,"
 	  "assert=(commit_timestamp=none,read_timestamp=none),"
 	  "block_allocation=best,block_compressor=,cache_resident=false,"
-	  "checkpoint=,checkpoint_lsn=,checkpoint_timestamp=,"
-	  "checksum=uncompressed,collator=,columns=,dictionary=0,"
-	  "encryption=(keyid=,name=),format=btree,huffman_key=,"
-	  "huffman_value=,id=,ignore_in_memory_cache_size=false,"
-	  "internal_item_max=0,internal_key_max=0,"
-	  "internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
-	  "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
-	  "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB,"
-	  "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,"
-	  "prefix_compression_min=4,split_deepen_min_child=0,"
-	  "split_deepen_per_child=0,split_pct=90,value_format=u,"
-	  "version=(major=0,minor=0)",
-	  confchk_file_meta, 41
+	  "checkpoint=,checkpoint_lsn=,checksum=uncompressed,collator=,"
+	  "columns=,dictionary=0,encryption=(keyid=,name=),format=btree,"
+	  "huffman_key=,huffman_value=,id=,"
+	  "ignore_in_memory_cache_size=false,internal_item_max=0,"
+	  "internal_key_max=0,internal_key_truncate=true,"
+	  "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,"
+	  "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0,"
+	  "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0,"
+	  "os_cache_max=0,prefix_compression=false,prefix_compression_min=4"
+	  ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90,"
+	  "value_format=u,version=(major=0,minor=0)",
+	  confchk_file_meta, 40
 	},
 	{ "index.meta",
 	  "app_metadata=,collator=,columns=,extractor=,immutable=false,"

src/third_party/wiredtiger/src/conn/conn_api.c

@@ -1055,9 +1055,10 @@ __conn_close(WT_CONNECTION *wt_conn, const char *config)
 	if (cval.val != 0)
 		F_SET(conn, WT_CONN_LEAK_MEMORY);
 	WT_TRET(__wt_config_gets(session, cfg, "use_timestamp", &cval));
-	if (cval.val != 0 && conn->txn_global.has_stable_timestamp) {
+	if (cval.val != 0) {
 		ckpt_cfg = "use_timestamp=true";
-		F_SET(conn, WT_CONN_CLOSING_TIMESTAMP);
+		if (conn->txn_global.has_stable_timestamp)
+			F_SET(conn, WT_CONN_CLOSING_TIMESTAMP);
 	}
 
 err:	/*

src/third_party/wiredtiger/src/conn/conn_dhandle.c

@@ -790,7 +790,8 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session)
 restart:
 	TAILQ_FOREACH(dhandle, &conn->dhqh, q) {
 		if (WT_IS_METADATA(dhandle) ||
-		    strcmp(dhandle->name, WT_LAS_URI) == 0)
+		    strcmp(dhandle->name, WT_LAS_URI) == 0 ||
+		    WT_PREFIX_MATCH(dhandle->name, WT_SYSTEM_PREFIX))
 			continue;
 
 		WT_WITH_DHANDLE(session, dhandle,

src/third_party/wiredtiger/src/cursor/cur_backup.c

@@ -470,12 +470,13 @@ __backup_list_uri_append(
 	    !WT_PREFIX_MATCH(name, "colgroup:") &&
 	    !WT_PREFIX_MATCH(name, "index:") &&
 	    !WT_PREFIX_MATCH(name, "lsm:") &&
+	    !WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX) &&
 	    !WT_PREFIX_MATCH(name, "table:"))
 		WT_RET_MSG(session, ENOTSUP,
 		    "hot backup is not supported for objects of type %s",
 		    name);
 
-	/* Ignore the lookaside table. */
+	/* Ignore the lookaside table or system info. */
 	if (strcmp(name, WT_LAS_URI) == 0)
 		return (0);
 
@@ -485,6 +486,13 @@ __backup_list_uri_append(
 	__wt_free(session, value);
 	WT_RET(ret);
 
+	/*
+	 * We want to retain the system information in the backup metadata
+	 * file above, but there is no file object to copy so return now.
+	 */
+	if (WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX))
+		return (0);
+
 	/* Add file type objects to the list of files to be copied. */
 	if (WT_PREFIX_MATCH(name, "file:"))
 		WT_RET(__backup_list_append(session, cb, name));

src/third_party/wiredtiger/src/evict/evict_page.c

@@ -121,16 +121,24 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing)
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_PAGE *page;
-	bool clean_page, inmem_split, tree_dead;
+	bool clean_page, inmem_split, local_gen, tree_dead;
 
 	conn = S2C(session);
 	page = ref->page;
+	local_gen = false;
 
 	__wt_verbose(session, WT_VERB_EVICT,
 	    "page %p (%s)", (void *)page, __wt_page_type_string(page->type));
 
-	/* Enter the eviction generation. */
-	__wt_session_gen_enter(session, WT_GEN_EVICT);
+	/*
+	 * Enter the eviction generation. If we re-enter eviction, leave the
+	 * previous eviction generation (which must be as low as the current
+	 * generation), untouched.
+	 */
+	if (__wt_session_gen(session, WT_GEN_EVICT) == 0) {
+		local_gen = true;
+		__wt_session_gen_enter(session, WT_GEN_EVICT);
+	}
 
 	/*
 	 * Get exclusive access to the page if our caller doesn't have the tree
@@ -221,8 +229,9 @@ err:		if (!closing)
 		WT_STAT_DATA_INCR(session, cache_eviction_fail);
 	}
 
-done:	/* Leave the eviction generation. */
-	__wt_session_gen_leave(session, WT_GEN_EVICT);
+done:	/* Leave any local eviction generation. */
+	if (local_gen)
+		__wt_session_gen_leave(session, WT_GEN_EVICT);
 
 	return (ret);
 }

src/third_party/wiredtiger/src/include/extern.h

@@ -509,6 +509,7 @@ extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, W
 extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep);
 extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt);
+extern int __wt_meta_sysinfo_set(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_ext_metadata_remove(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));

src/third_party/wiredtiger/src/include/meta.h

@@ -28,6 +28,9 @@
 
 #define	WT_LAS_URI		"file:WiredTigerLAS.wt"	/* Lookaside table URI*/
 
+#define	WT_SYSTEM_PREFIX	"system:"		/* System URI prefix */
+#define	WT_SYSTEM_CKPT_URI	"system:checkpoint"	/* Checkpoint URI */
+
 /*
  * Optimize comparisons against the metafile URI, flag handles that reference
  * the metadata file.

src/third_party/wiredtiger/src/include/txn.h

@@ -103,6 +103,7 @@ struct __wt_txn_global {
 
 	WT_DECL_TIMESTAMP(commit_timestamp)
 	WT_DECL_TIMESTAMP(last_ckpt_timestamp)
+	WT_DECL_TIMESTAMP(meta_ckpt_timestamp)
 	WT_DECL_TIMESTAMP(oldest_timestamp)
 	WT_DECL_TIMESTAMP(pinned_timestamp)
 	WT_DECL_TIMESTAMP(recovery_timestamp)

src/third_party/wiredtiger/src/meta/meta_ckpt.c

@@ -375,7 +375,6 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
 	time_t secs;
 	int64_t maxorder;
 	const char *sep;
-	char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 2];
 
 	WT_ERR(__wt_scr_alloc(session, 0, &buf));
 	maxorder = 0;
@@ -453,21 +452,6 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
 		WT_ERR(__wt_buf_catfmt(session, buf,
 		    ",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")",
 		    ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset));
-	hex_timestamp[0] = '0';
-	hex_timestamp[1] = '\0';
-#ifdef HAVE_TIMESTAMPS
-	/*
-	 * We need to record the timestamp of the checkpoint in the metadata's
-	 * checkpoint record. Although the read_timestamp remains set for the
-	 * duration of the checkpoint, we set and unset the flag based on the
-	 * file's durability. Record the timestamp if the flag is set.
-	 */
-	if (F_ISSET(&session->txn, WT_TXN_HAS_TS_READ))
-		WT_ERR(__wt_timestamp_to_hex_string(session, hex_timestamp,
-		    &session->txn.read_timestamp));
-#endif
-	WT_ERR(__wt_buf_catfmt(session, buf,
-	    ",checkpoint_timestamp=\"%s\"", hex_timestamp));
 	WT_ERR(__ckpt_set(session, fname, buf->mem));
 
 err:	__wt_scr_free(session, &buf);
@@ -509,6 +493,49 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
 	WT_CLEAR(*ckpt);		/* Clear to prepare for re-use. */
 }
 
+/*
+ * __wt_meta_sysinfo_set --
+ *	Set the system information in the metadata.
+ */
+int
+__wt_meta_sysinfo_set(WT_SESSION_IMPL *session)
+{
+	WT_DECL_ITEM(buf);
+	WT_DECL_RET;
+	char hex_timestamp[2 * WT_TIMESTAMP_SIZE + 2];
+
+	WT_ERR(__wt_scr_alloc(session, 0, &buf));
+	hex_timestamp[0] = '0';
+	hex_timestamp[1] = '\0';
+#ifdef HAVE_TIMESTAMPS
+	/*
+	 * We need to record the timestamp of the checkpoint in the metadata.
+	 * The timestamp value is set at a higher level, either in checkpoint
+	 * or in recovery.
+	 */
+	WT_ERR(__wt_timestamp_to_hex_string(session, hex_timestamp,
+	    &S2C(session)->txn_global.meta_ckpt_timestamp));
+#endif
+
+	/*
+	 * Don't leave a zero entry in the metadata: remove it.  This avoids
+	 * downgrade issues if the metadata is opened with an older version of
+	 * WiredTiger that does not understand the new entry.
+	 */
+	if (strcmp(hex_timestamp, "0") == 0)
+		WT_ERR_NOTFOUND_OK(
+		    __wt_metadata_remove(session, WT_SYSTEM_CKPT_URI));
+	else {
+		WT_ERR(__wt_buf_catfmt(session, buf,
+		    "checkpoint_timestamp=\"%s\"", hex_timestamp));
+		WT_ERR(__wt_metadata_update(
+		    session, WT_SYSTEM_CKPT_URI, buf->data));
+	}
+
+err:	__wt_scr_free(session, &buf);
+	return (ret);
+}
+
 /*
  * __ckpt_version_chk --
  *	Check the version major/minor numbers.

src/third_party/wiredtiger/src/support/generation.c

@@ -190,6 +190,12 @@ __wt_session_gen(WT_SESSION_IMPL *session, int which)
 void
 __wt_session_gen_enter(WT_SESSION_IMPL *session, int which)
 {
+	/*
+	 * Don't enter a generation we're already in, it will likely result in
+	 * code intended to be protected by a generation running outside one.
+	 */
+	WT_ASSERT(session, session->generations[which] == 0);
+
 	/*
 	 * Assign the thread's resource generation and publish it, ensuring
 	 * threads waiting on a resource to drain see the new value. Check we