refactor OID class a bit, cleanup, better id generation SERVER-2175

SConstruct

@@ -296,7 +296,7 @@ if has_option( "full" ):
 
 # ------    SOURCE FILE SETUP -----------
 
-commonFiles = Split( "pch.cpp buildinfo.cpp db/common.cpp db/jsobj.cpp db/json.cpp db/lasterror.cpp db/nonce.cpp db/queryutil.cpp db/projection.cpp shell/mongo.cpp" )
+commonFiles = Split( "pch.cpp buildinfo.cpp db/common.cpp db/jsobj.cpp bson/oid.cpp db/json.cpp db/lasterror.cpp db/nonce.cpp db/queryutil.cpp db/projection.cpp shell/mongo.cpp" )
 commonFiles += [ "util/background.cpp" , "util/mmap.cpp" , "util/sock.cpp" ,  "util/util.cpp" , "util/message.cpp" , 
                  "util/assert_util.cpp" , "util/log.cpp" , "util/httpclient.cpp" , "util/md5main.cpp" , "util/base64.cpp", "util/concurrency/vars.cpp", "util/concurrency/task.cpp", "util/debug_util.cpp",
                  "util/concurrency/thread_pool.cpp", "util/password.cpp", "util/version.cpp", "util/signal_handlers.cpp",  

bson/oid.cpp

@@ -0,0 +1,135 @@
+// @file oid.cpp
+
+/*    Copyright 2009 10gen Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#pragma once
+
+#include "pch.h"
+#include "oid.h"
+#include "util/atomic_int.h"
+#include "../db/nonce.h"
+
+namespace mongo {
+
+    static unsigned short ourPid() { 
+        unsigned short pid;
+#if defined(_WIN32)
+        pid = (unsigned short) GetCurrentProcessId();
+#elif defined(__linux__)
+        pid = (unsigned short) getpid();
+#else
+        pid = (unsigned short) security.getNonce();
+#endif
+        return pid;
+    }
+
+    // machine # before folding in the process id
+    static OID::MachineAndPid ourMachine;
+
+    static OID::MachineAndPid __gen() { 
+        unsigned long long n = security.getNonce();
+        OID::MachineAndPid x = ourMachine = (OID::MachineAndPid&) n;
+        x._pid = x._pid ^ ourPid();
+        return x;
+    }
+
+    // after folding in the process id
+    static OID::MachineAndPid ourMachineAndPid = __gen();
+
+    void OID::regenMachineId() { 
+        ourMachineAndPid = __gen();
+    }
+
+    inline bool operator!=(const OID::MachineAndPid& l, const OID::MachineAndPid& r) { 
+        return l._pid != r._pid || l._machineNumber != r._machineNumber;
+    }
+
+    unsigned OID::getMachineId() { 
+        unsigned char x[4];
+        x[0] = ourMachineAndPid._machineNumber[0];
+        x[1] = ourMachineAndPid._machineNumber[1];
+        x[2] = ourMachineAndPid._machineNumber[2];
+        x[3] = 0;
+        return (unsigned&) x[0];
+    }
+
+    void OID::justForked() {
+        MachineAndPid x = ourMachine;
+        // we let the random # for machine go into all 5 bytes of MachineAndPid, and then 
+        // xor in the pid into _pid.  this reduces the probability of collisions.
+        x._pid = x._pid ^ ourPid();
+        ourMachineAndPid = __gen();
+        assert( x != ourMachineAndPid );
+        ourMachineAndPid = x;
+    }
+
+    void OID::init() {
+        static AtomicUInt inc = security.getNonce();
+
+        {
+            unsigned t = (unsigned) time(0);
+            unsigned char *T = (unsigned char *) &t;
+            _time[0] = T[3]; // big endian order because we use memcmp() to compare OID's 
+            _time[1] = T[2];
+            _time[2] = T[1];
+            _time[3] = T[0];
+        }
+
+        _machineAndPid = ourMachineAndPid;
+
+        {
+            int new_inc = inc++;
+            unsigned char *T = (unsigned char *) &new_inc;
+            _inc[0] = T[2];
+            _inc[1] = T[1];
+            _inc[2] = T[0];
+        }
+    }
+
+    void OID::init( string s ){
+        assert( s.size() == 24 );
+        const char *p = s.c_str();
+        for( int i = 0; i < 12; i++ ) {
+            data[i] = fromHex(p);
+            p += 2;
+        }
+    }
+
+    void OID::init(Date_t date, bool max){
+        int time = (int) (date / 1000);
+        char* T = (char *) &time;
+        data[0] = T[3];
+        data[1] = T[2];
+        data[2] = T[1];
+        data[3] = T[0];
+
+        if (max)
+            *(long long*)(data + 4) = 0xFFFFFFFFFFFFFFFFll;
+        else
+            *(long long*)(data + 4) = 0x0000000000000000ll;
+    }
+
+    time_t OID::asTimeT(){
+        int time;
+        char* T = (char *) &time;
+        T[0] = data[3];
+        T[1] = data[2];
+        T[2] = data[1];
+        T[3] = data[0];
+        return time;
+    }
+
+}

bson/oid.h

@@ -30,20 +30,33 @@ namespace mongo {
         automatically in certain circumstances.
 
         Warning: You must call OID::newState() after a fork().
+
+        Typical contents of the BSON ObjectID is a 12-byte value consisting of a 4-byte timestamp (seconds since epoch), 
+        a 3-byte machine id, a 2-byte process id, and a 3-byte counter. Note that the timestamp and counter fields must 
+        be stored big endian unlike the rest of BSON. This is because they are compared byte-by-byte and we want to ensure 
+        a mostly increasing order.
     */
     class OID {
+    public:
+        struct MachineAndPid { 
+            unsigned char _machineNumber[3];
+            unsigned short _pid;
+        };
+    private:
         union {
-            struct{
+            struct {
+                // 12 bytes total
+                unsigned char _time[4];
+                MachineAndPid _machineAndPid;
+                unsigned char _inc[3];
+            };
+            struct {
                 long long a;
                 unsigned b;
             };
             unsigned char data[12];
         };
-        static unsigned _machine;
     public:
-        /** call this after a fork */
-        static void newState();
-
 		/** initialize to 'null' */
 		void clear() { a = 0; b = 0; }
 
@@ -65,8 +78,6 @@ namespace mongo {
 
         static OID gen() { OID o; o.init(); return o; }
 
-        static unsigned staticMachine(){ return _machine; }
-
         /** sets the contents to a new oid / randomized value */
         void init();
 
@@ -84,6 +95,12 @@ namespace mongo {
         int compare( const OID& other ) const { return memcmp( data , other.data , 12 ); }
 
         bool operator<( const OID& other ) const { return compare( other ) < 0; }
+
+        /** call this after a fork */
+        static void justForked();
+
+        static unsigned getMachineId();
+        static void regenMachineId();
     };
 #pragma pack()
 

db/db.vcxproj

@@ -196,6 +196,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\bson\oid.cpp" />
     <ClCompile Include="..\client\dbclientcursor.cpp" />
     <ClCompile Include="..\client\distlock.cpp" />
     <ClCompile Include="..\client\model.cpp" />

db/db.vcxproj.filters

@@ -439,6 +439,9 @@
     <ClCompile Include="..\util\alignedbuilder.cpp">
       <Filter>util</Filter>
     </ClCompile>
+    <ClCompile Include="..\bson\oid.cpp">
+      <Filter>bson</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="repl\rs_config.h">

db/dbcommands_generic.cpp

@@ -202,10 +202,10 @@ namespace mongo {
                 bb.done();
             }
             if ( cmdObj["oidReset"].trueValue() ){
-                result.append( "oidMachineOld" , OID::staticMachine() );
-                OID::newState();
+                result.append( "oidMachineOld" , OID::getMachineId() );
+                OID::regenMachineId();
             }
-            result.append( "oidMachine" , OID::staticMachine() );
+            result.append( "oidMachine" , OID::getMachineId() );
             return true;
         }
 

db/jsobj.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "pch.h"
+#include "../bson/oid.h"
 #include "jsobj.h"
 #include "nonce.h"
 #include "../bson/util/atomic_int.h"
@@ -1185,84 +1186,6 @@ namespace mongo {
         }
     } bson_unittest;
 
-/*
-    BSONObjBuilder& BSONObjBuilderValueStream::operator<<( const char * value ) {
-        _builder->append( _fieldName , value );
-        return *_builder;
-    }
-
-    BSONObjBuilder& BSONObjBuilderValueStream::operator<<( const int value ) {
-        _builder->append( _fieldName , value );
-        return *_builder;
-    }
-
-    BSONObjBuilder& BSONObjBuilderValueStream::operator<<( const double value ) {
-        _builder->append( _fieldName , value );
-        return *_builder;
-    }
-*/
-
-    void OID::init() {
-        static AtomicUInt inc = getRandomNumber();
-        unsigned t = (unsigned) time(0);
-        char *T = (char *) &t;
-        data[0] = T[3];
-        data[1] = T[2];
-        data[2] = T[1];
-        data[3] = T[0];
-
-        (unsigned&) data[4] = _machine;
-
-        int new_inc = inc++;
-        T = (char *) &new_inc;
-        char * raw = (char*)&b;
-        raw[0] = T[3];
-        raw[1] = T[2];
-        raw[2] = T[1];
-        raw[3] = T[0];
-    }
-
-    unsigned OID::_machine = (unsigned) security.getNonceInitSafe();
-    void OID::newState(){
-        unsigned before = _machine;
-        // using fresh Security object to avoid buffered devrandom
-        _machine = (unsigned)security.getNonce();
-        assert( _machine != before );
-    }
-
-    void OID::init( string s ){
-        assert( s.size() == 24 );
-        const char *p = s.c_str();
-        for( int i = 0; i < 12; i++ ) {
-            data[i] = fromHex(p);
-            p += 2;
-        }
-    }
-
-    void OID::init(Date_t date, bool max){
-        int time = (int) (date / 1000);
-        char* T = (char *) &time;
-        data[0] = T[3];
-        data[1] = T[2];
-        data[2] = T[1];
-        data[3] = T[0];
-
-        if (max)
-            *(long long*)(data + 4) = 0xFFFFFFFFFFFFFFFFll;
-        else
-            *(long long*)(data + 4) = 0x0000000000000000ll;
-    }
-
-    time_t OID::asTimeT(){
-        int time;
-        char* T = (char *) &time;
-        T[0] = data[3];
-        T[1] = data[2];
-        T[2] = data[1];
-        T[3] = data[0];
-        return time;
-    }
-
     Labeler::Label GT( "$gt" );
     Labeler::Label GTE( "$gte" );
     Labeler::Label LT( "$lt" );

dbtests/test.vcxproj

@@ -260,6 +260,7 @@
     <ClInclude Include="..\util\unittest.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\bson\oid.cpp" />
     <ClCompile Include="..\client\dbclientcursor.cpp" />
     <ClCompile Include="..\client\distlock.cpp" />
     <ClCompile Include="..\client\gridfs.cpp" />

dbtests/test.vcxproj.filters

@@ -731,6 +731,9 @@
     <ClCompile Include="..\util\alignedbuilder.cpp">
       <Filter>util</Filter>
     </ClCompile>
+    <ClCompile Include="..\bson\oid.cpp">
+      <Filter>db</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\SConstruct">

pch.h

@@ -39,6 +39,10 @@
 #endif
 
 #if defined(_WIN32)
+
+// for rand_s() usage:
+#define _CRT_RAND_S
+
 # ifndef NOMINMAX
 #  define NOMINMAX
 # endif

s/dbgrid.vcxproj

@@ -187,6 +187,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\bson\oid.cpp" />
     <ClCompile Include="..\client\dbclientcursor.cpp" />
     <ClCompile Include="..\client\distlock.cpp" />
     <ClCompile Include="..\db\dbwebserver.cpp" />

s/dbgrid.vcxproj.filters

@@ -311,6 +311,9 @@
     <ClCompile Include="..\util\alignedbuilder.cpp">
       <Filter>Shared Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\bson\oid.cpp">
+      <Filter>Shared Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="gridconfig.h">

shell/msvc/mongo.vcxproj

@@ -87,6 +87,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\bson\oid.cpp" />
     <ClCompile Include="..\..\client\clientOnly.cpp" />
     <ClCompile Include="..\..\client\connpool.cpp" />
     <ClCompile Include="..\..\client\syncclusterconnection.cpp" />

shell/msvc/mongo.vcxproj.filters

@@ -219,6 +219,9 @@
     <ClCompile Include="..\..\scripting\bench.cpp">
       <Filter>scripting</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\bson\oid.cpp">
+      <Filter>bson</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\SConstruct" />