make more commands work with mongos SERVER-1462

jstests/sharding/count1.js

@@ -36,7 +36,6 @@ assert.eq( 6 , db.foo.find().sort( { name : 1 } ).count() , "basic count after s
 
 s.adminCommand( { movechunk : "test.foo" , find : { name : "joe" } , to : secondary.getMongo().name } );
 
-
 assert.eq( 3 , primary.foo.find().toArray().length , "primary count" );
 assert.eq( 3 , secondary.foo.find().toArray().length , "secondary count" );
 assert.eq( 3 , primary.foo.find().sort( { name : 1 } ).toArray().length , "primary count sorted" );

jstests/slowNightly/run_sharding_passthrough.js

@@ -52,9 +52,19 @@ files.forEach(
 	// getnonce logout medianKey profile reIndex repairDatabase
 	// reseterror splitVector validate
 
-	// These all fail due to missing commands:
-	if (/[\/\\](apitest_db|apitest_dbcollection|auth1|auth2|basic1|basic2|capped1|capped3|capped4|copydb-auth|cursor6|datasize|datasize2|dbadmin|dbhash|drop|dropIndex|error3|evalb|find1|find3|in5|index1|index10|index2|index.*|jni1|jni2|jni3|jni4|jni8|median|multi2|profile1|recstore|remove|remove2|repair|sort1|sort2|sort4|sort5|sort_numeric|splitvector|unique2|update)\.js$/.test(x.name)) {
-	    print(" !!!!!!!!!!!!!!! skipping test that fails under sharding (missing command) " + x.name)	    
+     /* missing commands :
+     * forceerror and switchtoclienterrors
+     * cloneCollectionAsCapped
+     * splitvector
+     * profile (apitest_db, cursor6, evalb)
+     * copydbgetnonce
+     * dbhash
+     * medianKey
+     * clean (apitest_dbcollection)
+     * logout and getnonce
+     */
+	if (/[\/\\](error3|capped3|splitvector|apitest_db|cursor6|copydb-auth|profile1|dbhash|median|apitest_dbcollection|evalb|auth1|auth2)\.js$/.test(x.name)) {
+	    print(" !!!!!!!!!!!!!!! skipping test that has failed under sharding but might not anymore " + x.name)	    
 	    return;
 	}
 	// These are bugs (some might be fixed now):

s/chunk.cpp

@@ -744,6 +744,21 @@ namespace mongo {
         }
     }
 
+    void ChunkManager::getShardsForRange(set<Shard>& shards, const BSONObj& min, const BSONObj& max){
+        uassert(13405, "min must have shard key", hasShardKey(min));
+        uassert(13406, "max must have shard key", hasShardKey(max));
+
+        ChunkRangeMap::const_iterator it = _chunkRanges.upper_bound(min);
+        ChunkRangeMap::const_iterator end = _chunkRanges.lower_bound(max);
+
+        for (; it!=end; ++ it){
+            shards.insert(it->second->getShard());
+
+            // once we know we need to visit all shards no need to keep looping
+            if (shards.size() == _shards.size())
+                break;
+        }
+    }
 
     void ChunkManager::getAllShards( set<Shard>& all ){
         rwlock lk( _lock , false ); 

s/chunk.h

@@ -267,8 +267,8 @@ namespace mongo {
         void maybeChunkCollection();
 
         void getShardsForQuery( set<Shard>& shards , const BSONObj& query );
-
         void getAllShards( set<Shard>& all );
+        void getShardsForRange(set<Shard>& shards, const BSONObj& min, const BSONObj& max); // [min, max)
 
         void save();
 

s/commands_public.cpp

@@ -35,7 +35,7 @@ namespace mongo {
 
         class PublicGridCommand : public Command {
         public:
-            PublicGridCommand( const char * n ) : Command( n ){
+            PublicGridCommand( const char* n, const char* oldname=NULL ) : Command( n, false, oldname ){
             }
             virtual bool slaveOk() const {
                 return true;
@@ -65,6 +65,79 @@ namespace mongo {
                 return ok;
             }
         };
+
+        class RunOnAllShardsCommand : public Command {
+        public:
+            RunOnAllShardsCommand(const char* n, const char* oldname=NULL) : Command(n, false, oldname) {}
+
+            virtual bool slaveOk() const { return true; }
+            virtual bool adminOnly() const { return false; }
+
+            // all grid commands are designed not to lock
+            virtual LockType locktype() const { return NONE; } 
+
+
+            // default impl uses all shards for DB
+            virtual void getShards(const string& dbName , BSONObj& cmdObj, set<Shard>& shards){
+                DBConfigPtr conf = grid.getDBConfig( dbName , false );
+                conf->getAllShards(shards);
+            }
+
+            virtual void aggregateResults(const vector<BSONObj>& results, BSONObjBuilder& output) {}
+
+            // don't override
+            virtual bool run(const string& dbName , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& output, bool){
+                set<Shard> shards;
+                getShards(dbName, cmdObj, shards);
+
+                list< shared_ptr<Future::CommandResult> > futures;
+                for ( set<Shard>::const_iterator i=shards.begin(), end=shards.end() ; i != end ; i++ ){
+                    futures.push_back( Future::spawnCommand( i->getConnString() , dbName , cmdObj ) );
+                }
+
+                vector<BSONObj> results;
+                BSONObjBuilder subobj (output.subobjStart("raw"));
+                BSONObjBuilder errors;
+                for ( list< shared_ptr<Future::CommandResult> >::iterator i=futures.begin(); i!=futures.end(); i++ ){
+                    shared_ptr<Future::CommandResult> res = *i;
+                    if ( ! res->join() ){
+                        errors.appendAs(res->result()["errmsg"], res->getServer());
+                    }
+                    results.push_back( res->result() );
+                    subobj.append( res->getServer() , res->result() );
+                }
+
+                subobj.done();
+
+                BSONObj errobj = errors.done();
+                if (! errobj.isEmpty()){
+                    errmsg = errobj.toString(false, true);
+                    return false;
+                }
+
+                aggregateResults(results, output);
+                return true;
+            }
+
+        };
+
+        class AllShardsCollectionCommand : public RunOnAllShardsCommand {
+        public:
+            AllShardsCollectionCommand(const char* n, const char* oldname=NULL) : RunOnAllShardsCommand(n, oldname) {}
+
+            virtual void getShards(const string& dbName , BSONObj& cmdObj, set<Shard>& shards){
+                string fullns = dbName + '.' + cmdObj.firstElement().valuestrsafe();
+
+                DBConfigPtr conf = grid.getDBConfig( dbName , false );
+
+                if ( ! conf || ! conf->isShardingEnabled() || ! conf->isSharded( fullns ) ){
+                    shards.insert(conf->getShard(fullns));
+                } else {
+                    conf->getChunkManager(fullns)->getAllShards(shards);
+                }
+            }
+        };
+
 
         class NotAllowedOnShardedCollectionCmd : public PublicGridCommand {
         public:
@@ -87,6 +160,62 @@ namespace mongo {
 
         // ----
 
+        class DropIndexesCmd : public AllShardsCollectionCommand {
+        public:
+            DropIndexesCmd() :  AllShardsCollectionCommand("dropIndexes", "deleteIndexes") {}
+        } dropIndexesCmd;
+
+        class ReIndexCmd : public AllShardsCollectionCommand {
+        public:
+            ReIndexCmd() :  AllShardsCollectionCommand("reIndex") {}
+        } reIndexCmd;
+
+        class ValidateCmd : public AllShardsCollectionCommand {
+        public:
+            ValidateCmd() :  AllShardsCollectionCommand("validate") {}
+        } validateCmd;
+
+        class RepairDatabaseCmd : public RunOnAllShardsCommand {
+        public:
+            RepairDatabaseCmd() :  RunOnAllShardsCommand("repairDatabase") {}
+        } repairDatabaseCmd;
+
+        class DBStatsCmd : public RunOnAllShardsCommand {
+        public:
+            DBStatsCmd() :  RunOnAllShardsCommand("dbstats") {}
+
+            virtual void aggregateResults(const vector<BSONObj>& results, BSONObjBuilder& output) {
+                long long objects = 0;
+                long long dataSize = 0;
+                long long storageSize = 0;
+                long long numExtents = 0;
+                long long indexes = 0;
+                long long indexSize = 0;
+                long long fileSize = 0;
+
+                for (vector<BSONObj>::const_iterator it(results.begin()), end(results.end()); it != end; ++it){
+                    const BSONObj& b = *it;
+                    objects     += b["objects"].numberLong();
+                    dataSize    += b["dataSize"].numberLong();
+                    storageSize += b["storageSize"].numberLong();
+                    numExtents  += b["numExtents"].numberLong();
+                    indexes     += b["indexes"].numberLong();
+                    indexSize   += b["indexSize"].numberLong();
+                    fileSize    += b["fileSize"].numberLong();
+                }
+
+                //result.appendNumber( "collections" , ncollections ); //TODO: need to find a good way to get this
+                output.appendNumber( "objects" , objects );
+                output.append      ( "avgObjSize" , double(dataSize) / double(objects) );
+                output.appendNumber( "dataSize" , dataSize );
+                output.appendNumber( "storageSize" , storageSize);
+                output.appendNumber( "numExtents" , numExtents );
+                output.appendNumber( "indexes" , indexes );
+                output.appendNumber( "indexSize" , indexSize );
+                output.appendNumber( "fileSize" , fileSize );
+            }
+        } DBStatsCmdObj;
+
         class DropCmd : public PublicGridCommand {
         public:
             DropCmd() : PublicGridCommand( "drop" ){}
@@ -420,6 +549,59 @@ namespace mongo {
 
         } findAndModifyCmd;
 
+        class DataSizeCmd : public PublicGridCommand {
+        public:
+            DataSizeCmd() : PublicGridCommand("dataSize", "datasize") { }
+            bool run(const string& dbName, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+                string fullns = cmdObj.firstElement().String();
+
+                DBConfigPtr conf = grid.getDBConfig( dbName , false );
+
+                if ( ! conf || ! conf->isShardingEnabled() || ! conf->isSharded( fullns ) ){
+                    return passthrough( conf , cmdObj , result);
+                }
+
+                ChunkManagerPtr cm = conf->getChunkManager( fullns );
+                massert( 13407 ,  "how could chunk manager be null!" , cm );
+
+                BSONObj min = cmdObj.getObjectField( "min" );
+                BSONObj max = cmdObj.getObjectField( "max" );
+                BSONObj keyPattern = cmdObj.getObjectField( "keyPattern" );
+
+                uassert(13408,  "keyPattern must equal shard key", cm->getShardKey().key() == keyPattern);
+
+                // yes these are doubles...
+                double size = 0;
+                double numObjects = 0;
+                int millis = 0;
+
+                set<Shard> shards;
+                cm->getShardsForRange(shards, min, max);
+                for ( set<Shard>::iterator i=shards.begin(), end=shards.end() ; i != end; ++i ){
+                    ShardConnection conn( *i , fullns );
+                    BSONObj res;
+                    bool ok = conn->runCommand( conf->getName() , cmdObj , res );
+                    conn.done();
+
+                    if ( ! ok ){
+                        result.appendElements( res );
+                        return false;
+                    }
+
+                    size       += res["size"].number();
+                    numObjects += res["numObjects"].number();
+                    millis     += res["millis"].numberInt();
+
+                }
+
+                result.append( "size", size );
+                result.append( "numObjects" , numObjects );
+                result.append( "millis" , millis );
+                return true;
+            }
+
+        } DataSizeCmd;
+
         class ConvertToCappedCmd : public NotAllowedOnShardedCollectionCmd  {
         public:
             ConvertToCappedCmd() : NotAllowedOnShardedCollectionCmd("convertToCapped"){}

s/config.cpp

@@ -375,6 +375,15 @@ namespace mongo {
         return true;
     }
 
+    void DBConfig::getAllShards(set<Shard>& shards) const{
+        shards.insert(getPrimary());
+        for (Collections::const_iterator it(_collections.begin()), end(_collections.end()); it != end; ++it){
+            if (it->second.isSharded()){
+                it->second.getCM()->getAllShards(shards);
+            } // TODO: handle collections on non-primary shard
+        }
+    }
+
     /* --- Grid --- */
 
     DBConfigPtr Grid::getDBConfig( string database , bool create ){

s/config.h

@@ -160,6 +160,8 @@ namespace mongo {
          */
         bool unserialize(const BSONObj& from);
 
+        void getAllShards(set<Shard>& shards) const;
+
     protected:
 
         /** 

shell/collection.js

@@ -355,8 +355,10 @@ DBCollection.prototype.validate = function() {
 
     res.valid = false;
 
-    if ( res.result ){
-        var str = "-" + tojson( res.result );
+    var raw = res.result || res.raw;
+
+    if ( raw ){
+        var str = "-" + tojson( raw );
         res.valid = ! ( str.match( /exception/ ) || str.match( /corrupt/ ) );
 
         var p = /lastExtentSize:(\d+)/;

shell/mongo_vstudio.cpp

@@ -649,40 +649,34 @@ const char * jsconcatcode =
  "by = b[key];}\n"
  "return Math.sqrt( Math.pow( by - ay , 2 ) +\n"
  "Math.pow( bx - ax , 2 ) );}\n"
- "rs = function () { return \"try rs.help()\"; }\n"
+ "rs = function () { \"try rs.help()\" }\n"
  "rs.help = function () {\n"
  "print(\"\\trs.status()                     { replSetGetStatus : 1 } checks repl set status\");\n"
  "print(\"\\trs.initiate()                   { replSetInitiate : null } initiates set with default settings\");\n"
  "print(\"\\trs.initiate(cfg)                { replSetInitiate : cfg } initiates set with configuration cfg\");\n"
  "print(\"\\trs.add(hostportstr)             add a new member to the set with default attributes\");\n"
  "print(\"\\trs.add(membercfgobj)            add a new member to the set with extra attributes\");\n"
- "print(\"\\trs.addArb(hostportstr)          add a new member which is arbiterOnly:true\");\n"
- "print(\"\\trs.stepDown()                   step down as primary (momentarily)\");\n"
  "print(\"\\trs.conf()                       return configuration from local.system.replset\");\n"
  "print();\n"
  "print(\"\\tdb.isMaster()                   check who is primary\");\n"
  "print();\n"
- "print(\"\\tsee also http://<mongod_host>:28017/_replSet for additional diagnostic info\");}\n"
+ "print(\"\\tsee also http://<host>:28017/_replSet for additional diagnostic info\");}\n"
  "rs.status = function () { return db._adminCommand(\"replSetGetStatus\"); }\n"
  "rs.initiate = function (c) { return db._adminCommand({ replSetInitiate: c }); }\n"
- "rs.add = function (hostport, arb) {\n"
+ "rs.add = function (hostport) {\n"
  "var cfg = hostport;\n"
  "var local = db.getSisterDB(\"local\");\n"
- "assert(local.system.replset.count() <= 1, \"error: local.system.replset has unexpected contents\");\n"
+ "assert(local.system.replset.count() == 1, \"error: local.system.replset unexpected (or empty) contents\");\n"
  "var c = local.system.replset.findOne();\n"
  "assert(c, \"no config object retrievable from local.system.replset\");\n"
  "c.version++;\n"
  "var max = 0;\n"
  "for (var i in c.members)\n"
  "if (c.members[i]._id > max) max = c.members[i]._id;\n"
- "if (isString(hostport)) {\n"
+ "if (isString(hostport))\n"
  "cfg = { _id: max + 1, host: hostport };\n"
- "if (arb)\n"
- "cfg.arbiterOnly = true;}\n"
  "c.members.push(cfg);\n"
  "return db._adminCommand({ replSetReconfig: c });}\n"
- "rs.stepDown = function () { return db._adminCommand({ replSetStepDown: 1 }); }\n"
- "rs.addArb = function (hn) { return this.add(hn, true); }\n"
  "rs.conf = function () { return db.getSisterDB(\"local\").system.replset.findOne(); }\n"
  "help = shellHelper.help = function (x) {\n"
  "if (x == \"connect\") {\n"
@@ -725,10 +719,10 @@ const char * jsconcatcode =
  "return;}\n"
  "print(\"\\t\" + \"db.help()                    help on db methods\");\n"
  "print(\"\\t\" + \"db.mycoll.help()             help on collection methods\");\n"
- "print(\"\\t\" + \"rs.help()                    help on replica set methods\");\n"
  "print(\"\\t\" + \"help connect                 connecting to a db help\");\n"
  "print(\"\\t\" + \"help admin                   administrative help\");\n"
  "print(\"\\t\" + \"help misc                    misc things to know\");\n"
+ "print(\"\\t\" + \"rs.help()                    help on replica set methods\");\n"
  "print();\n"
  "print(\"\\t\" + \"show dbs                     show database names\");\n"
  "print(\"\\t\" + \"show collections             show collections in current database\");\n"
@@ -1288,12 +1282,6 @@ const char * jsconcatcode =
  "throw \"error: \" + tojson( ret );\n"
  "this._numReturned++;\n"
  "return ret;}\n"
- "DBQuery.prototype.objsLeftInBatch = function(){\n"
- "this._exec();\n"
- "var ret = this._cursor.objsLeftInBatch();\n"
- "if ( ret.$err )\n"
- "throw \"error: \" + tojson( ret );\n"
- "return ret;}\n"
  "DBQuery.prototype.toArray = function(){\n"
  "if ( this._arr )\n"
  "return this._arr;\n"