Facebook sync (facebookresearch#504)

* Facebook sync

* Update swig wrappers.

* Fix comment.

AutoTune.cpp

@@ -6,9 +6,11 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-/* Copyright 2004-present Facebook. All Rights Reserved.
-   implementation of Hyper-parameter auto-tuning
-*/
+// -*- c++ -*-
+
+/*
+ * implementation of Hyper-parameter auto-tuning
+ */
 
 #include "AutoTune.h"
 
@@ -25,7 +27,8 @@
 #include "MetaIndexes.h"
 #include "IndexScalarQuantizer.h"
 #include "IndexHNSW.h"
-
+#include "IndexBinaryFlat.h"
+#include "IndexBinaryIVF.h"
 
 namespace faiss {
 
@@ -734,11 +737,8 @@ Index *index_factory (int d, const char *description_in, MetricType metric)
             vt_1 = new OPQMatrix (d, opq_M);
         } else if (stok == "L2norm") {
             vt_1 = new NormalizationTransform (d, 2.0);
-        // coarse quantizers
-        } else if (!coarse_quantizer &&
-                   sscanf (tok, "IVF%d_HNSW%d", &ncentroids, &M) == 2) {
-            FAISS_THROW_IF_NOT (metric == METRIC_L2);
-            coarse_quantizer_1 = new IndexHNSWFlat (d, M);
+
+
         } else if (!coarse_quantizer &&
                    sscanf (tok, "IVF%d", &ncentroids) == 1) {
             if (metric == METRIC_L2) {
@@ -755,24 +755,31 @@ Index *index_factory (int d, const char *description_in, MetricType metric)
             add_idmap = true;
 
             // IVFs
-        } else if (!index && stok == "Flat") {
+        } else if (!index && (stok == "Flat" || stok == "FlatDedup")) {
             if (coarse_quantizer) {
                 // if there was an IVF in front, then it is an IVFFlat
-                IndexIVF *index_ivf = new IndexIVFFlat (
-                    coarse_quantizer, d, ncentroids, metric);
+                IndexIVF *index_ivf = stok == "Flat" ?
+                    new IndexIVFFlat (
+                          coarse_quantizer, d, ncentroids, metric) :
+                    new IndexIVFFlatDedup (
+                          coarse_quantizer, d, ncentroids, metric);
                 index_ivf->quantizer_trains_alone =
                     get_trains_alone (coarse_quantizer);
                 index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
                 del_coarse_quantizer.release ();
                 index_ivf->own_fields = true;
                 index_1 = index_ivf;
             } else {
+                FAISS_THROW_IF_NOT_MSG (stok != "FlatDedup",
+                                        "dedup supported only for IVFFlat");
                 index_1 = new IndexFlat (d, metric);
             }
-        } else if (!index && (stok == "SQ8" || stok == "SQ4")) {
+        } else if (!index && (stok == "SQ8" || stok == "SQ4" ||
+                              stok == "SQfp16")) {
             ScalarQuantizer::QuantizerType qt =
                 stok == "SQ8" ? ScalarQuantizer::QT_8bit :
                 stok == "SQ4" ? ScalarQuantizer::QT_4bit :
+                stok == "SQfp16" ? ScalarQuantizer::QT_fp16 :
                 ScalarQuantizer::QT_4bit;
             if (coarse_quantizer) {
                 IndexIVFScalarQuantizer *index_ivf =
@@ -905,7 +912,27 @@ Index *index_factory (int d, const char *description_in, MetricType metric)
     return index;
 }
 
+IndexBinary *index_binary_factory(int d, const char *description)
+{
+    IndexBinary *index = nullptr;
+
+    int ncentroids = -1;
 
+    if (sscanf(description, "BIVF%d", &ncentroids) == 1) {
+        IndexBinaryIVF *index_ivf = new IndexBinaryIVF(
+            new IndexBinaryFlat(d), d, ncentroids
+        );
+        index_ivf->own_fields = true;
+        index = index_ivf;
+    } else if (std::string(description) == "BFlat") {
+        index = new IndexBinaryFlat(d);
+    } else {
+        FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
+                               description);
+    }
+
+    return index;
+}
 
 
-}; // namespace faiss
+} // namespace faiss

AutoTune.h

@@ -6,7 +6,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved.
 // -*- c++ -*-
 
 #ifndef FAISS_AUTO_TUNE_H
@@ -15,6 +14,7 @@
 #include <vector>
 
 #include "Index.h"
+#include "IndexBinary.h"
 
 namespace faiss {
 
@@ -203,6 +203,8 @@ struct ParameterSpace {
 Index *index_factory (int d, const char *description,
                       MetricType metric = METRIC_L2);
 
+IndexBinary *index_binary_factory (int d, const char *description);
+
 
 
 } // namespace faiss

AuxIndexStructures.cpp

@@ -6,11 +6,12 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved
 // -*- c++ -*-
 
 #include "AuxIndexStructures.h"
 
+#include "FaissAssert.h"
+
 #include <cstring>
 
 namespace faiss {
@@ -208,9 +209,44 @@ bool IDSelectorBatch::is_member (idx_t i) const
 }
 
 
+/***********************************************************************
+ * IO functions
+ ***********************************************************************/
+
 
+int IOReader::fileno ()
+{
+    FAISS_THROW_MSG ("IOReader does not support memory mapping");
+}
 
+int IOWriter::fileno ()
+{
+    FAISS_THROW_MSG ("IOWriter does not support memory mapping");
+}
 
 
+size_t VectorIOWriter::operator()(
+                const void *ptr, size_t size, size_t nitems)
+{
+    size_t o = data.size();
+    data.resize(o + size * nitems);
+    memcpy (&data[o], ptr, size * nitems);
+    return nitems;
+}
 
+size_t VectorIOReader::operator()(
+                  void *ptr, size_t size, size_t nitems)
+{
+    if (rp >= data.size()) return 0;
+    size_t nremain = (data.size() - rp) / size;
+    if (nremain < nitems) nitems = nremain;
+    memcpy (ptr, &data[rp], size * nitems);
+    rp += size * nitems;
+    return nitems;
 }
+
+
+
+
+
+} // namespace faiss

AuxIndexStructures.h

@@ -6,14 +6,16 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved
 // -*- c++ -*-
+
 // Auxiliary index structures, that are used in indexes but that can
 // be forward-declared
 
 #ifndef FAISS_AUX_INDEX_STRUCTURES_H
 #define FAISS_AUX_INDEX_STRUCTURES_H
 
+#include <stdint.h>
+
 #include <vector>
 #include <unordered_set>
 
@@ -157,10 +159,8 @@ struct RangeSearchPartialResult: BufferList {
     /// begin a new result
     QueryResult & new_result (idx_t qno);
 
-
     void finalize ();
 
-
     /// called by range_search before do_allocation
     void set_lims ();
 
@@ -169,6 +169,46 @@ struct RangeSearchPartialResult: BufferList {
 
 };
 
+/***********************************************************
+ * Abstract I/O objects
+ ***********************************************************/
+
+
+struct IOReader {
+    // fread
+    virtual size_t operator()(
+         void *ptr, size_t size, size_t nitems) = 0;
+
+    // return a file number that can be memory-mapped
+    virtual int fileno ();
+
+    virtual ~IOReader() {}
+};
+
+struct IOWriter {
+    // fwrite
+    virtual size_t operator()(
+         const void *ptr, size_t size, size_t nitems) = 0;
+
+    // return a file number that can be memory-mapped
+    virtual int fileno ();
+
+    virtual ~IOWriter() {}
+};
+
+
+struct VectorIOReader:IOReader {
+    const std::vector<uint8_t> data;
+    size_t rp = 0;
+    size_t operator()(void *ptr, size_t size, size_t nitems) override;
+};
+
+struct VectorIOWriter:IOWriter {
+    std::vector<uint8_t> data;
+    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
+};
+
+
 
 }; // namespace faiss
 

Clustering.cpp

@@ -6,14 +6,11 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-/* Copyright 2004-present Facebook. All Rights Reserved.
-   kmeans clustering routines
-*/
+// -*- c++ -*-
 
 #include "Clustering.h"
 
 
-
 #include <cmath>
 #include <cstdio>
 #include <cstring>

Clustering.h

@@ -6,7 +6,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved
 // -*- c++ -*-
 
 #ifndef FAISS_CLUSTERING_H

FaissAssert.h

@@ -6,7 +6,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
 
 #ifndef FAISS_ASSERT_INCLUDED
 #define FAISS_ASSERT_INCLUDED

FaissException.cpp

@@ -6,7 +6,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
 
 #include "FaissException.h"
 

FaissException.h

@@ -6,7 +6,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
 
 #ifndef FAISS_EXCEPTION_INCLUDED
 #define FAISS_EXCEPTION_INCLUDED

Heap.cpp

@@ -6,7 +6,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-/* Copyright 2004-present Facebook. All Rights Reserved. */
+// -*- c++ -*-
+
 /* Function for soft heap */
 
 #include "Heap.h"
@@ -15,13 +16,6 @@
 namespace faiss {
 
 
-
-
-
-
-
-
-
 template <typename C>
 void HeapArray<C>::heapify ()
 {
@@ -126,7 +120,4 @@ template class HeapArray<CMin <int, long> >;
 template class HeapArray<CMax <int, long> >;
 
 
-
-
-
 }  // END namespace fasis

Heap.h

@@ -6,8 +6,9 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-/* Copyright 2004-present Facebook. All Rights Reserved.
- *
+// -*- c++ -*-
+
+/*
  * C++ support for heaps. The set of functions is tailored for
  * efficient similarity search.
  *
@@ -31,7 +32,6 @@
 #include <limits>
 
 
-
 namespace faiss {
 
 /*******************************************************************
@@ -490,8 +490,6 @@ void indirect_heap_push (size_t k,
 }
 
 
-
-
 } // namespace faiss
 
 #endif  /* FAISS_Heap_h */

Index.cpp

@@ -6,13 +6,14 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
 
 #include "IndexFlat.h"
 #include "FaissAssert.h"
 
 #include <cstring>
 
+
 namespace faiss {
 
 Index::~Index ()

Index.h

@@ -6,7 +6,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Copyright 2004-present Facebook. All Rights Reserved
 // -*- c++ -*-
 
 #ifndef FAISS_INDEX_H