Add check for struct sizes

Summary:
C++ Faiss is compiled with (at least) 3 compilers:
- the one compiling the main CPU Faiss
- the one called by nvcc to compile host code on GPU Faiss
- the one compiling the SWIG generated file

Sometimes the compilers are misconfigured and have different ideas of data alignment, size and packing. This is a hard to catch bug.

This test attemtps to check if one of these differences occur.

The test did catch the bug in one configuration in the Github version, see


#4136
https://github.com/facebookresearch/faiss/actions/runs/13177278514/job/36779465009?pr=4136

Differential Revision: D69243348

Author: mdouze

faiss/CMakeLists.txt

@@ -214,6 +214,7 @@ set(FAISS_HEADERS
   utils/simdlib_emulated.h
   utils/simdlib_neon.h
   utils/simdlib_ppc64.h
+  utils/struct_packing_test.h
   utils/utils.h
   utils/distances_fused/avx512.h
   utils/distances_fused/distances_fused.h

faiss/gpu/GpuIndex.h

@@ -194,5 +194,7 @@ bool isGpuIndex(faiss::Index* index);
 /// Does the given CPU index instance have a corresponding GPU implementation?
 bool isGpuIndexImplemented(faiss::Index* index);
 
+int struct_packing_test_cuda(int q);
+
 } // namespace gpu
 } // namespace faiss

faiss/gpu/impl/IndexUtils.cu

@@ -7,6 +7,7 @@
 
 #include <faiss/gpu/impl/IndexUtils.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/utils/struct_packing_test.h>
 #include <faiss/gpu/utils/DeviceDefs.cuh>
 #include <limits>
 
@@ -38,5 +39,7 @@ void validateNProbe(size_t nprobe) {
             nprobe);
 }
 
+int struct_packing_test_cuda(int q) STRUCT_PACKING_FUNCTION_BODY
+
 } // namespace gpu
 } // namespace faiss

faiss/gpu/test/test_gpu_basics.py

@@ -469,3 +469,26 @@ class TestGpuFlags(unittest.TestCase):
 
     def test_gpu_flag(self):
         assert "GPU" in faiss.get_compile_options().split()
+
+
+class TestStructPacking(unittest.TestCase): 
+    """ Verify if the size structures as seen from Cuda and C++ are the same """
+
+    def test_swig(self): 
+        " This test is redundant with the CPU tests, but let's check run it just in case"
+        sizes = np.array([
+            (faiss.struct_packing_test_cpp(q),
+            faiss.struct_packing_test_swig(q)) 
+            for q in range(20)
+        ])
+        print(sizes)
+        np.testing.assert_array_equal(sizes[:, 0], sizes[:, 1])
+
+    def test_cuda(self): 
+        sizes = np.array([
+            (faiss.struct_packing_test_cpp(q),
+            faiss.struct_packing_test_cuda(q)) 
+            for q in range(20)
+        ])
+        print(sizes)
+        np.testing.assert_array_equal(sizes[:, 0], sizes[:, 1])

faiss/python/swigfaiss.swig

@@ -679,6 +679,7 @@ struct faiss::simd16uint16 {};
 
 %include  <faiss/gpu/GpuIndicesOptions.h>
 %include  <faiss/gpu/GpuClonerOptions.h>
+
 %include  <faiss/gpu/GpuIndex.h>
 #ifdef FAISS_ENABLE_CUVS
 %include  <faiss/gpu/GpuIndexCagra.h>
@@ -1245,6 +1246,20 @@ REV_SWIG_PTR(uint64_t, NPY_UINT64);
 
 #endif
 
+namespace faiss {
+int struct_packing_test_swig (int q);
+}
+
+%{
+
+#include <faiss/utils/struct_packing_test.h>
+
+namespace faiss {
+int struct_packing_test_swig (int q) STRUCT_PACKING_FUNCTION_BODY
+}
+
+%}
+
 
 
 /*******************************************************************

faiss/utils/struct_packing_test.h

@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/* This file and macro is there to catch discrepancies on the different
+ * compilers used in Faiss align and pack structures. This is a difficult to
+ * catch bug. It works by declaring a structure and measuring its size in
+ * diffetent compilers from a function. The function can be called from a test.
+ * If the size as seen from different compilers changes, it's a guarantee for
+ * fireworks at runtime. */
+
+#pragma once
+#include <vector>
+
+namespace faiss {
+namespace struct_packing_test {
+
+/*******************************************************
+ * Fake structure to detect structure packing/alignment
+ * issues.
+ *******************************************************/
+
+struct StructPackingTestD {
+    int a;
+    bool b, c, d;
+    long e;
+    bool f, g, h, i, j;
+};
+
+struct StructPackingTestC : StructPackingTestD {
+    bool k;
+    int l;
+    bool m;
+};
+
+struct StructPackingTestA {
+    virtual void* operator()(int) {
+        return nullptr;
+    }
+
+    virtual ~StructPackingTestA() {}
+};
+
+struct StructPackingTestB : StructPackingTestA {
+    StructPackingTestC options;
+    std::vector<void*> vres;
+    std::vector<int> devices;
+    int ncall;
+
+    void* operator()(int) override {
+        return nullptr;
+    }
+    virtual ~StructPackingTestB() {}
+};
+
+// This is the object hierachy of GpuProgressiveDimIndexFactory that initially
+// triggered this error (see PR #4135 and #4136)
+
+enum IndicesOptionsB {
+    INDICES_CPU_B = 0,
+    INDICES_IVF_B = 1,
+    INDICES_32_BIT_B = 2,
+    INDICES_64_BIT_B = 3,
+};
+
+struct GpuClonerOptionsB {
+    IndicesOptionsB indicesOptions = INDICES_64_BIT_B;
+
+    bool useFloat16CoarseQuantizer = false;
+
+    bool useFloat16 = false;
+
+    bool usePrecomputed = false;
+
+    long reserveVecs = 0;
+
+    bool storeTransposed = false;
+
+    bool verbose = false;
+
+    bool use_cuvs = false;
+    bool allowCpuCoarseQuantizer = false;
+};
+
+struct GpuMultipleClonerOptionsB : public GpuClonerOptionsB {
+    bool shard = false;
+    int shard_type = 1;
+    bool common_ivf_quantizer = false;
+};
+
+struct ProgressiveDimIndexFactoryB {
+
+    virtual void* operator()(int dim) {
+        return nullptr;
+    }
+
+    virtual ~ProgressiveDimIndexFactoryB() {}
+};
+
+struct GpuProgressiveDimIndexFactoryB : ProgressiveDimIndexFactoryB {
+    GpuMultipleClonerOptionsB options;
+    std::vector<void*> vres;
+    std::vector<int> devices;
+    int ncall;
+
+    explicit GpuProgressiveDimIndexFactoryB(int ngpu) {}
+
+    void* operator()(int dim) override {
+        return nullptr;
+    }
+
+    virtual ~GpuProgressiveDimIndexFactoryB() override {}
+};
+
+} // namespace struct_packing_test
+
+} // namespace faiss
+
+// body of function should be
+// int function_name (int q) STRUCT_PACKING_FUNCTION_BODY
+
+#define STRUCT_PACKING_FUNCTION_BODY                                           \
+    {                                                                          \
+        struct_packing_test::StructPackingTestB sb;                            \
+        switch (q) {                                                           \
+            case 0:                                                            \
+                return sizeof(struct_packing_test::StructPackingTestB);        \
+            case 1:                                                            \
+                return (char*)&sb.ncall - (char*)&sb;                          \
+            case 2:                                                            \
+                return sizeof(struct_packing_test::StructPackingTestD);        \
+            case 3:                                                            \
+                return sizeof(struct_packing_test::StructPackingTestC);        \
+            case 4:                                                            \
+                return sizeof(struct_packing_test::StructPackingTestB);        \
+            case 5:                                                            \
+                return sizeof(struct_packing_test::StructPackingTestA);        \
+            case 6:                                                            \
+                return sizeof(struct_packing_test::IndicesOptionsB);           \
+            case 7:                                                            \
+                return sizeof(struct_packing_test::GpuMultipleClonerOptionsB); \
+            case 8:                                                            \
+                return sizeof(struct_packing_test::GpuClonerOptionsB);         \
+            case 9:                                                            \
+                return sizeof(                                                 \
+                        struct_packing_test::ProgressiveDimIndexFactoryB);     \
+            case 10:                                                           \
+                return sizeof(                                                 \
+                        struct_packing_test::GpuProgressiveDimIndexFactoryB);  \
+            default:                                                           \
+                return -1;                                                     \
+        }                                                                      \
+    }

faiss/utils/utils.cpp

@@ -8,6 +8,7 @@
 // -*- c++ -*-
 
 #include <faiss/Index.h>
+#include <faiss/utils/struct_packing_test.h>
 #include <faiss/utils/utils.h>
 
 #include <cassert>
@@ -641,4 +642,6 @@ void CodeSet::insert(size_t n, const uint8_t* codes, bool* inserted) {
     }
 }
 
+int struct_packing_test_cpp(int q) STRUCT_PACKING_FUNCTION_BODY;
+
 } // namespace faiss

faiss/utils/utils.h

@@ -204,6 +204,8 @@ struct CodeSet {
     void insert(size_t n, const uint8_t* codes, bool* inserted);
 };
 
+int struct_packing_test_cpp(int q);
+
 } // namespace faiss
 
 #endif /* FAISS_utils_h */

tests/test_build_blocks.py

@@ -562,3 +562,16 @@ def test_small(self):
     def xx_test_large(self):
         # don't run by default because it's slow
         self.do_test(2 ** 21, 10 ** 6)
+
+
+class TestStructPacking(unittest.TestCase): 
+    """ Verify if the size structures as seen from SWIG and C++ are the same """
+
+    def test_swig(self):   
+        sizes = np.array([
+            (faiss.struct_packing_test_cpp(q),
+            faiss.struct_packing_test_swig(q)) 
+            for q in range(20)
+        ])
+        print(sizes)
+        np.testing.assert_array_equal(sizes[:, 0], sizes[:, 1])