Initial commit

This commit is contained in:
yichuan520030910320
2025-06-30 09:05:05 +00:00
commit 46f6cc100b
1231 changed files with 278432 additions and 0 deletions

View File

@@ -0,0 +1,101 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexFlat.h>
// 64-bit int
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
faiss::IndexFlatL2 index(d); // call constructor
printf("is_trained = %s\n", index.is_trained ? "true" : "false");
index.add(nb, xb); // add vectors to the index
printf("ntotal = %zd\n", index.ntotal);
int k = 4;
{ // sanity check: search 5 first vectors of xb
idx_t* I = new idx_t[k * 5];
float* D = new float[k * 5];
index.search(5, xb, k, D, I);
// print results
printf("I=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
printf("D=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%7g ", D[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
{ // search xq
idx_t* I = new idx_t[k * nq];
float* D = new float[k * nq];
index.search(nq, xq, k, D, I);
// print results
printf("I (5 first results)=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
printf("D (5 last results)=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5f ", D[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,79 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexFlat.h>
#include <faiss/IndexIVFFlat.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
int nlist = 100;
int k = 4;
faiss::IndexFlatL2 quantizer(d); // the other index
faiss::IndexIVFFlat index(&quantizer, d, nlist);
assert(!index.is_trained);
index.train(nb, xb);
assert(index.is_trained);
index.add(nb, xb);
{ // search xq
idx_t* I = new idx_t[k * nq];
float* D = new float[k * nq];
index.search(nq, xq, k, D, I);
printf("I=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
printf("D=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5f ", D[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,95 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexFlat.h>
#include <faiss/IndexIVFPQ.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
int nlist = 100;
int k = 4;
int m = 8; // bytes per vector
faiss::IndexFlatL2 quantizer(d); // the other index
faiss::IndexIVFPQ index(&quantizer, d, nlist, m, 8);
index.train(nb, xb);
index.add(nb, xb);
{ // sanity check
idx_t* I = new idx_t[k * 5];
float* D = new float[k * 5];
index.search(5, xb, k, D, I);
printf("I=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
printf("D=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%7g ", D[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
{ // search xq
idx_t* I = new idx_t[k * nq];
float* D = new float[k * nq];
index.nprobe = 10;
index.search(nq, xq, k, D, I);
printf("I=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,120 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexFlat.h>
#include <faiss/gpu/GpuIndexFlat.h>
#include <faiss/gpu/GpuIndexIVFFlat.h>
#include <faiss/gpu/StandardGpuResources.h>
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
faiss::gpu::StandardGpuResources res;
// Using a flat index
faiss::gpu::GpuIndexFlatL2 index_flat(&res, d);
printf("is_trained = %s\n", index_flat.is_trained ? "true" : "false");
index_flat.add(nb, xb); // add vectors to the index
printf("ntotal = %ld\n", index_flat.ntotal);
int k = 4;
{ // search xq
long* I = new long[k * nq];
float* D = new float[k * nq];
index_flat.search(nq, xq, k, D, I);
// print results
printf("I (5 first results)=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
printf("I (5 last results)=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
// Using an IVF index
int nlist = 100;
faiss::gpu::GpuIndexIVFFlat index_ivf(&res, d, nlist, faiss::METRIC_L2);
assert(!index_ivf.is_trained);
index_ivf.train(nb, xb);
assert(index_ivf.is_trained);
index_ivf.add(nb, xb); // add vectors to the index
printf("is_trained = %s\n", index_ivf.is_trained ? "true" : "false");
printf("ntotal = %ld\n", index_ivf.ntotal);
{ // search xq
long* I = new long[k * nq];
float* D = new float[k * nq];
index_ivf.search(nq, xq, k, D, I);
// print results
printf("I (5 first results)=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
printf("I (5 last results)=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexFlat.h>
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/gpu/GpuIndexFlat.h>
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/gpu/utils/DeviceUtils.h>
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
int ngpus = faiss::gpu::getNumDevices();
printf("Number of GPUs: %d\n", ngpus);
std::vector<faiss::gpu::GpuResourcesProvider*> res;
std::vector<int> devs;
for (int i = 0; i < ngpus; i++) {
res.push_back(new faiss::gpu::StandardGpuResources);
devs.push_back(i);
}
faiss::IndexFlatL2 cpu_index(d);
faiss::Index* gpu_index =
faiss::gpu::index_cpu_to_gpu_multiple(res, devs, &cpu_index);
printf("is_trained = %s\n", gpu_index->is_trained ? "true" : "false");
gpu_index->add(nb, xb); // add vectors to the index
printf("ntotal = %ld\n", gpu_index->ntotal);
int k = 4;
{ // search xq
long* I = new long[k * nq];
float* D = new float[k * nq];
gpu_index->search(nq, xq, k, D, I);
// print results
printf("I (5 first results)=\n");
for (int i = 0; i < 5; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
printf("I (5 last results)=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5ld ", I[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete gpu_index;
for (int i = 0; i < ngpus; i++) {
delete res[i];
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,73 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexHNSW.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[d * nb];
float* xq = new float[d * nq];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++)
xb[d * i + j] = distrib(rng);
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++)
xq[d * i + j] = distrib(rng);
xq[d * i] += i / 1000.;
}
int k = 4;
faiss::IndexHNSWFlat index(d, 32);
index.add(nb, xb);
{ // search xq
idx_t* I = new idx_t[k * nq];
float* D = new float[k * nq];
index.search(nq, xq, k, D, I);
printf("I=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5zd ", I[i * k + j]);
printf("\n");
}
printf("D=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++)
printf("%5f ", D[i * k + j]);
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexPQFastScan.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[(int)(d * nb)];
float* xq = new float[(int)(d * nq)];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) {
xb[d * i + j] = distrib(rng);
}
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++) {
xq[d * i + j] = distrib(rng);
}
xq[d * i] += i / 1000.;
}
int m = 8;
int n_bit = 4;
faiss::IndexPQFastScan index(d, m, n_bit);
printf("Index is trained? %s\n", index.is_trained ? "true" : "false");
index.train(nb, xb);
printf("Index is trained? %s\n", index.is_trained ? "true" : "false");
index.add(nb, xb);
int k = 4;
{ // search xq
idx_t* I = new idx_t[(int)(k * nq)];
float* D = new float[(int)(k * nq)];
index.search(nq, xq, k, D, I);
printf("I=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++) {
printf("%5zd ", I[i * k + j]);
}
printf("\n");
}
delete[] I;
delete[] D;
}
delete[] xb;
delete[] xq;
return 0;
} // namespace facebook::detail

View File

@@ -0,0 +1,84 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexPQFastScan.h>
#include <faiss/IndexRefine.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[(int)(d * nb)];
float* xq = new float[(int)(d * nq)];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) {
xb[d * i + j] = distrib(rng);
}
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++) {
xq[d * i + j] = distrib(rng);
}
xq[d * i] += i / 1000.;
}
int m = 8;
int n_bit = 4;
faiss::IndexPQFastScan index(d, m, n_bit);
faiss::IndexRefineFlat index_refine(&index);
// refine index after PQFastScan
printf("Index is trained? %s\n",
index_refine.is_trained ? "true" : "false");
index_refine.train(nb, xb);
printf("Index is trained? %s\n",
index_refine.is_trained ? "true" : "false");
index_refine.add(nb, xb);
int k = 4;
{ // search xq
idx_t* I = new idx_t[(int)(k * nq)];
float* D = new float[(int)(k * nq)];
float k_factor = 3;
faiss::IndexRefineSearchParameters* params =
new faiss::IndexRefineSearchParameters();
params->k_factor = k_factor;
index_refine.search(nq, xq, k, D, I, params);
printf("I=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++) {
printf("%5zd ", I[i * k + j]);
}
printf("\n");
}
delete[] I;
delete[] D;
delete params;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,104 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <faiss/IndexPQFastScan.h>
#include <faiss/IndexRefine.h>
#include <faiss/index_factory.h>
using idx_t = faiss::idx_t;
int main() {
int d = 64; // dimension
int nb = 100000; // database size
int nq = 10000; // nb of queries
std::mt19937 rng;
std::uniform_real_distribution<> distrib;
float* xb = new float[(int)(d * nb)];
float* xq = new float[(int)(d * nq)];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) {
xb[d * i + j] = distrib(rng);
}
xb[d * i] += i / 1000.;
}
for (int i = 0; i < nq; i++) {
for (int j = 0; j < d; j++) {
xq[d * i + j] = distrib(rng);
}
xq[d * i] += i / 1000.;
}
// Constructing the refine PQ index with SQfp16 with index factory
faiss::Index* index_fp16;
index_fp16 = faiss::index_factory(
d, "PQ32x4fs,Refine(SQfp16)", faiss::METRIC_L2);
index_fp16->train(nb, xb);
index_fp16->add(nb, xb);
// Constructing the refine PQ index with SQ8
faiss::Index* index_sq8;
index_sq8 =
faiss::index_factory(d, "PQ32x4fs,Refine(SQ8)", faiss::METRIC_L2);
index_sq8->train(nb, xb);
index_sq8->add(nb, xb);
int k = 10;
{ // search xq
idx_t* I_fp16 = new idx_t[(int)(k * nq)];
float* D_fp16 = new float[(int)(k * nq)];
idx_t* I_sq8 = new idx_t[(int)(k * nq)];
float* D_sq8 = new float[(int)(k * nq)];
// Parameterization on k factor while doing search for index refinement
float k_factor = 3;
faiss::IndexRefineSearchParameters* params =
new faiss::IndexRefineSearchParameters();
params->k_factor = k_factor;
// Perform index search using different index refinement
index_fp16->search(nq, xq, k, D_fp16, I_fp16, params);
index_sq8->search(nq, xq, k, D_sq8, I_sq8, params);
printf("I_fp16=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++) {
printf("%5zd ", I_fp16[i * k + j]);
}
printf("\n");
}
printf("I_sq8=\n");
for (int i = nq - 5; i < nq; i++) {
for (int j = 0; j < k; j++) {
printf("%5zd ", I_sq8[i * k + j]);
}
printf("\n");
}
delete[] I_fp16;
delete[] D_fp16;
delete[] I_sq8;
delete[] D_sq8;
delete params;
delete index_fp16;
delete index_sq8;
}
delete[] xb;
delete[] xq;
return 0;
}

View File

@@ -0,0 +1,31 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
add_executable(1-Flat EXCLUDE_FROM_ALL 1-Flat.cpp)
target_link_libraries(1-Flat PRIVATE faiss)
add_executable(2-IVFFlat EXCLUDE_FROM_ALL 2-IVFFlat.cpp)
target_link_libraries(2-IVFFlat PRIVATE faiss)
add_executable(3-IVFPQ EXCLUDE_FROM_ALL 3-IVFPQ.cpp)
target_link_libraries(3-IVFPQ PRIVATE faiss)
add_executable(4-GPU EXCLUDE_FROM_ALL 4-GPU.cpp)
target_link_libraries(4-GPU PRIVATE faiss)
add_executable(5-Multiple-GPUs EXCLUDE_FROM_ALL 5-Multiple-GPUs.cpp)
target_link_libraries(5-Multiple-GPUs PRIVATE faiss)
add_executable(6-HNSW EXCLUDE_FROM_ALL 6-HNSW.cpp)
target_link_libraries(6-HNSW PRIVATE faiss)
add_executable(7-PQFastScan EXCLUDE_FROM_ALL 7-PQFastScan.cpp)
target_link_libraries(7-PQFastScan PRIVATE faiss)
add_executable(8-PQFastScanRefine EXCLUDE_FROM_ALL 8-PQFastScanRefine.cpp)
target_link_libraries(8-PQFastScanRefine PRIVATE faiss)
add_executable(9-RefineComparison EXCLUDE_FROM_ALL 9-RefineComparison.cpp)
target_link_libraries(9-RefineComparison PRIVATE faiss)