Initial commit
This commit is contained in:
101
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/1-Flat.cpp
vendored
Normal file
101
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/1-Flat.cpp
vendored
Normal file
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexFlat.h>
|
||||
|
||||
// 64-bit int
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
faiss::IndexFlatL2 index(d); // call constructor
|
||||
printf("is_trained = %s\n", index.is_trained ? "true" : "false");
|
||||
index.add(nb, xb); // add vectors to the index
|
||||
printf("ntotal = %zd\n", index.ntotal);
|
||||
|
||||
int k = 4;
|
||||
|
||||
{ // sanity check: search 5 first vectors of xb
|
||||
idx_t* I = new idx_t[k * 5];
|
||||
float* D = new float[k * 5];
|
||||
|
||||
index.search(5, xb, k, D, I);
|
||||
|
||||
// print results
|
||||
printf("I=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("D=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%7g ", D[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index.search(nq, xq, k, D, I);
|
||||
|
||||
// print results
|
||||
printf("I (5 first results)=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("D (5 last results)=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5f ", D[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
79
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/2-IVFFlat.cpp
vendored
Normal file
79
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/2-IVFFlat.cpp
vendored
Normal file
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexFlat.h>
|
||||
#include <faiss/IndexIVFFlat.h>
|
||||
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int nlist = 100;
|
||||
int k = 4;
|
||||
|
||||
faiss::IndexFlatL2 quantizer(d); // the other index
|
||||
faiss::IndexIVFFlat index(&quantizer, d, nlist);
|
||||
assert(!index.is_trained);
|
||||
index.train(nb, xb);
|
||||
assert(index.is_trained);
|
||||
index.add(nb, xb);
|
||||
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index.search(nq, xq, k, D, I);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("D=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5f ", D[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
95
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/3-IVFPQ.cpp
vendored
Normal file
95
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/3-IVFPQ.cpp
vendored
Normal file
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexFlat.h>
|
||||
#include <faiss/IndexIVFPQ.h>
|
||||
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int nlist = 100;
|
||||
int k = 4;
|
||||
int m = 8; // bytes per vector
|
||||
faiss::IndexFlatL2 quantizer(d); // the other index
|
||||
faiss::IndexIVFPQ index(&quantizer, d, nlist, m, 8);
|
||||
|
||||
index.train(nb, xb);
|
||||
index.add(nb, xb);
|
||||
|
||||
{ // sanity check
|
||||
idx_t* I = new idx_t[k * 5];
|
||||
float* D = new float[k * 5];
|
||||
|
||||
index.search(5, xb, k, D, I);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("D=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%7g ", D[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index.nprobe = 10;
|
||||
index.search(nq, xq, k, D, I);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
120
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/4-GPU.cpp
vendored
Normal file
120
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/4-GPU.cpp
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexFlat.h>
|
||||
#include <faiss/gpu/GpuIndexFlat.h>
|
||||
#include <faiss/gpu/GpuIndexIVFFlat.h>
|
||||
#include <faiss/gpu/StandardGpuResources.h>
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
faiss::gpu::StandardGpuResources res;
|
||||
|
||||
// Using a flat index
|
||||
|
||||
faiss::gpu::GpuIndexFlatL2 index_flat(&res, d);
|
||||
|
||||
printf("is_trained = %s\n", index_flat.is_trained ? "true" : "false");
|
||||
index_flat.add(nb, xb); // add vectors to the index
|
||||
printf("ntotal = %ld\n", index_flat.ntotal);
|
||||
|
||||
int k = 4;
|
||||
|
||||
{ // search xq
|
||||
long* I = new long[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index_flat.search(nq, xq, k, D, I);
|
||||
|
||||
// print results
|
||||
printf("I (5 first results)=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("I (5 last results)=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
// Using an IVF index
|
||||
|
||||
int nlist = 100;
|
||||
faiss::gpu::GpuIndexIVFFlat index_ivf(&res, d, nlist, faiss::METRIC_L2);
|
||||
|
||||
assert(!index_ivf.is_trained);
|
||||
index_ivf.train(nb, xb);
|
||||
assert(index_ivf.is_trained);
|
||||
index_ivf.add(nb, xb); // add vectors to the index
|
||||
|
||||
printf("is_trained = %s\n", index_ivf.is_trained ? "true" : "false");
|
||||
printf("ntotal = %ld\n", index_ivf.ntotal);
|
||||
|
||||
{ // search xq
|
||||
long* I = new long[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index_ivf.search(nq, xq, k, D, I);
|
||||
|
||||
// print results
|
||||
printf("I (5 first results)=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("I (5 last results)=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
99
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/5-Multiple-GPUs.cpp
vendored
Normal file
99
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/5-Multiple-GPUs.cpp
vendored
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexFlat.h>
|
||||
#include <faiss/gpu/GpuAutoTune.h>
|
||||
#include <faiss/gpu/GpuCloner.h>
|
||||
#include <faiss/gpu/GpuIndexFlat.h>
|
||||
#include <faiss/gpu/StandardGpuResources.h>
|
||||
#include <faiss/gpu/utils/DeviceUtils.h>
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int ngpus = faiss::gpu::getNumDevices();
|
||||
|
||||
printf("Number of GPUs: %d\n", ngpus);
|
||||
|
||||
std::vector<faiss::gpu::GpuResourcesProvider*> res;
|
||||
std::vector<int> devs;
|
||||
for (int i = 0; i < ngpus; i++) {
|
||||
res.push_back(new faiss::gpu::StandardGpuResources);
|
||||
devs.push_back(i);
|
||||
}
|
||||
|
||||
faiss::IndexFlatL2 cpu_index(d);
|
||||
|
||||
faiss::Index* gpu_index =
|
||||
faiss::gpu::index_cpu_to_gpu_multiple(res, devs, &cpu_index);
|
||||
|
||||
printf("is_trained = %s\n", gpu_index->is_trained ? "true" : "false");
|
||||
gpu_index->add(nb, xb); // add vectors to the index
|
||||
printf("ntotal = %ld\n", gpu_index->ntotal);
|
||||
|
||||
int k = 4;
|
||||
|
||||
{ // search xq
|
||||
long* I = new long[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
gpu_index->search(nq, xq, k, D, I);
|
||||
|
||||
// print results
|
||||
printf("I (5 first results)=\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("I (5 last results)=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5ld ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete gpu_index;
|
||||
|
||||
for (int i = 0; i < ngpus; i++) {
|
||||
delete res[i];
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
73
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/6-HNSW.cpp
vendored
Normal file
73
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/6-HNSW.cpp
vendored
Normal file
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexHNSW.h>
|
||||
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[d * nb];
|
||||
float* xq = new float[d * nq];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xb[d * i + j] = distrib(rng);
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++)
|
||||
xq[d * i + j] = distrib(rng);
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int k = 4;
|
||||
|
||||
faiss::IndexHNSWFlat index(d, 32);
|
||||
index.add(nb, xb);
|
||||
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[k * nq];
|
||||
float* D = new float[k * nq];
|
||||
|
||||
index.search(nq, xq, k, D, I);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("D=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++)
|
||||
printf("%5f ", D[i * k + j]);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
75
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/7-PQFastScan.cpp
vendored
Normal file
75
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/7-PQFastScan.cpp
vendored
Normal file
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexPQFastScan.h>
|
||||
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[(int)(d * nb)];
|
||||
float* xq = new float[(int)(d * nq)];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xb[d * i + j] = distrib(rng);
|
||||
}
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xq[d * i + j] = distrib(rng);
|
||||
}
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int m = 8;
|
||||
int n_bit = 4;
|
||||
|
||||
faiss::IndexPQFastScan index(d, m, n_bit);
|
||||
printf("Index is trained? %s\n", index.is_trained ? "true" : "false");
|
||||
index.train(nb, xb);
|
||||
printf("Index is trained? %s\n", index.is_trained ? "true" : "false");
|
||||
index.add(nb, xb);
|
||||
|
||||
int k = 4;
|
||||
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[(int)(k * nq)];
|
||||
float* D = new float[(int)(k * nq)];
|
||||
|
||||
index.search(nq, xq, k, D, I);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++) {
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
} // namespace facebook::detail
|
||||
84
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/8-PQFastScanRefine.cpp
vendored
Normal file
84
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/8-PQFastScanRefine.cpp
vendored
Normal file
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexPQFastScan.h>
|
||||
#include <faiss/IndexRefine.h>
|
||||
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[(int)(d * nb)];
|
||||
float* xq = new float[(int)(d * nq)];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xb[d * i + j] = distrib(rng);
|
||||
}
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xq[d * i + j] = distrib(rng);
|
||||
}
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
int m = 8;
|
||||
int n_bit = 4;
|
||||
|
||||
faiss::IndexPQFastScan index(d, m, n_bit);
|
||||
faiss::IndexRefineFlat index_refine(&index);
|
||||
// refine index after PQFastScan
|
||||
|
||||
printf("Index is trained? %s\n",
|
||||
index_refine.is_trained ? "true" : "false");
|
||||
index_refine.train(nb, xb);
|
||||
printf("Index is trained? %s\n",
|
||||
index_refine.is_trained ? "true" : "false");
|
||||
index_refine.add(nb, xb);
|
||||
|
||||
int k = 4;
|
||||
{ // search xq
|
||||
idx_t* I = new idx_t[(int)(k * nq)];
|
||||
float* D = new float[(int)(k * nq)];
|
||||
float k_factor = 3;
|
||||
faiss::IndexRefineSearchParameters* params =
|
||||
new faiss::IndexRefineSearchParameters();
|
||||
params->k_factor = k_factor;
|
||||
index_refine.search(nq, xq, k, D, I, params);
|
||||
|
||||
printf("I=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++) {
|
||||
printf("%5zd ", I[i * k + j]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I;
|
||||
delete[] D;
|
||||
delete params;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
104
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/9-RefineComparison.cpp
vendored
Normal file
104
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/9-RefineComparison.cpp
vendored
Normal file
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <random>
|
||||
|
||||
#include <faiss/IndexPQFastScan.h>
|
||||
#include <faiss/IndexRefine.h>
|
||||
#include <faiss/index_factory.h>
|
||||
using idx_t = faiss::idx_t;
|
||||
|
||||
int main() {
|
||||
int d = 64; // dimension
|
||||
int nb = 100000; // database size
|
||||
int nq = 10000; // nb of queries
|
||||
|
||||
std::mt19937 rng;
|
||||
std::uniform_real_distribution<> distrib;
|
||||
|
||||
float* xb = new float[(int)(d * nb)];
|
||||
float* xq = new float[(int)(d * nq)];
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xb[d * i + j] = distrib(rng);
|
||||
}
|
||||
xb[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nq; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
xq[d * i + j] = distrib(rng);
|
||||
}
|
||||
xq[d * i] += i / 1000.;
|
||||
}
|
||||
|
||||
// Constructing the refine PQ index with SQfp16 with index factory
|
||||
faiss::Index* index_fp16;
|
||||
index_fp16 = faiss::index_factory(
|
||||
d, "PQ32x4fs,Refine(SQfp16)", faiss::METRIC_L2);
|
||||
index_fp16->train(nb, xb);
|
||||
index_fp16->add(nb, xb);
|
||||
|
||||
// Constructing the refine PQ index with SQ8
|
||||
faiss::Index* index_sq8;
|
||||
index_sq8 =
|
||||
faiss::index_factory(d, "PQ32x4fs,Refine(SQ8)", faiss::METRIC_L2);
|
||||
index_sq8->train(nb, xb);
|
||||
index_sq8->add(nb, xb);
|
||||
|
||||
int k = 10;
|
||||
{ // search xq
|
||||
idx_t* I_fp16 = new idx_t[(int)(k * nq)];
|
||||
float* D_fp16 = new float[(int)(k * nq)];
|
||||
idx_t* I_sq8 = new idx_t[(int)(k * nq)];
|
||||
float* D_sq8 = new float[(int)(k * nq)];
|
||||
|
||||
// Parameterization on k factor while doing search for index refinement
|
||||
float k_factor = 3;
|
||||
faiss::IndexRefineSearchParameters* params =
|
||||
new faiss::IndexRefineSearchParameters();
|
||||
params->k_factor = k_factor;
|
||||
|
||||
// Perform index search using different index refinement
|
||||
index_fp16->search(nq, xq, k, D_fp16, I_fp16, params);
|
||||
index_sq8->search(nq, xq, k, D_sq8, I_sq8, params);
|
||||
|
||||
printf("I_fp16=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++) {
|
||||
printf("%5zd ", I_fp16[i * k + j]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("I_sq8=\n");
|
||||
for (int i = nq - 5; i < nq; i++) {
|
||||
for (int j = 0; j < k; j++) {
|
||||
printf("%5zd ", I_sq8[i * k + j]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
delete[] I_fp16;
|
||||
delete[] D_fp16;
|
||||
delete[] I_sq8;
|
||||
delete[] D_sq8;
|
||||
delete params;
|
||||
|
||||
delete index_fp16;
|
||||
delete index_sq8;
|
||||
}
|
||||
|
||||
delete[] xb;
|
||||
delete[] xq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
31
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/CMakeLists.txt
vendored
Normal file
31
packages/leann-backend-hnsw/third_party/faiss/tutorial/cpp/CMakeLists.txt
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
add_executable(1-Flat EXCLUDE_FROM_ALL 1-Flat.cpp)
|
||||
target_link_libraries(1-Flat PRIVATE faiss)
|
||||
|
||||
add_executable(2-IVFFlat EXCLUDE_FROM_ALL 2-IVFFlat.cpp)
|
||||
target_link_libraries(2-IVFFlat PRIVATE faiss)
|
||||
|
||||
add_executable(3-IVFPQ EXCLUDE_FROM_ALL 3-IVFPQ.cpp)
|
||||
target_link_libraries(3-IVFPQ PRIVATE faiss)
|
||||
|
||||
add_executable(4-GPU EXCLUDE_FROM_ALL 4-GPU.cpp)
|
||||
target_link_libraries(4-GPU PRIVATE faiss)
|
||||
|
||||
add_executable(5-Multiple-GPUs EXCLUDE_FROM_ALL 5-Multiple-GPUs.cpp)
|
||||
target_link_libraries(5-Multiple-GPUs PRIVATE faiss)
|
||||
|
||||
add_executable(6-HNSW EXCLUDE_FROM_ALL 6-HNSW.cpp)
|
||||
target_link_libraries(6-HNSW PRIVATE faiss)
|
||||
|
||||
add_executable(7-PQFastScan EXCLUDE_FROM_ALL 7-PQFastScan.cpp)
|
||||
target_link_libraries(7-PQFastScan PRIVATE faiss)
|
||||
|
||||
add_executable(8-PQFastScanRefine EXCLUDE_FROM_ALL 8-PQFastScanRefine.cpp)
|
||||
target_link_libraries(8-PQFastScanRefine PRIVATE faiss)
|
||||
|
||||
add_executable(9-RefineComparison EXCLUDE_FROM_ALL 9-RefineComparison.cpp)
|
||||
target_link_libraries(9-RefineComparison PRIVATE faiss)
|
||||
Reference in New Issue
Block a user