Initial commit
This commit is contained in:
yichuan520030910320
39
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/build/action.yml
vendored
Normal file
39
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/build/action.yml
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
name: 'DiskANN Build Bootstrap'
|
||||
description: 'Prepares DiskANN build environment and executes build'
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
# ------------ Linux Build ---------------
|
||||
- name: Prepare and Execute Build
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
run: |
|
||||
sudo scripts/dev/install-dev-deps-ubuntu.bash
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DUNIT_TEST=True
|
||||
cmake --build build -- -j
|
||||
cmake --install build --prefix="dist"
|
||||
shell: bash
|
||||
# ------------ End Linux Build ---------------
|
||||
# ------------ Windows Build ---------------
|
||||
- name: Add VisualStudio command line tools into path
|
||||
if: runner.os == 'Windows'
|
||||
uses: ilammy/msvc-dev-cmd@v1
|
||||
- name: Run configure and build for Windows
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
mkdir build && cd build && cmake .. -DUNIT_TEST=True && msbuild diskann.sln /m /nologo /t:Build /p:Configuration="Release" /property:Platform="x64" -consoleloggerparameters:"ErrorsOnly;Summary"
|
||||
cd ..
|
||||
mkdir dist
|
||||
mklink /j .\dist\bin .\x64\Release\
|
||||
shell: cmd
|
||||
# ------------ End Windows Build ---------------
|
||||
# ------------ Windows Build With EXEC_ENV_OLS and USE_BING_INFRA ---------------
|
||||
- name: Add VisualStudio command line tools into path
|
||||
if: runner.os == 'Windows'
|
||||
uses: ilammy/msvc-dev-cmd@v1
|
||||
- name: Run configure and build for Windows with Bing feature flags
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
mkdir build_bing && cd build_bing && cmake .. -DEXEC_ENV_OLS=1 -DUSE_BING_INFRA=1 -DUNIT_TEST=True && msbuild diskann.sln /m /nologo /t:Build /p:Configuration="Release" /property:Platform="x64" -consoleloggerparameters:"ErrorsOnly;Summary"
|
||||
cd ..
|
||||
shell: cmd
|
||||
# ------------ End Windows Build ---------------
|
||||
13
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/format-check/action.yml
vendored
Normal file
13
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/format-check/action.yml
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
name: 'Checking code formatting...'
|
||||
description: 'Ensures code complies with code formatting rules'
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Checking code formatting...
|
||||
run: |
|
||||
sudo apt install clang-format
|
||||
find include -name '*.h' -type f -print0 | xargs -0 -P 16 /usr/bin/clang-format --Werror --dry-run
|
||||
find src -name '*.cpp' -type f -print0 | xargs -0 -P 16 /usr/bin/clang-format --Werror --dry-run
|
||||
find apps -name '*.cpp' -type f -print0 | xargs -0 -P 16 /usr/bin/clang-format --Werror --dry-run
|
||||
find python -name '*.cpp' -type f -print0 | xargs -0 -P 16 /usr/bin/clang-format --Werror --dry-run
|
||||
shell: bash
|
||||
@@ -0,0 +1,28 @@
|
||||
name: 'Generating Random Data (Basic)'
|
||||
description: 'Generates the random data files used in acceptance tests'
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Generate Random Data (Basic)
|
||||
run: |
|
||||
mkdir data
|
||||
|
||||
echo "Generating random 1020,1024,1536D float and 4096 int8 vectors for index"
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1020D_5K_norm1.0.bin -D 1020 -N 5000 --norm 1.0
|
||||
#dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1024D_5K_norm1.0.bin -D 1024 -N 5000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1536D_5K_norm1.0.bin -D 1536 -N 5000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type int8 --output_file data/rand_int8_4096D_5K_norm1.0.bin -D 4096 -N 5000 --norm 1.0
|
||||
|
||||
echo "Generating random 1020,1024,1536D float and 4096D int8 avectors for query"
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1020D_1K_norm1.0.bin -D 1020 -N 1000 --norm 1.0
|
||||
#dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1024D_1K_norm1.0.bin -D 1024 -N 1000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_1536D_1K_norm1.0.bin -D 1536 -N 1000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type int8 --output_file data/rand_int8_4096D_1K_norm1.0.bin -D 4096 -N 1000 --norm 1.0
|
||||
|
||||
echo "Computing ground truth for 1020,1024,1536D float and 4096D int8 avectors for query"
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn l2 --base_file data/rand_float_1020D_5K_norm1.0.bin --query_file data/rand_float_1020D_1K_norm1.0.bin --gt_file data/l2_rand_float_1020D_5K_norm1.0_1020D_1K_norm1.0_gt100 --K 100
|
||||
#dist/bin/compute_groundtruth --data_type float --dist_fn l2 --base_file data/rand_float_1024D_5K_norm1.0.bin --query_file data/rand_float_1024D_1K_norm1.0.bin --gt_file data/l2_rand_float_1024D_5K_norm1.0_1024D_1K_norm1.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn l2 --base_file data/rand_float_1536D_5K_norm1.0.bin --query_file data/rand_float_1536D_1K_norm1.0.bin --gt_file data/l2_rand_float_1536D_5K_norm1.0_1536D_1K_norm1.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type int8 --dist_fn l2 --base_file data/rand_int8_4096D_5K_norm1.0.bin --query_file data/rand_int8_4096D_1K_norm1.0.bin --gt_file data/l2_rand_int8_4096D_5K_norm1.0_4096D_1K_norm1.0_gt100 --K 100
|
||||
|
||||
shell: bash
|
||||
38
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/generate-random/action.yml
vendored
Normal file
38
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/generate-random/action.yml
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
name: 'Generating Random Data (Basic)'
|
||||
description: 'Generates the random data files used in acceptance tests'
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Generate Random Data (Basic)
|
||||
run: |
|
||||
mkdir data
|
||||
|
||||
echo "Generating random vectors for index"
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_10D_10K_norm1.0.bin -D 10 -N 10000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_10D_10K_unnorm.bin -D 10 -N 10000 --rand_scaling 2.0
|
||||
dist/bin/rand_data_gen --data_type int8 --output_file data/rand_int8_10D_10K_norm50.0.bin -D 10 -N 10000 --norm 50.0
|
||||
dist/bin/rand_data_gen --data_type uint8 --output_file data/rand_uint8_10D_10K_norm50.0.bin -D 10 -N 10000 --norm 50.0
|
||||
|
||||
echo "Generating random vectors for query"
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_10D_1K_norm1.0.bin -D 10 -N 1000 --norm 1.0
|
||||
dist/bin/rand_data_gen --data_type float --output_file data/rand_float_10D_1K_unnorm.bin -D 10 -N 1000 --rand_scaling 2.0
|
||||
dist/bin/rand_data_gen --data_type int8 --output_file data/rand_int8_10D_1K_norm50.0.bin -D 10 -N 1000 --norm 50.0
|
||||
dist/bin/rand_data_gen --data_type uint8 --output_file data/rand_uint8_10D_1K_norm50.0.bin -D 10 -N 1000 --norm 50.0
|
||||
|
||||
echo "Computing ground truth for floats across l2, mips, and cosine distance functions"
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn l2 --base_file data/rand_float_10D_10K_norm1.0.bin --query_file data/rand_float_10D_1K_norm1.0.bin --gt_file data/l2_rand_float_10D_10K_norm1.0_10D_1K_norm1.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn mips --base_file data/rand_float_10D_10K_norm1.0.bin --query_file data/rand_float_10D_1K_norm1.0.bin --gt_file data/mips_rand_float_10D_10K_norm1.0_10D_1K_norm1.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn cosine --base_file data/rand_float_10D_10K_norm1.0.bin --query_file data/rand_float_10D_1K_norm1.0.bin --gt_file data/cosine_rand_float_10D_10K_norm1.0_10D_1K_norm1.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type float --dist_fn cosine --base_file data/rand_float_10D_10K_unnorm.bin --query_file data/rand_float_10D_1K_unnorm.bin --gt_file data/cosine_rand_float_10D_10K_unnorm_10D_1K_unnorm_gt100 --K 100
|
||||
|
||||
echo "Computing ground truth for int8s across l2, mips, and cosine distance functions"
|
||||
dist/bin/compute_groundtruth --data_type int8 --dist_fn l2 --base_file data/rand_int8_10D_10K_norm50.0.bin --query_file data/rand_int8_10D_1K_norm50.0.bin --gt_file data/l2_rand_int8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type int8 --dist_fn mips --base_file data/rand_int8_10D_10K_norm50.0.bin --query_file data/rand_int8_10D_1K_norm50.0.bin --gt_file data/mips_rand_int8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type int8 --dist_fn cosine --base_file data/rand_int8_10D_10K_norm50.0.bin --query_file data/rand_int8_10D_1K_norm50.0.bin --gt_file data/cosine_rand_int8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
|
||||
echo "Computing ground truth for uint8s across l2, mips, and cosine distance functions"
|
||||
dist/bin/compute_groundtruth --data_type uint8 --dist_fn l2 --base_file data/rand_uint8_10D_10K_norm50.0.bin --query_file data/rand_uint8_10D_1K_norm50.0.bin --gt_file data/l2_rand_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type uint8 --dist_fn mips --base_file data/rand_uint8_10D_10K_norm50.0.bin --query_file data/rand_uint8_10D_1K_norm50.0.bin --gt_file data/mips_rand_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
dist/bin/compute_groundtruth --data_type uint8 --dist_fn cosine --base_file data/rand_uint8_10D_10K_norm50.0.bin --query_file data/rand_uint8_10D_1K_norm50.0.bin --gt_file data/cosine_rand_uint8_10D_10K_norm50.0_10D_1K_norm50.0_gt100 --K 100
|
||||
|
||||
shell: bash
|
||||
22
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/python-wheel/action.yml
vendored
Normal file
22
packages/leann-backend-diskann/third_party/DiskANN/.github/actions/python-wheel/action.yml
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
name: Build Python Wheel
|
||||
description: Builds a python wheel with cibuildwheel
|
||||
inputs:
|
||||
cibw-identifier:
|
||||
description: "CI build wheel identifier to build"
|
||||
required: true
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- uses: actions/setup-python@v3
|
||||
- name: Install cibuildwheel
|
||||
run: python -m pip install cibuildwheel==2.11.3
|
||||
shell: bash
|
||||
- name: Building Python ${{inputs.cibw-identifier}} Wheel
|
||||
run: python -m cibuildwheel --output-dir dist
|
||||
env:
|
||||
CIBW_BUILD: ${{inputs.cibw-identifier}}
|
||||
shell: bash
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: wheels
|
||||
path: ./dist/*.whl
|
||||
Reference in New Issue
Block a user