add faiss

CMakeLists.txt

@@ -278,6 +278,12 @@ target_link_libraries(grpc_network_impl_test PUBLIC
  ${Protobuf_LIBRARY}
 )
 
+add_subdirectory(faiss)
+add_executable(faisslib_test 
+ src/faiss_tests.cc
+)
+target_link_libraries(faisslib_test PUBLIC faiss)
+
 # eraft-ctl
 add_executable(eraftkv-ctl 
  src/eraftkv_ctl.cc

Dockerfile

@@ -56,3 +56,7 @@ FROM eraft/eraftkv:v0.0.4
 # RUN ldconfig
 
 RUN apt-get install -y redis-tools telnet
+
+# install faiss
+
+RUN apt-get install libatlas-base-dev libatlas3-base -y && apt-get install swig -y

doc/eraft-vdb.md

@@ -18,10 +18,10 @@ Vec stores a single vector, include vector id, vector data and additional commen
 - vid:
 A unique identifier of your vector.
 
-- vdata:
+- vdata (vextra):
 Vectorized representation of information in the physical world such as sound, video, and pictures.
 
-- vextra:
+- vlabel:
 The description information of the vector datam which is a json structure.
 
 ### Storage Cluster

doc/vdb-server.md

@@ -0,0 +1,53 @@
+## vdb-server RESP protocol for vector design
+
+### 1.Create a Vecset
+
+- Syntax
+```
+VECSET ADD vecset_name vecset_size 
+```
+- Return
+vecset id
+
+### 2.Get a Vecset
+
+- Syntax
+```
+VECSET GET vecset_name
+```
+- Return
+
+Bulk string reply:
+
+```
+vecset_size
+vec_count
+vecdata_disk_size
+vecdata_mem_size
+c_time
+```
+
+### 3.Add Vecs to Vecset
+
+- Syntax
+```
+VEC ADD vecset_name vector_data vector_label [vector_data vector_label ...]
+```
+
+- Return
+The status of the add operation.
+
+### 4.Search from a vecset
+
+-Syntax
+```
+VEC SEARCH vecset_name vector_data DESC count
+```
+- Return 
+
+search result vector data similar with 
+vector_data.
+
+```
+vec_id score vec_label [vec_id score vec_label ...]
+```

doc/vector-similarity-search-in-eraftvdb.md

@@ -0,0 +1,5 @@
+
+https://colab.research.google.com/drive/1OLdDYk-TzVNhpyL3pAONtkHnKGI-GUsU?usp=sharing#scrollTo=420dPmKH8sTi
+
+https://medium.com/swlh/fine-grained-image-similarity-detection-using-facebook-ai-similarity-search-faiss-b357da4f1644
+

faiss/.circleci/Dockerfile.cpu

@@ -0,0 +1,11 @@
+FROM cimg/base:stable-20.04
+
+# Install python3, swig, and MKL.
+RUN sudo apt-get update && \
+sudo apt-get install -y python3-dev python3-pip swig libmkl-dev
+
+# Install recent CMake.
+RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | sudo tar xzf - --strip-components=1 -C /usr
+
+# Install numpy/scipy/pytorch for python tests.
+RUN pip3 install numpy scipy torch

faiss/.circleci/Dockerfile.faiss_gpu

@@ -0,0 +1,28 @@
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
+
+# Install python3, wget, and openblas.
+RUN apt-get update && \
+ apt-get install -y python3-dev python3-pip libopenblas-dev wget libpcre3-dev
+
+# Install swig 4.0.2.
+RUN wget -nv -O - https://sourceforge.net/projects/swig/files/swig/swig-4.0.2/swig-4.0.2.tar.gz/download | tar zxf - && cd swig-4.0.2 && ./configure && make -j && make install
+
+# Install recent CMake.
+RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf - --strip-components=1 -C /usr
+
+# Install numpy/scipy/pytorch for python tests.
+RUN pip3 install numpy scipy torch
+
+COPY . /faiss
+
+WORKDIR /faiss
+
+RUN cmake -B build \
+ -DFAISS_ENABLE_GPU=ON \
+ -DFAISS_ENABLE_C_API=ON \
+ -DFAISS_ENABLE_PYTHON=ON \
+ -DBUILD_TESTING=ON \
+ -DCMAKE_CUDA_FLAGS="-gencode arch=compute_75,code=sm_75" \
+ .
+
+RUN make -C build -j8