Merge branch 'main' into multinode-test

Signed-off-by: Andrew Klepchick <aklepchi@amazon.com>

Author: MrFlap

.github/actions/check-files/action.yml

@@ -0,0 +1,30 @@
+name: 'KNN Check Files Changed'
+description: 'Check if specific files have changed for KNN repo'
+inputs:
+  files:
+    description: 'Files to check for changes'
+outputs:
+  files_changed:
+    description: 'Whether any files changed'
+    value: ${{ steps.changed-files.outputs.any_changed }}
+runs:
+  using: 'composite'
+  steps:
+    - name: Combine files
+      id: combine-files
+      shell: bash
+      run: |
+        DEFAULT_FILES="build.gradle,settings.gradle,src/**,build-tools/**,buildSrc/**,gradle/**,jni/**"
+        COMBINED_FILES="$DEFAULT_FILES,${{ inputs.files }}"
+        echo "combined_files=$COMBINED_FILES" >> $GITHUB_OUTPUT
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@v47.0.0
+      with:
+        files: ${{ steps.combine-files.outputs.combined_files }}
+        files_separator: ","
+    - name: Debug output
+      shell: bash
+      run: |
+        echo "Files changed: ${{ steps.changed-files.outputs.any_changed }}"
+        echo "Combined files: ${{ steps.combine-files.outputs.combined_files }}"

.github/workflows/CI.yml

@@ -6,39 +6,37 @@ on:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/CI.yml'
+
   pull_request:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/CI.yml'
 
 jobs:
+  check-files:
+    name: Check file for Build and Test k-NN
+    runs-on: ubuntu-latest
+    outputs:
+      RUN_BUILD_AND_TEST: ${{ steps.check.outputs.files_changed }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check files
+        id: check
+        uses: ./.github/actions/check-files
+        with:
+          files: .github/workflows/CI.yml
+
   Get-CI-Image-Tag:
+    needs: check-files
+    if: needs.check-files.outputs.RUN_BUILD_AND_TEST == 'true'
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
     with:
       product: opensearch
 
   Build-k-NN-Linux:
     strategy:
       matrix:
-        java: [21, 24]
+        java: [21, 25]
 
     env:
       CC: gcc10-gcc
@@ -99,14 +97,15 @@ jobs:
 
 
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          use_pypi: true
 
   Build-k-NN-MacOS:
     strategy:
       matrix:
-        java: [21, 24]
+        java: [21, 25]
 
     name: Build and Test k-NN Plugin on MacOS
     needs: Get-CI-Image-Tag
@@ -149,7 +148,7 @@ jobs:
   Build-k-NN-Windows:
     strategy:
       matrix:
-        java: [21, 24]
+        java: [21, 25]
 
     name: Build and Test k-NN Plugin on Windows
     needs: Get-CI-Image-Tag
@@ -216,4 +215,16 @@ jobs:
       # TODO: Detect processor count and set the value of nproc.count
       - name: Run build
         run: |
-          ./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4'
+          ./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4'
+
+  check-results:
+    needs: [check-files, Build-k-NN-Windows, Build-k-NN-MacOS, Build-k-NN-Linux]
+    if: always()
+    name: Check results Build and Test k-NN
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check results
+        if: ${{ needs.check-files.outputs.RUN_BUILD_AND_TEST && needs.Build-k-NN-Linux.result == 'failure' }}
+        run: |
+          echo "Integration tests failed, failing the check-results step"
+          exit 1

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -36,7 +36,7 @@ jobs:
         java: [ 21 ]
         os: [ubuntu-latest]
         bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0","2.19.1", "2.20.0-SNAPSHOT"]
-        opensearch_version : [ "3.4.0-SNAPSHOT" ]
+        opensearch_version : [ "3.5.0-SNAPSHOT" ]
         exclude:
           - os: windows-latest
             bwc_version: "2.0.1"
@@ -131,7 +131,7 @@ jobs:
         java: [ 21 ]
         os: [ubuntu-latest]
         bwc_version: [ "2.20.0-SNAPSHOT" ]
-        opensearch_version: [ "3.4.0-SNAPSHOT" ]
+        opensearch_version: [ "3.5.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests
     runs-on: ${{ matrix.os }}

.github/workflows/remote_index_build.yml

@@ -6,31 +6,28 @@ on:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/remote_index_build.yml'
   pull_request:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/remote_index_build.yml'
 
 jobs:
+  check-files:
+    name: Check file for Build and Test k-NN using Remote Index Builder
+    runs-on: ubuntu-latest
+    outputs:
+      RUN_IT_TEST: ${{ steps.check.outputs.files_changed }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check files
+        id: check
+        uses: ./.github/actions/check-files
+        with:
+          files: .github/workflows/remote_index_build.yml
+
   Remote-Index-Build-IT-Tests:
+    needs: check-files
+    if: needs.check-files.outputs.RUN_IT_TEST == 'true'
     strategy:
       matrix:
         java: [21]
@@ -151,3 +148,14 @@ jobs:
           docker logout
           rm -rf ${{ github.workspace }}/*
 
+  check-results:
+    needs: [check-files, Remote-Index-Build-IT-Tests]
+    if: always()
+    name: Check results for Build and Test k-NN using Remote Index Builder
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check results
+        if: ${{ needs.check-files.outputs.RUN_IT_TEST && needs.Remote-Index-Build-IT-Tests.result == 'failure' }}
+        run: |
+          echo "Remote Integration tests failed, failing the check-results step"
+          exit 1

.github/workflows/test_security.yml

@@ -6,39 +6,36 @@ on:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/test_security.yml'
   pull_request:
     branches:
       - "*"
       - "feature/**"
-    paths:
-      - 'build.gradle'
-      - 'settings.gradle'
-      - 'src/**'
-      - 'build-tools/**'
-      - 'buildSrc/**'
-      - 'gradle/**'
-      - 'jni/**'
-      - '.github/workflows/test_security.yml'
 
 jobs:
+  check-files:
+    name: Check file for Test k-NN on Secure Cluster
+    runs-on: ubuntu-latest
+    outputs:
+      RUN_SECURE_IT_TEST: ${{ steps.check.outputs.files_changed }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check files
+        id: check
+        uses: ./.github/actions/check-files
+        with:
+          files: .github/workflows/test_security.yml
+
   Get-CI-Image-Tag:
+    needs: check-files
+    if: needs.check-files.outputs.RUN_SECURE_IT_TEST == 'true'
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
     with:
       product: opensearch
 
   integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [21, 24]
+        java: [21, 25]
     env:
        CC: gcc10-gcc
        CXX: gcc10-g++
@@ -92,4 +89,16 @@ jobs:
           else
             echo "avx512 and avx2 not available on system"
             su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Davx512_spr.enabled=false -Dnproc.count=`nproc`"
-          fi  
+          fi
+
+  check-results:
+    needs: [check-files, integ-test-with-security-linux]
+    if: always()
+    name: Check results Test k-NN on Secure Cluster
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check results
+        if: ${{ needs.check-files.outputs.RUN_SECURE_IT_TEST && needs.integ-test-with-security-linux.result == 'failure' }}
+        run: |
+          echo "Integration tests failed, failing the check-results step"
+          exit 1

CHANGELOG.md

@@ -4,21 +4,35 @@ All notable changes to this project are documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries.
 
-## [Unreleased 3.4](https://github.com/opensearch-project/k-NN/compare/main...HEAD)
+## [Unreleased 3.5](https://github.com/opensearch-project/k-NN/compare/main...HEAD)
 ### Features
-* Memory optimized search warmup ([#2954](https://github.com/opensearch-project/k-NN/pull/2954))
+
 ### Maintenance
-* Onboard to s3 snapshots ([#2943](https://github.com/opensearch-project/k-NN/pull/2943))
 * Adds testing for cosine similarity for multi shard and multi node setups [#2977](https://github.com/opensearch-project/k-NN/pull/2977)
+* Added gradle task to generate task dependency graph ([#3032](https://github.com/opensearch-project/k-NN/pull/3032))
+* Added new gradle task validateLibraryUsage so that System.loadLibrary cannot be run outside KNNLibraryLoader ([#3033](https://github.com/opensearch-project/k-NN/pull/3033))
+* Add IT and bwc test with indices containing both vector and non-vector docs ([#3064](https://github.com/opensearch-project/k-NN/pull/3064))
+
 
 ### Bug Fixes
-* Fix blocking old indices created before 2.18 to use memory optimized search. [#2918](https://github.com/opensearch-project/k-NN/pull/2918)
-* Fix NativeEngineKnnQuery to return all part results for valid totalHits in response [#2965](https://github.com/opensearch-project/k-NN/pull/2965)
-* Fix unsafe concurrent update query vector in KNNQueryBuilder [#2974](https://github.com/opensearch-project/k-NN/pull/2974)
+* Fix indexing for 16x and 8x compression [#3019](https://github.com/opensearch-project/k-NN/pull/3019)
+* Block index creation for Faiss engine with cosine similarity and byte vectors due to normalization incompatibility [#3002](https://github.com/opensearch-project/k-NN/pull/3002)
+* Update validation for cases when k is greater than total results [#3038](https://github.com/opensearch-project/k-NN/pull/3038)
+* Add regex support to derived source transformer include / exclude check [#3031](https://github.com/opensearch-project/k-NN/pull/3031)
+* Correct ef_search parameter for Lucene engine and override mergeLeafResults to return top K results [#3037](https://github.com/opensearch-project/k-NN/pull/3037)
+* Fix efficient filtering in nested k-NN queries [#2990](https://github.com/opensearch-project/k-NN/pull/2990)
+* Fix nested docs and exact search query when some docs has no vector field present. [#3051](https://github.com/opensearch-project/k-NN/pull/3051)
+* Changed warmup seek to use long instead of int to avoid overflow [#3067](https://github.com/opensearch-project/k-NN/pull/3067)
+* Fix memory-optimized-search reentrant search bug in byte index. [#3071](https://github.com/opensearch-project/k-NN/pull/3071)
+* Fix MemoryOptimizedKNNWeight to use expanded 'k' for disk based index. [#3061](https://github.com/opensearch-project/k-NN/pull/3061)
+* Added new exception type to signify expected warmup behavior [#3070](https://github.com/opensearch-project/k-NN/pull/3070)
+* Add derived source handling for when field is excluded in source on indexing [#3049](https://github.com/opensearch-project/k-NN/pull/3049)
+* Fix patch to have a valid score conversion for BinaryCagra. [#2983](https://github.com/opensearch-project/k-NN/pull/2983)
 
 ### Refactoring
-* Refactor to not use parallel for MMR rerank. [#2968](https://github.com/opensearch-project/k-NN/pull/2968)
+* Change ordering of build task and added tests to catch uninitialized libraries [#3024](https://github.com/opensearch-project/k-NN/pull/3024)
 
 ### Enhancements
-* Removed VectorSearchHolders map from NativeEngines990KnnVectorsReader [#2948](https://github.com/opensearch-project/k-NN/pull/2948)
-* Native scoring for FP16 [#2922](https://github.com/opensearch-project/k-NN/pull/2922)
+* Index setting to disable exact search after ANN Search with Faiss efficient filters [#3022](https://github.com/opensearch-project/k-NN/pull/3022)
+* Bulk SIMD for FP16 distance computation [#3071](https://github.com/opensearch-project/k-NN/pull/3071)
+* Include AdditionalCodecs argument to allow additional Codec registration [#3085](https://github.com/opensearch-project/k-NN/pull/3085)

DEVELOPER_GUIDE.md

@@ -63,6 +63,14 @@ One easy way to install on mac or linux is to use pip:
 pip install cmake==3.24.0
 ```
 
+On Mac it might be needed to use pip3 and add the cmake install location to path
+example for python3.9
+```bash
+pip3 install cmake==3.24.0
+export PATH="$HOME/Library/Python/3.9/bin:$PATH"
+```
+
+
 #### Faiss Dependencies
 
 To build the *faiss* JNI library, you need to have openmp, lapack, gflags and blas installed. For more information on *faiss*