feat: add advanced search using Couchbase FTS (#122)

* integrate couchbase FTS

* bump up go version in CI

* move query-parser to the project folder

* ignore parser gen tests

* update go version and api version

* bump up golangci-lint to 1.63

Author: JLL32

.github/workflows/ci.yaml

@@ -7,7 +7,7 @@ jobs:
     name: Build & Test Go Package
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -25,7 +25,7 @@ jobs:
           go build -v ./...
 
       - name: Test With Coverage
-        run: go test ./... -race -coverprofile=coverage -covermode=atomic
+        run: go test ./... -race -coverprofile=coverage -covermode=atomic -skip TestGenerate
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -35,7 +35,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.59
+          version: v1.63
 
   build-container:
     runs-on: ubuntu-20.04

Dockerfile

@@ -2,7 +2,7 @@
 # STEP 1 build executable binary
 ################################
 
-FROM golang:1.21-alpine AS build-stage
+FROM golang:1.23-alpine AS build-stage
 
 # Install git + SSL ca certificates.
 # Git is required for fetching the dependencies.
@@ -34,7 +34,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo \
 
 FROM alpine:latest
 LABEL maintainer="https://github.com/saferwall/saferwall-api"
-LABEL version="0.8.0"
+LABEL version="0.9.0"
 LABEL description="Saferwall web APIs service"
 
 ENV USER=saferwall

VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

cmd/main.go

@@ -13,6 +13,7 @@ import (
 	"os/signal"
 	"time"
 
+	"github.com/MicahParks/recaptcha"
 	"github.com/go-playground/locales/en"
 	ut "github.com/go-playground/universal-translator"
 	"github.com/go-playground/validator/v10"
@@ -29,11 +30,10 @@ import (
 	tpl "github.com/saferwall/saferwall-api/internal/template"
 	"github.com/saferwall/saferwall-api/pkg/log"
 	"github.com/yeka/zip"
-	"github.com/MicahParks/recaptcha"
 )
 
 // Version indicates the current version of the application.
-var Version = "0.8.0"
+var Version = "0.9.0"
 
 var flagConfig = flag.String("config", "./../configs/", "path to the config file")
 var flagN1QLFiles = flag.String("db", "./../db/", "path to the n1ql files")

go.mod

@@ -1,6 +1,6 @@
 module github.com/saferwall/saferwall-api
 
-go 1.21
+go 1.23.0
 
 require (
 	github.com/MicahParks/recaptcha v0.0.5
@@ -15,6 +15,7 @@ require (
 	github.com/labstack/echo/v4 v4.9.1
 	github.com/minio/minio-go/v7 v7.0.73
 	github.com/nsqio/go-nsq v1.1.0
+	github.com/saferwall/advanced-search v0.0.0-20250120184926-f1a096cecd50
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/swag v1.16.3

go.sum

@@ -8,12 +8,13 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/rand"
 	"strings"
 	"time"
 
 	gocb "github.com/couchbase/gocb/v2"
-	"github.com/couchbase/gocb/v2/search"
+	"github.com/saferwall/saferwall-api/internal/query-parser/gen"
 )
 
 const (
@@ -248,15 +249,18 @@ func shortID(length int) string {
 	return string(b)
 }
 
-func (db *DB) Search(ctx context.Context, val *interface{}, totalHits *uint64) error {
+func (db *DB) Search(ctx context.Context, stringQuery string, val *interface{}, totalHits *uint64) error {
 
-	queryOne := search.NewMatchQuery("spyware").Field("multiav.last_scan.avast.output")
-	queryTwo := search.NewTermQuery("exe").Field("file_extension")
-	query := search.NewConjunctionQuery(queryOne, queryTwo)
+	fmt.Printf("Query: %v", stringQuery)
+	query, err := gen.Generate(stringQuery)
+	if err != nil {
+		panic(err.Error())
+		// return err
+	}
 
 	// sfw._default.sfw_fts
 	result, err := db.Cluster.SearchQuery(
-		"multiav_fts", query,
+		"sfw._default.sfw_fts", query,
 		&gocb.SearchOptions{
 			Limit: 100,
 			Fields: []string{"size", "file_extension", "file_format", "first_seen", "last_scan", "tags.packer", "tags.pe",

internal/db/db.go

@@ -259,7 +259,7 @@ func (r repository) MetaUI(ctx context.Context, id string) (
 func (r repository) Search(ctx context.Context, input FileSearchRequest) (FileSearchResponse, error) {
 
 	resp := FileSearchResponse{}
-	err := r.db.Search(ctx, &resp.Results, &resp.TotalHits)
+	err := r.db.Search(ctx, input.Query, &resp.Results, &resp.TotalHits)
 	if err != nil {
 		return resp, err
 	}

internal/file/repository.go

@@ -0,0 +1,120 @@
+package gen
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/couchbase/gocb/v2/search"
+	"github.com/saferwall/advanced-search/gen"
+	"github.com/saferwall/advanced-search/parser"
+	"github.com/saferwall/advanced-search/token"
+	"github.com/saferwall/saferwall-api/internal/query-parser/lexer"
+)
+
+func Generate(input string) (search.Query, error) {
+	l := lexer.New(input)
+	var tokens []*token.Token
+	for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() {
+		tokCopy := tok
+		tokens = append(tokens, &tokCopy)
+	}
+
+	p := parser.New(tokens)
+	expr, err := p.Parse()
+	if err != nil {
+		return nil, err
+	}
+	result, err := gen.GenerateCouchbaseFTS(expr)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func GenerateCouchbaseFTS(expr parser.Expression) (search.Query, error) {
+	switch e := expr.(type) {
+	case *parser.BinaryExpression:
+		return generateBinaryCouchbase(e)
+	case *parser.ComparisonExpression:
+		return generateComparisonCouchbase(e)
+	default:
+		return nil, fmt.Errorf("unsupported expression type: %T", expr)
+	}
+}
+
+func generateBinaryCouchbase(expr *parser.BinaryExpression) (search.Query, error) {
+	left, err := GenerateCouchbaseFTS(expr.Left)
+	if err != nil {
+		return nil, err
+	}
+
+	right, err := GenerateCouchbaseFTS(expr.Right)
+	if err != nil {
+		return nil, err
+	}
+
+	switch expr.Operator.Type {
+	case token.AND:
+		return search.NewConjunctionQuery(left, right), nil
+
+	case token.OR:
+		return search.NewDisjunctionQuery(left, right), nil
+	default:
+		return nil, fmt.Errorf("unsupported operator type: %T", expr.Operator.Type)
+	}
+}
+
+func generateComparisonCouchbase(expr *parser.ComparisonExpression) (search.Query, error) {
+	// NOTE: might need to support term match query
+
+	switch expr.Operator.Type {
+	case token.ASSIGN:
+		return search.NewMatchQuery(expr.Right).Field(expr.Left), nil
+	case token.NOT_EQ:
+		return search.NewBooleanQuery().MustNot(search.NewMatchQuery(expr.Right).Field(expr.Left)), nil
+	case token.GT, token.GE, token.LT, token.LE:
+		return generateRangeQuery(expr)
+	default:
+		return nil, fmt.Errorf("unsupported comparison operator: %s", expr.Operator.Type)
+	}
+}
+
+func generateRangeQuery(expr *parser.ComparisonExpression) (search.Query, error) {
+	switch expr.Operator.Type {
+	case token.GT, token.GE:
+		isInclusive := expr.Operator.Type == token.GE
+		if v, ok := isValidF32(expr.Right); ok {
+			return search.NewNumericRangeQuery().Field(expr.Left).Min(v, isInclusive), nil
+		} else if lexer.IsISODate(expr.Right) {
+			return search.NewDateRangeQuery().Field(expr.Left).Start(expr.Right, isInclusive), nil
+		} else {
+			return search.NewTermRangeQuery(expr.Left).Min(expr.Right, isInclusive), nil
+		}
+
+	case token.LT, token.LE:
+		isInclusive := expr.Operator.Type == token.LE
+		if v, ok := isValidF32(expr.Right); ok {
+			return search.NewNumericRangeQuery().Field(expr.Left).Max(v, isInclusive), nil
+		} else if lexer.IsISODate(expr.Right) {
+			return search.NewDateRangeQuery().Field(expr.Left).End(expr.Right, isInclusive), nil
+		} else {
+			return search.NewTermRangeQuery(expr.Left).Max(expr.Right, isInclusive), nil
+		}
+	}
+
+	return nil, fmt.Errorf("unsupported range operator: %s", expr.Operator.Type)
+}
+
+func isValidF32(s string) (float32, bool) {
+	// Attempt to parse the string as a float64
+	value, err := strconv.ParseFloat(s, 32)
+	// Check for parsing errors and ensure it fits in float32 range
+	if err == nil {
+		// Convert the value to float32 and back to float64 for precision check
+		f32Value := float32(value)
+		if float64(f32Value) == value {
+			return f32Value, true
+		}
+	}
+	return 0, false
+}

internal/query-parser/gen/gen.go

@@ -0,0 +1,105 @@
+package gen
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/couchbase/gocb/v2/search"
+)
+
+func TestGenerate(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		wantErr  bool
+		expected search.Query
+	}{
+		{
+			name:    "simple equality",
+			input:   "size=100",
+			wantErr: false,
+			expected: search.NewMatchQuery("100").
+				Field("size"),
+		},
+		{
+			name:    "simple inequality",
+			input:   "size!=100",
+			wantErr: false,
+			expected: search.NewBooleanQuery().
+				MustNot(search.NewMatchQuery("100").Field("size")),
+		},
+		{
+			name:    "numeric greater than",
+			input:   "size>100",
+			wantErr: false,
+			expected: search.NewNumericRangeQuery().
+				Field("size").
+				Min(float32(100), false),
+		},
+		{
+			name:    "numeric greater than or equal",
+			input:   "size>=100",
+			wantErr: false,
+			expected: search.NewNumericRangeQuery().
+				Field("size").
+				Min(float32(100), true),
+		},
+		{
+			name:    "date comparison",
+			input:   "created>2024-01-01",
+			wantErr: false,
+			expected: search.NewDateRangeQuery().
+				Field("created").
+				Start("2024-01-01", false),
+		},
+		{
+			name:    "string range",
+			input:   "name>alice",
+			wantErr: false,
+			expected: search.NewTermRangeQuery("name").
+				Min("alice", false),
+		},
+		{
+			name:    "AND operation",
+			input:   "size=100 AND name=test",
+			wantErr: false,
+			expected: search.NewConjunctionQuery(
+				search.NewMatchQuery("100").Field("size"),
+				search.NewMatchQuery("test").Field("name"),
+			),
+		},
+		{
+			name:    "OR operation",
+			input:   "size=100 OR name=test",
+			wantErr: false,
+			expected: search.NewDisjunctionQuery(
+				search.NewMatchQuery("100").Field("size"),
+				search.NewMatchQuery("test").Field("name"),
+			),
+		},
+		{
+			name:    "invalid syntax",
+			input:   "size=",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			query, err := Generate(tt.input)
+			if tt.wantErr {
+				if err == nil {
+					t.Error("expected error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+			if !reflect.DeepEqual(query, tt.expected) {
+				t.Errorf("\nexpected: %#v\ngot: %#v", tt.expected, query)
+			}
+		})
+	}
+}