select: return error if field doesn't exist on tables with strict schema

internal/query/statement/explain_test.go

@@ -4,9 +4,10 @@ import (
  "encoding/json"
  "testing"
 
+ "github.com/stretchr/testify/require"
+
  "github.com/genjidb/genji"
  "github.com/genjidb/genji/internal/testutil/assert"
- "github.com/stretchr/testify/require"
 )
 
 func TestExplainStmt(t *testing.T) {
@@ -20,18 +21,19 @@ func TestExplainStmt(t *testing.T) {
  {"EXPLAIN SELECT * FROM test", false, `"table.Scan(\"test\")"`},
  {"EXPLAIN SELECT *, a FROM test", false, `"table.Scan(\"test\") | docs.Project(*, a)"`},
  {"EXPLAIN SELECT a + 1 FROM test", false, `"table.Scan(\"test\") | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10", false, `"table.Scan(\"test\") | docs.Filter(c > 10) | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 AND d > 20", false, `"table.Scan(\"test\") | docs.Filter(c > 10) | docs.Filter(d > 20) | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 OR d > 20", false, `"table.Scan(\"test\") | docs.Filter(c > 10 OR d > 20) | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c IN [1 + 1, 2 + 2]", false, `"table.Scan(\"test\") | docs.Filter(c IN [2, 4]) | docs.Project(a + 1)"`},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10 AND d > 20", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 OR d > 20", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c IN [1 + 1, 2 + 2]", true, ``},
  {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10", false, `"index.Scan(\"idx_a\", [{\"min\": [10], \"exclusive\": true}]) | docs.Project(a + 1)"`},
  {"EXPLAIN SELECT a + 1 FROM test WHERE x = 10 AND y > 5", false, `"index.Scan(\"idx_x_y\", [{\"min\": [10, 5], \"exclusive\": true}]) | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10 AND b > 20 AND c > 30", false, `"index.Scan(\"idx_b\", [{\"min\": [20], \"exclusive\": true}]) | docs.Filter(a > 10) | docs.Filter(c > 30) | docs.Project(a + 1)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d LIMIT 10 OFFSET 20", false, `"table.Scan(\"test\") | docs.Filter(c > 30) | docs.Project(a + 1) | docs.TempTreeSort(d) | docs.Skip(20) | docs.Take(10)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d DESC LIMIT 10 OFFSET 20", false, `"table.Scan(\"test\") | docs.Filter(c > 30) | docs.Project(a + 1) | docs.TempTreeSortReverse(d) | docs.Skip(20) | docs.Take(10)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"index.ScanReverse(\"idx_a\") | docs.Filter(c > 30) | docs.Project(a + 1) | docs.Skip(20) | docs.Take(10)"`},
- {"EXPLAIN SELECT a FROM test WHERE c > 30 GROUP BY a ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"index.ScanReverse(\"idx_a\") | docs.Filter(c > 30) | docs.GroupAggregate(a) | docs.Project(a) | docs.Skip(20) | docs.Take(10)"`},
- {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 GROUP BY a + 1 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"table.Scan(\"test\") | docs.Filter(c > 30) | docs.TempTreeSort(a + 1) | docs.GroupAggregate(a + 1) | docs.Project(a + 1) | docs.TempTreeSortReverse(a) | docs.Skip(20) | docs.Take(10)"`},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10 AND b > 20 AND c > 30", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d LIMIT 10 OFFSET 20", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d DESC LIMIT 10 OFFSET 20", true, ``},
+ {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", true, ``},
+ {"EXPLAIN SELECT a FROM test WHERE c > 30 GROUP BY a ORDER BY a DESC LIMIT 10 OFFSET 20", true, ``},
+ {"EXPLAIN SELECT a FROM test WHERE a > 30 GROUP BY c", true, ``},
+ {"EXPLAIN SELECT a FROM test WHERE a > 30 GROUP BY a ORDER BY c", true, ``},
  {"EXPLAIN UPDATE test SET a = 10", false, `"table.Scan(\"test\") | paths.Set(a, 10) | table.Validate(\"test\") | index.Delete(\"idx_a\") | index.Delete(\"idx_b\") | index.Delete(\"idx_x_y\") | table.Replace(\"test\") | index.Insert(\"idx_a\") | index.Insert(\"idx_b\") | index.Insert(\"idx_x_y\") | discard()"`},
  {"EXPLAIN UPDATE test SET a = 10 WHERE c > 10", false, `"table.Scan(\"test\") | docs.Filter(c > 10) | paths.Set(a, 10) | table.Validate(\"test\") | index.Delete(\"idx_a\") | index.Delete(\"idx_b\") | index.Delete(\"idx_x_y\") | table.Replace(\"test\") | index.Insert(\"idx_a\") | index.Insert(\"idx_b\") | index.Insert(\"idx_x_y\") | discard()"`},
  {"EXPLAIN UPDATE test SET a = 10 WHERE a > 10", false, `"index.Scan(\"idx_a\", [{\"min\": [10], \"exclusive\": true}]) | paths.Set(a, 10) | table.Validate(\"test\") | index.Delete(\"idx_a\") | index.Delete(\"idx_b\") | index.Delete(\"idx_x_y\") | table.Replace(\"test\") | index.Insert(\"idx_a\") | index.Insert(\"idx_b\") | index.Insert(\"idx_x_y\") | discard()"`},

internal/query/statement/select.go

@@ -2,14 +2,18 @@ package statement
 
 import (
  "fmt"
+ "strings"
 
  "github.com/cockroachdb/errors"
+
  "github.com/genjidb/genji/document"
+ "github.com/genjidb/genji/internal/database"
  "github.com/genjidb/genji/internal/expr"
  "github.com/genjidb/genji/internal/sql/scanner"
  "github.com/genjidb/genji/internal/stream"
  "github.com/genjidb/genji/internal/stream/docs"
  "github.com/genjidb/genji/internal/stream/table"
+ "github.com/genjidb/genji/types"
 )
 
 type SelectCoreStmt struct {
@@ -20,21 +24,115 @@ type SelectCoreStmt struct {
  ProjectionExprs []expr.Expr
 }
 
-func (stmt *SelectCoreStmt) Prepare(*Context) (*StreamStmt, error) {
+func (stmt *SelectCoreStmt) analyzeExpr(ti *database.TableInfo, e expr.Expr) error {
+ if ti.FieldConstraints.AllowExtraFields {
+ return nil
+ }
+
+ var (
+ prev scanner.Token
+ path string
+ )
+
+ scan := scanner.NewScanner(strings.NewReader(e.String()))
+ for {
+ tok, _, lit := scan.Scan()
+ switch tok {
+ case scanner.IDENT:
+ path += lit
+ case scanner.DOT:
+ path += "."
+ case scanner.WS:
+ if prev == scanner.IDENT {
+ err := analyzePath(strings.Split(path, "."), ti.FieldConstraints)
+ if err != nil {
+ return err
+ }
+ }
+ path = ""
+ case scanner.EOF:
+ if prev == scanner.IDENT {
+ return analyzePath(strings.Split(path, "."), ti.FieldConstraints)
+ }
+ return nil
+ }
+
+ prev = tok
+ }
+
+}
+
+func analyzePath(path []string, fc database.FieldConstraints) error {
+ if fc.AllowExtraFields {
+ return nil
+ }
+
+ f, ok := fc.ByField[path[0]]
+ if !ok {
+ return types.ErrFieldNotFound
+ }
+
+ if len(path) == 1 {
+ return nil
+ }
+
+ return analyzePath(path[1:], f.AnonymousType.FieldConstraints)
+}
+
+func (stmt *SelectCoreStmt) Prepare(ctx *Context) (*StreamStmt, error) {
  isReadOnly := true
 
- var s *stream.Stream
+ var (
+ s *stream.Stream
+ ti *database.TableInfo
+ err error
+ )
 
  if stmt.TableName != "" {
+ ti, err = ctx.Catalog.GetTableInfo(stmt.TableName)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, pe := range stmt.ProjectionExprs {
+ var err error
+ expr.Walk(pe, func(e expr.Expr) bool {
+ switch e.(type) {
+ case expr.Path:
+ err = analyzePath(strings.Split(e.String(), "."), ti.FieldConstraints)
+ if err != nil {
+ return false
+ }
+
+ return true
+ default:
+ return true
+ }
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+
  s = s.Pipe(table.Scan(stmt.TableName))
  }
 
  if stmt.WhereExpr != nil {
+ err := stmt.analyzeExpr(ti, stmt.WhereExpr)
+ if err != nil {
+ return nil, err
+ }
+
  s = s.Pipe(docs.Filter(stmt.WhereExpr))
  }
 
  // when using GROUP BY, only aggregation functions or GroupByExpr can be selected
  if stmt.GroupByExpr != nil {
+ err := stmt.analyzeExpr(ti, stmt.GroupByExpr)
+ if err != nil {
+ return nil, err
+ }
+
  var invalidProjectedField expr.Expr
  var aggregators []expr.AggregatorBuilder
 
@@ -167,19 +265,33 @@ func NewSelectStatement() *SelectStmt {
 
 // Prepare implements the Preparer interface.
 func (stmt *SelectStmt) Prepare(ctx *Context) (Statement, error) {
- var s *stream.Stream
-
- var prev scanner.Token
-
- var coreStmts []*stream.Stream
- var readOnly bool = true
-
+ var (
+ coreStmts []*stream.Stream
+ s *stream.Stream
+ prev scanner.Token
+ errStmt []error
+ readOnly bool
+ )
+
+ readOnly = true
  for i, coreSelect := range stmt.CompoundSelect {
  coreStmt, err := coreSelect.Prepare(ctx)
  if err != nil {
  return nil, err
  }
 
+ if stmt.OrderBy != nil {
+ ti, err := ctx.Catalog.GetTableInfo(coreSelect.TableName)
+ if err != nil {
+ return nil, err
+ }
+
+ err = coreSelect.analyzeExpr(ti, stmt.OrderBy)
+ if err != nil {
+ errStmt = append(errStmt, err)
+ }
+ }
+
  if len(stmt.CompoundSelect) == 1 {
  s = coreStmt.Stream
  readOnly = coreStmt.ReadOnly
@@ -212,6 +324,10 @@ func (stmt *SelectStmt) Prepare(ctx *Context) (Statement, error) {
  }
 
  if stmt.OrderBy != nil {
+ if len(errStmt) == len(stmt.CompoundSelect) {
+ return nil, errStmt[0]
+ }
+
  if stmt.OrderByDirection == scanner.DESC {
  s = s.Pipe(docs.TempTreeSortReverse(stmt.OrderBy))
  } else {

internal/query/statement/select_test.go

@@ -7,12 +7,13 @@ import (
  "strconv"
  "testing"
 
+ "github.com/stretchr/testify/require"
+
  "github.com/genjidb/genji"
  "github.com/genjidb/genji/document"
  "github.com/genjidb/genji/internal/testutil"
  "github.com/genjidb/genji/internal/testutil/assert"
  "github.com/genjidb/genji/types"
- "github.com/stretchr/testify/require"
 )
 
 func TestSelectStmt(t *testing.T) {
@@ -95,9 +96,9 @@ func TestSelectStmt(t *testing.T) {
  {"With multiple maxs", "SELECT MAX(color), MAX(weight) FROM test", false, `[{"MAX(color)": "red", "MAX(weight)": 200}]`, nil},
  {"With sum", "SELECT SUM(k) FROM test", false, `[{"SUM(k)": 6}]`, nil},
  {"With multiple sums", "SELECT SUM(color), SUM(weight) FROM test", false, `[{"SUM(color)": null, "SUM(weight)": 300}]`, nil},
- {"With two non existing idents, =", "SELECT * FROM test WHERE z = y", false, `[]`, nil},
- {"With two non existing idents, >", "SELECT * FROM test WHERE z > y", false, `[]`, nil},
- {"With two non existing idents, !=", "SELECT * FROM test WHERE z != y", false, `[]`, nil},
+ {"With two non existing idents, =", "SELECT * FROM test WHERE z = y", true, ``, nil},
+ {"With two non existing idents, >", "SELECT * FROM test WHERE z > y", true, ``, nil},
+ {"With two non existing idents, !=", "SELECT * FROM test WHERE z != y", true, ``, nil},
  // See issue https://github.com/genjidb/genji/issues/283
  {"With empty WHERE and IN", "SELECT * FROM test WHERE [] IN [];", false, `[]`, nil},
  {"Invalid use of MIN() aggregator", "SELECT * FROM test LIMIT min(0)", true, ``, nil},
@@ -231,6 +232,22 @@ func TestSelectStmt(t *testing.T) {
  assert.Error(t, err)
  })
 
+ t.Run("with unknown field", func(t *testing.T) {
+ db, err := genji.Open(":memory:")
+ assert.NoError(t, err)
+ defer db.Close()
+
+ err = db.Exec("CREATE TABLE test(a INTEGER);")
+ assert.NoError(t, err)
+
+ err = db.Exec(`INSERT INTO test (a) VALUES (1)`)
+ assert.NoError(t, err)
+
+ st, err := db.Query("SELECT a,b FROM test")
+ assert.Error(t, err)
+ defer st.Close()
+ })
+
  t.Run("with order by and indexes", func(t *testing.T) {
  db, err := genji.Open(":memory:")
  assert.NoError(t, err)