From 177d75fdb71b106f3c53c504be58a493c8b4cd0f Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 13:38:58 +0000 Subject: [PATCH 01/32] Red --- test/Table_Tests/src/Util_Spec.enso | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/Table_Tests/src/Util_Spec.enso b/test/Table_Tests/src/Util_Spec.enso index 6d68a14d6366..821d5ccdf0e1 100644 --- a/test/Table_Tests/src/Util_Spec.enso +++ b/test/Table_Tests/src/Util_Spec.enso @@ -94,6 +94,7 @@ add_specs suite_builder = actual_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] actual_column.should_equal expected_column group_builder.specify "Two Columns Containing Nothing Are Equal" <| + # This is somewhat of a special case, as Nothing != Nothing but for the purposes of testing we consider them equal expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column.should_equal expected_column @@ -101,22 +102,22 @@ add_specs suite_builder = expected_column = Column.from_vector "Col" [1.0, 2.0, 3.0] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : Nothing\n\t Expected: 3.0\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Expected Nothing Value should fail" <| expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column = Column.from_vector "Col" [1.0, 2.0, 3.0] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : 3.0\n\t Expected: Nothing\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Actual Nothing Value to Expected NaN Value should fail" <| expected_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : Nothing\n\t Expected: NaN\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Actual NaN Value to Expected Nothing Value should fail" <| - expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] - actual_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] + expected_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] + actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : NaN\n\t Expected: Nothing\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' group_builder.specify "Two Tables Are Equal" <| expected_table = Table.new [Column.from_vector "Col1" ["Quis", "custodiet", "ipsos", "custodes?"], Column.from_vector "Col2" ["Who", "guards", "the", "guards?"]] From 7db4acaf7faee467f5f7f6bbe411682b5922d3b7 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 13:47:19 +0000 Subject: [PATCH 02/32] Fix tests --- test/Table_Tests/src/Util_Spec.enso | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Table_Tests/src/Util_Spec.enso b/test/Table_Tests/src/Util_Spec.enso index 821d5ccdf0e1..7abf44a4711e 100644 --- a/test/Table_Tests/src/Util_Spec.enso +++ b/test/Table_Tests/src/Util_Spec.enso @@ -102,22 +102,22 @@ add_specs suite_builder = expected_column = Column.from_vector "Col" [1.0, 2.0, 3.0] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : Nothing\n\t Expected: 3.0\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Expected Nothing Value should fail" <| expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column = Column.from_vector "Col" [1.0, 2.0, 3.0] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : 3.0\n\t Expected: Nothing\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Actual Nothing Value to Expected NaN Value should fail" <| expected_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : Nothing\n\t Expected: NaN\n\t (at LOCATION_PATH).' group_builder.specify "Comparing to Actual NaN Value to Expected Nothing Value should fail" <| - expected_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] - actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] + expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] + actual_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] res = Panic.recover Test_Failure_Error (column_should_equal_impl actual_column expected_column "LOCATION_PATH") - res.catch.message.should_equal 'Column: Col differs at row 0.\n\t Actual : Who\n\t Expected: Quis\n\t (at LOCATION_PATH).' + res.catch.message.should_equal 'Column: Col differs at row 2.\n\t Actual : NaN\n\t Expected: Nothing\n\t (at LOCATION_PATH).' group_builder.specify "Two Tables Are Equal" <| expected_table = Table.new [Column.from_vector "Col1" ["Quis", "custodiet", "ipsos", "custodes?"], Column.from_vector "Col2" ["Who", "guards", "the", "guards?"]] From 95966163b0b9c234789e305698943b964b6bb24c Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 13:51:14 +0000 Subject: [PATCH 03/32] Remove comment --- test/Table_Tests/src/Util_Spec.enso | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Table_Tests/src/Util_Spec.enso b/test/Table_Tests/src/Util_Spec.enso index 7abf44a4711e..6d68a14d6366 100644 --- a/test/Table_Tests/src/Util_Spec.enso +++ b/test/Table_Tests/src/Util_Spec.enso @@ -94,7 +94,6 @@ add_specs suite_builder = actual_column = Column.from_vector "Col" [1.0, 2.0, Number.nan] actual_column.should_equal expected_column group_builder.specify "Two Columns Containing Nothing Are Equal" <| - # This is somewhat of a special case, as Nothing != Nothing but for the purposes of testing we consider them equal expected_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column = Column.from_vector "Col" [1.0, 2.0, Nothing] actual_column.should_equal expected_column From 7da328ea5f5c8c9c94f401c6081a2ee0a794ed4e Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 10:46:27 +0000 Subject: [PATCH 04/32] checkpoint --- .../Database/0.0.0-dev/src/DB_Table.enso | 40 +++++++++++++++++-- .../src/Internal/Base_Generator.enso | 18 ++++++++- .../src/Internal/Common/Offset_Helpers.enso | 13 ++++++ .../src/Internal/SQLServer_Dialect.enso | 1 + .../Common_Table_Operations/Offset_Spec.enso | 12 +++--- .../src/Common_Table_Operations/Util.enso | 2 +- 6 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 3e190cf8899f..da158b63aa2d 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -57,6 +57,7 @@ import project.Internal.Aggregate_Helper import project.Internal.Base_Generator import project.Internal.Common.Database_Join_Helper import project.Internal.Common.Lookup_Query_Helper +import project.Internal.Common.Offset_Helpers import project.Internal.Common.Row_Number_Helpers import project.Internal.DB_Data_Link_Helpers import project.Internal.Helpers @@ -3194,9 +3195,42 @@ type DB_Table @default (self-> Widget_Helpers.make_fill_default_value_selector2) @group_by Widget_Helpers.make_column_name_multi_selector @order_by Widget_Helpers.make_order_by_selector - offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> Table = - _ = [columns, n, fill_with, group_by, order_by, set_mode, on_problems] - Error.throw (Unsupported_Database_Operation.Error "offset") + offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> DB_Table = + Feature.Offset.if_supported_else_throw self.connection.dialect "offset" <| + if columns.is_empty then self else + problem_builder = Problem_Builder.new error_on_missing_columns=True + grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder + grouping_columns.each column-> + if column.value_type.is_floating_point then + problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) + ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder + problem_builder.attach_problems_before on_problems <| + order_descriptors = case ordering.is_empty of + False -> ordering.map element-> + column = element.column + associated_selector = element.associated_selector + self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing + True -> case self.default_ordering of + Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") + descriptors -> descriptors + grouping_expressions = (grouping_columns.map _.as_internal).map .expression + resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder + new_expr = Offset_Helpers.make_offset n (resolved_columns.at 0 . as_internal) order_descriptors grouping_expressions + + type_mapping = self.connection.dialect.get_type_mapping + infer_from_database_callback expression = + SQL_Type_Reference.new self.connection self.context expression + new_type_ref = type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] new_expr + + name = if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [resolved_columns.at 0, n, fill_with] else (resolved_columns.at 0).name + new_column = Internal_Column.Value name new_type_ref new_expr + + rebuild_table columns = + self.updated_columns (columns.map .as_internal) + renamed_table = Add_Row_Number.rename_columns_if_needed self name on_problems rebuild_table + + updated_table = self.set (self.make_column new_column) + updated_table.as_subquery ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 0e829a6994fc..a52773c17fed 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -195,7 +195,7 @@ type SQL_Generator order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY " limit_part = case ctx.limit of - Nothing -> "" + Nothing -> "TOP 100 PERCENT " _ : Integer -> dialect.get_limit_sql_modifier ctx.limit extensions = ctx.extensions.map extension-> @@ -467,7 +467,7 @@ base_dialect_operations = nulls = [["IS_NULL", make_right_unary_op "IS NULL"], ["FILL_NULL", make_function "COALESCE"]] contains = [["IS_IN", make_is_in], ["IS_IN_COLUMN", make_is_in_column]] types = [simple_cast] - windows = [["ROW_NUMBER", make_row_number], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group]] + windows = [["ROW_NUMBER", make_row_number], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group], ["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"]] base_dict = Dictionary.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains + types + windows) Dialect_Operations.Value base_dict @@ -554,6 +554,20 @@ make_row_number (arguments : Vector) (metadata : Row_Number_Metadata) = if argum SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping SQL_Builder.code "(row_number() OVER (" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ ") * " ++ step.paren ++ " + " ++ offset.paren ++ ")" +## PRIVATE +make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 3 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else + n = arguments.at 0 + colName = arguments.at 1 + + ordering_and_grouping = arguments.drop 2 + ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) + grouping = ordering_and_grouping.take (..Last metadata.groupings_count) + + group_part = if grouping.length == 0 then "" else + SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping + + SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) OVER (" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ "))" + ## PRIVATE A helper for `lookup_and_replace`, and perhaps other operation. It creates an expression that returns a row number within a group. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso new file mode 100644 index 000000000000..f90bfbb9c34c --- /dev/null +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -0,0 +1,13 @@ +from Standard.Base import all + +import project.Internal.IR.Operation_Metadata +import project.Internal.IR.Order_Descriptor.Order_Descriptor +import project.Internal.IR.SQL_Expression.SQL_Expression +import project.Internal.IR.Internal_Column.Internal_Column + +## PRIVATE +make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) -> SQL_Expression = + params = [SQL_Expression.Literal n.abs.to_text, colName.expression] + order_descriptors + grouping_expressions + metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length + lead_lag = if n<0 then "LAG" else "LEAD" + SQL_Expression.Operation lead_lag params metadata diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index db420cb41ed8..17c1475d307d 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -292,6 +292,7 @@ type SQLServer_Dialect Feature.Join -> True Feature.Union -> True Feature.Aggregate -> True + Feature.Offset -> True _ -> False ## PRIVATE diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 589334a93b8b..035a62218c7f 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -32,7 +32,7 @@ add_offset_specs suite_builder setup = c = t.at 0 c_nothings = t.at 1 c_zero_rows = t.take 0 . at 0 - suite_builder.group prefix+"Column.Offset with default fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Column.Offset with default fill strategy" group_builder-> group_builder.specify "Works with default values" <| r = c.offset r.to_vector . should_equal [Nothing, 1, 2] @@ -55,7 +55,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Works with zero rows" <| r = c_zero_rows.offset r.to_vector . should_equal [] - suite_builder.group prefix+"Column.Offset with closest value fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Column.Offset with closest value fill strategy" group_builder-> group_builder.specify "Negative n shifts the values down" <| r = c.offset -1 ..Closest_Value r.to_vector . should_equal [1, 1, 2] @@ -80,7 +80,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Works with positive n and column of nothings" <| r = c_nothings.offset 1 ..Closest_Value r.to_vector . should_equal [Nothing, Nothing, Nothing] - suite_builder.group prefix+"Column.Offset with wrap around fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Column.Offset with wrap around fill strategy" group_builder-> group_builder.specify "Negative n shifts the values down" <| r = c.offset -1 ..Wrap_Around r.to_vector . should_equal [3, 1, 2] @@ -111,7 +111,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Works with positive n and column of nothings" <| r = c_nothings.offset 1 ..Wrap_Around r.to_vector . should_equal [Nothing, Nothing, Nothing] - suite_builder.group prefix+"Column.Offset with constant fill strategy" pending="TODO - constant fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Column.Offset with constant fill strategy" pending="TODO - constant fill strategy" group_builder-> group_builder.specify "Negative n shifts the values down" <| r = c.offset -1 42 r.to_vector . should_equal [42, 1, 2] @@ -139,7 +139,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Can create mixed colums" <| r = c.offset -1 "42" r.to_vector . should_equal ["42", 1, 2] - suite_builder.group prefix+"Works in Table.set expressions with default" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Works in Table.set expressions with default" group_builder-> group_builder.specify "Works with default values" <| r = t.set (expr 'offset([A])') . at "offset([A])" r.to_vector . should_equal [Nothing, 1, 2] @@ -347,7 +347,7 @@ add_offset_specs suite_builder setup = orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] - group_builder.specify "Negative n shifts the values down" <| + group_builder.specify "Negative n shifts the values downX" <| t2.offset ["Col"] -1 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database Table.input [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, "C", "A", "D", "B"]]] group_builder.specify "Positive n shifts the values up" <| diff --git a/test/Table_Tests/src/Common_Table_Operations/Util.enso b/test/Table_Tests/src/Common_Table_Operations/Util.enso index 8c63931ae62e..bf7dfc3b58bb 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Util.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Util.enso @@ -83,7 +83,7 @@ Error.should_equal_tz_agnostic self other = Builds a table ensuring that the rows are in the order as given. build_sorted_table setup table_structure = # Workaround for https://github.com/enso-org/enso/issues/10321 - if setup.prefix.contains "Snowflake" . not then setup.table_builder table_structure else + if setup.prefix.contains "Snowflake" then setup.table_builder table_structure else row_count = case table_structure.first of def : Vector -> def.second.length col : Column -> col.length From bb4c113febf3537b4d901391b6197d9e7be5b750 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 11:22:15 +0000 Subject: [PATCH 05/32] Checkpoint --- .../Standard/Database/0.0.0-dev/src/DB_Table.enso | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index da158b63aa2d..ac8ccb8cb953 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -3215,21 +3215,18 @@ type DB_Table descriptors -> descriptors grouping_expressions = (grouping_columns.map _.as_internal).map .expression resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder - new_expr = Offset_Helpers.make_offset n (resolved_columns.at 0 . as_internal) order_descriptors grouping_expressions + new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions) type_mapping = self.connection.dialect.get_type_mapping infer_from_database_callback expression = SQL_Type_Reference.new self.connection self.context expression - new_type_ref = type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] new_expr + new_type_refs = new_exprs.map expr->(type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] expr) - name = if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [resolved_columns.at 0, n, fill_with] else (resolved_columns.at 0).name - new_column = Internal_Column.Value name new_type_ref new_expr + new_names = resolved_columns.map c->(if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name) + new_columns = (new_names.zip new_type_refs).zip new_exprs a->b->(Internal_Column.Value a.first a.second b) - rebuild_table columns = - self.updated_columns (columns.map .as_internal) - renamed_table = Add_Row_Number.rename_columns_if_needed self name on_problems rebuild_table - - updated_table = self.set (self.make_column new_column) + updated_table = new_columns.fold self t-> col-> + t.set (self.make_column col) updated_table.as_subquery ## PRIVATE From 1ac2603ed91604ca862adaffbb55c82aa00ff8af Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 11:37:52 +0000 Subject: [PATCH 06/32] 6 red --- .../lib/Standard/Table/0.0.0-dev/src/Table.enso | 7 ++++--- .../src/Common_Table_Operations/Offset_Spec.enso | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso index ac3f2c4760df..0954e2f22005 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso @@ -149,9 +149,10 @@ type Table Error.throw (Illegal_Argument.Error "Each column must be represented by a pair whose first element is the column name and the second element is a vector of elements that will constitute that column, or an existing column. Got: "+columns.to_text) cols = columns.map on_problems=No_Wrap.Value c-> case c of - v : Vector -> - if v.length != 2 then invalid_input_shape else - Column.from_vector (v.at 0) (v.at 1) . java_column + v : Vector -> case v.length of + 2 -> Column.from_vector (v.at 0) (v.at 1) . java_column + 3 -> Column.from_vector (v.at 0) (v.at 1) (v.at 2) . java_column + _ -> invalid_input_shape col : Column -> col.java_column _ -> invalid_input_shape Panic.recover Illegal_Argument <| diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 035a62218c7f..9c92cf4375a5 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -202,9 +202,9 @@ add_offset_specs suite_builder setup = Table.input [colText, ["offset([Text Values], 2, Fill_With.Nothing)", ["C", Nothing, Nothing]]] suite_builder.group prefix+"Table.Offset with closest value fill strategy" group_builder-> colA = ["A", [1, 2, 3]] - colB = ["B", [Nothing, Nothing, Nothing]] + colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] t1 = build_sorted_table [colA, colB] - group_builder.specify "Negative n shifts the values down" <| + group_builder.specify "Negative n shifts the values downX" <| t1.offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([A], -1, Fill_With.Closest_Value)", [1, 1, 2]]] group_builder.specify "Positive n shifts the values up" <| @@ -224,13 +224,13 @@ add_offset_specs suite_builder setup = Table.input [["A", [], Value_Type.Integer], ["B", []], ["offset([A], -1, Fill_With.Closest_Value)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], -1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing]]] + Table.input [colA, colB, ["offset([B], -1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing]]] + Table.input [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> colA = ["A", [1, 2, 3]] - colB = ["B", [Nothing, Nothing, Nothing]] + colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] t1 = build_sorted_table [colA, colB] group_builder.specify "Negative n shifts the values down" <| t1.offset ["A"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database @@ -264,10 +264,10 @@ add_offset_specs suite_builder setup = Table.input [["A", [], Value_Type.Integer], ["B", []], ["offset([A], -1, Fill_With.Wrap_Around)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], -1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing]]] + Table.input [colA, colB, ["offset([B], -1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], 1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing]]] + Table.input [colA, colB, ["offset([B], 1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset works with grouping - default fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] From 1aa2ceefa04fc7a649be7744b39758b9c11f6977 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 11:52:34 +0000 Subject: [PATCH 07/32] 3 red --- .../src/Common_Table_Operations/Offset_Spec.enso | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 9c92cf4375a5..466100dce642 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -183,8 +183,8 @@ add_offset_specs suite_builder setup = t1.offset ["A"] 1024 . should_equal ignore_order=setup.is_database Table.input [colA, ["offset([A], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with zero rows" <| - t1.take 0 . offset ["A"] . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Nothing)", [], Value_Type.Integer]] + build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 . should_equal ignore_order=setup.is_database + Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Nothing)", [], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset with default fill strategy (Text Values)" group_builder-> colText = ["Text Values", ["A", "B", "C"]] t1 = build_sorted_table [colText] @@ -220,8 +220,8 @@ add_offset_specs suite_builder setup = t1.offset ["A"] 1024 ..Closest_Value . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([A], 1024, Fill_With.Closest_Value)", [3, 3, 3]]] group_builder.specify "Works with zero rows" <| - t1.take 0 . offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["B", []], ["offset([A], -1, Fill_With.Closest_Value)", [], Value_Type.Integer]] + build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database + Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Closest_Value)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([B], -1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] @@ -260,8 +260,8 @@ add_offset_specs suite_builder setup = t1.offset ["A"] 1024 ..Wrap_Around . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([A], 1024, Fill_With.Wrap_Around)", [2, 3, 1]]] group_builder.specify "Works with zero rows" <| - t1.take 0 . offset ["A"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["B", []], ["offset([A], -1, Fill_With.Wrap_Around)", [], Value_Type.Integer]] + build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database + Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Wrap_Around)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([B], -1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] From 05a9b5fb82b17b2a781b2dfc5045d19c535ea066 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 12:03:23 +0000 Subject: [PATCH 08/32] Green --- .../Database/0.0.0-dev/src/DB_Table.enso | 61 +++++++++---------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index ac8ccb8cb953..76a6efb26dbf 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -3197,37 +3197,36 @@ type DB_Table @order_by Widget_Helpers.make_order_by_selector offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> DB_Table = Feature.Offset.if_supported_else_throw self.connection.dialect "offset" <| - if columns.is_empty then self else - problem_builder = Problem_Builder.new error_on_missing_columns=True - grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder - grouping_columns.each column-> - if column.value_type.is_floating_point then - problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) - ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder - problem_builder.attach_problems_before on_problems <| - order_descriptors = case ordering.is_empty of - False -> ordering.map element-> - column = element.column - associated_selector = element.associated_selector - self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing - True -> case self.default_ordering of - Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") - descriptors -> descriptors - grouping_expressions = (grouping_columns.map _.as_internal).map .expression - resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder - new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions) - - type_mapping = self.connection.dialect.get_type_mapping - infer_from_database_callback expression = - SQL_Type_Reference.new self.connection self.context expression - new_type_refs = new_exprs.map expr->(type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] expr) - - new_names = resolved_columns.map c->(if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name) - new_columns = (new_names.zip new_type_refs).zip new_exprs a->b->(Internal_Column.Value a.first a.second b) - - updated_table = new_columns.fold self t-> col-> - t.set (self.make_column col) - updated_table.as_subquery + problem_builder = Problem_Builder.new error_on_missing_columns=True + grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder + grouping_columns.each column-> + if column.value_type.is_floating_point then + problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) + ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder + resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder + problem_builder.attach_problems_before on_problems <| if columns.is_empty then self else + order_descriptors = case ordering.is_empty of + False -> ordering.map element-> + column = element.column + associated_selector = element.associated_selector + self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing + True -> case self.default_ordering of + Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") + descriptors -> descriptors + grouping_expressions = (grouping_columns.map _.as_internal).map .expression + new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions) + + type_mapping = self.connection.dialect.get_type_mapping + infer_from_database_callback expression = + SQL_Type_Reference.new self.connection self.context expression + new_type_refs = new_exprs.map expr->(type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] expr) + + new_names = resolved_columns.map c->(if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name) + new_columns = (new_names.zip new_type_refs).zip new_exprs a->b->(Internal_Column.Value a.first a.second b) + + updated_table = new_columns.fold self t-> col-> + t.set (self.make_column col) + updated_table.as_subquery ## PRIVATE From e703358e7022afa1c1b31b4adeddd2ada8901531 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 31 Jan 2025 16:31:31 +0000 Subject: [PATCH 09/32] Checkpoint --- .../Standard/Database/0.0.0-dev/src/DB_Table.enso | 2 +- .../0.0.0-dev/src/Internal/Base_Generator.enso | 12 +++++++++--- .../src/Internal/Common/Offset_Helpers.enso | 11 ++++++++--- .../src/Common_Table_Operations/Offset_Spec.enso | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 76a6efb26dbf..5860faae0890 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -3214,7 +3214,7 @@ type DB_Table Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") descriptors -> descriptors grouping_expressions = (grouping_columns.map _.as_internal).map .expression - new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions) + new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions fill_with) type_mapping = self.connection.dialect.get_type_mapping infer_from_database_callback expression = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index a52773c17fed..bd6526b59eba 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -555,18 +555,24 @@ make_row_number (arguments : Vector) (metadata : Row_Number_Metadata) = if argum SQL_Builder.code "(row_number() OVER (" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ ") * " ++ step.paren ++ " + " ++ offset.paren ++ ")" ## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 3 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 + fill = arguments.at 2 - ordering_and_grouping = arguments.drop 2 + ordering_and_grouping = arguments.drop 3 ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) grouping = ordering_and_grouping.take (..Last metadata.groupings_count) group_part = if grouping.length == 0 then "" else SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping - SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) OVER (" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ "))" + partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ ")" + lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + fill_sql = if fill.is_empty then SQL_Builder.code "NULL" else (SQL_Builder.code "" ++ fill ++ "(" ++ colName ++ ") " ++ partition_sql) + + if fill.is_empty then lead_lag_sql else + SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" ## PRIVATE A helper for `lookup_and_replace`, and perhaps other operation. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index f90bfbb9c34c..b19cb8b21cde 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -1,4 +1,5 @@ from Standard.Base import all +from Standard.Table import Fill_With import project.Internal.IR.Operation_Metadata import project.Internal.IR.Order_Descriptor.Order_Descriptor @@ -6,8 +7,12 @@ import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.Internal_Column.Internal_Column ## PRIVATE -make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) -> SQL_Expression = - params = [SQL_Expression.Literal n.abs.to_text, colName.expression] + order_descriptors + grouping_expressions - metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length +make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) fill_with:Fill_With -> SQL_Expression = lead_lag = if n<0 then "LAG" else "LEAD" + fill = case fill_with of + Fill_With.Nothing -> "" + Fill_With.Closest_Value -> if n<0 then "FIRST_VALUE" else "LAST_VALUE" + _ -> "" + params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill] + order_descriptors + grouping_expressions + metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length SQL_Expression.Operation lead_lag params metadata diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 466100dce642..34372a5289da 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -347,7 +347,7 @@ add_offset_specs suite_builder setup = orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] - group_builder.specify "Negative n shifts the values downX" <| + group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database Table.input [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, "C", "A", "D", "B"]]] group_builder.specify "Positive n shifts the values up" <| From 76447e0a468835dce26715e882730c81ab60bcf7 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Tue, 4 Feb 2025 09:09:30 +0000 Subject: [PATCH 10/32] Remove wrap around tests --- .../src/Common_Table_Operations/Offset_Spec.enso | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 34372a5289da..24e47df96fb1 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -228,7 +228,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Closest_Value . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] - suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> colA = ["A", [1, 2, 3]] colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] t1 = build_sorted_table [colA, colB] @@ -318,7 +318,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", [3, 3, 3, 4, 4, 4, 4, 2, 2]]] - suite_builder.group prefix+"Table.Offset works with grouping - wrap around fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Table.Offset works with grouping - wrap around fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] t2 = build_sorted_table [groupColumn, dataCol] @@ -393,7 +393,7 @@ add_offset_specs suite_builder setup = group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database Table.input [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", ["E", "E", "E", "E", "E"]]] - suite_builder.group prefix+"Table.Offset works with ordering - wrap around fill strategy" group_builder-> + if setup.is_database.not then suite_builder.group prefix+"Table.Offset works with ordering - wrap around fill strategy" group_builder-> orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] From d4067653cf6f7d2c4885e425bea2448c7c6cbb0c Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Tue, 4 Feb 2025 11:35:40 +0000 Subject: [PATCH 11/32] Green --- .../src/Internal/Base_Generator.enso | 12 +++++++---- .../src/Internal/Common/Offset_Helpers.enso | 5 +++-- .../src/Internal/IR/Order_Descriptor.enso | 10 ++++++++++ .../Common_Table_Operations/Offset_Spec.enso | 20 +++++++++---------- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index bd6526b59eba..f443f650b653 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -564,15 +564,19 @@ make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) grouping = ordering_and_grouping.take (..Last metadata.groupings_count) + ordering_for_lead_lag = ordering.take (..First (ordering.length / 2).truncate) + ordering_for_row_number = ordering.take (..Last (ordering.length / 2).truncate) + group_part = if grouping.length == 0 then "" else SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping - partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ ")" + partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_lead_lag ++ ")" lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" - fill_sql = if fill.is_empty then SQL_Builder.code "NULL" else (SQL_Builder.code "" ++ fill ++ "(" ++ colName ++ ") " ++ partition_sql) - + if fill.is_empty then lead_lag_sql else - SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" + partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_row_number ++ ")" + fill_sql = if fill.is_empty then SQL_Builder.code "NULL" else (SQL_Builder.code "" ++ fill ++ "(" ++ colName ++ ") " ++ partition_sql_for_row_number) + SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" ## PRIVATE A helper for `lookup_and_replace`, and perhaps other operation. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index b19cb8b21cde..0eca6d215c5c 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -11,8 +11,9 @@ make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector lead_lag = if n<0 then "LAG" else "LEAD" fill = case fill_with of Fill_With.Nothing -> "" - Fill_With.Closest_Value -> if n<0 then "FIRST_VALUE" else "LAST_VALUE" + Fill_With.Closest_Value -> if n<0 then "FIRST_VALUE" else "FIRST_VALUE" _ -> "" - params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill] + order_descriptors + grouping_expressions + order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse + params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill] + order_descriptors + order_descriptors_for_row_number + grouping_expressions metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length SQL_Expression.Operation lead_lag params metadata diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Order_Descriptor.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Order_Descriptor.enso index 19e7ace76912..34ae32bce576 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Order_Descriptor.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Order_Descriptor.enso @@ -8,6 +8,16 @@ type Order_Descriptor ## PRIVATE Value (expression : SQL_Expression) (direction : Sort_Direction) (nulls_order : Nothing | Nulls_Order = Nothing) (collation : Nothing | Text = Nothing) + reverse self -> Order_Descriptor = + new_direction = case self.direction of + Sort_Direction.Ascending -> Sort_Direction.Descending + Sort_Direction.Descending -> Sort_Direction.Ascending + new_nulls_order = case self.nulls_order of + Nothing -> Nothing + Nulls_Order.First -> Nulls_Order.Last + Nulls_Order.Last -> Nulls_Order.First + Order_Descriptor.Value self.expression new_direction new_nulls_order self.collation + ## PRIVATE A bottom-up, depth-first traversal of this IR node and its children. Each node is passed to the provided function, and the return value of the diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 24e47df96fb1..00d189f4c77f 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -295,29 +295,29 @@ add_offset_specs suite_builder setup = Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset works with grouping - closest fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] - dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] + dataCol = ["Col", [1, 2, 3, 10, 20, 30, 40, 100, 200]] t2 = build_sorted_table [groupColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", [1, 1, 2, 1, 1, 2, 3, 1, 1]]] + Table.input [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", [1, 1, 2, 10, 10, 20, 30, 100, 100]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", [2, 3, 3, 2, 3, 4, 4, 2, 2]]] - group_builder.specify "Negative n shifts the values down (n=2)" <| + Table.input [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", [2, 3, 3, 20, 30, 40, 40, 200, 200]]] + group_builder.specify "WWW2" <| t2.offset ["Col"] -2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", [1, 1, 1, 1, 1, 1, 2, 1, 1]]] - group_builder.specify "Positive n shifts the values up (n=2)" <| + Table.input [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 20, 100, 100]]] + group_builder.specify "WWW" <| t2.offset ["Col"] 2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", [3, 3, 3, 3, 4, 4, 4, 2, 2]]] + Table.input [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", [3, 3, 3, 30, 40, 40, 40, 200, 200]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", [1, 2, 3, 1, 2, 3, 4, 1, 2]]] + Table.input [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", [1, 2, 3, 10, 20, 30, 40, 100, 200]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", [1, 1, 1, 1, 1, 1, 1, 1, 1]]] + Table.input [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 10, 100, 100]]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", [3, 3, 3, 4, 4, 4, 4, 2, 2]]] + Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", [3, 3, 3, 40, 40, 40, 40, 200, 200]]] if setup.is_database.not then suite_builder.group prefix+"Table.Offset works with grouping - wrap around fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] From 1b1be94fa99f0f1ace40246c5f440b3641ebf411 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Tue, 4 Feb 2025 11:48:58 +0000 Subject: [PATCH 12/32] Refactor --- .../Database/0.0.0-dev/src/Internal/Base_Generator.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index f443f650b653..1e18db51d74f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -575,7 +575,7 @@ make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL if fill.is_empty then lead_lag_sql else partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_row_number ++ ")" - fill_sql = if fill.is_empty then SQL_Builder.code "NULL" else (SQL_Builder.code "" ++ fill ++ "(" ++ colName ++ ") " ++ partition_sql_for_row_number) + fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" ## PRIVATE From 5d3b6f0b2044a193d594ae000fa2e025d7fa190b Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Wed, 5 Feb 2025 16:45:21 +0000 Subject: [PATCH 13/32] Adjust limit --- .../src/Database/Redshift/Internal/Redshift_Dialect.enso | 6 ++++-- .../Database/0.0.0-dev/src/Internal/Base_Generator.enso | 4 +--- .../0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso | 4 +++- .../0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso | 4 +++- .../Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso | 4 +++- .../Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso | 4 +++- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Internal/Redshift_Dialect.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Internal/Redshift_Dialect.enso index 6a2f145fc738..c26785909d35 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Internal/Redshift_Dialect.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Database/Redshift/Internal/Redshift_Dialect.enso @@ -76,8 +76,10 @@ type Redshift_Dialect Generates SQL modifier for limiting the number of rows and its position in the query get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = - SQL_Builder.code (" LIMIT " + limit.to_text) - + case limit of + Nothing -> "" + _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text) + ## PRIVATE Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output If you add extensions using Context_Extension you have to provide an order for them here diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 1e18db51d74f..1da3cb603b51 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -194,9 +194,7 @@ type SQL_Generator orders = ctx.orders.map (self.generate_order dialect) order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY " - limit_part = case ctx.limit of - Nothing -> "TOP 100 PERCENT " - _ : Integer -> dialect.get_limit_sql_modifier ctx.limit + limit_part = dialect.get_limit_sql_modifier ctx.limit extensions = ctx.extensions.map extension-> part = extension.run_generator (gen_exprs extension.expressions) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index d33b1ccfdb0c..9b61d2a11e3b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -87,7 +87,9 @@ type Postgres_Dialect Generates SQL modifier for limiting the number of rows and its position in the query get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = - SQL_Builder.code (" LIMIT " + limit.to_text) + case limit of + Nothing -> "" + _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text) ## PRIVATE Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 7aa5a9ea1023..4ebc212d10d4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -82,7 +82,9 @@ type SQLite_Dialect Generates SQL modifier for limiting the number of rows and its position in the query get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = - SQL_Builder.code (" LIMIT " + limit.to_text) + case limit of + Nothing -> "" + _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text) ## PRIVATE Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index 17c1475d307d..b1587b856dc0 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -92,7 +92,9 @@ type SQLServer_Dialect Generates SQL modifier for limiting the number of rows and its position in the query get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = - SQL_Builder.code (" TOP " + limit.to_text + " ") + case limit of + Nothing -> "TOP 100 PERCENT " + _ : Integer -> SQL_Builder.code (" TOP " + limit.to_text + " ") ## PRIVATE Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output diff --git a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso index 39ff41988669..fbdc0d14c7b5 100644 --- a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso +++ b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso @@ -95,7 +95,9 @@ type Snowflake_Dialect Generates SQL modifier for limiting the number of rows and its position in the query get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = - SQL_Builder.code (" LIMIT " + limit.to_text) + case limit of + Nothing -> "" + _ : Integer -> SQL_Builder.code (" LIMIT " + limit.to_text) ## PRIVATE Returns an ordering of SQL_Part's that determine what order each part gets written in for the final SQL output From 3252bca5c849bd195f2ebad95577da314d36fa35 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Wed, 5 Feb 2025 16:49:46 +0000 Subject: [PATCH 14/32] Clean up --- test/Table_Tests/src/Common_Table_Operations/Util.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Util.enso b/test/Table_Tests/src/Common_Table_Operations/Util.enso index bf7dfc3b58bb..8cc75caa4fc1 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Util.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Util.enso @@ -83,7 +83,7 @@ Error.should_equal_tz_agnostic self other = Builds a table ensuring that the rows are in the order as given. build_sorted_table setup table_structure = # Workaround for https://github.com/enso-org/enso/issues/10321 - if setup.prefix.contains "Snowflake" then setup.table_builder table_structure else + if setup.prefix.contains "Snowflake" . not && setup.prefix.contains "SQLServer" . not then setup.table_builder table_structure else row_count = case table_structure.first of def : Vector -> def.second.length col : Column -> col.length From 8c240dbef6f425d3922ccdf91c9da56f33de8a28 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Wed, 5 Feb 2025 17:05:53 +0000 Subject: [PATCH 15/32] Refactor --- .../src/Internal/Base_Generator.enso | 26 +---------------- .../src/Internal/SQLServer_Dialect.enso | 29 ++++++++++++++++++- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 1da3cb603b51..1ec13e41864b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -465,7 +465,7 @@ base_dialect_operations = nulls = [["IS_NULL", make_right_unary_op "IS NULL"], ["FILL_NULL", make_function "COALESCE"]] contains = [["IS_IN", make_is_in], ["IS_IN_COLUMN", make_is_in_column]] types = [simple_cast] - windows = [["ROW_NUMBER", make_row_number], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group], ["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"]] + windows = [["ROW_NUMBER", make_row_number], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group]] base_dict = Dictionary.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains + types + windows) Dialect_Operations.Value base_dict @@ -552,30 +552,6 @@ make_row_number (arguments : Vector) (metadata : Row_Number_Metadata) = if argum SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping SQL_Builder.code "(row_number() OVER (" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering ++ ") * " ++ step.paren ++ " + " ++ offset.paren ++ ")" -## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else - n = arguments.at 0 - colName = arguments.at 1 - fill = arguments.at 2 - - ordering_and_grouping = arguments.drop 3 - ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) - grouping = ordering_and_grouping.take (..Last metadata.groupings_count) - - ordering_for_lead_lag = ordering.take (..First (ordering.length / 2).truncate) - ordering_for_row_number = ordering.take (..Last (ordering.length / 2).truncate) - - group_part = if grouping.length == 0 then "" else - SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping - - partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_lead_lag ++ ")" - lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" - - if fill.is_empty then lead_lag_sql else - partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_row_number ++ ")" - fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number - SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" - ## PRIVATE A helper for `lookup_and_replace`, and perhaps other operation. It creates an expression that returns a row number within a group. diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index b1587b856dc0..63e0ad788697 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -27,6 +27,7 @@ import Standard.Database.Internal.IR.Context.Context_Extension import Standard.Database.Internal.IR.From_Spec.From_Spec import Standard.Database.Internal.IR.Internal_Column.Internal_Column import Standard.Database.Internal.IR.Nulls_Order.Nulls_Order +import Standard.Database.Internal.IR.Operation_Metadata.Row_Number_Metadata import Standard.Database.Internal.IR.Order_Descriptor.Order_Descriptor import Standard.Database.Internal.IR.Query.Query import Standard.Database.Internal.IR.SQL_Expression.SQL_Expression @@ -50,6 +51,7 @@ from Standard.Database.Internal.Statement_Setter import fill_hole_default import project.Internal.SQLServer_Error_Mapper.SQLServer_Error_Mapper import project.Internal.SQLServer_Type_Mapping.SQLServer_Type_Mapping + polyglot java import java.sql.Types as Java_Types polyglot java import org.enso.database.JDBCUtils @@ -506,7 +508,8 @@ make_dialect_operations = date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]] special_overrides = [is_empty, ["IIF", _make_iif]] other = [["RUNTIME_ERROR", make_runtime_error_op]] - my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other + windows = [["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"]] + my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other + windows base = Base_Generator.base_dialect_operations . extend_with my_mappings Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN") @@ -620,6 +623,30 @@ left = Base_Generator.lift_binary_op "LEFT" str-> n-> right = Base_Generator.lift_binary_op "RIGHT" str-> n-> SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))" +## PRIVATE +make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else + n = arguments.at 0 + colName = arguments.at 1 + fill = arguments.at 2 + + ordering_and_grouping = arguments.drop 3 + ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) + grouping = ordering_and_grouping.take (..Last metadata.groupings_count) + + ordering_for_lead_lag = ordering.take (..First (ordering.length / 2).truncate) + ordering_for_row_number = ordering.take (..Last (ordering.length / 2).truncate) + + group_part = if grouping.length == 0 then "" else + SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping + + partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_lead_lag ++ ")" + lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + + if fill.is_empty then lead_lag_sql else + partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_row_number ++ ")" + fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number + SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" + ## PRIVATE make_order_descriptor internal_column sort_direction text_ordering = nulls = Nothing From d3a5a6129506f8d92bb0c91d0c8a9c49d990f3d1 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 15:11:19 +0000 Subject: [PATCH 16/32] Refactor --- .../src/Internal/Base_Generator.enso | 3 +++ .../src/Internal/Common/Offset_Helpers.enso | 2 +- .../src/Internal/IR/SQL_Expression.enso | 4 +++ .../src/Internal/SQLServer_Dialect.enso | 25 ++++++++++--------- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 1ec13e41864b..27b43971dbe9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -100,6 +100,9 @@ type SQL_Generator wrapped_binder ++ "." ++ "x" False -> self.generate_expression dialect standalone_expression + SQL_Expression.List values -> + parsed_values = values.map (self.generate_expression dialect) + SQL_Builder.join ", " parsed_values query : Query -> self.generate_sub_query dialect query descriptor : Order_Descriptor -> self.generate_order dialect descriptor diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index 0eca6d215c5c..ff1d7618bd7b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -14,6 +14,6 @@ make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Fill_With.Closest_Value -> if n<0 then "FIRST_VALUE" else "FIRST_VALUE" _ -> "" order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse - params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill] + order_descriptors + order_descriptors_for_row_number + grouping_expressions + params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number] metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length SQL_Expression.Operation lead_lag params metadata diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso index f6edb3f63c5f..972a39cb57db 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso @@ -103,6 +103,8 @@ type SQL_Expression enclosing `Let` value. Let_Ref (name : Text) (binder : Text) (standalone_expression : SQL_Expression) + List value:Vector + ## PRIVATE A bottom-up, depth-first traversal of this IR node and its children. Each node is passed to the provided function, and the return value of the @@ -121,4 +123,6 @@ type SQL_Expression SQL_Expression.Let name binder (rec bindee) (rec body) SQL_Expression.Let_Ref name binder standalone_expression -> SQL_Expression.Let_Ref name binder (rec standalone_expression) + SQL_Expression.List value -> + SQL_Expression.List (value.map rec) f new_sql_expression diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index 63e0ad788697..a0c50da65886 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -454,6 +454,11 @@ private _generate_expression dialect base_gen expr expression_kind:Expression_Ki converted_expr pair final_expr null_checks_result + SQL_Expression.List values -> + parsed_args_and_null_checks = values.map (v-> _generate_expression dialect base_gen v expression_kind materialize_null_check) + final_expr = SQL_Builder.join ", " (parsed_args_and_null_checks.map p->p.first) + null_checks = parsed_args_and_null_checks.map .second . flatten + pair final_expr null_checks query : Query -> pair (base_gen.generate_sub_query dialect query) [] descriptor : Order_Descriptor -> pair (base_gen.generate_order dialect descriptor) [] @@ -624,26 +629,22 @@ right = Base_Generator.lift_binary_op "RIGHT" str-> n-> SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))" ## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length < 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length != 6 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 fill = arguments.at 2 + grouping = arguments.at 3 + ordering_for_lead_lag = arguments.at 4 + ordering_for_row_number = arguments.at 5 - ordering_and_grouping = arguments.drop 3 - ordering = ordering_and_grouping.drop (..Last metadata.groupings_count) - grouping = ordering_and_grouping.take (..Last metadata.groupings_count) - - ordering_for_lead_lag = ordering.take (..First (ordering.length / 2).truncate) - ordering_for_row_number = ordering.take (..Last (ordering.length / 2).truncate) - - group_part = if grouping.length == 0 then "" else - SQL_Builder.code "PARTITION BY " ++ SQL_Builder.join ", " grouping + group_part = if grouping.is_empty then "" else + SQL_Builder.code "PARTITION BY " ++ grouping - partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_lead_lag ++ ")" + partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_lead_lag ++ ")" lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" if fill.is_empty then lead_lag_sql else - partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ SQL_Builder.join ", " ordering_for_row_number ++ ")" + partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_row_number ++ ")" fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" From 965ee86f7c6f38dfab1a0bc47713ea4b300ecd29 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 15:24:18 +0000 Subject: [PATCH 17/32] refactor --- .../Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso | 3 +-- .../Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index ff1d7618bd7b..e32792255d54 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -15,5 +15,4 @@ make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector _ -> "" order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number] - metadata = Operation_Metadata.Row_Number_Metadata.Value grouping_expressions.length - SQL_Expression.Operation lead_lag params metadata + SQL_Expression.Operation lead_lag params diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index a0c50da65886..6010d6e055bf 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -629,7 +629,7 @@ right = Base_Generator.lift_binary_op "RIGHT" str-> n-> SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))" ## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector metadata:Row_Number_Metadata -> SQL_Builder = if arguments.length != 6 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 6 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 fill = arguments.at 2 From 256dfa92fef4d4a0e3516063a45ce3b7448b3b28 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 15:45:52 +0000 Subject: [PATCH 18/32] Refactor --- .../src/Internal/Common/Offset_Helpers.enso | 14 ++++----- .../src/Internal/SQLServer_Dialect.enso | 30 ++++++++++++------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index e32792255d54..a69edc5cd98a 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -9,10 +9,10 @@ import project.Internal.IR.Internal_Column.Internal_Column ## PRIVATE make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) fill_with:Fill_With -> SQL_Expression = lead_lag = if n<0 then "LAG" else "LEAD" - fill = case fill_with of - Fill_With.Nothing -> "" - Fill_With.Closest_Value -> if n<0 then "FIRST_VALUE" else "FIRST_VALUE" - _ -> "" - order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse - params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.Literal fill, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number] - SQL_Expression.Operation lead_lag params + case fill_with of + Fill_With.Closest_Value -> + order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse + params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number] + SQL_Expression.Operation lead_lag+"_CLOSEST" params + _ -> SQL_Expression.Operation lead_lag [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors] + diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index 6010d6e055bf..f2dfe6aee8ca 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -513,7 +513,7 @@ make_dialect_operations = date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]] special_overrides = [is_empty, ["IIF", _make_iif]] other = [["RUNTIME_ERROR", make_runtime_error_op]] - windows = [["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"]] + windows = [["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"], ["LEAD_CLOSEST", make_lead_lag_closest_value "LEAD"], ["LAG_CLOSEST", make_lead_lag_closest_value "LAG"]] my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other + windows base = Base_Generator.base_dialect_operations . extend_with my_mappings Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN") @@ -629,13 +629,24 @@ right = Base_Generator.lift_binary_op "RIGHT" str-> n-> SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))" ## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 6 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 - fill = arguments.at 2 - grouping = arguments.at 3 - ordering_for_lead_lag = arguments.at 4 - ordering_for_row_number = arguments.at 5 + grouping = arguments.at 2 + ordering = arguments.at 3 + + group_part = if grouping.is_empty then "" else + SQL_Builder.code "PARTITION BY " ++ grouping + partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering ++ ")" + SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + +## PRIVATE +make_lead_lag_closest_value lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 5 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else + n = arguments.at 0 + colName = arguments.at 1 + grouping = arguments.at 2 + ordering_for_lead_lag = arguments.at 3 + ordering_for_row_number = arguments.at 4 group_part = if grouping.is_empty then "" else SQL_Builder.code "PARTITION BY " ++ grouping @@ -643,10 +654,9 @@ make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.lengt partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_lead_lag ++ ")" lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" - if fill.is_empty then lead_lag_sql else - partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_row_number ++ ")" - fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number - SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" + partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_row_number ++ ")" + fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number + SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" ## PRIVATE make_order_descriptor internal_column sort_direction text_ordering = From 07122e72f9c05eb74e0b952e5d13bb6f5520dc3a Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 15:53:14 +0000 Subject: [PATCH 19/32] Refactor --- .../src/Internal/SQLServer_Dialect.enso | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index f2dfe6aee8ca..f1b6dcafbadc 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -513,7 +513,7 @@ make_dialect_operations = date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]] special_overrides = [is_empty, ["IIF", _make_iif]] other = [["RUNTIME_ERROR", make_runtime_error_op]] - windows = [["LEAD", make_lead_lag "LEAD"], ["LAG", make_lead_lag "LAG"], ["LEAD_CLOSEST", make_lead_lag_closest_value "LEAD"], ["LAG_CLOSEST", make_lead_lag_closest_value "LAG"]] + windows = [["LEAD", _make_lead_lag "LEAD"], ["LAG", _make_lead_lag "LAG"], ["LEAD_CLOSEST", _make_lead_lag_closest_value "LEAD"], ["LAG_CLOSEST", _make_lead_lag_closest_value "LAG"]] my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other + windows base = Base_Generator.base_dialect_operations . extend_with my_mappings Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN") @@ -628,33 +628,34 @@ left = Base_Generator.lift_binary_op "LEFT" str-> n-> right = Base_Generator.lift_binary_op "RIGHT" str-> n-> SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))" +_build_partition_sql grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder = + group_part = if grouping.is_empty then "" else + SQL_Builder.code "PARTITION BY " ++ grouping + SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering ++ ")" + ## PRIVATE -make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +_build_lead_lag_sql lead_lag:Text n:SQL_Builder colName:SQL_Builder grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder = + partition_sql = _build_partition_sql grouping ordering + SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + +## PRIVATE +_make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 grouping = arguments.at 2 ordering = arguments.at 3 - - group_part = if grouping.is_empty then "" else - SQL_Builder.code "PARTITION BY " ++ grouping - partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering ++ ")" - SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + _build_lead_lag_sql lead_lag n colName grouping ordering ## PRIVATE -make_lead_lag_closest_value lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 5 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else +_make_lead_lag_closest_value lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 5 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else n = arguments.at 0 colName = arguments.at 1 grouping = arguments.at 2 ordering_for_lead_lag = arguments.at 3 ordering_for_row_number = arguments.at 4 - group_part = if grouping.is_empty then "" else - SQL_Builder.code "PARTITION BY " ++ grouping - - partition_sql = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_lead_lag ++ ")" - lead_lag_sql = SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" - - partition_sql_for_row_number = SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering_for_row_number ++ ")" + lead_lag_sql = _build_lead_lag_sql lead_lag n colName grouping ordering_for_lead_lag + partition_sql_for_row_number = _build_partition_sql grouping ordering_for_row_number fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" From 41066bf11484284798dade8c9580dd9a862fa485 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:00:42 +0000 Subject: [PATCH 20/32] Documentation --- .../0.0.0-dev/src/Internal/IR/SQL_Expression.enso | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso index 972a39cb57db..f9acbe04bcca 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso @@ -103,7 +103,13 @@ type SQL_Expression enclosing `Let` value. Let_Ref (name : Text) (binder : Text) (standalone_expression : SQL_Expression) - List value:Vector + ## PRIVATE + The internal representation of an list of SQL expressions which will be + separated by commas in a list in the final SQL query. + + Arguments: + - value: the values thatt should be built into a comma separated list. + List values:Vector ## PRIVATE A bottom-up, depth-first traversal of this IR node and its children. Each @@ -123,6 +129,6 @@ type SQL_Expression SQL_Expression.Let name binder (rec bindee) (rec body) SQL_Expression.Let_Ref name binder standalone_expression -> SQL_Expression.Let_Ref name binder (rec standalone_expression) - SQL_Expression.List value -> - SQL_Expression.List (value.map rec) + SQL_Expression.List values -> + SQL_Expression.List (values.map rec) f new_sql_expression From 446f46a494e45d892442dedc65ccb884e1a7827d Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:01:30 +0000 Subject: [PATCH 21/32] Cleanup --- .../Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso | 2 -- 1 file changed, 2 deletions(-) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index f1b6dcafbadc..84840139d74c 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -27,7 +27,6 @@ import Standard.Database.Internal.IR.Context.Context_Extension import Standard.Database.Internal.IR.From_Spec.From_Spec import Standard.Database.Internal.IR.Internal_Column.Internal_Column import Standard.Database.Internal.IR.Nulls_Order.Nulls_Order -import Standard.Database.Internal.IR.Operation_Metadata.Row_Number_Metadata import Standard.Database.Internal.IR.Order_Descriptor.Order_Descriptor import Standard.Database.Internal.IR.Query.Query import Standard.Database.Internal.IR.SQL_Expression.SQL_Expression @@ -51,7 +50,6 @@ from Standard.Database.Internal.Statement_Setter import fill_hole_default import project.Internal.SQLServer_Error_Mapper.SQLServer_Error_Mapper import project.Internal.SQLServer_Type_Mapping.SQLServer_Type_Mapping - polyglot java import java.sql.Types as Java_Types polyglot java import org.enso.database.JDBCUtils From 83af296379b13327a7af6aec9f1c97984b1fbe75 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:03:34 +0000 Subject: [PATCH 22/32] Comment --- .../Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso index 84840139d74c..b6c36ee45e7e 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso @@ -93,7 +93,9 @@ type SQLServer_Dialect get_limit_sql_modifier : Integer -> Any get_limit_sql_modifier self limit = case limit of - Nothing -> "TOP 100 PERCENT " + Nothing -> + ## SQLServer doesn't allow order by in sub queries unless we add in the below. We should consider removing order bys from subqueries in the future. + "TOP 100 PERCENT " _ : Integer -> SQL_Builder.code (" TOP " + limit.to_text + " ") ## PRIVATE From d6d5ce589f4f5ce37e7cafee3341e81a7aaa67aa Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:35:25 +0000 Subject: [PATCH 23/32] Checkpoint --- .../Database/0.0.0-dev/src/DB_Table.enso | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 5860faae0890..7d4cb005af45 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -906,20 +906,10 @@ type DB_Table add_row_number self (name:Text="Row") (from:Integer=0) (step:Integer=1) (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (on_problems:Problem_Behavior=..Report_Warning) = Feature.Add_Row_Number.if_supported_else_throw self.connection.dialect "add_row_number" <| problem_builder = Problem_Builder.new error_on_missing_columns=True - grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder - grouping_columns.each column-> - if column.value_type.is_floating_point then - problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) + grouping_columns = _resolve_grouping_columns self.columns_helper group_by problem_builder ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder problem_builder.attach_problems_before on_problems <| - order_descriptors = case ordering.is_empty of - False -> ordering.map element-> - column = element.column - associated_selector = element.associated_selector - self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing - True -> case self.default_ordering of - Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `add_row_number` requires an ordering in database.") - descriptors -> descriptors + order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering grouping_expressions = (grouping_columns.map _.as_internal).map .expression new_expr = Row_Number_Helpers.make_row_number from step order_descriptors grouping_expressions @@ -3198,21 +3188,11 @@ type DB_Table offset self columns:(Vector (Integer | Text | Regex | By_Type))=(Missing_Argument.throw "columns") n:Integer=-1 fill_with:Fill_With=..Nothing (group_by:(Vector | Text | Integer | Regex)=[]) (order_by:(Vector | Text)=[]) (set_mode:Set_Mode=..Add) (on_problems:Problem_Behavior=..Report_Warning) -> DB_Table = Feature.Offset.if_supported_else_throw self.connection.dialect "offset" <| problem_builder = Problem_Builder.new error_on_missing_columns=True - grouping_columns = self.columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder - grouping_columns.each column-> - if column.value_type.is_floating_point then - problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) + grouping_columns = _resolve_grouping_columns self.columns_helper group_by problem_builder ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder problem_builder.attach_problems_before on_problems <| if columns.is_empty then self else - order_descriptors = case ordering.is_empty of - False -> ordering.map element-> - column = element.column - associated_selector = element.associated_selector - self.connection.dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing - True -> case self.default_ordering of - Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") - descriptors -> descriptors + order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering grouping_expressions = (grouping_columns.map _.as_internal).map .expression new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions fill_with) @@ -3221,7 +3201,9 @@ type DB_Table SQL_Type_Reference.new self.connection self.context expression new_type_refs = new_exprs.map expr->(type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] expr) - new_names = resolved_columns.map c->(if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name) + new_column_name c:DB_Column = + if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name + new_names = resolved_columns.map c->(new_column_name c) new_columns = (new_names.zip new_type_refs).zip new_exprs a->b->(Internal_Column.Value a.first a.second b) updated_table = new_columns.fold self t-> col-> @@ -3317,6 +3299,25 @@ make_literal_table connection column_vectors column_names alias = DB_Table.new alias connection internal_columns context +## PRIVATE +private _resolve_grouping_columns columns_helper:Table_Column_Helper group_by:Vector problem_builder:Problem_Builder -> Vector = + grouping_columns = columns_helper.select_columns_helper group_by Case_Sensitivity.Default True problem_builder + grouping_columns.each column-> + if column.value_type.is_floating_point then + problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) + grouping_columns + +# PRIVATE +private _resolve_order_descriptors dialect ordering default_ordering = + case ordering.is_empty of + False -> ordering.map element-> + column = element.column + associated_selector = element.associated_selector + dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing + True -> case default_ordering of + Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") + descriptors -> descriptors + ## PRIVATE Many_Files_List.from (that : DB_Table) = _ = that From 7ac3022067cfc9b156667c17127a86d6a0d98963 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:49:07 +0000 Subject: [PATCH 24/32] Refactor --- .../Standard/Database/0.0.0-dev/src/DB_Table.enso | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 7d4cb005af45..3865e827c530 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -3194,17 +3194,13 @@ type DB_Table problem_builder.attach_problems_before on_problems <| if columns.is_empty then self else order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering grouping_expressions = (grouping_columns.map _.as_internal).map .expression - new_exprs = resolved_columns.map c->(Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions fill_with) - - type_mapping = self.connection.dialect.get_type_mapping - infer_from_database_callback expression = - SQL_Type_Reference.new self.connection self.context expression - new_type_refs = new_exprs.map expr->(type_mapping.infer_return_type infer_from_database_callback "OFFSET" [] expr) - new_column_name c:DB_Column = + offset_expr c:DB_Column -> SQL_Expression = + Offset_Helpers.make_offset n (c.as_internal) order_descriptors grouping_expressions fill_with + new_column_name c:DB_Column -> Text = if set_mode==Set_Mode.Add then self.column_naming_helper.function_name "offset" [c, n, fill_with] else c.name - new_names = resolved_columns.map c->(new_column_name c) - new_columns = (new_names.zip new_type_refs).zip new_exprs a->b->(Internal_Column.Value a.first a.second b) + + new_columns = resolved_columns.map c->(Internal_Column.Value (new_column_name c) c.sql_type_reference (offset_expr c)) updated_table = new_columns.fold self t-> col-> t.set (self.make_column col) From 448b64cacd082683e4d7ae32ca0e71f422645ab1 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 16:50:22 +0000 Subject: [PATCH 25/32] fix --- distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 3865e827c530..ecdf70df6a6c 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -3303,7 +3303,7 @@ private _resolve_grouping_columns columns_helper:Table_Column_Helper group_by:Ve problem_builder.report_other_warning (Floating_Point_Equality.Error column.name) grouping_columns -# PRIVATE +## PRIVATE private _resolve_order_descriptors dialect ordering default_ordering = case ordering.is_empty of False -> ordering.map element-> From f640eb1c7cb2d87962ff04b83bbd8988edc6a989 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 18:46:52 +0000 Subject: [PATCH 26/32] More tests --- .../src/Internal/Common/Offset_Helpers.enso | 4 +++- .../src/Common_Table_Operations/Offset_Spec.enso | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso index a69edc5cd98a..b1fbbf8b7444 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Offset_Helpers.enso @@ -1,4 +1,5 @@ from Standard.Base import all +from Standard.Database.Errors import Unsupported_Database_Operation from Standard.Table import Fill_With import project.Internal.IR.Operation_Metadata @@ -10,9 +11,10 @@ import project.Internal.IR.Internal_Column.Internal_Column make_offset (n : Integer) (colName: Internal_Column) (order_descriptors : Vector Order_Descriptor) (grouping_expressions : Vector SQL_Expression) fill_with:Fill_With -> SQL_Expression = lead_lag = if n<0 then "LAG" else "LEAD" case fill_with of + Fill_With.Nothing -> SQL_Expression.Operation lead_lag [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors] Fill_With.Closest_Value -> order_descriptors_for_row_number = if n<0 then order_descriptors else order_descriptors.map o->o.reverse params = [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors, SQL_Expression.List order_descriptors_for_row_number] SQL_Expression.Operation lead_lag+"_CLOSEST" params - _ -> SQL_Expression.Operation lead_lag [SQL_Expression.Literal n.abs.to_text, colName.expression, SQL_Expression.List grouping_expressions, SQL_Expression.List order_descriptors] + Fill_With.Wrap_Around -> Error.throw (Unsupported_Database_Operation.Error "offset with Fill_With.Wrap_Around") diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 00d189f4c77f..0f9c923ce9e2 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -28,10 +28,15 @@ add_specs suite_builder setup = add_offset_specs suite_builder setup = prefix = setup.prefix build_sorted_table = Util.build_sorted_table setup - t = build_sorted_table [["A", [1, 2, 3]], ["B", [Nothing, Nothing, Nothing]], ["Text Values", ["A", "B", "C"]]] + t = build_sorted_table [["A", [1, 2, 3]], ["B", [Nothing, Nothing, Nothing], Value_Type.Integer], ["Text Values", ["A", "B", "C"]]] c = t.at 0 c_nothings = t.at 1 c_zero_rows = t.take 0 . at 0 + if setup.is_database then suite_builder.group prefix+"Column.Offset does not work in database" group_builder-> + group_builder.specify "Gives an error message" <| + r = c.offset + ## TODO - Ideally this should have a custome error message telling users to use Table.offset + r.should_fail_with (Unsupported_Database_Operation.Error "offset") if setup.is_database.not then suite_builder.group prefix+"Column.Offset with default fill strategy" group_builder-> group_builder.specify "Works with default values" <| r = c.offset @@ -228,6 +233,13 @@ add_offset_specs suite_builder setup = group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Closest_Value . should_equal ignore_order=setup.is_database Table.input [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + if setup.is_database then suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> + colA = ["A", [1, 2, 3]] + colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] + t1 = build_sorted_table [colA, colB] + group_builder.specify "Reports unsupported for in database" <| + r = t1.offset ["A"] -1 ..Wrap_Around + r.should_fail_with (Unsupported_Database_Operation.Error "offset with Fill_With.Wrap_Around") if setup.is_database.not then suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> colA = ["A", [1, 2, 3]] colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] From 0da96b93a6d8a930a7896de30e742cc87d352a0f Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 18:47:54 +0000 Subject: [PATCH 27/32] Table.new --- .../Common_Table_Operations/Offset_Spec.enso | 178 +++++++++--------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index 0f9c923ce9e2..de8e3237c99d 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -168,71 +168,71 @@ add_offset_specs suite_builder setup = t1 = build_sorted_table [colA] group_builder.specify "Works with default values" <| t1.offset ["A"] . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], -1, Fill_With.Nothing)", [Nothing, 1, 2]]] + Table.new [colA, ["offset([A], -1, Fill_With.Nothing)", [Nothing, 1, 2]]] group_builder.specify "Works with negative n values" <| t1.offset ["A"] -2 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], -2, Fill_With.Nothing)", [Nothing, Nothing, 1]]] + Table.new [colA, ["offset([A], -2, Fill_With.Nothing)", [Nothing, Nothing, 1]]] group_builder.specify "Works with positive n values (n=1)" <| t1.offset ["A"] 1 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], 1, Fill_With.Nothing)", [2, 3, Nothing]]] + Table.new [colA, ["offset([A], 1, Fill_With.Nothing)", [2, 3, Nothing]]] group_builder.specify "Works with positive n values (n=2)" <| t1.offset ["A"] 2 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], 2, Fill_With.Nothing)", [3, Nothing, Nothing]]] + Table.new [colA, ["offset([A], 2, Fill_With.Nothing)", [3, Nothing, Nothing]]] group_builder.specify "Zero n is a no-op" <| t1.offset ["A"] 0 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], 0, Fill_With.Nothing)" , [1, 2, 3]]] + Table.new [colA, ["offset([A], 0, Fill_With.Nothing)" , [1, 2, 3]]] group_builder.specify "Large negative n values work" <| t1.offset ["A"] -1024 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, ["offset([A], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Large positive n values work" <| t1.offset ["A"] 1024 . should_equal ignore_order=setup.is_database - Table.input [colA, ["offset([A], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, ["offset([A], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with zero rows" <| build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Nothing)", [], Value_Type.Integer]] + Table.new [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Nothing)", [], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset with default fill strategy (Text Values)" group_builder-> colText = ["Text Values", ["A", "B", "C"]] t1 = build_sorted_table [colText] group_builder.specify "Works with default values" <| t1.offset ["Text Values"] . should_equal ignore_order=setup.is_database - Table.input [colText, ["offset([Text Values], -1, Fill_With.Nothing)", [Nothing, "A", "B"]]] + Table.new [colText, ["offset([Text Values], -1, Fill_With.Nothing)", [Nothing, "A", "B"]]] group_builder.specify "Works with negative n values" <| t1.offset ["Text Values"] -2 . should_equal ignore_order=setup.is_database - Table.input [colText, ["offset([Text Values], -2, Fill_With.Nothing)", [Nothing, Nothing, "A"]]] + Table.new [colText, ["offset([Text Values], -2, Fill_With.Nothing)", [Nothing, Nothing, "A"]]] group_builder.specify "Works with positive n values (n=1)" <| t1.offset ["Text Values"] 1 . should_equal ignore_order=setup.is_database - Table.input [colText, ["offset([Text Values], 1, Fill_With.Nothing)", ["B", "C", Nothing]]] + Table.new [colText, ["offset([Text Values], 1, Fill_With.Nothing)", ["B", "C", Nothing]]] group_builder.specify "Works with positive n values (n=2)" <| t1.offset ["Text Values"] 2 . should_equal ignore_order=setup.is_database - Table.input [colText, ["offset([Text Values], 2, Fill_With.Nothing)", ["C", Nothing, Nothing]]] + Table.new [colText, ["offset([Text Values], 2, Fill_With.Nothing)", ["C", Nothing, Nothing]]] suite_builder.group prefix+"Table.Offset with closest value fill strategy" group_builder-> colA = ["A", [1, 2, 3]] colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] t1 = build_sorted_table [colA, colB] group_builder.specify "Negative n shifts the values downX" <| t1.offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -1, Fill_With.Closest_Value)", [1, 1, 2]]] + Table.new [colA, colB, ["offset([A], -1, Fill_With.Closest_Value)", [1, 1, 2]]] group_builder.specify "Positive n shifts the values up" <| t1.offset ["A"] 1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 1, Fill_With.Closest_Value)", [2, 3, 3]]] + Table.new [colA, colB, ["offset([A], 1, Fill_With.Closest_Value)", [2, 3, 3]]] group_builder.specify "Zero n is a no-op" <| t1.offset ["A"] 0 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 0, Fill_With.Closest_Value)", [1, 2, 3]]] + Table.new [colA, colB, ["offset([A], 0, Fill_With.Closest_Value)", [1, 2, 3]]] group_builder.specify "Large negative n values work" <| t1.offset ["A"] -1024 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -1024, Fill_With.Closest_Value)", [1, 1, 1]]] + Table.new [colA, colB, ["offset([A], -1024, Fill_With.Closest_Value)", [1, 1, 1]]] group_builder.specify "Large positive n values work" <| t1.offset ["A"] 1024 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 1024, Fill_With.Closest_Value)", [3, 3, 3]]] + Table.new [colA, colB, ["offset([A], 1024, Fill_With.Closest_Value)", [3, 3, 3]]] group_builder.specify "Works with zero rows" <| build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Closest_Value)", [], Value_Type.Integer]] + Table.new [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Closest_Value)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], -1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, colB, ["offset([B], -1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Closest_Value . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, colB, ["offset([B], 1, Fill_With.Closest_Value)", [Nothing, Nothing, Nothing], Value_Type.Integer]] if setup.is_database then suite_builder.group prefix+"Table.Offset with wrap around fill strategy" group_builder-> colA = ["A", [1, 2, 3]] colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] @@ -246,243 +246,243 @@ add_offset_specs suite_builder setup = t1 = build_sorted_table [colA, colB] group_builder.specify "Negative n shifts the values down" <| t1.offset ["A"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -1, Fill_With.Wrap_Around)", [3, 1, 2]]] + Table.new [colA, colB, ["offset([A], -1, Fill_With.Wrap_Around)", [3, 1, 2]]] group_builder.specify "Positive n shifts the values up" <| t1.offset ["A"] 1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 1, Fill_With.Wrap_Around)", [2, 3, 1]]] + Table.new [colA, colB, ["offset([A], 1, Fill_With.Wrap_Around)", [2, 3, 1]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t1.offset ["A"] -2 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -2, Fill_With.Wrap_Around)", [2, 3, 1]]] + Table.new [colA, colB, ["offset([A], -2, Fill_With.Wrap_Around)", [2, 3, 1]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t1.offset ["A"] 2 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 2, Fill_With.Wrap_Around)", [3, 1, 2]]] + Table.new [colA, colB, ["offset([A], 2, Fill_With.Wrap_Around)", [3, 1, 2]]] group_builder.specify "Zero n is a no-op" <| t1.offset ["A"] 0 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 0, Fill_With.Wrap_Around)", [1, 2, 3]]] + Table.new [colA, colB, ["offset([A], 0, Fill_With.Wrap_Around)", [1, 2, 3]]] group_builder.specify "Larger than num rows negative n values work" <| t1.offset ["A"] -4 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -4, Fill_With.Wrap_Around)", [3, 1, 2]]] + Table.new [colA, colB, ["offset([A], -4, Fill_With.Wrap_Around)", [3, 1, 2]]] group_builder.specify "Larger than num rows positive n values work" <| t1.offset ["A"] 4 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 4, Fill_With.Wrap_Around)", [2, 3, 1]]] + Table.new [colA, colB, ["offset([A], 4, Fill_With.Wrap_Around)", [2, 3, 1]]] group_builder.specify "Large negative n values work" <| t1.offset ["A"] -1024 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], -1024, Fill_With.Wrap_Around)", [3, 1, 2]]] + Table.new [colA, colB, ["offset([A], -1024, Fill_With.Wrap_Around)", [3, 1, 2]]] group_builder.specify "Large positive n values work" <| t1.offset ["A"] 1024 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([A], 1024, Fill_With.Wrap_Around)", [2, 3, 1]]] + Table.new [colA, colB, ["offset([A], 1024, Fill_With.Wrap_Around)", [2, 3, 1]]] group_builder.specify "Works with zero rows" <| build_sorted_table [["A",[], Value_Type.Integer],["B",[], Value_Type.Integer]] . sort ["B"] . offset ["A"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Wrap_Around)", [], Value_Type.Integer]] + Table.new [["A", [], Value_Type.Integer], ["B", [], Value_Type.Integer], ["offset([A], -1, Fill_With.Wrap_Around)", [], Value_Type.Integer]] group_builder.specify "Works with negative n and column of nothings" <| t1.offset ["B"] -1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], -1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, colB, ["offset([B], -1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Works with positive n and column of nothings" <| t1.offset ["B"] 1 ..Wrap_Around . should_equal ignore_order=setup.is_database - Table.input [colA, colB, ["offset([B], 1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [colA, colB, ["offset([B], 1, Fill_With.Wrap_Around)", [Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset works with grouping - default fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] t2 = build_sorted_table [groupColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, 1, 2, Nothing, 1, 2, 3, Nothing, 1]]] + Table.new [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, 1, 2, Nothing, 1, 2, 3, Nothing, 1]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Nothing)", [2, 3, Nothing, 2, 3, 4, Nothing, 2, Nothing]]] + Table.new [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Nothing)", [2, 3, Nothing, 2, 3, 4, Nothing, 2, Nothing]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Nothing)", [Nothing, Nothing, 1, Nothing, Nothing, 1, 2, Nothing, Nothing]]] + Table.new [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Nothing)", [Nothing, Nothing, 1, Nothing, Nothing, 1, 2, Nothing, Nothing]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Nothing)", [3, Nothing, Nothing, 3, 4, Nothing, Nothing, Nothing, Nothing]]] + Table.new [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Nothing)", [3, Nothing, Nothing, 3, 4, Nothing, Nothing, Nothing, Nothing]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Nothing)", [1, 2, 3, 1, 2, 3, 4, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Nothing)", [1, 2, 3, 1, 2, 3, 4, 1, 2]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Nothing group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset works with grouping - closest fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 10, 20, 30, 40, 100, 200]] t2 = build_sorted_table [groupColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", [1, 1, 2, 10, 10, 20, 30, 100, 100]]] + Table.new [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", [1, 1, 2, 10, 10, 20, 30, 100, 100]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", [2, 3, 3, 20, 30, 40, 40, 200, 200]]] + Table.new [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", [2, 3, 3, 20, 30, 40, 40, 200, 200]]] group_builder.specify "WWW2" <| t2.offset ["Col"] -2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 20, 100, 100]]] + Table.new [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 20, 100, 100]]] group_builder.specify "WWW" <| t2.offset ["Col"] 2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", [3, 3, 3, 30, 40, 40, 40, 200, 200]]] + Table.new [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", [3, 3, 3, 30, 40, 40, 40, 200, 200]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", [1, 2, 3, 10, 20, 30, 40, 100, 200]]] + Table.new [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", [1, 2, 3, 10, 20, 30, 40, 100, 200]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 10, 100, 100]]] + Table.new [groupColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 10, 100, 100]]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", [3, 3, 3, 40, 40, 40, 40, 200, 200]]] + Table.new [groupColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", [3, 3, 3, 40, 40, 40, 40, 200, 200]]] if setup.is_database.not then suite_builder.group prefix+"Table.Offset works with grouping - wrap around fill strategy" group_builder-> groupColumn = ["Group", ["A", "A", "A", "B", "B", "B", "B", "C", "C"]] dataCol = ["Col", [1, 2, 3, 1, 2, 3, 4, 1, 2]] t2 = build_sorted_table [groupColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Wrap_Around)", [3, 1, 2, 4, 1, 2, 3, 2, 1]]] + Table.new [groupColumn, dataCol, ["offset([Col], -1, Fill_With.Wrap_Around)", [3, 1, 2, 4, 1, 2, 3, 2, 1]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Wrap_Around)", [2, 3, 1, 2, 3, 4, 1, 2, 1]]] + Table.new [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Wrap_Around)", [2, 3, 1, 2, 3, 4, 1, 2, 1]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Wrap_Around)", [2, 3, 1, 3, 4, 1, 2, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Wrap_Around)", [2, 3, 1, 3, 4, 1, 2, 1, 2]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Wrap_Around)", [3, 1, 2, 3, 4, 1, 2, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Wrap_Around)", [3, 1, 2, 3, 4, 1, 2, 1, 2]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Wrap_Around)", [1, 2, 3, 1, 2, 3, 4, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], 0, Fill_With.Wrap_Around)", [1, 2, 3, 1, 2, 3, 4, 1, 2]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1022 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], -1022, Fill_With.Wrap_Around)", [2, 3, 1, 3, 4, 1, 2, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], -1022, Fill_With.Wrap_Around)", [2, 3, 1, 3, 4, 1, 2, 1, 2]]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1022 ..Wrap_Around group_by=["Group"] . should_equal ignore_order=setup.is_database - Table.input [groupColumn, dataCol, ["offset([Col], 1022, Fill_With.Wrap_Around)", [3, 1, 2, 3, 4, 1, 2, 1, 2]]] + Table.new [groupColumn, dataCol, ["offset([Col], 1022, Fill_With.Wrap_Around)", [3, 1, 2, 3, 4, 1, 2, 1, 2]]] suite_builder.group prefix+"Table.Offset works with ordering - default fill strategy" group_builder-> orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, "C", "A", "D", "B"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Nothing)", [Nothing, "C", "A", "D", "B"]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Nothing)", ["B", "E", "C", Nothing, "D"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Nothing)", ["B", "E", "C", Nothing, "D"]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Nothing)", [Nothing, "B", Nothing, "C", "A"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Nothing)", [Nothing, "B", Nothing, "C", "A"]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Nothing)", ["C", Nothing, "D", Nothing, "E"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Nothing)", ["C", Nothing, "D", Nothing, "E"]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Nothing)", ["A", "D", "B", "E", "C"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Nothing)", ["A", "D", "B", "E", "C"]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Char]] + Table.new [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Char]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Nothing order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Char]] + Table.new [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing, Nothing], Value_Type.Char]] suite_builder.group prefix+"Table.Offset works with ordering - closest fill strategy" group_builder-> orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", ["A", "C", "A", "D", "B"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Closest_Value)", ["A", "C", "A", "D", "B"]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", ["B", "E", "C", "E", "D"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", ["B", "E", "C", "E", "D"]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", ["A", "B", "A", "C", "A"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", ["A", "B", "A", "C", "A"]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", ["C", "E", "D", "E", "E"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", ["C", "E", "D", "E", "E"]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", ["A", "D", "B", "E", "C"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Closest_Value)", ["A", "D", "B", "E", "C"]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", ["A", "A", "A", "A", "A"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Closest_Value)", ["A", "A", "A", "A", "A"]]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Closest_Value order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", ["E", "E", "E", "E", "E"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Closest_Value)", ["E", "E", "E", "E", "E"]]] if setup.is_database.not then suite_builder.group prefix+"Table.Offset works with ordering - wrap around fill strategy" group_builder-> orderColumn = ["Order", [1, 4, 2, 5, 3]] dataCol = ["Col", ["A", "D", "B", "E", "C"]] t2 = build_sorted_table [orderColumn, dataCol] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col"] -1 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Wrap_Around)", ["E", "C", "A", "D", "B"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -1, Fill_With.Wrap_Around)", ["E", "C", "A", "D", "B"]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Wrap_Around)", ["B", "E", "C", "A", "D"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 1, Fill_With.Wrap_Around)", ["B", "E", "C", "A", "D"]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Wrap_Around)", ["D", "B", "E", "C", "A"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -2, Fill_With.Wrap_Around)", ["D", "B", "E", "C", "A"]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Wrap_Around)", ["C", "A", "D", "B", "E"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 2, Fill_With.Wrap_Around)", ["C", "A", "D", "B", "E"]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col"] 0 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Wrap_Around)", ["A", "D", "B", "E", "C"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 0, Fill_With.Wrap_Around)", ["A", "D", "B", "E", "C"]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col"] -1024 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Wrap_Around)", ["B", "E", "C", "A", "D"]]] + Table.new [orderColumn, dataCol, ["offset([Col], -1024, Fill_With.Wrap_Around)", ["B", "E", "C", "A", "D"]]] group_builder.specify "Large positive n values work" <| t2.offset ["Col"] 1024 ..Wrap_Around order_by=["Order"] . should_equal ignore_order=setup.is_database - Table.input [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Wrap_Around)", ["E", "C", "A", "D", "B"]]] + Table.new [orderColumn, dataCol, ["offset([Col], 1024, Fill_With.Wrap_Around)", ["E", "C", "A", "D", "B"]]] suite_builder.group prefix+"Table.Offset works with multiple columns" group_builder-> col1 = ["Col1", ["A", "B", "C", "D"]] col2 = ["Col2", [1, 2, 3, 4]] t2 = build_sorted_table [col1, col2] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col1", "Col2"] -1 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], -1, Fill_With.Nothing)", [Nothing, "A", "B", "C"]], ["offset([Col2], -1, Fill_With.Nothing)", [Nothing, 1, 2, 3]]] + Table.new [col1, col2, ["offset([Col1], -1, Fill_With.Nothing)", [Nothing, "A", "B", "C"]], ["offset([Col2], -1, Fill_With.Nothing)", [Nothing, 1, 2, 3]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col1", "Col2"] 1 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], 1, Fill_With.Nothing)", ["B", "C", "D", Nothing]], ["offset([Col2], 1, Fill_With.Nothing)", [2, 3, 4, Nothing]]] + Table.new [col1, col2, ["offset([Col1], 1, Fill_With.Nothing)", ["B", "C", "D", Nothing]], ["offset([Col2], 1, Fill_With.Nothing)", [2, 3, 4, Nothing]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col1", "Col2"] -2 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], -2, Fill_With.Nothing)", [Nothing, Nothing, "A", "B"]], ["offset([Col2], -2, Fill_With.Nothing)", [Nothing, Nothing, 1, 2]]] + Table.new [col1, col2, ["offset([Col1], -2, Fill_With.Nothing)", [Nothing, Nothing, "A", "B"]], ["offset([Col2], -2, Fill_With.Nothing)", [Nothing, Nothing, 1, 2]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col1", "Col2"] 2 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], 2, Fill_With.Nothing)", ["C", "D", Nothing, Nothing]], ["offset([Col2], 2, Fill_With.Nothing)", [3, 4, Nothing, Nothing]]] + Table.new [col1, col2, ["offset([Col1], 2, Fill_With.Nothing)", ["C", "D", Nothing, Nothing]], ["offset([Col2], 2, Fill_With.Nothing)", [3, 4, Nothing, Nothing]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col1", "Col2"] 0 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], 0, Fill_With.Nothing)", ["A", "B", "C", "D"]], ["offset([Col2], 0, Fill_With.Nothing)", [1, 2, 3, 4]]] + Table.new [col1, col2, ["offset([Col1], 0, Fill_With.Nothing)", ["A", "B", "C", "D"]], ["offset([Col2], 0, Fill_With.Nothing)", [1, 2, 3, 4]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col1", "Col2"] -1024 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["offset([Col2], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [col1, col2, ["offset([Col1], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["offset([Col2], -1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Large positive n values work" <| t2.offset ["Col1", "Col2"] 1024 ..Nothing . should_equal ignore_order=setup.is_database - Table.input [col1, col2, ["offset([Col1], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["offset([Col2], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [col1, col2, ["offset([Col1], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["offset([Col2], 1024, Fill_With.Nothing)", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] suite_builder.group prefix+"Table.Offset works with multiple columns - updating in place" group_builder-> col1 = ["Col1", ["A", "B", "C", "D"]] col2 = ["Col2", [1, 2, 3, 4]] t2 = build_sorted_table [col1, col2] group_builder.specify "Negative n shifts the values down" <| t2.offset ["Col1", "Col2"] -1 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", [Nothing, "A", "B", "C"]], ["Col2", [Nothing, 1, 2, 3]]] + Table.new [["Col1", [Nothing, "A", "B", "C"]], ["Col2", [Nothing, 1, 2, 3]]] group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col1", "Col2"] 1 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", ["B", "C", "D", Nothing]], ["Col2", [2, 3, 4, Nothing]]] + Table.new [["Col1", ["B", "C", "D", Nothing]], ["Col2", [2, 3, 4, Nothing]]] group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col1", "Col2"] -2 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", [Nothing, Nothing, "A", "B"]], ["Col2", [Nothing, Nothing, 1, 2]]] + Table.new [["Col1", [Nothing, Nothing, "A", "B"]], ["Col2", [Nothing, Nothing, 1, 2]]] group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col1", "Col2"] 2 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", ["C", "D", Nothing, Nothing]], ["Col2", [3, 4, Nothing, Nothing]]] + Table.new [["Col1", ["C", "D", Nothing, Nothing]], ["Col2", [3, 4, Nothing, Nothing]]] group_builder.specify "Zero n is a no-op" <| t2.offset ["Col1", "Col2"] 0 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", ["A", "B", "C", "D"]], ["Col2", [1, 2, 3, 4]]] + Table.new [["Col1", ["A", "B", "C", "D"]], ["Col2", [1, 2, 3, 4]]] group_builder.specify "Large negative n values work" <| t2.offset ["Col1", "Col2"] -1024 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["Col2", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [["Col1", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["Col2", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Large positive n values work" <| t2.offset ["Col1", "Col2"] 1024 ..Nothing set_mode=..Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["Col2", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] + Table.new [["Col1", [Nothing, Nothing, Nothing, Nothing], Value_Type.Char], ["Col2", [Nothing, Nothing, Nothing, Nothing], Value_Type.Integer]] group_builder.specify "Add_Or_Update equivalent to update" <| t2.offset ["Col1", "Col2"] -1 ..Nothing set_mode=..Add_Or_Update . should_equal ignore_order=setup.is_database - Table.input [["Col1", [Nothing, "A", "B", "C"]], ["Col2", [Nothing, 1, 2, 3]]] + Table.new [["Col1", [Nothing, "A", "B", "C"]], ["Col2", [Nothing, 1, 2, 3]]] suite_builder.group prefix+"Table.Offset handles bad inputs gracefully" group_builder-> t2 = build_sorted_table [["Col1", ["A", "B", "C", "D"]], ["Col2", [1, 2, 3, 4]]] t3 = build_sorted_table [["Group", [1.1, 1.1, 1.1, 1.1]], ["Col2", [1, 2, 3, 4]]] From 23f620710805cd45534f97925903279036cb678c Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 18:51:21 +0000 Subject: [PATCH 28/32] Cleanup --- .../src/Common_Table_Operations/Offset_Spec.enso | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso index de8e3237c99d..c4b0b4e0a5bc 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Offset_Spec.enso @@ -209,7 +209,7 @@ add_offset_specs suite_builder setup = colA = ["A", [1, 2, 3]] colB = ["B", [Nothing, Nothing, Nothing], Value_Type.Integer] t1 = build_sorted_table [colA, colB] - group_builder.specify "Negative n shifts the values downX" <| + group_builder.specify "Negative n shifts the values down" <| t1.offset ["A"] -1 ..Closest_Value . should_equal ignore_order=setup.is_database Table.new [colA, colB, ["offset([A], -1, Fill_With.Closest_Value)", [1, 1, 2]]] group_builder.specify "Positive n shifts the values up" <| @@ -315,10 +315,10 @@ add_offset_specs suite_builder setup = group_builder.specify "Positive n shifts the values up" <| t2.offset ["Col"] 1 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database Table.new [groupColumn, dataCol, ["offset([Col], 1, Fill_With.Closest_Value)", [2, 3, 3, 20, 30, 40, 40, 200, 200]]] - group_builder.specify "WWW2" <| + group_builder.specify "Negative n shifts the values down (n=2)" <| t2.offset ["Col"] -2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database Table.new [groupColumn, dataCol, ["offset([Col], -2, Fill_With.Closest_Value)", [1, 1, 1, 10, 10, 10, 20, 100, 100]]] - group_builder.specify "WWW" <| + group_builder.specify "Positive n shifts the values up (n=2)" <| t2.offset ["Col"] 2 ..Closest_Value group_by=["Group"] . should_equal ignore_order=setup.is_database Table.new [groupColumn, dataCol, ["offset([Col], 2, Fill_With.Closest_Value)", [3, 3, 3, 30, 40, 40, 40, 200, 200]]] group_builder.specify "Zero n is a no-op" <| From b048a6f0cdd65f00e64a07fa706474651bfb2225 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 18:52:54 +0000 Subject: [PATCH 29/32] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9a7e90003ed..d994a3745c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,12 +47,14 @@ - In `Delimited` format, the `keep_invalid_rows` setting has been renamed to `on_invalid_rows`. The default behaviour was also changed to add any extra columns instead of discarding them. +- [Added DB_Table.Offset for SQLServer][12206] [11926]: https://github.com/enso-org/enso/pull/11926 [12031]: https://github.com/enso-org/enso/pull/12031 [12071]: https://github.com/enso-org/enso/pull/12071 [12092]: https://github.com/enso-org/enso/pull/12092 [12231]: https://github.com/enso-org/enso/pull/12231 +[12206]: https://github.com/enso-org/enso/pull/12206 #### Enso Language & Runtime From bab1648ef424f56121bfce09b029c3b914f3e1dc Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Thu, 6 Feb 2025 21:04:47 +0000 Subject: [PATCH 30/32] Fix test --- test/Table_Tests/src/In_Memory/Table_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/In_Memory/Table_Spec.enso b/test/Table_Tests/src/In_Memory/Table_Spec.enso index 3a6b9230c999..d0ae14b7809f 100644 --- a/test/Table_Tests/src/In_Memory/Table_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Table_Spec.enso @@ -148,7 +148,7 @@ add_specs suite_builder = group_builder.specify "should handle error scenarios gracefully" <| Table.new [["X", [1,2,3]], ["Y", [4]]] . should_fail_with Illegal_Argument Table.new [["X", [1]], ["X", [2]]] . should_fail_with Illegal_Argument - Table.new [["X", [1]], ["Y", [2], "Z"]] . should_fail_with Illegal_Argument + Table.new [["X", [1]], ["Y", [2], Value_Type.Integer, "Z"]] . should_fail_with Illegal_Argument Table.new [["X"]] . should_fail_with Illegal_Argument Table.new ["X", "Y", "Z"] . should_fail_with Illegal_Argument Table.new [] . should_fail_with Illegal_Argument From c0362cb78a0e3a649016eaf8ef2feb70117764ac Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 7 Feb 2025 13:47:59 +0000 Subject: [PATCH 31/32] Code review changes --- .../Standard/Table/0.0.0-dev/src/Table.enso | 2 +- .../Table_Tests/src/In_Memory/Table_Spec.enso | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso index 0954e2f22005..0e4028b5d59e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso @@ -146,7 +146,7 @@ type Table new : Vector (Vector | Column) -> Table new columns = invalid_input_shape = - Error.throw (Illegal_Argument.Error "Each column must be represented by a pair whose first element is the column name and the second element is a vector of elements that will constitute that column, or an existing column. Got: "+columns.to_text) + Error.throw (Illegal_Argument.Error "Each column must be represented by a vector whose first element is the column name, second element is a vector of elements that will constitute that column (or an existing column) and an optional third element that is the Value_Type for the column. Got: "+columns.to_text) cols = columns.map on_problems=No_Wrap.Value c-> case c of v : Vector -> case v.length of diff --git a/test/Table_Tests/src/In_Memory/Table_Spec.enso b/test/Table_Tests/src/In_Memory/Table_Spec.enso index d0ae14b7809f..36e922007ae6 100644 --- a/test/Table_Tests/src/In_Memory/Table_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Table_Spec.enso @@ -128,6 +128,23 @@ add_specs suite_builder = r_4.column_names . should_equal ["foo", "bar", "baz"] r_4.columns.map .value_type . should_equal [Value_Type.Date, Value_Type.Char, Value_Type.Time] + r_5 = Table.input [["foo", [Nothing], Value_Type.Integer]] + r_5.column_names . should_equal ["foo"] + r_5.columns.map .value_type . should_equal [Value_Type.Integer] + + group_builder.specify "Table.new should allow inputting a table with types" <| + r = Table.new [["foo", [1, 2, 3], Value_Type.Float], ["bar", [False, True, False]]] + r.column_names . should_equal ["foo", "bar"] + r.columns.map .value_type . should_equal [Value_Type.Float, Value_Type.Boolean] + + r_2 = Table.new [["foo", ["False", "True", "false"], Value_Type.Char]] + r_2.column_names . should_equal ["foo"] + r_2.columns.map .value_type . should_equal [Value_Type.Char] + + r_3 = Table.new [["foo", [Nothing], Value_Type.Integer]] + r_3.column_names . should_equal ["foo"] + r_3.columns.map .value_type . should_equal [Value_Type.Integer] + group_builder.specify "should allow creating Enso code from a Table" <| r = Table.new [["foo", [1, 2, 3]], ["bar", [False, True, False]]] r.pretty . should_equal "Table.new [['foo', [1, 2, 3]], ['bar', [False, True, False]]]" @@ -145,9 +162,13 @@ add_specs suite_builder = r4.pretty . should_equal "Table.new [['foo', [1, 2, 3]], ['bar', [False, True, False]], ['date', [Date.new 2022 8 27, Date.new 1999 1 1, Date.new 2012 1 23]], ['time', [Time_Of_Day.new 18, Time_Of_Day.new 1 2 34, Time_Of_Day.new 12]]]" Debug.eval r4.pretty . should_equal r4 + group_builder.specify "allows setting the column type" <| + + group_builder.specify "should handle error scenarios gracefully" <| Table.new [["X", [1,2,3]], ["Y", [4]]] . should_fail_with Illegal_Argument Table.new [["X", [1]], ["X", [2]]] . should_fail_with Illegal_Argument + Test.expect_panic Type_Error <| Table.new [["Y", [2], "Z"]] Table.new [["X", [1]], ["Y", [2], Value_Type.Integer, "Z"]] . should_fail_with Illegal_Argument Table.new [["X"]] . should_fail_with Illegal_Argument Table.new ["X", "Y", "Z"] . should_fail_with Illegal_Argument From 6c07f5967b1227f04a43257dc35b438338167b31 Mon Sep 17 00:00:00 2001 From: Adam Riley Date: Fri, 7 Feb 2025 16:23:01 +0000 Subject: [PATCH 32/32] Code review --- .../lib/Standard/Database/0.0.0-dev/src/DB_Table.enso | 8 ++++---- .../0.0.0-dev/src/Internal/IR/SQL_Expression.enso | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index ecdf70df6a6c..f80b3dce5c7e 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -909,7 +909,7 @@ type DB_Table grouping_columns = _resolve_grouping_columns self.columns_helper group_by problem_builder ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder problem_builder.attach_problems_before on_problems <| - order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering + order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering "add_row_number" grouping_expressions = (grouping_columns.map _.as_internal).map .expression new_expr = Row_Number_Helpers.make_row_number from step order_descriptors grouping_expressions @@ -3192,7 +3192,7 @@ type DB_Table ordering = Table_Helpers.resolve_order_by self.columns order_by problem_builder resolved_columns = self.columns_helper.select_columns_helper columns Case_Sensitivity.Default True problem_builder problem_builder.attach_problems_before on_problems <| if columns.is_empty then self else - order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering + order_descriptors = _resolve_order_descriptors self.connection.dialect ordering self.default_ordering "offset" grouping_expressions = (grouping_columns.map _.as_internal).map .expression offset_expr c:DB_Column -> SQL_Expression = @@ -3304,14 +3304,14 @@ private _resolve_grouping_columns columns_helper:Table_Column_Helper group_by:Ve grouping_columns ## PRIVATE -private _resolve_order_descriptors dialect ordering default_ordering = +private _resolve_order_descriptors dialect ordering default_ordering operation_name:Text = case ordering.is_empty of False -> ordering.map element-> column = element.column associated_selector = element.associated_selector dialect.prepare_order_descriptor column associated_selector.direction text_ordering=Nothing True -> case default_ordering of - Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `offset` requires an ordering in database.") + Nothing -> Error.throw (Illegal_Argument.Error "The table has no existing ordering (e.g. from a `sort` operation or primary key). `"+operation_name+"` requires an ordering in database.") descriptors -> descriptors ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso index f9acbe04bcca..780c81c533b3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso @@ -107,8 +107,10 @@ type SQL_Expression The internal representation of an list of SQL expressions which will be separated by commas in a list in the final SQL query. + NOTE - Only valid in certain locations so should be used carefully + Arguments: - - value: the values thatt should be built into a comma separated list. + - value: the values that should be built into a comma separated list. List values:Vector ## PRIVATE