test: Add integration tests (#11)

* test: Add integration tests

Author: max-sixty

Taskfile.yml

@@ -0,0 +1,23 @@
+version: "3"
+
+tasks:
+  # There are a few problems with a standard test loop, such that we need to
+  # completely blow away the python environment on each loop:
+  # - Having the dbt-prql dependency as develop=True doesn't seem to copy
+  #   `zzz_dbt_prql.pth` with that name; instead using `dbt_prql.pth`. This then
+  #   doesn't load early enough. (Seems weird, might be worth confirming)
+  # - Having the dbt-prql dependency as develop=False doesn't seem to update
+  #   when reinstalled.
+  reset-test-env:
+    dir: tests
+
+    cmds:
+      - cmd: poetry env remove 3.9
+        ignore_error: true
+      - poetry env use 3.9
+      - poetry install
+
+  test:
+    dir: tests/dbt_test_project
+    cmds:
+      - dbt compile --profiles-dir=..

dbt_prql/patch.py

@@ -1,30 +1,43 @@
 from jinja2.ext import Extension
+from jinja2.parser import Parser
 
 
 class PrqlExtension(Extension):
     tags = {"prql"}
 
-    def parse(self, parser):
+    def parse(self, parser: Parser):
+        import logging
+
         from jinja2 import nodes
         from jinja2.nodes import Const
 
+        logger = logging.getLogger(__name__)
+
         line_number = next(parser.stream).lineno
-        prql = parser.parse_statements(["name:endprql"], drop_needle=True)
+        prql_jinja = parser.parse_statements(["name:endprql"], drop_needle=True)
+        logger.info(f"Parsing statement:\n{prql_jinja}")
         return nodes.CallBlock(
-            self.call_method("_to_sql", [Const("")]), [], [], prql
+            self.call_method("_to_sql", [Const("")]), [], [], prql_jinja
         ).set_lineno(line_number)
 
     def _to_sql(self, args, caller):
+        _ = args
+        import logging
+
         import prql_python
 
+        logger = logging.getLogger(__name__)
+
         prql = caller()
+        logger.info(f"Parsing PRQL:\n{prql}")
         sql = prql_python.to_sql(prql)
         output = f"""
 -- SQL created from PRQL. Original PRQL:
 {chr(10).join(f'-- {line}' for line in prql.splitlines())}
 
 {sql}
 """
+        logger.info(f"Parsed into SQL:\n{sql}")
         return output
 
 
@@ -42,10 +55,10 @@ def patch_dbt_environment() -> None:
 
     logger = logging.getLogger(__name__)
 
-    log_level = os.environ.get("DBT_PRQL_LOG_LEVEL")
-    if log_level is not None and log_level != "":
+    log_level = os.environ.get("DBT_PRQL_LOG_LEVEL", "")
+    if log_level != "":
         logging.basicConfig()
-        logger.setLevel(int(log_level))
+        logger.setLevel(log_level)
         logger.warning(
             f"Setting PRQL log level to {log_level}. Please note that logging anything "
             "from this module can affect the reliability of some libraries which "
@@ -78,10 +91,10 @@ def with_prql_extension(*args, **kwargs):
     jinja.get_environment = env_with_prql
 
     if os.environ.get("DBT_PRQL_TEST"):
-        test_prql_parse()
+        test_jinja_parse()
 
 
-def test_prql_parse() -> None:
+def test_jinja_parse() -> None:
     import logging
 
     from dbt.clients.jinja import get_environment

poetry.lock

@@ -16,7 +16,12 @@ python = ">=3.7.3"
 # We'll try and quite quickly on a new _.X._ release from dbt.
 # And we can unblock this when we have upstream tests https://github.com/prql/dbt-prql/issues/6
 dbt-core = ">=1.1.0,<1.3.0"
-prql_python = ">=0.2.0"
+prql_python = "0.2.6"
+
+[tool.poetry.dev-dependencies]
+black = "*"
+flake8 = "*"
+mypy = "*"
 
 [build-system]
 build-backend = "poetry.core.masonry.api"

pyproject.toml

@@ -0,0 +1 @@
+.user.yml

tests/.gitignore

@@ -0,0 +1,19 @@
+# dbt-prql tests
+
+These are some early integration tests. Currently they're run from the
+command-line with `task`:
+
+```
+task test
+```
+
+Some severe caveats:
+
+- Because of <https://github.com/prql/prql/issues/918>, the postgres tests
+  demonstrate that this doesn't currently work with postgres.
+- The tests may fail _after_ passing the compilation, depending on whether
+  there's a valid Google Cloud auth setup. Unfortunately it doesn't seem
+  possible to use standard dbt commands without querying a DB. So we may need to
+  work with [dbt's test
+  fixtures](https://github.com/dbt-labs/dbt-core/tree/main/tests); at first
+  glance they don't seem well-described.

tests/README.md

@@ -0,0 +1,4 @@
+
+target/
+dbt_modules/
+logs/

tests/dbt_test_project/.gitignore

@@ -0,0 +1,5 @@
+name: "dbt_test_projct"
+config-version: 2
+# Unused
+version: "0.0.0"
+profile: prql-test

tests/dbt_test_project/dbt_project.yml

@@ -0,0 +1,11 @@
+version: 2
+
+# Source template from https://docs.getdbt.com/docs/building-a-dbt-project/using-sources
+
+sources:
+  - name: salesforce
+    tables:
+      - name: in_process
+  - name: team
+    tables:
+      - name: team_sales

tests/dbt_test_project/models/src_sources.yml

@@ -0,0 +1,8 @@
+{% prql %}
+from in_process = {{ source('salesforce', 'in_process') }}
+derive expected_sales = probability * value
+join {{ source('team', 'team_sales') }} [name]
+group name (
+  aggregate (sum expected_sales)
+)
+{% endprql %}