feat: add kernel computation and update to v0.2.0 (#36)

* wip: kernel computation feature (#27)

* fix: improve data parser

* feat: init kernel computing feature powered by duckdb

* fixes in kernel computation (#33)

* fix: timezone diff issue

* fix: userKernel -> kernelComputation

* fix: global var & stopApp (#34)

* docs: add kernel mode (#35)

Author: bruceyyu

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: GWalkR
 Title: Interactive Exploratory Data Analysis Tool
-Version: 0.1.5
+Version: 0.2.0
 Authors@R: c(
     person("Yue", "Yu", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0002-9302-0793")),
@@ -17,4 +17,7 @@ Imports:
     htmlwidgets,
     jsonlite,
     openssl,
-    shiny
+    shiny,
+    shinycssloaders,
+    DBI,
+    duckdb

NAMESPACE

@@ -3,6 +3,10 @@
 export(gwalkr)
 export(gwalkrOutput)
 export(renderGwalkr)
+import(DBI)
+import(duckdb)
 import(htmlwidgets)
 import(openssl)
 import(shiny)
+import(shinycssloaders)
+importFrom(jsonlite,toJSON)

R/duckdb_utils.R

@@ -0,0 +1,59 @@
+library(DBI)
+library(duckdb)
+
+my_env <- new.env()
+
+duckdb_register_con <- function(df) {
+  my_env$con <- dbConnect(duckdb::duckdb(), ":memory:")
+  dbExecute(my_env$con, "INSTALL icu")
+  dbExecute(my_env$con, "LOAD icu")
+  dbExecute(my_env$con, "SET GLOBAL TimeZone = 'UTC'")
+  DBI::dbWriteTable(my_env$con, "gwalkr_mid_table", as.data.frame(df), overwrite = FALSE)
+}
+
+duckdb_unregister_con <- function(df) {
+  if (!is.null(my_env$con)) {
+    dbDisconnect(my_env$con)
+    my_env$con <- NULL  # Set to NULL after disconnecting
+  }
+}
+
+duckdb_get_field_meta <- function() {
+  if (exists("con", envir = my_env)) {
+    result <- dbGetQuery(my_env$con, 'SELECT * FROM gwalkr_mid_table LIMIT 1')
+    if (nrow(result) > 0) {
+      return(get_data_meta_type(result))
+    }
+  } else {
+    stop("Database connection not found.")
+  }
+}
+
+duckdb_get_data <- function(sql) {
+  if (exists("con", envir = my_env)) {
+    result <- dbGetQuery(my_env$con, sql)
+    if (nrow(result) > 0) {
+      return(result)
+    }
+  } else {
+    stop("Database connection not found.")
+  }
+}
+
+get_data_meta_type <- function(data) {
+  meta_types <- list()
+
+  for (key in names(data)) {
+    value <- data[[key]]
+    field_meta_type <- if (inherits(value, "POSIXct")) {
+      if (!is.null(attr(value, "tzone"))) "datetime_tz" else "datetime"
+    } else if (is.numeric(value)) {
+      "number"
+    } else {
+      "string"
+    }
+    meta_types <- append(meta_types, list(list(key = key, type = field_meta_type)))
+  }
+
+  return(meta_types)
+}

R/gwalkr.R

@@ -4,14 +4,19 @@
 #'
 #' @import htmlwidgets
 #' @import openssl
+#' @importFrom jsonlite toJSON
+#' @import shiny
+#' @import shinycssloaders
+#' @import DBI
+#' @import duckdb
 #'
 #' @param data A data frame to be visualized in the GWalkR. The data frame should not be empty.
 #' @param lang A character string specifying the language for the widget. Possible values are "en" (default), "ja", "zh".
 #' @param dark A character string specifying the dark mode preference. Possible values are "light" (default), "dark", "media".
-#' @param columnSpecs An optional list of lists to manually specify the types of some columns in the data frame. 
-#' Each top level element in the list corresponds to a column, and the list assigned to each column should have 
-#' two elements: `analyticalType` and `semanticType`. `analyticalType` can 
-#' only be one of "measure" or "dimension". `semanticType` can only be one of 
+#' @param columnSpecs An optional list of lists to manually specify the types of some columns in the data frame.
+#' Each top level element in the list corresponds to a column, and the list assigned to each column should have
+#' two elements: `analyticalType` and `semanticType`. `analyticalType` can
+#' only be one of "measure" or "dimension". `semanticType` can only be one of
 #' "quantitative", "temporal", "nominal" or "ordinal". For example:
 #' \code{list(
 #'   "gender" = list(analyticalType = "dimension", semanticType = "nominal"),
@@ -20,6 +25,7 @@
 #' @param visConfig An optional config string to reproduce your chart. You can copy the string by clicking "export config" button on the GWalkR interface.
 #' @param visConfigFile An optional config file path to reproduce your chart. You can download the file by clicking "export config" button then "download" button on the GWalkR interface.
 #' @param toolbarExclude An optional list of strings to exclude the tools from toolbar UI. However, Kanaries brand info is not allowed to be removed or changed unless you are granted with special permission.
+#' @param kernelComputation An optional boolean to enable the kernel mode computation which is much more efficient. Default is FALSE.
 #'
 #' @return An \code{htmlwidget} object that can be rendered in R environments
 #'
@@ -28,42 +34,47 @@
 #' gwalkr(mtcars)
 #'
 #' @export
-gwalkr <- function(data, lang = "en", dark = "light", columnSpecs = list(), visConfig = NULL, visConfigFile = NULL, toolbarExclude = list()) {
+gwalkr <- function(data, lang = "en", dark = "light", columnSpecs = list(), visConfig = NULL, visConfigFile = NULL, toolbarExclude = list(), kernelComputation = FALSE) {
   if (!is.data.frame(data)) stop("data must be a data frame")
   if (!is.null(visConfig) && !is.null(visConfigFile)) stop("visConfig and visConfigFile are mutually exclusive")
   lang <- match.arg(lang, choices = c("en", "ja", "zh"))
 
   rawFields <- raw_fields(data, columnSpecs)
   colnames(data) <- sapply(colnames(data), fname_encode)
-  
+
   if (!is.null(visConfigFile)) {
     visConfig <- readLines(visConfigFile, warn=FALSE)
   }
-  # forward options using x
-  x = list(
-    dataSource = jsonlite::toJSON(data),
-    rawFields = rawFields,
-    i18nLang = lang,
-    visSpec = visConfig,
-    dark = dark,
-    toolbarExclude = toolbarExclude
-  )
 
-  # create widget
-  htmlwidgets::createWidget(
-    name = 'gwalkr',
-    x,
-    package = 'GWalkR',
-    width='100%',
-    height='100%'
-  )
+  if (kernelComputation) {
+    gwalkr_kernel(data, lang, dark, rawFields, visConfig, toolbarExclude)
+  } else {
+    x = list(
+      dataSource = toJSON(data),
+      rawFields = rawFields,
+      i18nLang = lang,
+      visSpec = visConfig,
+      dark = dark,
+      toolbarExclude = toolbarExclude,
+      useKernel = FALSE
+    )
+
+    # create widget
+    htmlwidgets::createWidget(
+      name = 'gwalkr',
+      x,
+      package = 'GWalkR',
+      width='100%',
+      height='100%'
+    )
+  }
 }
 
 #' Shiny bindings for gwalkr
 #'
 #' Output and render functions for using gwalkr within Shiny
 #' applications and interactive Rmd documents.
-#' 
+#'
 #' @import shiny
 #'
 #' @param outputId output variable to read from

R/gwalkr_kernel.R

@@ -0,0 +1,89 @@
+utils::globalVariables(c(".rs.invokeShinyPaneViewer"))
+
+convert_timestamps_in_df <- function(df) {
+  for (colname in colnames(df)) {
+    if (inherits(df[[colname]], "POSIXt")) {
+      df[[colname]] <- as.numeric(as.POSIXct(df[[colname]], tz = "UTC")) * 1000
+    }
+  }
+  return(df)
+}
+
+gwalkr_kernel <- function(data, lang, dark, rawFields, visConfig, toolbarExclude) {
+  cat("GWalkR kernel mode initialized...\n")
+  cat("Note: The console is unavailable while running a Shiny app. You can stop the app to use the console.\n")
+
+  filter_func <- function(data, req) {
+    query <- parseQueryString(req$QUERY_STRING)
+
+    res <- duckdb_get_data(query$sql)
+    res <- convert_timestamps_in_df(res)
+
+    json <- toJSON(
+      res,
+      auto_unbox = TRUE
+    )
+
+    httpResponse(
+      status = 200L,
+      content_type = "application/json",
+      content = json
+    )
+  }
+
+  app_options <- if (exists(".rs.invokeShinyPaneViewer")) {
+    c(launch.browser = .rs.invokeShinyPaneViewer)
+  } else {
+    list()
+  }
+
+  app <- shinyApp(
+    ui = fluidPage(
+      shinycssloaders::withSpinner(
+        gwalkrOutput("gwalkr_kernel"),
+        proxy.height="400px"
+      )
+    ),
+
+    server = function(input, output, session) {
+      path <- session$registerDataObj(
+        "GWALKR",
+        NULL,
+        filter_func
+      )
+
+      duckdb_register_con(data)
+      fieldMetas <- duckdb_get_field_meta()
+
+      x = list(
+        rawFields = rawFields,
+        i18nLang = lang,
+        visSpec = visConfig,
+        dark = dark,
+        toolbarExclude = toolbarExclude,
+        useKernel = TRUE,
+        fieldMetas = fieldMetas,
+        endpointPath = path
+      )
+
+      output$gwalkr_kernel = renderGwalkr({
+        htmlwidgets::createWidget(
+          name = 'gwalkr',
+          x,
+          package = 'GWalkR',
+          width = '100%',
+          height = '100%'
+        )
+      })
+      session$onSessionEnded(function() {
+        cat("GwalkR closed")
+        duckdb_unregister_con()
+        stopApp()
+      })
+    },
+
+    options = app_options
+  )
+
+  if (interactive()) app
+}

README.md

@@ -24,20 +24,30 @@ It can simplify your R data analysis and data visualization workflow, by turning
 
 ## Getting Started
 
-### Setup GWalkR
+### 📦 Setup GWalkR
 
 ```R
 install.packages("GWalkR")
 library(GWalkR)
 ```
 
-### Start Your Data Exploration in a Single Line of Code
+### 📈 Start Your Data Exploration in a Single Line of Code
 
 ```R
 data(iris)
 gwalkr(iris)
 ```
 
+### 🚀 Switch to Kernel Computation for Large Datasets
+
+```R
+gwalkr(large_df, kernelComputation = TRUE)
+```
+
+Here is a [tutorial](https://medium.com/@bruceyu0416/eda-reimagined-in-r-gwalkr-duckdb-for-lightning-fast-visualizations-05b011e8ae39) with more details.
+
+Please note that the kernel mode will be running in a Shiny app which will block your R console. You can stop the app to use the console.
+
 ## Main Features
 ### Get an overview of your data frame under 'Data' tab.
 <img width="700" alt="image" src="https://github.com/bruceyyu/GWalkR/assets/33870780/67131cfa-a25b-44ae-90a0-95902ea5edb1">

man/gwalkr.Rd

@@ -11,14 +11,16 @@
   },
   "dependencies": {
     "@kanaries/graphic-walker": "^0.4.70",
+    "@kanaries/gw-dsl-parser": "^0.1.49",
     "@rollup/plugin-commonjs": "^25.0.2",
     "@rollup/plugin-replace": "^5.0.2",
     "@rollup/plugin-terser": "^0.4.3",
     "@rollup/plugin-typescript": "^11.1.2",
     "mobx-react-lite": "^3.4.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "styled-components": "^5.3.6"
+    "styled-components": "^5.3.6",
+    "vite-plugin-wasm": "^3.3.0"
   },
   "devDependencies": {
     "@types/react": "^18.2.14",

web_app/package.json

@@ -0,0 +1,31 @@
+import type { IDataQueryPayload, IRow } from "@kanaries/graphic-walker/interfaces";
+import { parser_dsl_with_meta } from "@kanaries/gw-dsl-parser";
+
+const DEFAULT_LIMIT = 50_000;
+
+const sendHTTPData = (sql: string, endpointPath: string) => {
+    return new Promise((resolve, reject) => {
+        fetch(`${endpointPath}&sql=${encodeURIComponent(sql)}`)
+            .then((response) => response.json())
+            .then((data) => {
+                console.log("Processed data from R:", data);
+                resolve(data);
+            })
+            .catch((error) => {
+                console.error("Error:", error);
+                reject(error);
+            });
+    });
+};
+
+export function getDataFromKernelBySql(fieldMetas: { key: string; type: string }[], endpointPath: string) {
+    return async (payload: IDataQueryPayload) => {
+        const sql = parser_dsl_with_meta(
+            "gwalkr_mid_table",
+            JSON.stringify({ ...payload, limit: payload.limit ?? DEFAULT_LIMIT }),
+            JSON.stringify({ gwalkr_mid_table: fieldMetas })
+        );
+        const result = (await sendHTTPData(sql, endpointPath)) ?? [];
+        return result as IRow[];
+    };
+}