Fix select columns with comma (#216) + add telemetry

README.md

@@ -144,6 +144,10 @@ Every contribution, big or small, is celebrated. Join us in our mission to refin
 
 <br/>
 
+## Telemetry
+
+Amphi collects **anonymous telemetry data** to help us understand users and their use-cases better to improve the product. You can of course **opt out** in the settings and disable any telemetry collection.
+
 ## 🛣️ Ecosystem
 
 Amphi is available as an extension for Jupyterlab, and Amphi ETL is based on Jupyterlab. Therefore Jupyterlab extensions can be installed on Amphi ETL.

amphi-etl/Dockerfile.ubuntu

@@ -27,7 +27,7 @@ RUN chown -R $NB_UID:$NB_GID /home/amphi
 RUN apt-get update && apt-get install -y --no-install-recommends \
     && python3 -m pip install --upgrade pip setuptools wheel \
     && python3 -m pip install --no-cache-dir pandas==2.2.1 numpy \
-    jupyterlab==4.2.5 jupyterlab-amphi==0.8.21  \
+    jupyterlab==4.2.5 jupyterlab-amphi==0.8.22  \
     sqlalchemy==2.0.4 python-dotenv \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 

amphi-etl/amphi/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.8.21'
+__version__ = VERSION = '0.8.22'

amphi-etl/config/labconfig/page_config.json

@@ -17,6 +17,9 @@
       "@jupyterlab/celltags-extension:plugin": true,
       "@jupyterlab/tooltip-extension:notebooks": true,
       "@jupyterlab/workspaces-extension:sidebar": true,
-      "@jupyterlab/apputils-extension:announcements": true
+      "@jupyterlab/apputils-extension:announcements": true,
+      "@jupyterlab/console-extension": true,
+      "@jupyterlab/theme-dark-high-contrast-extension": true,
+      "@jupyterlab/logconsole-extension:plugin": true
     }
 }

amphi-etl/config/settings/overrides.json

@@ -3,6 +3,9 @@
     "theme": "Amphi Light",
     "theme-scrollbars": true
   },
+  "@jupyterlab/statusbar-extension:plugin": {
+    "visible": false
+  },
   "@jupyterlab/mainmenu-extension:plugin": {
     "menus": [
       {

amphi-etl/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.8.21"
+  "version": "0.8.22"
 }

amphi-etl/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/amphi-etl",
-    "version": "0.8.21",
+    "version": "0.8.22",
     "keywords": [
         "amphi",
         "etl",

amphi-etl/requirements.txt

@@ -1,5 +1,5 @@
 jupyterlab==4.3.2
-jupyterlab-amphi==0.8.21
+jupyterlab-amphi==0.8.22
 pandas==2.2.1
 # ../jupyterlab-amphi
 .

amphi-etl/setup.py

@@ -23,14 +23,14 @@ def collect_files(src_dir, dest_dir):
 
 setup(
     name='amphi-etl',
-    version='0.8.21',
+    version='0.8.22',
     description='Open-source and Python-based ETL',
     author='Thibaut Gourdel',
     author_email='[email protected]',
     license='ELv2',
     install_requires=[
         'jupyterlab==4.3.2',
-        'jupyterlab-amphi==0.8.21',
+        'jupyterlab-amphi==0.8.22',
         'pandas==2.2.1'
     ],
     keywords=[],  # Added an empty list for keywords to resolve the dynamic 'keywords' issue.

jupyterlab-amphi/_version.py

@@ -1,2 +1,2 @@
 """source of truth for ``amphi``` version."""
-__version__ = "0.8.21"
+__version__ = "0.8.22"

jupyterlab-amphi/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.8.21"
+  "version": "0.8.22"
 }

jupyterlab-amphi/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/jupyterlab-amphi",
-    "version": "0.8.21",
+    "version": "0.8.22",
     "keywords": [
         "amphi",
         "etl",

jupyterlab-amphi/packages/pipeline-components-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-core",
-  "version": "0.8.21",
+  "version": "0.8.22",
   "description": "Amphi Pipeline Core Components",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {
@@ -52,7 +52,7 @@
     "watch:src": "tsc -w"
   },
   "dependencies": {
-    "@amphi/pipeline-components-manager": "^0.8.21",
+    "@amphi/pipeline-components-manager": "^0.8.22",
     "@jupyter-widgets/base": "^4.1.6",
     "@jupyterlab/application": "^4.1.5",
     "@lumino/widgets": "^2.0.0",

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/files/ExcelFileInput.tsx

@@ -119,8 +119,6 @@ export class ExcelFileInput extends BaseCoreComponent {
   public provideDependencies({ config }): string[] {
     let deps: string[] = [];
 
-    console.log("config/data %o", config)
-
     const engine = config.excelOptions.engine;
 
     if (engine === 'None' || engine === 'openpyxl') {

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/files/LocalFileInput.tsx

@@ -63,8 +63,6 @@ export class LocalFileInput extends BaseCoreComponent {
       ]
     };
 
-    console.log("form %o", form)
-
     const description = "Use File Input to read data from a local file. Supports CSV, JSON, Excel, Parquet, and XML formats.";
 
     super("File Input", "localFileInput", description, "pandas_df_input", [], "inputs", fileTextIcon, defaultConfig, form);

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/files/ParquetFileInput.tsx

@@ -132,8 +132,6 @@ export class ParquetFileInput extends BaseCoreComponent {
     const options = Object.entries(parquetOptions)
       .filter(([key, value]) => value !== null && value !== '')
       .map(([key, value]) => {
-        console.log("key ??? %o", key)
-        console.log("value rhhh %o", value)
         if (key === 'storage_options' && typeof value === 'object') {
           return `${key}=${JSON.stringify(value)}`;
         } else if (typeof value === 'string') {

jupyterlab-amphi/packages/pipeline-components-local/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-local",
-  "version": "0.8.21",
+  "version": "0.8.22",
   "description": "Amphi Pipeline Local Components",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {
@@ -49,7 +49,7 @@
     "watch:src": "tsc -w"
   },
   "dependencies": {
-    "@amphi/pipeline-components-manager": "^0.8.21",
+    "@amphi/pipeline-components-manager": "^0.8.22",
     "@jupyter-widgets/base": "^4.1.6",
     "@jupyterlab/application": "^4.1.5",
     "@lumino/widgets": "^2.0.0",

jupyterlab-amphi/packages/pipeline-components-manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-manager",
-  "version": "0.8.21",
+  "version": "0.8.22",
   "description": "Amphi Pipeline Components Manager",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {

jupyterlab-amphi/packages/pipeline-components-manager/src/CodeGenerator.tsx

@@ -83,7 +83,6 @@ export class CodeGenerator {
 
         // Handle target node
         if (nodeObject.id === targetNodeId) {
-          console.log("nodeObject %o", nodeObject)
           let displayCode = '';
           if (nodeObject.type.includes('processor') || nodeObject.type.includes('input')) {
             if (nodeObject.type.includes('documents')) {

jupyterlab-amphi/packages/pipeline-components-manager/src/RequestService.tsx

@@ -28,10 +28,8 @@ export class RequestService {
 
       // Check if the previous node has already been executed recently
       const refNode = nodesMap.get(refNodeId);
-      console.log("refNode %o", refNode)
       if (refNode) {
         const { lastUpdated, lastExecuted, nameId } = refNode.data || {};
-        console.log("nameId %o", nameId)
 
         // If the node has been executed after its last update, skip code generation
         // console.log("lastExecuted %o, type: %s", lastExecuted, typeof lastExecuted);
@@ -41,16 +39,14 @@ export class RequestService {
           console.log("Skip generation")
           const dataframeVar = nameId || refNodeId;
           const codeToFetchContent = `print(_amphi_metadatapanel_getcontentof(${dataframeVar}))`;
-          console.log("codeToFetchContent %o", codeToFetchContent)
 
           const future = context.sessionContext.session.kernel!.requestExecute({ code: codeToFetchContent });
           future.onIOPub = msg => {
             if (msg.header.msg_type === 'stream') {
               const streamMsg = msg as KernelMessage.IStreamMsg;
               const output = streamMsg.content.text;
-              console.log("output %o", output)
 
-              const regex = /([^,]+)\s+\(([^,]+),\s*(named|unnamed)\)/g;
+              const regex = /([^,]+)\s+\(([^,]+(?:\[[^\]]+\])?),\s*(named|unnamed)\)/g;
               const newItems = [];
 
               let match;
@@ -390,8 +386,6 @@ ${pythonExtraction}
 `;
 
 
-    // console.log("CODE %o", code)
-
     // Format any remaining variables in the code
     code = CodeGenerator.formatVariables(code);
 
@@ -468,19 +462,13 @@ ${pythonExtraction}
         componentService
       );
 
-      console.log("nodeId %o", nodeId);
-      console.log("componentService %o", componentService);
-      console.log("context.model.toString() %o", context.model.toString());
-
       // Get component and data for the node
       const { component, data } = CodeGenerator.getComponentAndDataForNode(
         nodeId,
         componentService,
         context.model.toString()
       );
 
-      console.log("data %o", data)
-
 
       if (!component) {
         throw new Error("Component or data not found.");
@@ -522,7 +510,6 @@ sheet_names = excel_obj.sheet_names
 print(", ".join(sheet_names))
 `;
 
-      console.log("code %o", code)
       // Format variables in the code
       code = CodeGenerator.formatVariables(code);
 
@@ -544,9 +531,7 @@ print(", ".join(sheet_names))
 
           if (streamMsg.content.name === "stdout") {
             const output = streamMsg.content.text;
-            console.log("output %o", output)
             const sheets = output.split(", ");
-            console.log("sheets %o", sheets)
 
             const newItems = sheets.map((sheetName) => ({
               input: {},

jupyterlab-amphi/packages/pipeline-components-manager/src/configUtils.tsx

@@ -102,7 +102,6 @@ export const GenerateUIFormComponent = React.memo(({
   modalOpen,
   setModalOpen
 }: FormComponentProps) => {
-  console.log("GenerateUIFormComponent")
   const stopPropagation = useCallback((e: React.MouseEvent) => {
     e.stopPropagation();
   }, []);
@@ -169,13 +168,11 @@ export const GenerateUIInputs = React.memo(({
   advanced,
   formValues
 }: UIInputsProps) => {
-  console.log("GenerateUIInputs")
 
   const [connections, setConnections] = useState([]);
   const [optionsConnections, setOptionsConnections] = useState<Record<string, any[]>>({});
 
   const fetchConnections = useCallback(() => {
-    console.log("fetchConnections")
 
     const allConnections = PipelineService.getConnections(context.model.toString());
     setConnections(allConnections);
@@ -194,7 +191,6 @@ export const GenerateUIInputs = React.memo(({
 
   // Function to check if a field should be displayed based on its condition
   const shouldDisplayField = useCallback((field, values) => {
-    console.log("shouldDisplay")
     if (!field.condition) {
       return true;
     }

jupyterlab-amphi/packages/pipeline-components-manager/src/formUtils.tsx

@@ -2,7 +2,6 @@ import { Form } from 'antd';
 import React from 'react';
 
 export const renderFormItem = (field: any, content: any) => {
-  console.log("renderFormItem");
 
   return (
     <Form.Item

jupyterlab-amphi/packages/pipeline-components-manager/src/forms/InputFiles.tsx

@@ -54,8 +54,6 @@ export const InputFiles = ({ field, values, handleChange, context, advanced, man
         filter: (model) => model.path !== context.path,
       });
 
-      console.log("res %o", res);
-
       if (res.value && res.value.length > 0) {
         const relativePaths = res.value.map(file => PipelineService.getRelativePath(context.path, file.path));
         const newSelectedFiles = relativePaths.map(path => ({ label: path, value: path }));

jupyterlab-amphi/packages/pipeline-components-manager/src/forms/selectColumn.tsx

@@ -22,7 +22,6 @@ interface SelectColumnsProps {
 export const SelectColumn: React.FC<SelectColumnsProps> = ({
   field, handleChange, defaultValue, context, componentService, commands, nodeId, advanced
 }) => {
-  console.log("SelectColumn");
 
   const findOptionByValue = (value: any) => {
     if (value === undefined) {

jupyterlab-amphi/packages/pipeline-components-manager/src/forms/selectColumns.tsx

@@ -24,7 +24,6 @@ interface SelectColumnsProps {
 export const SelectColumns: React.FC<SelectColumnsProps> = ({
   field, handleChange, defaultValues, context, componentService, commands, nodeId, advanced
 }) => {
-  console.log("SelectColumns");
 
   const [items, setItems] = useState(field.options || []);
   const [selectedOptions, setSelectedOptions] = useState(defaultValues);

jupyterlab-amphi/packages/pipeline-console/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-log-console",
-  "version": "0.8.21",
+  "version": "0.8.22",
   "description": "Amphi Pipeline Log Console",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {
@@ -49,7 +49,7 @@
     "watch:src": "tsc -w"
   },
   "dependencies": {
-    "@amphi/pipeline-editor": "^0.8.21",
+    "@amphi/pipeline-editor": "^0.8.22",
     "@jupyterlab/application": "^4.0.0",
     "@jupyterlab/apputils": "^4.0.0",
     "@jupyterlab/console": "^4.0.0",

jupyterlab-amphi/packages/pipeline-editor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-editor",
-  "version": "0.8.21",
+  "version": "0.8.22",
   "description": "Amphi Pipeline Editor",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {
@@ -52,7 +52,7 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@amphi/pipeline-components-manager": "^0.8.21",
+    "@amphi/pipeline-components-manager": "^0.8.22",
     "@dagrejs/dagre": "^1.0.4",
     "@jupyter-widgets/jupyterlab-manager": "3.1.9",
     "@jupyterlab/application": "^4.2.2",
@@ -68,6 +68,7 @@
     "@lumino/dragdrop": "^2.0.0",
     "antd": "5.16.4",
     "html-to-image": "^1.11.11",
+    "posthog-js": "^1.194.3",
     "reactflow": "^11.11.4"
   },
   "resolutions": {

jupyterlab-amphi/packages/pipeline-editor/schema/extension.json

@@ -16,6 +16,12 @@
       "description": "Enable debug mode provide additional messages in the console.",
       "default": false
     },
+    "enableTelemetry": {
+      "type": "boolean",
+      "title": "Enable Telemetry",
+      "description": "Enable telemetry used to improve user experience and product performance.",
+      "default": true
+    },
     "defaultEngineBackend": {
       "type": "string",
       "title": "Default Engine for File Browser files",