owpythonscript: Offer to run kernel in-process

Author: irgolic

Orange/widgets/data/owpythonscript.py

@@ -18,6 +18,7 @@
 from orangewidget.widget import Msg
 from qtconsole import styles
 from qtconsole.client import QtKernelClient
+from qtconsole.inprocess import QtInProcessKernelManager
 from qtconsole.manager import QtKernelManager
 
 from AnyQt.QtWidgets import (
@@ -430,6 +431,7 @@ class Outputs:
     splitterState: Optional[bytes] = Setting(None)
 
     vimModeEnabled = Setting(False)
+    useInProcessKernel = Setting(False)
 
     class Warning(OWWidget.Warning):
         illegal_var_type = Msg('{} should be of type {}, not {}.')
@@ -592,6 +594,14 @@ def _(color, text):
             self.vim_indicator.indicator_text = text
             self.vim_indicator.update()
 
+        # Kernel type
+
+        gui.checkBox(
+            self.editor_controls, self, 'useInProcessKernel', 'Use in-process kernel',
+            tooltip="Avoids initializing data, but freezes Orange during computation.",
+            callback=self.init_kernel
+        )
+
         # Library
 
         self.libraryListSource = []
@@ -704,9 +714,14 @@ def init_kernel(self):
         ident = str(uuid.uuid4()).split('-')[-1]
         cf = os.path.join(self._temp_connection_dir, 'kernel-%s.json' % ident)
 
-        self.kernel_manager = QtKernelManager(
-            connection_file=cf
-        )
+        if self.useInProcessKernel:
+            self.kernel_manager = QtInProcessKernelManager(
+                connection_file=cf
+            )
+        else:
+            self.kernel_manager = QtKernelManager(
+                connection_file=cf
+            )
 
         self.kernel_manager.start_kernel(
             extra_arguments=[
@@ -723,9 +738,11 @@ def init_kernel(self):
             self.editor.kernel_manager = self.kernel_manager
             self.editor.kernel_client = self.kernel_client
         if self.console is not None:
+            self.console.set_in_process(self.useInProcessKernel)
             self.console.kernel_manager = self.kernel_manager
             self.console.kernel_client = self.kernel_client
             self.console.set_kernel_id(ident)
+            self.update_variables_in_console()
 
     def shutdown_kernel(self):
         self.kernel_client.stop_channels()
@@ -815,7 +832,9 @@ def handleNewSignals(self):
         self.func_sig.update_signal_text({
             n: len(getattr(self, n)) for n in self.signal_names
         })
+        self.update_variables_in_console()
 
+    def update_variables_in_console(self):
         self.set_status('Injecting variables...')
         vars = self.initial_locals_state()
         self.console.set_vars(vars)

Orange/widgets/data/utils/python_console.py

@@ -6,6 +6,7 @@
 import threading
 
 from AnyQt.QtCore import Qt, Signal
+from Orange.widgets.data.utils.python_kernel import update_kernel_vars, collect_kernel_vars
 from Orange.widgets.data.utils.python_serialize import OrangeZMQMixin
 from qtconsole.client import QtKernelClient
 from qtconsole.rich_jupyter_widget import RichJupyterWidget
@@ -33,6 +34,7 @@ class OrangeConsoleWidget(OrangeZMQMixin, RichJupyterWidget):
 
     def __init__(self, *args, style_sheet='', **kwargs):
         super().__init__(*args, **kwargs)
+        self.__is_in_process = False
         self.__is_ready = False
 
         self.__queued_broadcast = None
@@ -43,19 +45,20 @@ def __init__(self, *args, style_sheet='', **kwargs):
         self.__broadcasting = False
         self.__threads = []
 
-        self.inject_vars_comm = None
-        self.collect_vars_comm = None
-
         self.style_sheet = style_sheet + \
                            '.run-prompt { color: #aa22ff; }'
 
+        self.queue_init_client()
+
+    def queue_init_client(self):
         # Let the widget/kernel start up before trying to run a script,
         # by storing a queued execution payload when the widget's commit
         # method is invoked before <In [0]:> appears.
         @self.becomes_ready.connect
         def _():
             self.becomes_ready.disconnect(_)  # reset callback
-            self.init_client()
+            if not self.is_in_process():
+                self.init_client()
             self.becomes_ready.connect(self.__on_ready)
             self.__on_ready()
 
@@ -78,6 +81,24 @@ def __run_queued_payload(self):
         self.__queued_execution = None
         self.run_script(*qe)
 
+    def set_in_process(self, enabled):
+        if self.__is_in_process == enabled:
+            return
+        self.__is_in_process = enabled
+        self.__is_ready = False
+        self.__executing = False
+        self.__broadcasting = False
+        self.__prompt_num = 1
+
+        try:
+            self.becomes_ready.disconnect(self.__on_ready)
+        except:
+            pass
+        self.queue_init_client()
+
+    def is_in_process(self):
+        return self.__is_in_process
+
     def run_script(self, script):
         """
         Inject the in vars, run the script,
@@ -127,7 +148,12 @@ def set_vars(self, vars):
         self.in_prompt = "Injecting variables..."
         self._update_prompt(self.__prompt_num)
 
-        super().set_vars(vars)
+        if self.is_in_process():
+            kernel = self.kernel_manager.kernel
+            update_kernel_vars(kernel, vars, self.signals)
+            self.on_variables_injected()
+        else:
+            super().set_vars(vars)
 
     def on_variables_injected(self):
         log.debug('Cleared injecting variables')
@@ -186,6 +212,13 @@ def _handle_execute_reply(self, msg):
         self._update_prompt(self.__prompt_num)
         self.execution_finished.emit(True)
 
+        # collect variables manually, handle_new_vars will not be called
+        if self.is_in_process():
+            kernel = self.kernel_manager.kernel
+            self.results_ready.emit(
+                collect_kernel_vars(kernel, self.signals)
+            )
+
     # override
     def _handle_kernel_died(self, since_last_heartbeat):
         super()._handle_kernel_died(since_last_heartbeat)

Orange/widgets/data/utils/python_kernel.py

@@ -11,53 +11,58 @@
 
 class OrangeIPythonKernel(OrangeZMQMixin, IPythonKernel):
 
-    signals = ("data", "learner", "classifier", "object")
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.variables = defaultdict(list)
+        self.variables = {}
         self.init_comms_kernel()
-        self.handle_new_vars({})
 
     def handle_new_vars(self, vars):
-        default_vars = defaultdict(list)
-        default_vars.update(vars)
-
-        input_vars = {}
-
-        for signal in self.signals:
-            # remove old out_ vars
-            out_name = 'out_' + signal
-            if out_name in self.shell.user_ns:
-                del self.shell.user_ns[out_name]
-                self.shell.user_ns_hidden.pop(out_name, None)
-
-            if signal + 's' in vars and vars[signal + 's']:
-                input_vars['in_' + signal + 's'] = vars[signal + 's']
-
-                # prepend script to set single signal values,
-                # e.g. in_data = in_datas[0]
-                input_vars['in_' + signal] = input_vars['in_' + signal + 's'][0]
-            else:
-                input_vars['in_' + signal] = None
-                input_vars['in_' + signal + 's'] = []
-
-        self.shell.push(input_vars)
+        input_vars = update_kernel_vars(self, vars, self.signals)
         self.variables.update(input_vars)
 
     def execute_request(self, *args, **kwargs):
         result = super().execute_request(*args, **kwargs)
         if not self.is_initialized():
             return result
 
-        vars = defaultdict(list)
-        for signal in self.signals:
-            key = signal + 's'
-            name = 'out_' + signal
-            if name in self.shell.user_ns:
-                var = self.shell.user_ns[name]
-                vars[key].append(var)
-
-        self.set_vars(vars)
+        variables = collect_kernel_vars(self, self.signals)
+        prepared_variables = {
+            k[4:] + 's': [v]
+            for k, v in variables.items()
+        }
+        self.set_vars(prepared_variables)
 
         return result
+
+
+def update_kernel_vars(kernel, vars, signals):
+    input_vars = {}
+
+    for signal in signals:
+        # remove old out_ vars
+        out_name = 'out_' + signal
+        if out_name in kernel.shell.user_ns:
+            del kernel.shell.user_ns[out_name]
+            kernel.shell.user_ns_hidden.pop(out_name, None)
+
+        if signal + 's' in vars and vars[signal + 's']:
+            input_vars['in_' + signal + 's'] = vars[signal + 's']
+
+            # prepend script to set single signal values,
+            # e.g. in_data = in_datas[0]
+            input_vars['in_' + signal] = input_vars['in_' + signal + 's'][0]
+        else:
+            input_vars['in_' + signal] = None
+            input_vars['in_' + signal + 's'] = []
+    kernel.shell.push(input_vars)
+    return input_vars
+
+
+def collect_kernel_vars(kernel, signals):
+    variables = {}
+    for signal in signals:
+        name = 'out_' + signal
+        if name in kernel.shell.user_ns:
+            var = kernel.shell.user_ns[name]
+            variables[name] = var
+    return variables