From 76d0fae7201dd0667ced5314d98842ef2461e88c Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 13:34:36 +0100
Subject: [PATCH 01/50] Custom script to click on text fragments

---
 atest/01_Editor.robot                         | 37 +++++-------
 atest/Keywords.resource                       | 13 ++--
 atest/mouse_over_extension.py                 | 60 +++++++++++++++++++
 .../jupyterlab-lsp/src/features/jump_to.ts    |  2 +-
 4 files changed, 86 insertions(+), 26 deletions(-)

diff --git a/atest/01_Editor.robot b/atest/01_Editor.robot
index 8162206e5..8350e0448 100644
--- a/atest/01_Editor.robot
+++ b/atest/01_Editor.robot
@@ -9,7 +9,7 @@ Test Tags       ui:editor    aspect:ls:features
 
 *** Test Cases ***
 Bash
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'HelloWorld')])[last()]
+    ${def} =    Set Variable    lastToken:HelloWorld
     Editor Shows Features for Language
     ...    Bash
     ...    example.sh
@@ -17,19 +17,18 @@ Bash
     ...    Jump to Definition=${def}
 
 CSS
-    ${def} =    Set Variable
-    ...    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., '--some-var')])[last()]
+    ${def} =    Set Variable    lastToken:--some-var
     Editor Shows Features for Language    CSS    example.css    Diagnostics=Do not use empty rulesets
     ...    Jump to Definition=${def}    Rename=${def}
 
 Docker
-    ${def} =    Set Variable    xpath://div[contains(@class, 'cm-line')]//text()[contains(., 'PLANET')]
+    ${def} =    Set Variable    lastToken:PLANET
     Wait Until Keyword Succeeds    3x    100ms    Editor Shows Features for Language    Docker    Dockerfile
     ...    Diagnostics=Instructions should be written in uppercase letters    Jump to Definition=${def}
     ...    Rename=${def}
 
 JS
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'fib')])[last()]
+    ${def} =    Set Variable    lastToken:fib
     Editor Shows Features for Language    JS    example.js    Diagnostics=Expression expected
     ...    Jump to Definition=${def}    Rename=${def}
 
@@ -37,20 +36,20 @@ JSON
     Editor Shows Features for Language    JSON    example.json    Diagnostics=Duplicate object key
 
 JSX
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'hello')])[last()]
+    ${def} =    Set Variable    lastToken:hello
     Editor Shows Features for Language    JSX    example.jsx    Diagnostics=Expression expected
     ...    Jump to Definition=${def}    Rename=${def}
 # Julia
-#    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'add_together')])[last()]
+#    ${def} =    Set Variable    lastToken:add_together
 #    Editor Shows Features for Language    Julia    example.jl    Jump to Definition=${def}    Rename=${def}
 
 LaTeX
     [Tags]    language:latex
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//span[contains(text(), 'foo')])[last()]
+    ${def} =    Set Variable    lastToken:foo
     Editor Shows Features for Language    LaTeX    example.tex    Jump to Definition=${def}    Rename=${def}
 
 Less
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., '@width')])[last()]
+    ${def} =    Set Variable    lastToken:@width
     Editor Shows Features for Language    Less    example.less    Diagnostics=Do not use empty rulesets
     ...    Jump to Definition=${def}
 
@@ -58,7 +57,7 @@ Markdown
     Editor Shows Features for Language    Markdown    example.md    Diagnostics=`Color` is misspelt
 
 Python (pylsp)
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'fib')])[last()]
+    ${def} =    Set Variable    lastToken:fib
     Editor Shows Features for Server
     ...    pylsp
     ...    Python
@@ -68,35 +67,33 @@ Python (pylsp)
     ...    Rename=${def}
 
 Python (pyright)
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'fib')])[last()]
+    ${def} =    Set Variable    lastToken:fib
     Editor Shows Features for Server    pyright    Python    example.py    Diagnostics=is not defined (Pyright)
     ...    Jump to Definition=${def}
 
 R
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'fib')])[last()]
+    ${def} =    Set Variable    lastToken:fib
     Editor Shows Features for Language    R    example.R    Diagnostics=Put spaces around all infix operators
     ...    Jump to Definition=${def}
 
 Robot Framework
     [Tags]    gh:332
-    ${def} =    Set Variable
-    ...    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'Special Log')])[last()]
+    ${def} =    Set Variable    lastToken:Special Log
     Editor Shows Features for Language    Robot Framework    example.robot    Diagnostics=Undefined keyword
     ...    Jump to Definition=${def}
 
 SCSS
-    ${def} =    Set Variable
-    ...    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'primary-color')])[last()]
+    ${def} =    Set Variable    lastToken:primary-color
     Editor Shows Features for Language    SCSS    example.scss    Diagnostics=Do not use empty rulesets
     ...    Jump to Definition=${def}
 
 TSX
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'HelloWorld')])[last()]
+    ${def} =    Set Variable    lastToken:HelloWorld
     Editor Shows Features for Language    TSX    example.tsx
     ...    Diagnostics='hello' is declared but its value is never read.    Jump to Definition=${def}    Rename=${def}
 
 TypeScript
-    ${def} =    Set Variable    xpath:(//div[contains(@class, 'cm-line')]//text()[contains(., 'inc')])[last()]
+    ${def} =    Set Variable    lastToken:inc
     Editor Shows Features for Language    TypeScript    example.ts    Diagnostics=The left-hand side of an arithmetic
     ...    Jump to Definition=${def}    Rename=${def}
 
@@ -155,9 +152,7 @@ Editor Content Changed
 Editor Should Rename
     [Arguments]    ${symbol}
     Set Tags    feature:rename
-    ${sel} =    Set Variable If    "${symbol}".startswith(("xpath", "css"))    ${symbol}
-    ...    xpath:(//span[@role="presentation"][contains(., "${symbol}")])[last()]
-    Open Context Menu Over    ${sel}
+    Open Context Menu Over Token    ${symbol}
     ${old_content} =    Get Editor Content
     Capture Page Screenshot    03-rename-0.png
     Mouse Over    ${MENU RENAME}
diff --git a/atest/Keywords.resource b/atest/Keywords.resource
index a02563c19..71336841a 100644
--- a/atest/Keywords.resource
+++ b/atest/Keywords.resource
@@ -10,6 +10,7 @@ Library     ./ports.py
 Library     ./config.py
 Library     ./paths.py
 Library     ./diagnostics.py
+Library     ./mouse_over_extension.py
 
 
 *** Keywords ***
@@ -80,6 +81,7 @@ Read Page Config
     Set Global Variable    ${PAGE CONFIG}    ${config}
     Set Global Variable    ${LAB VERSION}    ${config["appVersion"]}
 
+
 Setup Suite For Screenshots
     [Arguments]    ${folder}
     Set Screenshot Directory    ${SCREENSHOTS DIR}${/}${folder}
@@ -324,6 +326,11 @@ Enter Cell Editor
     [Arguments]    ${cell_nr}    ${line}=1
     Click Element    css:.jp-Cell:nth-child(${cell_nr}) .cm-line:nth-child(${line})
     Wait Until Page Contains Element    css:.jp-Cell:nth-child(${cell_nr}) .cm-focused
+    # clicking on line does not appear sufficient in CM6 (but still useful because it ensures we have it in view
+    Execute JavaScript
+    ...    var view = document.querySelector('.jp-Cell:nth-child(${cell_nr}) .cm-content').cmView.view;
+    ...    var pos = view.state.doc.line(${line}).from;
+    ...    return view.dispatch({selection: {anchor: pos, head: pos}});
 
 Open Context Menu Over Cell Editor
     [Arguments]    ${cell_nr}    ${line}=1
@@ -433,11 +440,9 @@ Clean Up After Working with File and Settings
 
 Jump To Definition
     [Arguments]    ${symbol}
-    ${sel} =    Set Variable If    "${symbol}".startswith(("xpath", "css"))    ${symbol}
-    ...    xpath:(//span[@role="presentation"][contains(., "${symbol}")])[last()]
-    Click Element    ${sel}
+    Click Token    ${symbol}
     ${cursor} =    Measure Cursor Position
-    Open Context Menu Over    ${sel}
+    Open Context Menu Over Token    ${symbol}
     Capture Page Screenshot    02-jump-to-definition-0.png
     Wait Until Element Is Visible    ${MENU JUMP}    timeout=5s
     Mouse Over    ${MENU JUMP}
diff --git a/atest/mouse_over_extension.py b/atest/mouse_over_extension.py
index cfb867454..f0360ccdb 100644
--- a/atest/mouse_over_extension.py
+++ b/atest/mouse_over_extension.py
@@ -26,3 +26,63 @@ def mouse_over_and_wiggle(locator, x_wiggle=5):
     action_chains.move_to_element(sl.find_element(locator))
     wiggle(action_chains, x_wiggle)
     return action_chains.perform()
+
+
+def _over_text_in_line(token_locator: str, event = None):
+    which, text = token_locator.split(':', maxsplit=1)
+    assert which == 'lastToken'
+
+    sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
+    sl.driver.execute_script(
+        """
+        window.breadthFirstSearchReverse = function (nodes, text) {
+          for (let n of nodes.toReversed()) {
+            if (n.childNodes.length) {
+              var result = window.breadthFirstSearchReverse(
+                [...n.childNodes],
+               text
+              );
+              if (result) {
+               return result
+              }
+            } else if (n.textContent.includes(text) && n.nodeType === 3) {
+              return n;
+            }
+          }
+        }
+        const text = arguments[0];
+        const eventType = arguments[1];
+
+        const textNode = window.breadthFirstSearchReverse(
+          [...document.querySelectorAll('.cm-line')],
+          text
+        );
+
+        const offset = textNode.textContent.indexOf(text);
+        const subNode = (
+          textNode
+          .splitText(offset)
+          .splitText(text.length)
+          .previousSibling
+        );
+        const range = document.createRange();
+        range.selectNode(subNode);
+        const rect = range.getBoundingClientRect();
+
+        const e = new Event(eventType, {bubbles: true});
+        e.clientX = (rect.left + rect.right) / 2;
+        e.clientY = (rect.top + rect.bottom) / 2;
+
+        subNode.parentElement.dispatchEvent(e);
+        """,
+        text,
+        event
+    );
+
+
+def click_token(token_locator: str):
+    return _over_text_in_line(token_locator, event='click')
+
+
+def open_context_menu_over_token(token_locator: str):
+    return _over_text_in_line(token_locator, event='contextmenu')
diff --git a/packages/jupyterlab-lsp/src/features/jump_to.ts b/packages/jupyterlab-lsp/src/features/jump_to.ts
index a143f0d02..492713485 100644
--- a/packages/jupyterlab-lsp/src/features/jump_to.ts
+++ b/packages/jupyterlab-lsp/src/features/jump_to.ts
@@ -312,7 +312,7 @@ export class NavigationFeature extends Feature {
         return path + ', line: ' + location.range.start.line;
       });
 
-      // TODO: use selector with preview, basically needes the ui-component
+      // TODO: use selector with preview, basically needs the ui-component
       // from jupyterlab-citation-manager; let's try to move it to JupyterLab core
       // (and re-implement command palette with it)
       // the preview should use this.jumper.document_manager.services.contents

From 8f771a8a14894d910e6dad7965d82301e98e274c Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 17:06:47 +0100
Subject: [PATCH 02/50] Fix signature tests failing due to change in cursor
 positioning

---
 atest/Keywords.resource | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atest/Keywords.resource b/atest/Keywords.resource
index 71336841a..49f348ec7 100644
--- a/atest/Keywords.resource
+++ b/atest/Keywords.resource
@@ -329,7 +329,7 @@ Enter Cell Editor
     # clicking on line does not appear sufficient in CM6 (but still useful because it ensures we have it in view
     Execute JavaScript
     ...    var view = document.querySelector('.jp-Cell:nth-child(${cell_nr}) .cm-content').cmView.view;
-    ...    var pos = view.state.doc.line(${line}).from;
+    ...    var pos = view.state.doc.line(${line}).to;
     ...    return view.dispatch({selection: {anchor: pos, head: pos}});
 
 Open Context Menu Over Cell Editor

From 90977704030efc00f498017e5b53db90a911cc09 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 17:07:18 +0100
Subject: [PATCH 03/50] Fix signature hover box hiding on click

---
 packages/jupyterlab-lsp/src/features/signature.ts | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/signature.ts b/packages/jupyterlab-lsp/src/features/signature.ts
index c511fc9d7..ff01422b2 100644
--- a/packages/jupyterlab-lsp/src/features/signature.ts
+++ b/packages/jupyterlab-lsp/src/features/signature.ts
@@ -193,7 +193,7 @@ export function highlightCode(
       language,
       (token: string, className: string, from: number, to: number) => {
         let populated = false;
-        // In CodeMirror6 variables are not necessairly tokenized,
+        // In CodeMirror6 variables are not necessarily tokenized,
         // we need to split them manually
         if (from <= end && start <= to) {
           const a = Math.max(start, from);
@@ -326,13 +326,16 @@ export class SignatureFeature extends Feature {
           // Delay handling by moving on top of the stack
           // so that virtual document is updated.
           setTimeout(() => {
+            // be careful: updateListener also fires after blur, so we
+            // need to carefully check what changed to avoid invalidating
+            // user clicking on the hover box.
             if (viewUpdate.docChanged) {
               this.afterChange(
                 viewUpdate.changes,
                 adapter,
                 editorPosition
               ).catch(this.console.warn);
-            } else {
+            } else if (viewUpdate.selectionSet) {
               this.onCursorActivity(adapter, editorPosition).catch(
                 this.console.warn
               );
@@ -564,7 +567,7 @@ export class SignatureFeature extends Feature {
       response
     );
     if (displayPosition === null) {
-      // try to find last occurrance of trigger character to position the tooltip
+      // try to find last occurrence of trigger character to position the tooltip
       const content = editor.model.sharedModel.getSource();
       const lines = content.split('\n');
       const offset = offsetAtPosition(
@@ -595,7 +598,7 @@ export class SignatureFeature extends Feature {
       tooltip: {
         privilege: 'forceAbove',
         // do not move the tooltip to match the token to avoid drift of the
-        // tooltip due the simplicty of token matching rules; instead we keep
+        // tooltip due the simplicity of token matching rules; instead we keep
         // the position constant manually via `displayPosition`.
         alignment: undefined,
         hideOnKeyPress: false

From b823ee3026f44a0503a637788eb37c99a8e1d33e Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 17:09:20 +0100
Subject: [PATCH 04/50] Fix hover tests

---
 atest/05_Features/Hover.robot | 38 +++++----------
 atest/mouse_over_extension.py | 87 ++++++++++++++++++++++++-----------
 2 files changed, 73 insertions(+), 52 deletions(-)

diff --git a/atest/05_Features/Hover.robot b/atest/05_Features/Hover.robot
index 6430f3e9d..67bb7320c 100644
--- a/atest/05_Features/Hover.robot
+++ b/atest/05_Features/Hover.robot
@@ -17,7 +17,7 @@ ${HOVER_SIGNAL}     css:.cm-lsp-hover-available
 *** Test Cases ***
 Hover Does Not Trigger Automatically
     Enter Cell Editor    1
-    ${sel} =    Last Occurrence    python_add
+    ${sel} =    Set Variable    lastToken:python_add
     Configure JupyterLab Plugin    {"autoActivate": false}
     ...    plugin id=${HOVER PLUGIN ID}
     Trigger Automatically By Hover    ${sel}
@@ -27,7 +27,7 @@ Hover Does Not Trigger Automatically
 
 Hover Triggers Automatically
     Enter Cell Editor    1
-    ${sel} =    Last Occurrence    python_add
+    ${sel} =    Set Variable    lastToken:python_add
     Configure JupyterLab Plugin    {"delay": 100, "autoActivate": true}
     ...    plugin id=${HOVER PLUGIN ID}
     Trigger Automatically By Hover    ${sel}
@@ -48,7 +48,7 @@ Hover works in notebooks
 
 Hover can be triggered via modifier key once cursor stopped moving
     Enter Cell Editor    1
-    ${element} =    Last Occurrence    python_add
+    ${element} =    Set Variable    lastToken:python_add
     Wait Until Keyword Succeeds    5x    0.1 s    Trigger Via Modifier Key Press    ${element}
 
 Hover works in foreign code (javascript)
@@ -81,49 +81,35 @@ Update hover after character deletion
 
 
 *** Keywords ***
-Last Occurrence
-    [Arguments]    ${symbol}
-    ${sel} =    Set Variable If    "${symbol}".startswith(("xpath", "css"))    ${symbol}
-    ...    xpath:(//div[@class="cm-content"]//text()[contains(., "${symbol}")])[last()]
-    RETURN    ${sel}
-
 Trigger Automatically By Hover
     [Arguments]    ${sel}
     # bring the cursor to the element
-    Wokraround Visibility Problem    ${sel}
-    Mouse Over    ${sel}
+    Mouse Over Token    ${sel}
     Wait Until Page Contains Element    ${HOVER_SIGNAL}    timeout=10s
-    Mouse Over And Wiggle    ${sel}    5
+    Mouse Over Token And Wiggle    ${sel}    5
 
 Trigger Via Hover With Modifier
     [Arguments]    ${sel}
     # bring the cursor to the element
-    Wokraround Visibility Problem    ${sel}
-    Mouse Over    ${sel}
-    # move it back and forth (wiggle) while hodling the ctrl modifier
-    Mouse Over With Control    ${sel}    x_wiggle=5
+    Mouse Over Token    ${sel}
+    # move it back and forth (wiggle) while holding the ctrl modifier
+    Mouse Over Token With Control    ${sel}    x_wiggle=5
     Wait Until Keyword Succeeds    4x    0.1s    Page Should Contain Element    ${HOVER_BOX}
 
 Trigger Via Modifier Key Press
     [Arguments]    ${sel}
     # bring the cursor to the element
-    Wokraround Visibility Problem    ${sel}
-    Mouse Over    ${sel}
+    Mouse Over Token    ${sel}
     Wait Until Page Contains Element    ${HOVER_SIGNAL}    timeout=10s
-    Mouse Over And Wiggle    ${sel}    5
-    Press Keys    ${sel}    CTRL
+    Mouse Over Token And Wiggle    ${sel}    5
+    Press Keys    None    CTRL
     Wait Until Keyword Succeeds    4x    0.1s    Page Should Contain Element    ${HOVER_BOX}
 
 Trigger Tooltip
     [Documentation]    The default way to trigger the hover tooltip
     [Arguments]    ${symbol}
-    ${sel} =    Last Occurrence    ${symbol}
+    ${sel} =    Set Variable    lastToken:${symbol}
     Wait Until Keyword Succeeds    4x    0.1 s    Trigger Via Hover With Modifier    ${sel}
 
 Setup Hover Test
     Setup Notebook    Python    Hover.ipynb
-
-Wokraround Visibility Problem
-    [Arguments]    ${sel}
-    ${width}    ${height} =    Get Element Size    ${sel}
-    IF    ${width} == 0    Cover Element    ${sel}
diff --git a/atest/mouse_over_extension.py b/atest/mouse_over_extension.py
index f0360ccdb..88f020500 100644
--- a/atest/mouse_over_extension.py
+++ b/atest/mouse_over_extension.py
@@ -1,37 +1,43 @@
 from robot.libraries.BuiltIn import BuiltIn
-from selenium.webdriver import ActionChains
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
 from selenium.webdriver.common.keys import Keys
 from SeleniumLibrary import SeleniumLibrary
 
 
-def wiggle(action_chains, x_wiggle):
+def wiggle(action, x_wiggle):
     if x_wiggle:
-        action_chains.move_by_offset(xoffset=x_wiggle, yoffset=0)
-        action_chains.move_by_offset(xoffset=-x_wiggle, yoffset=0)
+        #action.pointer_action.move_by(xoffset=x_wiggle, yoffset=0)
+        action.key_action.pause()
+        #action.pointer_action.move_by(xoffset=-x_wiggle, yoffset=0)
+        action.key_action.pause()
 
 
-def mouse_over_with_control(locator, x_wiggle=0):
+def mouse_over_token_with_control(token_locator, x_wiggle=0):
     sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
-    action_chains = ActionChains(sl.driver)
-    action_chains.key_down(Keys.CONTROL)
-    action_chains.move_to_element(sl.find_element(locator))
-    wiggle(action_chains, x_wiggle)
-    action_chains.key_up(Keys.CONTROL)
-    return action_chains.perform()
+    location = _find_text_in_line(token_locator)
+    action = ActionBuilder(sl.driver)
 
+    action.key_action.key_down(Keys.CONTROL)
+    #action.pointer_action.pause()
+    action.pointer_action.move_to_location(location['x'], location['y'])
+    wiggle(action, x_wiggle)
+    action.key_action.key_up(Keys.CONTROL)
 
-def mouse_over_and_wiggle(locator, x_wiggle=5):
+    return action.perform()
+
+
+def mouse_over_token_and_wiggle(token_locator, x_wiggle=5):
     sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
-    action_chains = ActionChains(sl.driver)
-    action_chains.move_to_element(sl.find_element(locator))
-    wiggle(action_chains, x_wiggle)
-    return action_chains.perform()
+    action = ActionBuilder(sl.driver)
+    location = _find_text_in_line(token_locator)
+    action.pointer_action.move_to_location(location['x'], location['y'])
+    wiggle(action, x_wiggle)
+    return action.perform()
 
 
-def _over_text_in_line(token_locator: str, event = None):
+def _find_text_in_line(token_locator: str):
     which, text = token_locator.split(':', maxsplit=1)
     assert which == 'lastToken'
-
     sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
     sl.driver.execute_script(
         """
@@ -50,8 +56,11 @@ def _over_text_in_line(token_locator: str, event = None):
             }
           }
         }
+        """
+    )
+    return sl.driver.execute_script(
+        """
         const text = arguments[0];
-        const eventType = arguments[1];
 
         const textNode = window.breadthFirstSearchReverse(
           [...document.querySelectorAll('.cm-line')],
@@ -69,20 +78,46 @@ def _over_text_in_line(token_locator: str, event = None):
         range.selectNode(subNode);
         const rect = range.getBoundingClientRect();
 
+        return {
+            // selenium does not allow passing bare nodes, only elements
+            parentElement: subNode.parentElement,
+            x: (rect.left + rect.right) / 2,
+            y: (rect.top + rect.bottom) / 2
+        }
+        """,
+        text
+    )
+
+def _emit_over_text_in_line(token_locator: str, event: str):
+    sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
+    location = _find_text_in_line(token_locator)
+    sl.driver.execute_script(
+        """
+        const location = arguments[0];
+        const eventType = arguments[1];
+
         const e = new Event(eventType, {bubbles: true});
-        e.clientX = (rect.left + rect.right) / 2;
-        e.clientY = (rect.top + rect.bottom) / 2;
+        e.clientX = location.x;
+        e.clientY = location.y;
 
-        subNode.parentElement.dispatchEvent(e);
+        location.parentElement.dispatchEvent(e);
         """,
-        text,
+        location,
         event
-    );
+    )
+
+
+def mouse_over_token(token_locator: str):
+    sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
+    action = ActionBuilder(sl.driver)
+    location = _find_text_in_line(token_locator)
+    action.pointer_action.move_to_location(location['x'], location['y'])
+    return action.perform()
 
 
 def click_token(token_locator: str):
-    return _over_text_in_line(token_locator, event='click')
+    return _emit_over_text_in_line(token_locator, event='click')
 
 
 def open_context_menu_over_token(token_locator: str):
-    return _over_text_in_line(token_locator, event='contextmenu')
+    return _emit_over_text_in_line(token_locator, event='contextmenu')

From 61cf8a6803a4c955cd612cafba9b9b6228617e0d Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 22:12:17 +0100
Subject: [PATCH 05/50] Fix typos

---
 packages/jupyterlab-lsp/src/settings.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/settings.ts b/packages/jupyterlab-lsp/src/settings.ts
index 4f382ad5b..224a83092 100644
--- a/packages/jupyterlab-lsp/src/settings.ts
+++ b/packages/jupyterlab-lsp/src/settings.ts
@@ -77,9 +77,9 @@ interface IValidationData {
 }
 
 /**
- * Conflicts encounteredn when dot-collapsing settings
+ * Conflicts encountered when dot-collapsing settings
  * organised by server ID, and then as a mapping between
- * (dotted) setting ID and list of encoutnered values.
+ * (dotted) setting ID and list of encountered values.
  * The last encountered values is preferred for use.
  */
 type SettingsMergeConflicts = Record<string, Record<string, any[]>>;
@@ -187,7 +187,7 @@ export class SettingsSchemaManager {
         // 1.8% spent on `deepCopy()`
         // 1.79% spend on other tasks in `populate()`
         // There is a limit on the transformation time, and failing to transform
-        // in the default 1 second means that no settigns whatsoever are available.
+        // in the default 1 second means that no settings whatsoever are available.
         // Therefore validation in `populate()` was moved into an async function;
         // this means that we need to trigger re-load of settings
         // if there validation errors.
@@ -333,7 +333,7 @@ export class SettingsSchemaManager {
           configSchema.properties[key].default = value;
         }
       }
-      // add server-speficic default overrides from overrides.json (and pre-defined in schema)
+      // add server-specific default overrides from overrides.json (and pre-defined in schema)
       const serverDefaultsOverrides =
         defaultsOverrides && defaultsOverrides.hasOwnProperty(serverKey)
           ? defaultsOverrides[serverKey]
@@ -449,7 +449,7 @@ export class SettingsSchemaManager {
   /**
    * Validate user settings from plugin against provided schema,
    * asynchronously to avoid blocking the main thread.
-   * Stores validation reult in `this._validationErrors`.
+   * Stores validation result in `this._validationErrors`.
    */
   private async _validateSchemaLater(
     plugin: ISettingRegistry.IPlugin,

From 335a76d365a86da3c57234f093433c27bfd3f158 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 22:12:30 +0100
Subject: [PATCH 06/50] Fix `priority` not being respected

---
 packages/jupyterlab-lsp/src/index.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/jupyterlab-lsp/src/index.ts b/packages/jupyterlab-lsp/src/index.ts
index 51e482ab4..6b2a4dc97 100644
--- a/packages/jupyterlab-lsp/src/index.ts
+++ b/packages/jupyterlab-lsp/src/index.ts
@@ -156,9 +156,11 @@ export class LSPExtension {
       {}) as TLanguageServerConfigurations;
 
     // Add `configuration` as a copy of `serverSettings` to work with changed name upstream
+    // Add `rank` as a copy of priority fot the same reason.
     languageServerSettings = Object.fromEntries(
       Object.entries(languageServerSettings).map(([key, value]) => {
         value.configuration = value.serverSettings;
+        value.rank = value.priority;
         return [key, value];
       })
     );

From 97c8b45c491cb154a6ffd420efe51c2503eeb299 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 8 Sep 2023 22:15:34 +0100
Subject: [PATCH 07/50] Remove diagnostics response await (other than first)

which block diagnostics from being shown in file editor due to CM6
document mismatch due to delay by one version, and seems to work
fine without it in both editor and notebook
---
 packages/jupyterlab-lsp/src/features/diagnostics/feature.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
index 8a36c2418..3c5f03db2 100644
--- a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
+++ b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
@@ -56,7 +56,6 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
   };
   protected settings: IFeatureSettings<LSPDiagnosticsSettings>;
   protected console = new BrowserConsole().scope('Diagnostics');
-  private _responseReceived: PromiseDelegate<void> = new PromiseDelegate();
   private _firstResponseReceived: PromiseDelegate<void> = new PromiseDelegate();
   private _diagnosticsDatabases = new WeakMap<
     WidgetLSPAdapter<any>,
@@ -151,13 +150,12 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
             // do not wait for next update, show what we already know.
 
             // TODO: this still fails when scrolling down fast and then
-            // scrolling up to the skipped cells because the state invlidation
+            // scrolling up to the skipped cells because the state invalidation
             // counter kicks in but diagnostics does not get rendered yet before
             // we leave..
             await this._firstResponseReceived.promise;
           } else {
             await adapter.updateFinished;
-            await this._responseReceived.promise;
           }
 
           const database = this.getDiagnosticsDB(adapter);
@@ -568,8 +566,6 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
       const done = new Promise<void>(resolve => {
         setTimeout(() => {
           this._firstResponseReceived.resolve();
-          this._responseReceived.resolve();
-          this._responseReceived = new PromiseDelegate();
           resolve();
         }, 0);
       });

From 8e1fa2799273e3937a24463af8dce945a0204a32 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 01:50:47 +0100
Subject: [PATCH 08/50] Restore correct target for bash token test

---
 atest/01_Editor.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atest/01_Editor.robot b/atest/01_Editor.robot
index 8350e0448..541f4c63d 100644
--- a/atest/01_Editor.robot
+++ b/atest/01_Editor.robot
@@ -9,7 +9,7 @@ Test Tags       ui:editor    aspect:ls:features
 
 *** Test Cases ***
 Bash
-    ${def} =    Set Variable    lastToken:HelloWorld
+    ${def} =    Set Variable    lastToken:fib
     Editor Shows Features for Language
     ...    Bash
     ...    example.sh

From 6b57bca3e3650bce5fc387f6d5cd2612f05d87db Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 01:52:31 +0100
Subject: [PATCH 09/50] Do not create a temporary editor to update underline
 color

---
 .../src/features/diagnostics/feature.ts       | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
index 3c5f03db2..75b4a1b68 100644
--- a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
+++ b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
@@ -18,6 +18,7 @@ import {
 } from '@jupyterlab/lsp';
 import { TranslationBundle } from '@jupyterlab/translation';
 import { PromiseDelegate } from '@lumino/coreutils';
+import { StyleModule } from 'style-mod';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 
 import { CodeDiagnostics as LSPDiagnosticsSettings } from '../../_diagnostics';
@@ -240,29 +241,29 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
 
   private _reconfigureTheme() {
     const style = getComputedStyle(document.body);
-    const baseTheme = EditorView.baseTheme({
-      '.cm-lintRange-error': {
+    const lintTheme = new StyleModule({
+      '.cm-editor .cm-lintRange-error': {
         backgroundImage: underline(
           style.getPropertyValue(
             '--jp-editor-mirror-lsp-diagnostic-error-decoration-color'
           )
         )
       },
-      '.cm-lintRange-warning': {
+      '.cm-editor .cm-lintRange-warning': {
         backgroundImage: underline(
           style.getPropertyValue(
             '--jp-editor-mirror-lsp-diagnostic-warning-decoration-color'
           )
         )
       },
-      '.cm-lintRange-info': {
+      '.cm-editor .cm-lintRange-info': {
         backgroundImage: underline(
           style.getPropertyValue(
             '--jp-editor-mirror-lsp-diagnostic-information-decoration-color'
           )
         )
       },
-      '.cm-lintRange-hint': {
+      '.cm-editor .cm-lintRange-hint': {
         backgroundImage: underline(
           style.getPropertyValue(
             '--jp-editor-mirror-lsp-diagnostic-hint-decoration-color'
@@ -270,13 +271,7 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
         )
       }
     });
-    const tempEditor = new EditorView({ extensions: baseTheme });
-    const styleModules = tempEditor.state.facet(EditorView.styleModule);
-    const ourRules = styleModules.find(styleModule => {
-      const rules = styleModule.getRules().split('\n');
-      return rules.every(rule => rule.includes('cm-lintRange'));
-    })!;
-    this._styleElement.innerHTML = ourRules.getRules();
+    this._styleElement.innerHTML = lintTheme.getRules();
   }
 
   clearDocumentDiagnostics(

From 6ebf1d9d8c718045fa4bb1fbf558be422c5199e1 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:09:09 +0100
Subject: [PATCH 10/50] Fix jump between files test

---
 atest/05_Features/Jump.robot | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/atest/05_Features/Jump.robot b/atest/05_Features/Jump.robot
index 69ddf2cfa..2145aa3a7 100644
--- a/atest/05_Features/Jump.robot
+++ b/atest/05_Features/Jump.robot
@@ -15,8 +15,7 @@ Python Jumps Between Files
     Copy Files to Folder With Spaces    jump_a.py    jump_b.py
     Open ${FOLDER WITH SPACE}/jump_b.py in ${MENU EDITOR}
     Wait Until Fully Initialized
-    ${sel} =    Select Token Occurrence    a_function_definition
-    Jump To Definition    ${sel}
+    Jump To Definition    lastToken:a_function_definition
     Wait Until Page Contains    ANOTHER_CONSTANT
     Capture Page Screenshot    10-jumped.png
     [Teardown]    Clean Up Folder With Spaces    jump_a.py    jump_b.py

From 8f0c9d721ac194b8b2fadec22f029fe811e2c448 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:23:47 +0100
Subject: [PATCH 11/50] Fix bug in syntax highlight not switching for JS cells

---
 packages/jupyterlab-lsp/src/features/syntax_highlighting.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/jupyterlab-lsp/src/features/syntax_highlighting.ts b/packages/jupyterlab-lsp/src/features/syntax_highlighting.ts
index fc3bad706..bbba89d87 100644
--- a/packages/jupyterlab-lsp/src/features/syntax_highlighting.ts
+++ b/packages/jupyterlab-lsp/src/features/syntax_highlighting.ts
@@ -138,6 +138,12 @@ export class SyntaxHighlightingFeature extends Feature {
       return;
     }
 
+    if (Array.isArray(mimetype)) {
+      // Contrarily to what types say, mimetype can be an array.
+      // https://github.com/jupyterlab/jupyterlab/issues/15100
+      return mimetype[0];
+    }
+
     return mimetype;
   }
 

From 146f561073918c22da96d8426e809aaf4eb3a7aa Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:24:43 +0100
Subject: [PATCH 12/50] Remove JupyterLab 3 compatibility code

---
 atest/Keywords.resource | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/atest/Keywords.resource b/atest/Keywords.resource
index 49f348ec7..57e911768 100644
--- a/atest/Keywords.resource
+++ b/atest/Keywords.resource
@@ -398,11 +398,7 @@ Open File
 
 Open in Advanced Settings
     [Arguments]    ${plugin id}
-    IF    '${LAB VERSION}'.startswith('3.')
-        Lab Command    Advanced JSON Settings Editor
-    ELSE
-        Lab Command    Advanced Settings Editor
-    END
+    Lab Command    Advanced Settings Editor
     ${sel} =    Set Variable    css:[data-id="${plugin id}"]
     Wait Until Page Contains Element    ${sel}    timeout=10s
     Click Element    ${sel}

From 1d144903fe777823284a5b8b4e5a4d07d825c647 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:31:33 +0100
Subject: [PATCH 13/50] Remove pyright fixture pin

---
 atest/fixtures/overrides.json | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/atest/fixtures/overrides.json b/atest/fixtures/overrides.json
index c4cb73ca9..726fe9625 100644
--- a/atest/fixtures/overrides.json
+++ b/atest/fixtures/overrides.json
@@ -9,12 +9,6 @@
           "pylsp.plugins.flake8.enabled": true,
           "pylsp.plugins.pyflakes.enabled": false
         }
-      },
-      "pyright": {
-        "priority": 500,
-        "serverSettings": {
-          "python.analysis.useLibraryCodeForTypes": false
-        }
       }
     }
   }

From 5495bf2c6a4f06e0dbb5dcc54e7ebbbcb9b6d265 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:32:22 +0100
Subject: [PATCH 14/50] Fix a typo

---
 packages/jupyterlab-lsp/src/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/jupyterlab-lsp/src/index.ts b/packages/jupyterlab-lsp/src/index.ts
index 6b2a4dc97..379414f88 100644
--- a/packages/jupyterlab-lsp/src/index.ts
+++ b/packages/jupyterlab-lsp/src/index.ts
@@ -156,7 +156,7 @@ export class LSPExtension {
       {}) as TLanguageServerConfigurations;
 
     // Add `configuration` as a copy of `serverSettings` to work with changed name upstream
-    // Add `rank` as a copy of priority fot the same reason.
+    // Add `rank` as a copy of priority for the same reason.
     languageServerSettings = Object.fromEntries(
       Object.entries(languageServerSettings).map(([key, value]) => {
         value.configuration = value.serverSettings;

From 5c4a42b2d39cb8211fb9f2e98533d979bbd33021 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:43:59 +0100
Subject: [PATCH 15/50] Fix hover+wiggle keywords

---
 atest/mouse_over_extension.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/atest/mouse_over_extension.py b/atest/mouse_over_extension.py
index 88f020500..0c18125be 100644
--- a/atest/mouse_over_extension.py
+++ b/atest/mouse_over_extension.py
@@ -6,10 +6,8 @@
 
 def wiggle(action, x_wiggle):
     if x_wiggle:
-        #action.pointer_action.move_by(xoffset=x_wiggle, yoffset=0)
-        action.key_action.pause()
-        #action.pointer_action.move_by(xoffset=-x_wiggle, yoffset=0)
-        action.key_action.pause()
+        action.pointer_action.move_by(x_wiggle, 0)
+        action.pointer_action.move_by(-x_wiggle, 0)
 
 
 def mouse_over_token_with_control(token_locator, x_wiggle=0):
@@ -18,7 +16,7 @@ def mouse_over_token_with_control(token_locator, x_wiggle=0):
     action = ActionBuilder(sl.driver)
 
     action.key_action.key_down(Keys.CONTROL)
-    #action.pointer_action.pause()
+
     action.pointer_action.move_to_location(location['x'], location['y'])
     wiggle(action, x_wiggle)
     action.key_action.key_up(Keys.CONTROL)

From b87cf453cad147fd203abcd1dbc21a082984606f Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:51:56 +0100
Subject: [PATCH 16/50] Update hover js case to pass foreign threshold

---
 atest/examples/Hover.ipynb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/atest/examples/Hover.ipynb b/atest/examples/Hover.ipynb
index e68905259..4c77b42cd 100644
--- a/atest/examples/Hover.ipynb
+++ b/atest/examples/Hover.ipynb
@@ -35,7 +35,9 @@
    "outputs": [],
    "source": [
     "%%javascript\n",
-    "Math"
+    "function range(x) {\n",
+    "    Math.max(x) - Math.min(x)\n",
+    "}"
    ]
   },
   {
@@ -51,7 +53,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -65,7 +67,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.9"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

From 1133ff8401da6b0a84ecbc2e81fb69743e0d4ad5 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 02:52:38 +0100
Subject: [PATCH 17/50] Fix typo

---
 atest/05_Features/Hover.robot | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/atest/05_Features/Hover.robot b/atest/05_Features/Hover.robot
index 67bb7320c..7c7117e30 100644
--- a/atest/05_Features/Hover.robot
+++ b/atest/05_Features/Hover.robot
@@ -70,13 +70,13 @@ Update hover after character deletion
     Enter Cell Editor    4
     Trigger Tooltip    atan2
     Element Text Should Be    ${HOVER_SIGNAL}    atan2
-    Capture Page Screenshot    01-hover-before-delection.png
+    Capture Page Screenshot    01-hover-before-deletion.png
     Element Should Contain    ${HOVER_BOX}    atan2(y: SupportsFloat, x: SupportsFloat, /)
     Place Cursor In Cell Editor At    4    line=2    character=13
     Press Keys    None    DELETE
     Trigger Tooltip    atan
     Element Text Should Be    ${HOVER_SIGNAL}    atan
-    Capture Page Screenshot    02-hover-after-delection.png
+    Capture Page Screenshot    02-hover-after-deletion.png
     Element Should Contain    ${HOVER_BOX}    atan(x: SupportsFloat, /)
 
 

From dc58ac76555ce8ae2f6066a4549d1c3c660c93ec Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 11:37:02 +0100
Subject: [PATCH 18/50] Update expectation on copy notification which is now in
 a toast

---
 atest/04_Interface/DiagnosticsPanel.robot | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index 25fbb3975..8f9511e49 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -101,7 +101,7 @@ Diagnostic Message Can Be Copied
     Open Context Menu Over    css:.lsp-diagnostics-listing tbody tr
     Select Menu Entry    Copy diagnostic
     Close Diagnostics Panel
-    Wait Until Element Contains    css:.lsp-statusbar-item    Successfully copied    timeout=10s
+    Wait Until Element Contains    css:.jp-toast-message    Successfully copied    timeout=10s
 
 Diagnostics Panel Works After Removing Foreign Document
     Enter Cell Editor    2
@@ -134,7 +134,7 @@ Open Context Menu Over W291
 Open Notebook And Panel
     [Arguments]    ${notebook}
     Setup Notebook    Python    ${notebook}
-    Capture Page Screenshot    00-notebook-and-panel-openeing.png
+    Capture Page Screenshot    00-notebook-and-panel-opening.png
     Wait Until Page Contains Diagnostic    [title*="${DIAGNOSTIC}"]    timeout=20s
     Open Diagnostics Panel
     Capture Page Screenshot    00-notebook-and-panel-opened.png

From 113e0b548f982309dc6e88797c96c0bc18b8fa1e Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 12:09:49 +0100
Subject: [PATCH 19/50] Try updating pins for python linting

---
 requirements/github-actions.yml | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/requirements/github-actions.yml b/requirements/github-actions.yml
index e01cfff56..c131f33e6 100644
--- a/requirements/github-actions.yml
+++ b/requirements/github-actions.yml
@@ -12,9 +12,10 @@ dependencies:
   - pip
   - nodejs {nodejs}
   # for python language server (and development)
-  - python-lsp-server
-  - flake8 >=3.5
-  - autopep8 <1.6.0
+  - python-lsp-server >=1.8.0
+  - flake8 >=6.1.0
+  - autopep8 >=1.6.0
+  - pyflakes >=3.1.0
   # robotframework for testing and language server
   - jupyterlab_robotmode
   - robotframework >=4
@@ -36,12 +37,8 @@ dependencies:
   # test tools
   - pytest-asyncio
   - pytest-cov
-  # temporary pin added on 2022-12-28, if editing this file try to remove it:
-  # pytoolconfig 1.2.4 depends on packaging>=22.0 which breaks `pip check`
-  - pytoolconfig <1.2.4
-  # see https://github.com/conda-forge/pytest-flake8-feedstock/issues/20
-  # and https://github.com/tholo/pytest-flake8/issues/86
-  - pytest-flake8 <1.1.1
+  - pytoolconfig
+  - pytest-flake8
   - pytest-runner
   - ruamel_yaml
   - pytest-github-actions-annotate-failures

From f3a3db3fa11ce85afd0dc8130eb05f3a235052ed Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 13:16:46 +0100
Subject: [PATCH 20/50] Downgrade pin since pylsp 1.8 is not yet on conda-forge

---
 requirements/github-actions.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements/github-actions.yml b/requirements/github-actions.yml
index c131f33e6..fadbd4f90 100644
--- a/requirements/github-actions.yml
+++ b/requirements/github-actions.yml
@@ -12,10 +12,10 @@ dependencies:
   - pip
   - nodejs {nodejs}
   # for python language server (and development)
-  - python-lsp-server >=1.8.0
-  - flake8 >=6.1.0
-  - autopep8 >=1.6.0
-  - pyflakes >=3.1.0
+  - python-lsp-server >=1.7.4
+  - flake8
+  - autopep8
+  - pyflakes
   # robotframework for testing and language server
   - jupyterlab_robotmode
   - robotframework >=4

From edd88e264f59f2ce3d30e072ecf248d009fb0cb2 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 9 Sep 2023 13:54:49 +0100
Subject: [PATCH 21/50] Transition from comment type annotations not supported
 in newer pyflakes

Reference: https://github.com/PyCQA/pyflakes/issues/747
---
 .../jupyter_lsp/jupyter_lsp/handlers.py        |  2 +-
 .../jupyter_lsp/jupyter_lsp/manager.py         | 18 +++++++++---------
 .../jupyter_lsp/jupyter_lsp/types.py           |  5 ++---
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/python_packages/jupyter_lsp/jupyter_lsp/handlers.py b/python_packages/jupyter_lsp/jupyter_lsp/handlers.py
index a7312f4a4..704852916 100644
--- a/python_packages/jupyter_lsp/jupyter_lsp/handlers.py
+++ b/python_packages/jupyter_lsp/jupyter_lsp/handlers.py
@@ -23,7 +23,7 @@ class LanguageServerWebSocketHandler(  # type: ignore
 ):
     """Setup tornado websocket to route to language server sessions"""
 
-    language_server = None  # type: Optional[Text]
+    language_server: Optional[Text] = None
 
     async def open(self, language_server):
         await self.manager.ready()
diff --git a/python_packages/jupyter_lsp/jupyter_lsp/manager.py b/python_packages/jupyter_lsp/jupyter_lsp/manager.py
index 28e05eb05..2d9597e0c 100644
--- a/python_packages/jupyter_lsp/jupyter_lsp/manager.py
+++ b/python_packages/jupyter_lsp/jupyter_lsp/manager.py
@@ -61,17 +61,17 @@ class LanguageServerManager(LanguageServerManagerAPI):
         config=True
     )  # type: KeyedLanguageServerSpecs
 
-    autodetect = Bool(
+    autodetect: bool = Bool(
         True, help=_("try to find known language servers in sys.prefix (and elsewhere)")
     ).tag(
         config=True
-    )  # type: bool
+    )
 
-    sessions = Dict_(  # type:ignore[assignment]
+    sessions: Dict[Tuple[Text], LanguageServerSession] = Dict_(  # type:ignore[assignment]
         trait=Instance(LanguageServerSession),
         default_value={},
         help="sessions keyed by language server name",
-    )  # type: Dict[Tuple[Text], LanguageServerSession]
+    )
 
     virtual_documents_dir = Unicode(
         help="""Path to virtual documents relative to the content manager root
@@ -99,8 +99,8 @@ def _default_virtual_documents_dir(self):
         return os.getenv("JP_LSP_VIRTUAL_DIR", ".virtual_documents")
 
     @default("conf_d_language_servers")
-    def _default_conf_d_language_servers(self):
-        language_servers = {}  # type: KeyedLanguageServerSpecs
+    def _default_conf_d_language_servers(self) -> KeyedLanguageServerSpecs:
+        language_servers: KeyedLanguageServerSpecs = {}
 
         manager = ConfigManager(read_config_path=jupyter_config_path())
 
@@ -113,10 +113,10 @@ def _default_conf_d_language_servers(self):
 
         return language_servers
 
-    def __init__(self, **kwargs):
+    def __init__(self, **kwargs: Dict):
         """Before starting, perform all necessary configuration"""
         self.all_language_servers: KeyedLanguageServerSpecs = {}
-        self._language_servers_from_config = {}
+        self._language_servers_from_config: KeyedLanguageServerSpecs = {}
         super().__init__(**kwargs)
 
     def initialize(self, *args, **kwargs):
@@ -253,7 +253,7 @@ def _autodetect_language_servers(self, only_installed: bool):
 
         for ep in _entry_points or []:
             try:
-                spec_finder = ep.load()  # type: SpecMaker
+                spec_finder: SpecMaker = ep.load()
             except Exception as err:  # pragma: no cover
                 self.log.warning(
                     _("Failed to load language server spec finder `{}`: \n{}").format(
diff --git a/python_packages/jupyter_lsp/jupyter_lsp/types.py b/python_packages/jupyter_lsp/jupyter_lsp/types.py
index d9581f809..ea10592f6 100644
--- a/python_packages/jupyter_lsp/jupyter_lsp/types.py
+++ b/python_packages/jupyter_lsp/jupyter_lsp/types.py
@@ -72,9 +72,8 @@ class MessageScope(enum.Enum):
 class MessageListener(object):
     """A base listener implementation"""
 
-    listener = None  # type: HandlerListenerCallback
-    language_server = None  # type: Optional[Pattern[Text]]
-    method = None  # type: Optional[Pattern[Text]]
+    language_server: Optional[Pattern[Text]] = None
+    method: Optional[Pattern[Text]] = None
 
     def __init__(
         self,

From 3fb2bdef2ca666a998231c01972102df5bef787f Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 05:14:03 +0100
Subject: [PATCH 22/50] Fix defaults population which requires schema to be
 available,

and a workaround of sending the update again because the connection
manager is now a separate plugin which starts and can sent initial
configuration before our plugin gets to setting transformation.

Also use promise delegate over custom method to simplify code.
---
 packages/jupyterlab-lsp/src/index.ts    | 21 +++++++++++++++++++
 packages/jupyterlab-lsp/src/settings.ts | 27 ++++++++++++-------------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/index.ts b/packages/jupyterlab-lsp/src/index.ts
index 379414f88..f004be14d 100644
--- a/packages/jupyterlab-lsp/src/index.ts
+++ b/packages/jupyterlab-lsp/src/index.ts
@@ -165,6 +165,7 @@ export class LSPExtension {
       })
     );
 
+    const previousInitialConfig = this._connectionManager.initialConfigurations;
     this._connectionManager.initialConfigurations = languageServerSettings;
     // TODO: if priorities changed reset connections
 
@@ -172,6 +173,26 @@ export class LSPExtension {
     this.connectionManager.updateConfiguration(languageServerSettings);
     if (afterInitialization) {
       this.connectionManager.updateServerConfigurations(languageServerSettings);
+    } else {
+      // This would not be needed if we controlled the connection manager, but because
+      // it is now an independent plugin it may have started and finished initialization
+      // earlier, so it potentially sent wrong `initialConfigurations` and we have no way
+      // to avoid this, so we need to send a reconfiguration request instead.
+      //
+      // TODO: this is comparing objects which is always false,
+      // we should have a proper comparison to avoid redundant calls,
+      // but in practice because upstream never populates defaults and
+      // we always do, this means it would be false anyways,
+      // so for now the comparison serves more as a comment than logic.
+      if (previousInitialConfig != languageServerSettings) {
+        this.connectionManager.ready
+          .then(() => {
+            this.connectionManager.updateServerConfigurations(
+              languageServerSettings
+            );
+          })
+          .catch(console.error);
+      }
     }
     this._connectionManager.updateLogging(
       options.logAllCommunication,
diff --git a/packages/jupyterlab-lsp/src/settings.ts b/packages/jupyterlab-lsp/src/settings.ts
index 224a83092..6da7ba0c2 100644
--- a/packages/jupyterlab-lsp/src/settings.ts
+++ b/packages/jupyterlab-lsp/src/settings.ts
@@ -8,7 +8,8 @@ import { TranslationBundle } from '@jupyterlab/translation';
 import {
   JSONExt,
   ReadonlyPartialJSONObject,
-  ReadonlyJSONObject
+  ReadonlyJSONObject,
+  PromiseDelegate
 } from '@lumino/coreutils';
 import { Signal, ISignal } from '@lumino/signaling';
 import { FieldProps } from '@rjsf/utils';
@@ -145,18 +146,9 @@ export class SettingsSchemaManager {
     this._lastValidation = null;
     this._lastUserServerSettings = null;
     this._lastUserServerSettingsDoted = null;
-    this._defaultsPopulated = this._createDefaultsPromise();
     this._validationErrors = [];
   }
 
-  private _defaultsPopulated: Promise<void>;
-  private _populatedAccept: (value: unknown) => void;
-  private _createDefaultsPromise(): Promise<void> {
-    return new Promise(accept => {
-      this._populatedAccept = accept;
-    });
-  }
-
   get schemaValidated(): ISignal<
     SettingsSchemaManager,
     ISchemaValidator.IError[]
@@ -175,6 +167,9 @@ export class SettingsSchemaManager {
    */
   async setupSchemaTransform(pluginId: string): Promise<void> {
     const languageServerManager = this.options.languageServerManager;
+    // To populate defaults we require specs to be available, so we need to
+    // wait for until after the `languageServerManager` is ready.
+    await languageServerManager.ready;
 
     // Transform the plugin object to return different schema than the default.
     this.options.settingRegistry.transform(pluginId, {
@@ -200,7 +195,7 @@ export class SettingsSchemaManager {
         if (!this._canonical) {
           this._canonical = JSONExt.deepCopy(plugin.schema);
           this._populate(plugin, this._canonical);
-          this._populatedAccept(void 0);
+          this._defaultsPopulated.resolve(void 0);
         }
 
         return {
@@ -219,7 +214,7 @@ export class SettingsSchemaManager {
     // race condition, see https://github.com/jupyterlab/jupyterlab/issues/12978
     languageServerManager.sessionsChanged.connect(async () => {
       this._canonical = null;
-      this._defaultsPopulated = this._createDefaultsPromise();
+      this._defaultsPopulated = new PromiseDelegate();
       await this.options.settingRegistry.reload(pluginId);
     });
   }
@@ -272,7 +267,7 @@ export class SettingsSchemaManager {
       }
       if (!configSchema.properties) {
         this.console.warn(
-          'No properites in config schema - skipping transformation for',
+          'No properties in config schema - skipping transformation for',
           serverKey
         );
         continue;
@@ -373,10 +368,13 @@ export class SettingsSchemaManager {
     this._defaults = defaults;
   }
 
+  /**
+   * Normalize settings by dotted and nested specs, and merging with defaults.
+   */
   async normalizeSettings(
     composite: Required<LanguageServers>
   ): Promise<Required<LanguageServers>> {
-    await this._defaultsPopulated;
+    await this._defaultsPopulated.promise;
     // Cache collapsed settings for speed and to only show dialog once.
     // Note that JupyterLab attempts to transform in "preload" step (before splash screen end)
     // and then again for deferred extensions if the initial transform in preload timed out.
@@ -583,6 +581,7 @@ export class SettingsSchemaManager {
   }
 
   private _defaults: LanguageServerSettings;
+  private _defaultsPopulated = new PromiseDelegate();
   private _validationErrors: ISchemaValidator.IError[];
   private _schemaValidated: Signal<
     SettingsSchemaManager,

From c6de5a5e88f72e77ebcbe9f5d6d177c5e6247a05 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 06:22:12 +0100
Subject: [PATCH 23/50] Restore the default diagnostics timeout to 5s

as this is the default for `Wait Until Page Contains Element`
---
 atest/diagnostics.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/atest/diagnostics.py b/atest/diagnostics.py
index 4bd37cd24..85224491d 100644
--- a/atest/diagnostics.py
+++ b/atest/diagnostics.py
@@ -27,7 +27,7 @@ def page_contains_diagnostic(driver: WebDriver, selector, negate=False):
     return False if negate else True
 
 
-def wait_until_page_contains_diagnostic(selector, timeout='3s'):
+def wait_until_page_contains_diagnostic(selector, timeout='5s'):
     sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
     wait = WebDriverWait(sl.driver, timestr_to_secs(timeout))
     try:
@@ -50,7 +50,7 @@ def wait_until_page_contains_diagnostic(selector, timeout='3s'):
         )
 
 
-def wait_until_page_does_not_contain_diagnostic(selector, timeout='3s'):
+def wait_until_page_does_not_contain_diagnostic(selector, timeout='5s'):
     sl: SeleniumLibrary = BuiltIn().get_library_instance("SeleniumLibrary")
     wait = WebDriverWait(sl.driver, timestr_to_secs(timeout))
     return wait.until(

From 1cbf40184624b65e32f699854db8a59bfd0083ca Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 11:19:26 +0100
Subject: [PATCH 24/50] Update expectation for R lint message

---
 atest/04_Interface/DiagnosticsPanel.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index 8f9511e49..c2f51ac29 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -9,7 +9,7 @@ Test Tags           ui:notebook    aspect:ls:features
 
 
 *** Variables ***
-${DIAGNOSTIC MESSAGE R}     Closing curly-braces should always be on their own line
+${DIAGNOSTIC MESSAGE R}     Trailing blank lines are superfluous.
 ${DIAGNOSTIC MESSAGE}       trailing whitespace
 ${DIAGNOSTIC}               W291 trailing whitespace (pycodestyle)
 ${EXPECTED_COUNT}           4

From 9fce3accf4fe846e5091da158ab52631aba0c489 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 11:23:25 +0100
Subject: [PATCH 25/50] Update command name to singular which is used for n=1
 in lab 4.0

---
 atest/04_Interface/DiagnosticsPanel.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index c2f51ac29..a71c5cd80 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -112,7 +112,7 @@ Diagnostics Panel Works After Removing Foreign Document
     ...    ${DIAGNOSTIC MESSAGE}
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
     ...    ${DIAGNOSTIC MESSAGE R}
-    Lab Command    Delete Cells
+    Lab Command    Delete Cell
     # regain focus by entering cell
     Enter Cell Editor    2
     # trigger 7 document updates to trigger the garbage collector that removes unused documents

From 7203db3cb0ea04f695f36db0121927695b969f59 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 12:16:00 +0100
Subject: [PATCH 26/50] Improve styling of diagnostics placeholder

---
 .../src/features/diagnostics/listing.tsx               |  5 ++++-
 packages/jupyterlab-lsp/style/diagnostics_listing.css  | 10 ++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/packages/jupyterlab-lsp/src/features/diagnostics/listing.tsx b/packages/jupyterlab-lsp/src/features/diagnostics/listing.tsx
index 0cda4d357..651efc6aa 100644
--- a/packages/jupyterlab-lsp/src/features/diagnostics/listing.tsx
+++ b/packages/jupyterlab-lsp/src/features/diagnostics/listing.tsx
@@ -265,7 +265,10 @@ export class DiagnosticsListing extends VDomRenderer<DiagnosticsListing.Model> {
     if (diagnosticsDatabase == null || !adapter) {
       return (
         <div className={DIAGNOSTICS_PLACEHOLDER_CLASS}>
-          {this.trans.__('Diagnostics are not available')}
+          <h3>No diagnostics</h3>
+          {this.trans.__(
+            'Diagnostics panel shows linting results in notebooks and files connected to a language server.'
+          )}
         </div>
       );
     }
diff --git a/packages/jupyterlab-lsp/style/diagnostics_listing.css b/packages/jupyterlab-lsp/style/diagnostics_listing.css
index 841fb5410..ddc46e19d 100644
--- a/packages/jupyterlab-lsp/style/diagnostics_listing.css
+++ b/packages/jupyterlab-lsp/style/diagnostics_listing.css
@@ -2,6 +2,16 @@
   overflow-y: auto;
 }
 
+.lsp-diagnostics-placeholder {
+  padding: 8px;
+  color: var(--jp-content-font-color2);
+  text-align: center;
+}
+
+.lsp-diagnostics-placeholder > h3 {
+  margin-bottom: var(--jp-content-heading-margin-bottom);
+}
+
 .lsp-diagnostics-listing {
   color: var(--jp-ui-font-color1);
   background: var(--jp-layout-color1);

From 900ab24dca76508ae8df0c3a89fe906be8be6381 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 12:25:34 +0100
Subject: [PATCH 27/50] Improve diagnostics panel connection and disconnection
 logic

---
 .../src/features/diagnostics/feature.ts       | 47 +++++++++++++++----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
index 75b4a1b68..0a40bd8f7 100644
--- a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
+++ b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
@@ -78,6 +78,7 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
       );
       virtualDocument.foreignDocumentClosed.connect((document, context) => {
         // TODO: check if we need to cast
+        console.log('foreing closed');
         this.clearDocumentDiagnostics(adapter, context.foreignDocument);
       });
     });
@@ -101,12 +102,38 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
     const connectionManager = options.connectionManager;
     // https://github.com/jupyterlab/jupyterlab/issues/14783
     options.shell.currentChanged.connect(shell => {
+      if (shell.currentWidget == diagnosticsPanel.widget) {
+        // allow focusing on the panel
+        return;
+      }
+
       const adapter = [...connectionManager.adapters.values()].find(
         adapter => adapter.widget == shell.currentWidget
       );
 
       if (!adapter) {
-        this.console.debug('No adapter');
+        this.switchDiagnosticsPanelSource(null);
+        // this dance should not be needed once https://github.com/jupyterlab/jupyterlab/pull/14920 is in,
+        // but we will need to continue listening to `currentChanged` signal anyways to make sure we show
+        // empty indicator in launcher or other widget which does not support linting.
+        let attemptsLeft = 3;
+        const retry = () => {
+          const adapter = [...connectionManager.adapters.values()].find(
+            adapter => adapter.widget == shell.currentWidget
+          );
+          attemptsLeft -= 1;
+          if (adapter) {
+            this.switchDiagnosticsPanelSource(adapter);
+            attemptsLeft = 0;
+          }
+          if (attemptsLeft == 0) {
+            connectionManager.connected.disconnect(retry);
+          }
+        };
+        connectionManager.connected.connect(retry);
+        this.console.debug(
+          'No adapter (yet?), will retry on next connected document'
+        );
       } else {
         this.switchDiagnosticsPanelSource(adapter);
       }
@@ -298,16 +325,20 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
     return this._diagnosticsDatabases.get(adapter)!;
   }
 
-  switchDiagnosticsPanelSource = (adapter: WidgetLSPAdapter<any>) => {
+  switchDiagnosticsPanelSource = (adapter: WidgetLSPAdapter<any> | null) => {
     diagnosticsPanel.trans = this._trans;
-    const diagnostics = this.getDiagnosticsDB(adapter);
-    if (diagnosticsPanel.content.model.diagnostics == diagnostics) {
-      return;
+    if (adapter !== null) {
+      const diagnostics = this.getDiagnosticsDB(adapter);
+      if (diagnosticsPanel.content.model.diagnostics == diagnostics) {
+        return;
+      }
+      diagnosticsPanel.content.model.diagnostics = diagnostics;
+      diagnosticsPanel.content.model.settings = this.settings;
+      diagnosticsPanel.feature = this;
+    } else {
+      diagnosticsPanel.content.model.diagnostics = null;
     }
-    diagnosticsPanel.content.model.diagnostics = diagnostics;
     diagnosticsPanel.content.model.adapter = adapter;
-    diagnosticsPanel.content.model.settings = this.settings;
-    diagnosticsPanel.feature = this;
     diagnosticsPanel.update();
   };
 

From 7de2052de52ddacc062fda54d952744ad1942d63 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 10 Sep 2023 14:52:10 +0100
Subject: [PATCH 28/50] Fix removal of outdated diagnostics from foreign
 documents

---
 atest/04_Interface/DiagnosticsPanel.robot     | 20 +++++++---
 .../jupyterlab-lsp/src/adapters/notebook.ts   |  8 ++--
 .../jupyterlab-lsp/src/virtual/document.ts    | 38 +++++++++++++++++++
 3 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index a71c5cd80..0c9e5e642 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -9,12 +9,11 @@ Test Tags           ui:notebook    aspect:ls:features
 
 
 *** Variables ***
-${DIAGNOSTIC MESSAGE R}     Trailing blank lines are superfluous.
+${DIAGNOSTIC MESSAGE R}     Closing curly-braces should always be on their own line
 ${DIAGNOSTIC MESSAGE}       trailing whitespace
 ${DIAGNOSTIC}               W291 trailing whitespace (pycodestyle)
 ${EXPECTED_COUNT}           4
 ${MENU COLUMNS}             xpath://div[contains(@class, 'lm-Menu-itemLabel')][contains(text(), "columns")]
-${R CELL}                   %%R\n{}
 
 
 *** Test Cases ***
@@ -107,7 +106,11 @@ Diagnostics Panel Works After Removing Foreign Document
     Enter Cell Editor    2
     Lab Command    Insert Cell Below
     Enter Cell Editor    3
-    Press Keys    None    ${R CELL}
+    Press Keys    None    %%R\n
+    # these two steps ideally would not be needed (they show that for slow-starting server
+    # update may not be triggered until user manually makes another action).
+    Wait Until Fully Initialized
+    Press Keys    None    {}
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
     ...    ${DIAGNOSTIC MESSAGE}
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
@@ -116,13 +119,20 @@ Diagnostics Panel Works After Removing Foreign Document
     # regain focus by entering cell
     Enter Cell Editor    2
     # trigger 7 document updates to trigger the garbage collector that removes unused documents
-    # (search for VirtualDocument.remainining_lifetime for more)
+    # (search for VirtualDocument.remainingLifetime for more)
     Press Keys    None    1234567
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
     ...    ${DIAGNOSTIC MESSAGE}
     Wait Until Keyword Succeeds    10 x    1s    Element Should Not Contain    ${DIAGNOSTICS PANEL}
     ...    ${DIAGNOSTIC MESSAGE R}
-
+    # it should be possible to get the diagnostic back after re-creatign the cell
+    Lab Command    Insert Cell Below
+    Enter Cell Editor    3
+    Press Keys    None    %%R\n{}
+    Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
+    ...    ${DIAGNOSTIC MESSAGE}
+    Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}
+    ...    ${DIAGNOSTIC MESSAGE R}
 
 *** Keywords ***
 Open Context Menu Over W291
diff --git a/packages/jupyterlab-lsp/src/adapters/notebook.ts b/packages/jupyterlab-lsp/src/adapters/notebook.ts
index 89723263f..e76ee1d0a 100644
--- a/packages/jupyterlab-lsp/src/adapters/notebook.ts
+++ b/packages/jupyterlab-lsp/src/adapters/notebook.ts
@@ -48,21 +48,23 @@ export class NotebookAdapter extends UpstreamNotebookAdapter {
   }
 
   protected async onForeignDocumentOpened(
-    _: VirtualDocument,
+    virtualDocument: VirtualDocument,
     context: Document.IForeignContext
   ): Promise<void> {
     /*
-    Opening a standalone foreign document can result in an inifnite loop,
+    Opening a standalone foreign document can result in an infinite loop,
     as a new connection gets opened for these, and once that is ready
     `updateDocuments()` gets called.
 
     To avoid the problem, `updateDocuments()` gets suppressed for standalone
     documents. It does not affect non-standalone documents, because no new
     connection gets opened for these.
+
+    https://github.com/jupyter-lsp/jupyterlab-lsp/issues/959
     */
     try {
       this._blockUpdates = true;
-      await super.onForeignDocumentOpened(_, context);
+      await super.onForeignDocumentOpened(virtualDocument, context);
     } finally {
       this._blockUpdates = false;
     }
diff --git a/packages/jupyterlab-lsp/src/virtual/document.ts b/packages/jupyterlab-lsp/src/virtual/document.ts
index f42ddae55..550d404b5 100644
--- a/packages/jupyterlab-lsp/src/virtual/document.ts
+++ b/packages/jupyterlab-lsp/src/virtual/document.ts
@@ -141,6 +141,44 @@ export class VirtualDocument extends VirtualDocumentBase {
     this.lastSourceLine += sourceCellLines.length;
   }
 
+  /**
+   * Close all expired documents.
+   */
+  closeExpiredDocuments(): void {
+    const usedDocuments = new Set<VirtualDocument>();
+    for (const line of this.sourceLines.values()) {
+      for (const block of line.foreignDocumentsMap.values()) {
+        usedDocuments.add(block.virtualDocument as VirtualDocument);
+      }
+    }
+
+    const documentIDs = new Map<VirtualDocument, string[]>();
+    for (const [id, document] of (
+      this.foreignDocuments as Map<string, VirtualDocument>
+    ).entries()) {
+      const ids = documentIDs.get(document);
+      if (typeof ids !== 'undefined') {
+        documentIDs.set(document, [...ids, id]);
+      }
+      documentIDs.set(document, [id]);
+    }
+    const allDocuments = new Set<VirtualDocument>(documentIDs.keys());
+    const unusedVirtualDocuments = new Set(
+      [...allDocuments].filter(x => !usedDocuments.has(x))
+    );
+
+    for (let document of unusedVirtualDocuments.values()) {
+      document.remainingLifetime -= 1;
+      if (document.remainingLifetime <= 0) {
+        document.dispose();
+        const ids = documentIDs.get(document)!;
+        for (const id of ids) {
+          this.foreignDocuments.delete(id);
+        }
+      }
+    }
+  }
+
   /**
    * @experimental
    */

From b8c5dfaaef63bbfafbe25da6dd10952e50fe3a4b Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Mon, 11 Sep 2023 00:10:05 +0100
Subject: [PATCH 29/50] Check diagnostic count before opening settings

as settings can occlude the diagnostics panel
---
 atest/04_Interface/DiagnosticsPanel.robot | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index 0c9e5e642..4541cc239 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -78,9 +78,9 @@ Diagnostics Can Be Ignored By Code
     Open Context Menu Over W291
     Expand Menu Entry    Ignore diagnostics
     Select Menu Entry    code
+    Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
     Open in Advanced Settings    ${DIAGNOSTICS PLUGIN ID}
     Capture Page Screenshot    02-code-pressed.png
-    Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
 
 Diagnostics Can Be Ignored By Message
     Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_COUNT}
@@ -90,9 +90,9 @@ Diagnostics Can Be Ignored By Message
     Expand Menu Entry    Ignore diagnostics
     Capture Page Screenshot    02-menu-visible.png
     Select Menu Entry    Ignore diagnostics with "W291 trailing whitespace" message
+    Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
     Open in Advanced Settings    ${DIAGNOSTICS PLUGIN ID}
     Capture Page Screenshot    02-message-pressed.png
-    Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
 
 Diagnostic Message Can Be Copied
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}

From 5f9fab5878e4f88f4cadcf9a0e2a436f13b603c7 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Mon, 11 Sep 2023 00:16:21 +0100
Subject: [PATCH 30/50] Clean settings after testing ignoring diagnostics

which is what was causing the remaining tests to be flaky
---
 atest/04_Interface/DiagnosticsPanel.robot | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot
index 4541cc239..42a7b1cf8 100644
--- a/atest/04_Interface/DiagnosticsPanel.robot
+++ b/atest/04_Interface/DiagnosticsPanel.robot
@@ -79,8 +79,7 @@ Diagnostics Can Be Ignored By Code
     Expand Menu Entry    Ignore diagnostics
     Select Menu Entry    code
     Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
-    Open in Advanced Settings    ${DIAGNOSTICS PLUGIN ID}
-    Capture Page Screenshot    02-code-pressed.png
+    Configure JupyterLab Plugin    {}    plugin id=${DIAGNOSTICS PLUGIN ID}
 
 Diagnostics Can Be Ignored By Message
     Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_COUNT}
@@ -91,8 +90,7 @@ Diagnostics Can Be Ignored By Message
     Capture Page Screenshot    02-menu-visible.png
     Select Menu Entry    Ignore diagnostics with "W291 trailing whitespace" message
     Wait Until Keyword Succeeds    10 x    1s    Should Have Expected Rows Count    ${EXPECTED_AFTER}
-    Open in Advanced Settings    ${DIAGNOSTICS PLUGIN ID}
-    Capture Page Screenshot    02-message-pressed.png
+    Configure JupyterLab Plugin    {}    plugin id=${DIAGNOSTICS PLUGIN ID}
 
 Diagnostic Message Can Be Copied
     Wait Until Keyword Succeeds    10 x    1s    Element Should Contain    ${DIAGNOSTICS PANEL}

From 1adef070a63fc85f8bc4c1ceb2aac6909988b889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Krassowski?=
 <5832902+krassowski@users.noreply.github.com>
Date: Mon, 11 Sep 2023 18:59:39 +0100
Subject: [PATCH 31/50] Cleanup/link to upstream PR

---
 packages/jupyterlab-lsp/src/features/diagnostics/feature.ts | 1 -
 packages/jupyterlab-lsp/src/virtual/document.ts             | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
index 0a40bd8f7..fa102f6eb 100644
--- a/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
+++ b/packages/jupyterlab-lsp/src/features/diagnostics/feature.ts
@@ -78,7 +78,6 @@ export class DiagnosticsFeature extends Feature implements IDiagnosticsFeature {
       );
       virtualDocument.foreignDocumentClosed.connect((document, context) => {
         // TODO: check if we need to cast
-        console.log('foreing closed');
         this.clearDocumentDiagnostics(adapter, context.foreignDocument);
       });
     });
diff --git a/packages/jupyterlab-lsp/src/virtual/document.ts b/packages/jupyterlab-lsp/src/virtual/document.ts
index 550d404b5..72447ab29 100644
--- a/packages/jupyterlab-lsp/src/virtual/document.ts
+++ b/packages/jupyterlab-lsp/src/virtual/document.ts
@@ -145,6 +145,7 @@ export class VirtualDocument extends VirtualDocumentBase {
    * Close all expired documents.
    */
   closeExpiredDocuments(): void {
+    // TODO: remove once https://github.com/jupyterlab/jupyterlab/pull/15105 is in
     const usedDocuments = new Set<VirtualDocument>();
     for (const line of this.sourceLines.values()) {
       for (const block of line.foreignDocumentsMap.values()) {

From d08a9d8c1464bafe7ead4ffb01c0ec1522d03fb6 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Thu, 14 Sep 2023 20:37:59 +0100
Subject: [PATCH 32/50] Fix error when pasting multiple cells

---
 packages/jupyterlab-lsp/src/features/signature.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/jupyterlab-lsp/src/features/signature.ts b/packages/jupyterlab-lsp/src/features/signature.ts
index ff01422b2..e6084d042 100644
--- a/packages/jupyterlab-lsp/src/features/signature.ts
+++ b/packages/jupyterlab-lsp/src/features/signature.ts
@@ -313,7 +313,13 @@ export class SignatureFeature extends Feature {
 
           // TODO: the assumption that updated editor = active editor will fail on RTC. How to get `CodeEditor.IEditor` and `Document.IEditor` from `EditorView`? we got `CodeEditor.IModel` from `options.model` but may need more context here.
           const editorAccessor = adapter.activeEditor;
-          const editor = editorAccessor!.getEditor()!;
+          const editor = editorAccessor!.getEditor();
+
+          if (!editor) {
+            // see https://github.com/jupyter-lsp/jupyterlab-lsp/issues/984
+            // TODO: should not be needed once https://github.com/jupyterlab/jupyterlab/pull/14920 is in
+            return;
+          }
 
           // TODO: or should it come from viewUpdate instead?!
           // especially on copy paste this can be problematic.

From ab7de8ac46bc2dac421769eb59d3c20f3c71a24f Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Thu, 14 Sep 2023 22:43:14 +0100
Subject: [PATCH 33/50] Bump required JupyterLab version, switch to RC

---
 .github/workflows/job.test.yml           |  2 +-
 CHANGELOG.md                             | 10 ++++++++++
 CONTRIBUTING.md                          |  2 +-
 README.md                                |  2 +-
 binder/environment.yml                   |  2 +-
 docs/rtd.yml                             |  2 +-
 packages/jupyterlab-lsp/package.json     |  2 +-
 packages/metapackage/package.json        |  2 +-
 python_packages/jupyterlab_lsp/setup.cfg |  2 +-
 requirements/lab.txt                     |  2 +-
 10 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/job.test.yml b/.github/workflows/job.test.yml
index 1221ab3bc..2a8af681a 100644
--- a/.github/workflows/job.test.yml
+++ b/.github/workflows/job.test.yml
@@ -44,7 +44,7 @@ jobs:
       matrix:
         os: [ubuntu]
         nodejs: ['>=16,<17.0.0a0']
-        lab: ['>=4.0.5,<5.0.0a0']
+        lab: ['>=4.0.6,<5.0.0a0']
         r: ['>=4']
     steps:
       - uses: actions/checkout@v3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fafb1295..2124f4ae7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,15 @@
 ## Changelog
 
+### `@jupyter-lsp/jupyterlab-lsp 5.0.0-rc.0`
+
+- fixes diagnostics not showing up in text editor in certain circumstances
+- fixes signature hover box not being clickable
+- improves performance by not creating a temporary editor to setup linter underline style
+- JSON overrides work again
+- fixes issue with syntax highlighting breaking when pasting multiple cells
+
+Requires JupyterLab `>=4.0.6,<5.0.0a0`
+
 ### `@jupyter-lsp/jupyterlab-lsp 5.0.0-beta.1`
 
 - fix highlights conflict with selection
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 94571c134..01eaee9eb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -33,7 +33,7 @@ Development requires, at a minimum:
 
 - `nodejs >=16,!=17,<19`
 - `python >=3.8,<3.11.0a0`
-- `jupyterlab >=4.0.5,<5.0.0a0`
+- `jupyterlab >=4.0.6,<5.0.0a0`
 
 It is recommended to use a virtual environment (e.g. `virtualenv` or `conda env`)
 for development.
diff --git a/README.md b/README.md
index beaf2ebd6..130faa2e8 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,7 @@ Use context menu on rows in the panel to filter out diagnostics or copy their me
 
 You will need to have both of the following installed:
 
-- JupyterLab >=4.0.5,<5.0.0a0
+- JupyterLab >=4.0.6,<5.0.0a0
 - Python 3.8+
 
 In addition, if you wish to use javascript, html, markdown or any other NodeJS-based language server you will need to have appropriate NodeJS version installed.
diff --git a/binder/environment.yml b/binder/environment.yml
index dcb621bdf..0284add73 100644
--- a/binder/environment.yml
+++ b/binder/environment.yml
@@ -7,7 +7,7 @@ channels:
 dependencies:
   # runtime dependencies
   - python >=3.8,<3.11.0a0
-  - jupyterlab >=4.0.5,<5.0.0a0
+  - jupyterlab >=4.0.6,<5.0.0a0
   - jupyter_server >=1.1.2
   # TODO: uncomment once Notebook 7 is released
   #- notebook >= 7.0
diff --git a/docs/rtd.yml b/docs/rtd.yml
index 3748c5101..d0e56f53e 100644
--- a/docs/rtd.yml
+++ b/docs/rtd.yml
@@ -7,7 +7,7 @@ channels:
 
 dependencies:
   - importlib_metadata
-  - jupyterlab >=4.0.5,<5.0.0a0
+  - jupyterlab >=4.0.6,<5.0.0a0
   - myst-nb
   - nodejs >=16,!=17,<19
   - pandas
diff --git a/packages/jupyterlab-lsp/package.json b/packages/jupyterlab-lsp/package.json
index 2d9333ff8..336041e2c 100644
--- a/packages/jupyterlab-lsp/package.json
+++ b/packages/jupyterlab-lsp/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/jupyterlab-lsp",
-  "version": "5.0.0-beta.1",
+  "version": "5.0.0-rc.0",
   "description": "Language Server Protocol integration for JupyterLab",
   "keywords": [
     "jupyter",
diff --git a/packages/metapackage/package.json b/packages/metapackage/package.json
index 4a94698ac..520219f66 100644
--- a/packages/metapackage/package.json
+++ b/packages/metapackage/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/jupyterlab-lsp-metapackage",
-  "version": "5.0.0-beta.1",
+  "version": "5.0.0-rc.0",
   "description": "JupyterLab LSP - Meta Package. All of the packages used by JupyterLab LSP",
   "homepage": "https://github.com/jupyter-lsp/jupyterlab-lsp",
   "bugs": {
diff --git a/python_packages/jupyterlab_lsp/setup.cfg b/python_packages/jupyterlab_lsp/setup.cfg
index e1af2c659..90b7d42db 100644
--- a/python_packages/jupyterlab_lsp/setup.cfg
+++ b/python_packages/jupyterlab_lsp/setup.cfg
@@ -37,4 +37,4 @@ python_requires = >=3.8
 
 install_requires =
     jupyter_lsp >=2.0.0
-    jupyterlab >=4.0.5,<5.0.0a0
+    jupyterlab >=4.0.6,<5.0.0a0
diff --git a/requirements/lab.txt b/requirements/lab.txt
index a2d2631aa..85771d5af 100644
--- a/requirements/lab.txt
+++ b/requirements/lab.txt
@@ -1,3 +1,3 @@
 # the version of jupyterlab
 -r ./prod.txt
-jupyterlab >=4.0.5,<5.0.0a0
+jupyterlab >=4.0.6,<5.0.0a0

From 6d03b43b2d7e44688c27da814883c4e72cca86e9 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Thu, 14 Sep 2023 23:18:24 +0100
Subject: [PATCH 34/50] Bump JupyterLab versions in packages and lockfile

---
 packages/_example-extractor/package.json   |   4 +-
 packages/_klingon-integration/package.json |   6 +-
 packages/code-jumpers/package.json         |  20 +-
 packages/completion-theme/package.json     |   4 +-
 packages/jupyterlab-lsp/package.json       |  42 +-
 yarn.lock                                  | 754 ++++++++++++++-------
 6 files changed, 555 insertions(+), 275 deletions(-)

diff --git a/packages/_example-extractor/package.json b/packages/_example-extractor/package.json
index e7baa12a4..9af538354 100644
--- a/packages/_example-extractor/package.json
+++ b/packages/_example-extractor/package.json
@@ -14,8 +14,8 @@
     "@jupyter-lsp/jupyterlab-lsp": "^5.0.0-alpha.0"
   },
   "devDependencies": {
-    "@jupyterlab/application": "^4.0.5",
-    "@jupyterlab/testing": "^4.0.5",
+    "@jupyterlab/application": "^4.0.6",
+    "@jupyterlab/testing": "^4.0.6",
     "@types/jest": "^29.5.4",
     "jest": "^29.2.0",
     "rimraf": "^3.0.0",
diff --git a/packages/_klingon-integration/package.json b/packages/_klingon-integration/package.json
index bb7d42ca9..f6d8c6003 100644
--- a/packages/_klingon-integration/package.json
+++ b/packages/_klingon-integration/package.json
@@ -11,11 +11,11 @@
     "LICENSE"
   ],
   "dependencies": {
-    "@jupyterlab/application": "^4.0.5"
+    "@jupyterlab/application": "^4.0.6"
   },
   "devDependencies": {
-    "@jupyterlab/application": "^4.0.5",
-    "@jupyterlab/builder": "^4.0.5",
+    "@jupyterlab/application": "^4.0.6",
+    "@jupyterlab/builder": "^4.0.6",
     "typescript": "~5.0.4"
   },
   "jupyterlab": {
diff --git a/packages/code-jumpers/package.json b/packages/code-jumpers/package.json
index ff1faed19..df2aefca0 100644
--- a/packages/code-jumpers/package.json
+++ b/packages/code-jumpers/package.json
@@ -41,16 +41,16 @@
     "@jupyterlab/translation": "^4.0.0"
   },
   "devDependencies": {
-    "@jupyterlab/apputils": "^4.1.5",
-    "@jupyterlab/codeeditor": "^4.0.5",
-    "@jupyterlab/coreutils": "^6.0.5",
-    "@jupyterlab/docmanager": "^4.0.5",
-    "@jupyterlab/docregistry": "^4.0.5",
-    "@jupyterlab/fileeditor": "^4.0.5",
-    "@jupyterlab/notebook": "^4.0.5",
-    "@jupyterlab/observables": "^5.0.5",
-    "@jupyterlab/testing": "^4.0.5",
-    "@jupyterlab/translation": "^4.0.5",
+    "@jupyterlab/apputils": "^4.1.6",
+    "@jupyterlab/codeeditor": "^4.0.6",
+    "@jupyterlab/coreutils": "^6.0.6",
+    "@jupyterlab/docmanager": "^4.0.6",
+    "@jupyterlab/docregistry": "^4.0.6",
+    "@jupyterlab/fileeditor": "^4.0.6",
+    "@jupyterlab/notebook": "^4.0.6",
+    "@jupyterlab/observables": "^5.0.6",
+    "@jupyterlab/testing": "^4.0.6",
+    "@jupyterlab/translation": "^4.0.6",
     "rimraf": "^3.0.0",
     "typescript": "~5.0.4"
   },
diff --git a/packages/completion-theme/package.json b/packages/completion-theme/package.json
index a693cc0f1..9e1e380e6 100644
--- a/packages/completion-theme/package.json
+++ b/packages/completion-theme/package.json
@@ -33,8 +33,8 @@
     "clean": "rimraf lib"
   },
   "devDependencies": {
-    "@jupyterlab/builder": "^4.0.5",
-    "@jupyterlab/ui-components": "^4.0.5",
+    "@jupyterlab/builder": "^4.0.6",
+    "@jupyterlab/ui-components": "^4.0.6",
     "react": "^18.2.0",
     "rimraf": "^3.0.0",
     "typescript": "~5.0.4"
diff --git a/packages/jupyterlab-lsp/package.json b/packages/jupyterlab-lsp/package.json
index 336041e2c..962e391d2 100644
--- a/packages/jupyterlab-lsp/package.json
+++ b/packages/jupyterlab-lsp/package.json
@@ -64,32 +64,32 @@
     "@jupyter-lsp/completion-theme": "~3.3.0",
     "@jupyter-lsp/theme-material": "~2.1.1",
     "@jupyter-lsp/theme-vscode": "~2.1.1",
-    "@jupyterlab/lsp": "^4.0.5",
+    "@jupyterlab/lsp": "^4.0.6",
     "@rjsf/validator-ajv8": "^5.12.1",
     "lodash.mergewith": "^4.6.1"
   },
   "devDependencies": {
     "@codemirror/lint": "^6.4.0",
-    "@jupyter-notebook/application": "^7.0.2",
-    "@jupyterlab/application": "^4.0.5",
-    "@jupyterlab/apputils": "^4.1.5",
-    "@jupyterlab/builder": "^4.0.5",
-    "@jupyterlab/cells": "^4.0.5",
-    "@jupyterlab/codeeditor": "^4.0.5",
-    "@jupyterlab/codemirror": "^4.0.5",
-    "@jupyterlab/completer": "^4.0.5",
-    "@jupyterlab/coreutils": "^6.0.5",
-    "@jupyterlab/docmanager": "^4.0.5",
-    "@jupyterlab/docregistry": "^4.0.5",
-    "@jupyterlab/fileeditor": "^4.0.5",
-    "@jupyterlab/logconsole": "^4.0.5",
-    "@jupyterlab/notebook": "^4.0.5",
-    "@jupyterlab/rendermime": "^4.0.5",
-    "@jupyterlab/services": "^7.0.5",
-    "@jupyterlab/statusbar": "^4.0.5",
-    "@jupyterlab/testing": "^4.0.5",
-    "@jupyterlab/tooltip": "^4.0.5",
-    "@jupyterlab/ui-components": "^4.0.5",
+    "@jupyter-notebook/application": "^7.0.3",
+    "@jupyterlab/application": "^4.0.6",
+    "@jupyterlab/apputils": "^4.1.6",
+    "@jupyterlab/builder": "^4.0.6",
+    "@jupyterlab/cells": "^4.0.6",
+    "@jupyterlab/codeeditor": "^4.0.6",
+    "@jupyterlab/codemirror": "^4.0.6",
+    "@jupyterlab/completer": "^4.0.6",
+    "@jupyterlab/coreutils": "^6.0.6",
+    "@jupyterlab/docmanager": "^4.0.6",
+    "@jupyterlab/docregistry": "^4.0.6",
+    "@jupyterlab/fileeditor": "^4.0.6",
+    "@jupyterlab/logconsole": "^4.0.6",
+    "@jupyterlab/notebook": "^4.0.6",
+    "@jupyterlab/rendermime": "^4.0.6",
+    "@jupyterlab/services": "^7.0.6",
+    "@jupyterlab/statusbar": "^4.0.6",
+    "@jupyterlab/testing": "^4.0.6",
+    "@jupyterlab/tooltip": "^4.0.6",
+    "@jupyterlab/ui-components": "^4.0.6",
     "@lumino/algorithm": "*",
     "@lumino/widgets": "^2.3.0",
     "@types/jest": "^29.5.4",
diff --git a/yarn.lock b/yarn.lock
index fc4d344db..b82f3bd22 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2094,16 +2094,16 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/code-jumpers@workspace:packages/code-jumpers"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docmanager": ^4.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/fileeditor": ^4.0.5
-    "@jupyterlab/notebook": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/testing": ^4.0.5
-    "@jupyterlab/translation": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docmanager": ^4.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/fileeditor": ^4.0.6
+    "@jupyterlab/notebook": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/testing": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
     rimraf: ^3.0.0
     typescript: ~5.0.4
   peerDependencies:
@@ -2123,8 +2123,8 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/completion-theme@workspace:packages/completion-theme"
   dependencies:
-    "@jupyterlab/builder": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/builder": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     react: ^18.2.0
     rimraf: ^3.0.0
     typescript: ~5.0.4
@@ -2136,8 +2136,8 @@ __metadata:
   resolution: "@jupyter-lsp/jupyterlab-lsp-example-extractor@workspace:packages/_example-extractor"
   dependencies:
     "@jupyter-lsp/jupyterlab-lsp": ^5.0.0-alpha.0
-    "@jupyterlab/application": ^4.0.5
-    "@jupyterlab/testing": ^4.0.5
+    "@jupyterlab/application": ^4.0.6
+    "@jupyterlab/testing": ^4.0.6
     "@types/jest": ^29.5.4
     jest: ^29.2.0
     rimraf: ^3.0.0
@@ -2149,8 +2149,8 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/jupyterlab-lsp-klingon-integration@workspace:packages/_klingon-integration"
   dependencies:
-    "@jupyterlab/application": ^4.0.5
-    "@jupyterlab/builder": ^4.0.5
+    "@jupyterlab/application": ^4.0.6
+    "@jupyterlab/builder": ^4.0.6
     typescript: ~5.0.4
   languageName: unknown
   linkType: soft
@@ -2180,27 +2180,27 @@ __metadata:
     "@jupyter-lsp/completion-theme": ~3.3.0
     "@jupyter-lsp/theme-material": ~2.1.1
     "@jupyter-lsp/theme-vscode": ~2.1.1
-    "@jupyter-notebook/application": ^7.0.2
-    "@jupyterlab/application": ^4.0.5
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/builder": ^4.0.5
-    "@jupyterlab/cells": ^4.0.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/codemirror": ^4.0.5
-    "@jupyterlab/completer": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docmanager": ^4.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/fileeditor": ^4.0.5
-    "@jupyterlab/logconsole": ^4.0.5
-    "@jupyterlab/lsp": ^4.0.5
-    "@jupyterlab/notebook": ^4.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/statusbar": ^4.0.5
-    "@jupyterlab/testing": ^4.0.5
-    "@jupyterlab/tooltip": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyter-notebook/application": ^7.0.3
+    "@jupyterlab/application": ^4.0.6
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/builder": ^4.0.6
+    "@jupyterlab/cells": ^4.0.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/codemirror": ^4.0.6
+    "@jupyterlab/completer": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docmanager": ^4.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/fileeditor": ^4.0.6
+    "@jupyterlab/logconsole": ^4.0.6
+    "@jupyterlab/lsp": ^4.0.6
+    "@jupyterlab/notebook": ^4.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/testing": ^4.0.6
+    "@jupyterlab/tooltip": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": "*"
     "@lumino/widgets": ^2.3.0
     "@rjsf/validator-ajv8": ^5.12.1
@@ -2257,22 +2257,22 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@jupyter-notebook/application@npm:^7.0.2":
-  version: 7.0.2
-  resolution: "@jupyter-notebook/application@npm:7.0.2"
+"@jupyter-notebook/application@npm:^7.0.3":
+  version: 7.0.3
+  resolution: "@jupyter-notebook/application@npm:7.0.3"
   dependencies:
-    "@jupyterlab/application": ^4.0.4
-    "@jupyterlab/coreutils": ^6.0.4
-    "@jupyterlab/docregistry": ^4.0.4
-    "@jupyterlab/rendermime-interfaces": ^3.8.4
-    "@jupyterlab/ui-components": ^4.0.4
-    "@lumino/algorithm": ^2.0.0
-    "@lumino/coreutils": ^2.1.1
-    "@lumino/messaging": ^2.0.0
-    "@lumino/polling": ^2.1.1
-    "@lumino/signaling": ^2.1.1
-    "@lumino/widgets": ^2.1.1
-  checksum: dd3d2592baa458881265107b8c985a02a8d137a2e6fcd362c4ff43500ed99144c46f9871bfb8d44e7a23168298a440a830f87431177dd954674f89ff944ca148
+    "@jupyterlab/application": ^4.0.5
+    "@jupyterlab/coreutils": ^6.0.5
+    "@jupyterlab/docregistry": ^4.0.5
+    "@jupyterlab/rendermime-interfaces": ^3.8.5
+    "@jupyterlab/ui-components": ^4.0.5
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/polling": ^2.1.2
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+  checksum: 0f24dbac34b6008360609d1d2fc2637739d2897c695591385bfb8b5f505a179dc195dbbbe55e64e351d4d2c56120c1e0928eb67575ced47fd1ae794526a9a77b
   languageName: node
   linkType: hard
 
@@ -2290,7 +2290,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/application@npm:^4.0.4, @jupyterlab/application@npm:^4.0.5":
+"@jupyterlab/application@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/application@npm:4.0.5"
   dependencies:
@@ -2318,6 +2318,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/application@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/application@npm:4.0.6"
+  dependencies:
+    "@fortawesome/fontawesome-free": ^5.12.0
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/application": ^2.2.1
+    "@lumino/commands": ^2.1.3
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/polling": ^2.1.2
+    "@lumino/properties": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+  checksum: 1212b71d3717bc02543b3eee74e69be799634421bd9b291b7adf07ba27bf6f9c7db860c423c824eaced9c2524db2f6b58de2c58e7edd5de2f0d7fabbb2c94b8c
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/apputils@npm:^4.1.5":
   version: 4.1.5
   resolution: "@jupyterlab/apputils@npm:4.1.5"
@@ -2347,23 +2375,52 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/attachments@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/attachments@npm:4.0.5"
+"@jupyterlab/apputils@npm:^4.1.6":
+  version: 4.1.6
+  resolution: "@jupyterlab/apputils@npm:4.1.6"
+  dependencies:
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/settingregistry": ^4.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/commands": ^2.1.3
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/domutils": ^2.0.1
+    "@lumino/messaging": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/virtualdom": ^2.0.1
+    "@lumino/widgets": ^2.3.0
+    "@types/react": ^18.0.26
+    react: ^18.2.0
+    sanitize-html: ~2.7.3
+  checksum: 40fb43f5a6464c665f1b941d164f3366ab8ea906fed72894ccf026ebeebf0734409edb6546a151ac267cbd4f945e23474251aed644f7f0f0dbf2548b9230ae22
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/attachments@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/attachments@npm:4.0.6"
   dependencies:
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/rendermime-interfaces": ^3.8.5
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
     "@lumino/disposable": ^2.1.2
     "@lumino/signaling": ^2.1.2
-  checksum: bb0a5dc7e830fc42824743cc817cf59a43c43b6f3979b3d6214619baf69f77bb70606241b39a92da21788348eb1144a0914e3683f0b2b8d01a530e8aeaf6f01e
+  checksum: b7efd01d6a0b7f28a4ee8114723cdc33267f1c1f84763b71a04e783024a87f639e442be74eda5afecff7eea5750d094f4da8a46ee711ec1298645eb92252c7ee
   languageName: node
   linkType: hard
 
-"@jupyterlab/builder@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/builder@npm:4.0.5"
+"@jupyterlab/builder@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/builder@npm:4.0.6"
   dependencies:
     "@lumino/algorithm": ^2.0.1
     "@lumino/application": ^2.2.1
@@ -2398,32 +2455,32 @@ __metadata:
     worker-loader: ^3.0.2
   bin:
     build-labextension: lib/build-labextension.js
-  checksum: 60b12e784881a16a3d2c794b0edfaea85e5da0b84f1a751564741df665c0bfcea8baabb91e5c061461fc431a8a5570e837cbf7692b39935b0df7fe87e1c0f213
+  checksum: 8ee8db483e07bcc99c45133616a60f57eb2f323898217961cecf82ef875343879327ad7e74adaa860577742d946e8325f16dfcb54845930db41faa9f4bdad70c
   languageName: node
   linkType: hard
 
-"@jupyterlab/cells@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/cells@npm:4.0.5"
+"@jupyterlab/cells@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/cells@npm:4.0.6"
   dependencies:
     "@codemirror/state": ^6.2.0
     "@codemirror/view": ^6.9.6
     "@jupyter/ydoc": ^1.0.2
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/attachments": ^4.0.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/codemirror": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/documentsearch": ^4.0.5
-    "@jupyterlab/filebrowser": ^4.0.5
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/outputarea": ^4.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/toc": ^6.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/attachments": ^4.0.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/codemirror": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/documentsearch": ^4.0.6
+    "@jupyterlab/filebrowser": ^4.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/outputarea": ^4.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/toc": ^6.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/domutils": ^2.0.1
@@ -2434,7 +2491,7 @@ __metadata:
     "@lumino/virtualdom": ^2.0.1
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: d674a15ddf870bea876d8b40ec598bbe9ba6d59b653223b381beec7e4e1e18c1b2c623585a9edc24e186dc666d73c63c55cee76ec83f975183f17bb5a56a8573
+  checksum: b0bb039c05ee0d83f40b5ccb0efa27d90723808c50821f8b8abedc8770387916bbc46d8a886102bf353b58e434cf33849981f782f1dbb3658835cce219940f33
   languageName: node
   linkType: hard
 
@@ -2461,9 +2518,32 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/codemirror@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/codemirror@npm:4.0.5"
+"@jupyterlab/codeeditor@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/codeeditor@npm:4.0.6"
+  dependencies:
+    "@codemirror/state": ^6.2.0
+    "@jupyter/ydoc": ^1.0.2
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/dragdrop": ^2.1.3
+    "@lumino/messaging": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+    react: ^18.2.0
+  checksum: 831d330273280781dbdc223325d575ac373c0db17fab208f327bce4e1c2286c62f8264d1d612da1a762bc006cd81dfb0eb5108dd3bd8f8298f9be8ecac98b2ca
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/codemirror@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/codemirror@npm:4.0.6"
   dependencies:
     "@codemirror/autocomplete": ^6.5.1
     "@codemirror/commands": ^6.2.3
@@ -2486,11 +2566,11 @@ __metadata:
     "@codemirror/state": ^6.2.0
     "@codemirror/view": ^6.9.6
     "@jupyter/ydoc": ^1.0.2
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/documentsearch": ^4.0.5
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/translation": ^4.0.5
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/documentsearch": ^4.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
     "@lezer/common": ^1.0.2
     "@lezer/generator": ^1.2.2
     "@lezer/highlight": ^1.1.4
@@ -2499,22 +2579,22 @@ __metadata:
     "@lumino/disposable": ^2.1.2
     "@lumino/signaling": ^2.1.2
     yjs: ^13.5.40
-  checksum: 840d9abd7c34ce7fb09446eff235e056e2d04da290f83380c020a9c3e2a1a27c0d3fc7ffcbd54a1f6de6325a57cc18d350d30c61a0f27d9810d8d2ec32aa5cf2
+  checksum: fdd0c4655e8597b1beb985b84b82dcfe29b4c8d0ae7e7ff3b0aecdbc94bc2b94ec0b617b3d59d7739e271e76433b2e624672d94ec64cfa4efc628cab92418175
   languageName: node
   linkType: hard
 
-"@jupyterlab/completer@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/completer@npm:4.0.5"
+"@jupyterlab/completer@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/completer@npm:4.0.6"
   dependencies:
     "@jupyter/ydoc": ^1.0.2
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/statedb": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
@@ -2522,11 +2602,11 @@ __metadata:
     "@lumino/messaging": ^2.0.1
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
-  checksum: bca52950049bfad7a03a2a36d8ed0ac0c430bb4b7cbd833be5ece0d03293edd474349ff462dfe25a5dd76c9de77dd1c68d42d1444f43faec6dd4d1f79cefb162
+  checksum: 3fcd07e52a753a0e8e4cc3a27220fb8a8d64167097a73f76343fbbb40ce69681a7be4244b8c7f21f826dab91af4e25591d15245b2f41237d56d83d3aab3db39e
   languageName: node
   linkType: hard
 
-"@jupyterlab/coreutils@npm:^6.0.4, @jupyterlab/coreutils@npm:^6.0.5":
+"@jupyterlab/coreutils@npm:^6.0.5":
   version: 6.0.5
   resolution: "@jupyterlab/coreutils@npm:6.0.5"
   dependencies:
@@ -2540,17 +2620,31 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/docmanager@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/docmanager@npm:4.0.5"
+"@jupyterlab/coreutils@npm:^6.0.6":
+  version: 6.0.6
+  resolution: "@jupyterlab/coreutils@npm:6.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/statusbar": ^4.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/signaling": ^2.1.2
+    minimist: ~1.2.0
+    path-browserify: ^1.0.0
+    url-parse: ~1.5.4
+  checksum: cf3cfbc7c48cae20549f5514a949b253c2f9d67c79db107ab0a81c2b7a9c08e28f9fd264e3d944a05a8cb1bbb9676c6b4163b75c28788d1cb3a3cc523d44d802
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/docmanager@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/docmanager@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
@@ -2559,11 +2653,11 @@ __metadata:
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: 16627833d9d540e9569bd27e3464c6c9a5cf9f628265b5018a4f63e05f351c4891494b8c731f83bb279da3bb42d0da23cb1d1b536c0b1b4422e4f6f250377ca5
+  checksum: 25d3f694ae8664ca6c54bfcd36d8913caba9455fea68ed3df23963ce9723254c1f2c38fb6a93e267187f095392507d40cd2a4181c30173306c1c0b962e001b93
   languageName: node
   linkType: hard
 
-"@jupyterlab/docregistry@npm:^4.0.4, @jupyterlab/docregistry@npm:^4.0.5":
+"@jupyterlab/docregistry@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/docregistry@npm:4.0.5"
   dependencies:
@@ -2588,13 +2682,38 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/documentsearch@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/documentsearch@npm:4.0.5"
+"@jupyterlab/docregistry@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/docregistry@npm:4.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyter/ydoc": ^1.0.2
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/properties": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+  checksum: 57de3751ea04037f27596ffe782392fb4840f3fba1776a64bb7b0dc5936a3cee4de115b2133147cda23a697d3da7802daaec0effae10be329d6c774f102091ee
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/documentsearch@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/documentsearch@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
     "@lumino/messaging": ^2.0.1
@@ -2602,23 +2721,23 @@ __metadata:
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: d7fe83a57562e9f90555c8b938f77edff21f7204b52a7cdd4a0cd21f5382fd5a7906e5d7c2ec661802b5d9bada42f80fcaa5d129931aeac949e8655d290d9adf
+  checksum: e6cf3533cdae29ca2f81147b26b056718df16998f6d89274d90cffcc70eab705634a7e36d353e9dcaea38640b490315b4ae683e937755547f42b8a5623bc914a
   languageName: node
   linkType: hard
 
-"@jupyterlab/filebrowser@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/filebrowser@npm:4.0.5"
+"@jupyterlab/filebrowser@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/filebrowser@npm:4.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docmanager": ^4.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/statedb": ^4.0.5
-    "@jupyterlab/statusbar": ^4.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docmanager": ^4.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
@@ -2630,64 +2749,64 @@ __metadata:
     "@lumino/virtualdom": ^2.0.1
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: f47d55cc8ff246efe65fdbf1f0fc09e227eca9bafcf0f1e45e1973612ad13e0853f1393882decddc2f1df015f11097b6d751bdbcdc255ed438adc96598b376a8
+  checksum: abe7eca4072a9c3d1f7a756840d0ad209403928b958fe09dfd81fbb693cb18c91c64027157babe1e7353a556b11c070716326b16ee2eb629089399906a3467be
   languageName: node
   linkType: hard
 
-"@jupyterlab/fileeditor@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/fileeditor@npm:4.0.5"
+"@jupyterlab/fileeditor@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/fileeditor@npm:4.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/codemirror": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/documentsearch": ^4.0.5
-    "@jupyterlab/lsp": ^4.0.5
-    "@jupyterlab/statusbar": ^4.0.5
-    "@jupyterlab/toc": ^6.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/codemirror": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/documentsearch": ^4.0.6
+    "@jupyterlab/lsp": ^4.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/toc": ^6.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/commands": ^2.1.3
     "@lumino/coreutils": ^2.1.2
     "@lumino/messaging": ^2.0.1
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
     regexp-match-indices: ^1.0.2
-  checksum: 7598dec866704fb664223b805a3fa7db4eb6506f10b4c59a831404d1462e2d993955b259095ea7d35258bb1be9147860d261f11e48c493331bb77746807565ac
+  checksum: cc70beea6dffe131574a73106e9381c80fe05b440f00f37312e4c12a0995dd22989f414916f4b450d455db20bf28d9ea3c2ef339e5ecfd45e61ce433c0bb8a0a
   languageName: node
   linkType: hard
 
-"@jupyterlab/logconsole@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/logconsole@npm:4.0.5"
+"@jupyterlab/logconsole@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/logconsole@npm:4.0.6"
   dependencies:
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/outputarea": ^4.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/translation": ^4.0.5
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/outputarea": ^4.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/translation": ^4.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
     "@lumino/messaging": ^2.0.1
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
-  checksum: e18e395e38ce3994c2a5ccc637a6a2bf2beeb3e8b727f0aa5d8c816e47a1ffbb681af592596e55fb67cf2e66f7d05ce6e969cf7983d9d41d5a98832dcfed2ec5
+  checksum: 7d6e56c019819452a4fadbacdc02eef3a4ca2192c96d40f49b2336fffa9f00a3fd0981ee9b0498ce8e7609619221e20f0bf9e5bd4c05e16228c3c5524eccfcd0
   languageName: node
   linkType: hard
 
-"@jupyterlab/lsp@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/lsp@npm:4.0.5"
+"@jupyterlab/lsp@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/lsp@npm:4.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/translation": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/translation": ^4.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
     "@lumino/signaling": ^2.1.2
@@ -2695,7 +2814,7 @@ __metadata:
     vscode-jsonrpc: ^6.0.0
     vscode-languageserver-protocol: ^3.17.0
     vscode-ws-jsonrpc: ~1.0.2
-  checksum: b59d21c9df84963c354422134e525acabab7f7fe2930e4bb5b5b81edd3e8397772ce5c395bc1faa7c79cddb6bfefc9e1c41edfd939241681da483ae3238be00d
+  checksum: 7a20f402bd2777e5ec36efe3193357ae59a8d619516ccf34bb569afe68d32df2af2c77479216826a08fd6dc16952e1430f2465e1fd9878bf092ca8773e5e2d1e
   languageName: node
   linkType: hard
 
@@ -2708,28 +2827,37 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/notebook@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/notebook@npm:4.0.5"
+"@jupyterlab/nbformat@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/nbformat@npm:4.0.6"
+  dependencies:
+    "@lumino/coreutils": ^2.1.2
+  checksum: 43ace863be98a82875a55a947828b9b987cff35bb484e6cb6474c97f60aadbf31027c5f2fdf81b4ee2d108dc735b92c15c9b35cded765b0e476ebf0c8fd670f6
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/notebook@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/notebook@npm:4.0.6"
   dependencies:
     "@jupyter/ydoc": ^1.0.2
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/cells": ^4.0.5
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/codemirror": ^4.0.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/documentsearch": ^4.0.5
-    "@jupyterlab/lsp": ^4.0.5
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/settingregistry": ^4.0.5
-    "@jupyterlab/statusbar": ^4.0.5
-    "@jupyterlab/toc": ^6.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/cells": ^4.0.6
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/codemirror": ^4.0.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/documentsearch": ^4.0.6
+    "@jupyterlab/lsp": ^4.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/settingregistry": ^4.0.6
+    "@jupyterlab/statusbar": ^4.0.6
+    "@jupyterlab/toc": ^6.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/domutils": ^2.0.1
@@ -2740,7 +2868,7 @@ __metadata:
     "@lumino/virtualdom": ^2.0.1
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: c6979a1b3cc1a6e4eb82176d97bc2109f8f3bcf6b281853a6fb8d350e66fa443dcd34981d46b0aebb03356e6533956dd4ad233e6dee9198acbd62b9c6f027bcd
+  checksum: 9ffb5f39b5e6d34fc2df2662c790121fda3880a271f0606bd56f3bcff416b43b1697640d43f0a30231fcf24a935e193ca9b5bf016d34675ede0e02e1185afffb
   languageName: node
   linkType: hard
 
@@ -2757,17 +2885,30 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/outputarea@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/outputarea@npm:4.0.5"
+"@jupyterlab/observables@npm:^5.0.6":
+  version: 5.0.6
+  resolution: "@jupyterlab/observables@npm:5.0.6"
   dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/nbformat": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/rendermime-interfaces": ^3.8.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/translation": ^4.0.5
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+  checksum: 23232972e6a049b2addeae1d445bc3a10bb6c9a3dd4794225a0344aaa1ff62421ee300ef8803a19a3f068d2bba2de8b9a8dec719a7f47019fbd77c8d5dafb7a0
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/outputarea@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/outputarea@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/translation": ^4.0.6
     "@lumino/algorithm": ^2.0.1
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
@@ -2775,11 +2916,11 @@ __metadata:
     "@lumino/properties": ^2.0.1
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
-  checksum: fc7f49b09ad8104fd0ac022366877eee228beb63f237afa76e785e170cb17e9ae18a686e7ac09f5f74bf25735ebc089812ea9374cc7920f4a0a641b9d565a046
+  checksum: 2691fe7e4bdff895c821e970edbc19867107dbd8150aa7f32c7f3a4a7608f9ae9c7862dc5887fdca6983b9d6a947e05f23bbf5160c7c99eef93a0abf20d085a4
   languageName: node
   linkType: hard
 
-"@jupyterlab/rendermime-interfaces@npm:^3.8.4, @jupyterlab/rendermime-interfaces@npm:^3.8.5":
+"@jupyterlab/rendermime-interfaces@npm:^3.8.5":
   version: 3.8.5
   resolution: "@jupyterlab/rendermime-interfaces@npm:3.8.5"
   dependencies:
@@ -2789,6 +2930,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/rendermime-interfaces@npm:^3.8.6":
+  version: 3.8.6
+  resolution: "@jupyterlab/rendermime-interfaces@npm:3.8.6"
+  dependencies:
+    "@lumino/coreutils": ^1.11.0 || ^2.1.2
+    "@lumino/widgets": ^1.37.2 || ^2.3.0
+  checksum: 84ba0c3979e6ace6246e00248d1248075afb112f55be202257bb89a553b235d36ca82879c56f46f58285a5fc6d39914e93fea203c53245e0ac8d1b5ef838bb50
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/rendermime@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/rendermime@npm:4.0.5"
@@ -2809,6 +2960,26 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/rendermime@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/rendermime@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+    lodash.escape: ^4.0.1
+  checksum: 8f44601ccd6abe9985f8f713dcabf48ae38246b8b5a17a3963ebfe298dc2a03cc49d1f99c6d3bfeadbe1eb74803d0b3138c01347693a99166d7d70cb832c400b
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/services@npm:^7.0.5":
   version: 7.0.5
   resolution: "@jupyterlab/services@npm:7.0.5"
@@ -2828,6 +2999,25 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/services@npm:^7.0.6":
+  version: 7.0.6
+  resolution: "@jupyterlab/services@npm:7.0.6"
+  dependencies:
+    "@jupyter/ydoc": ^1.0.2
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/settingregistry": ^4.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/polling": ^2.1.2
+    "@lumino/properties": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    ws: ^8.11.0
+  checksum: 6e12ef309559977209e61ce3ec8ca74aa486d54f50d8f38211b684055fb2335a21c2ae6e846d2863e48524bffd7d6ac4d36dfc9f7ca610ae4b1c60752fa6c3a8
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/settingregistry@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/settingregistry@npm:4.0.5"
@@ -2847,6 +3037,25 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/settingregistry@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/settingregistry@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/nbformat": ^4.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@lumino/commands": ^2.1.3
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/signaling": ^2.1.2
+    "@rjsf/utils": ^5.1.0
+    ajv: ^8.12.0
+    json5: ^2.2.3
+  peerDependencies:
+    react: ">=16"
+  checksum: 70b6fc44a25e0d4ec36501c1418fe2764b9a9415f892df0901c43480b608a1621141ec8045183dfbca5aedf11ebaf1518dd76e2e96373b9ebe0efa6586ce856d
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/statedb@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/statedb@npm:4.0.5"
@@ -2860,6 +3069,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/statedb@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/statedb@npm:4.0.6"
+  dependencies:
+    "@lumino/commands": ^2.1.3
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/properties": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+  checksum: de507d094afcce7f7d12f9dc846788765616140b2f75ea22997f780056baaaadae0cf9683471a1d96c61d448b38860640c823302aeef0d5e5d7c9cf598074328
+  languageName: node
+  linkType: hard
+
 "@jupyterlab/statusbar@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/statusbar@npm:4.0.5"
@@ -2876,13 +3098,29 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/testing@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/testing@npm:4.0.5"
+"@jupyterlab/statusbar@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/statusbar@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/ui-components": ^4.0.6
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/widgets": ^2.3.0
+    react: ^18.2.0
+  checksum: c5d579b101e19670182d87de0d601fc9c73c40b5a81120e18e6cd7853ee9fd744fa31524f24b2c92cb587bb2d6aa54c08f6e257d868426a73d968e48b1101b7c
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/testing@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/testing@npm:4.0.6"
   dependencies:
     "@babel/core": ^7.10.2
     "@babel/preset-env": ^7.10.2
-    "@jupyterlab/coreutils": ^6.0.5
+    "@jupyterlab/coreutils": ^6.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/signaling": ^2.1.2
     child_process: ~1.0.2
@@ -2897,43 +3135,43 @@ __metadata:
     ts-jest: ^29.1.0
   peerDependencies:
     typescript: ">=4.3"
-  checksum: 5f242263f879bb075db6ff5125dbdb495589703ae2e287c171b7680954db16eb6fd3ab66d09d77ca27686a1d1ec3a0736c78f86808dd222949c834ebd13910fb
+  checksum: b6e7326d90ca2a7d36a825ea32557f2ce4436dc4c8965f3603fa430aad138b703f0e206c432509c48c72294a959ab47881a81c8c88b1e378b53fe9108861791e
   languageName: node
   linkType: hard
 
-"@jupyterlab/toc@npm:^6.0.5":
-  version: 6.0.5
-  resolution: "@jupyterlab/toc@npm:6.0.5"
-  dependencies:
-    "@jupyterlab/apputils": ^4.1.5
-    "@jupyterlab/coreutils": ^6.0.5
-    "@jupyterlab/docregistry": ^4.0.5
-    "@jupyterlab/observables": ^5.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/translation": ^4.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+"@jupyterlab/toc@npm:^6.0.6":
+  version: 6.0.6
+  resolution: "@jupyterlab/toc@npm:6.0.6"
+  dependencies:
+    "@jupyterlab/apputils": ^4.1.6
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/docregistry": ^4.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/translation": ^4.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/disposable": ^2.1.2
     "@lumino/messaging": ^2.0.1
     "@lumino/signaling": ^2.1.2
     "@lumino/widgets": ^2.3.0
     react: ^18.2.0
-  checksum: 4b688fdd2aa0d14db02394bafcbae5e0ce632681e8541ff3ca6153ba0e219dc20cb99f03ef4ac25f849b4b7b23f3e168e50a450bf952f42b0418e2e42aaeb546
+  checksum: d8d955a00e6678c50f73f18205dd79d77c752c3b0d33699554cdb77c3501657708c699642889975c97b58a85704c3bca40de01019ce087f188681bb843f35c3a
   languageName: node
   linkType: hard
 
-"@jupyterlab/tooltip@npm:^4.0.5":
-  version: 4.0.5
-  resolution: "@jupyterlab/tooltip@npm:4.0.5"
+"@jupyterlab/tooltip@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/tooltip@npm:4.0.6"
   dependencies:
-    "@jupyterlab/codeeditor": ^4.0.5
-    "@jupyterlab/rendermime": ^4.0.5
-    "@jupyterlab/services": ^7.0.5
-    "@jupyterlab/ui-components": ^4.0.5
+    "@jupyterlab/codeeditor": ^4.0.6
+    "@jupyterlab/rendermime": ^4.0.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/ui-components": ^4.0.6
     "@lumino/coreutils": ^2.1.2
     "@lumino/messaging": ^2.0.1
     "@lumino/widgets": ^2.3.0
-  checksum: 8a096fe65fb6887e47f12f20109b83f71249bb435eb552341958fd78a3f2d8d0979b900f143a4889e361ba722cb6f7cbd6831ac49d8ad17ce0710b5bea2c921e
+  checksum: ba65f9bda2ce1ecada6d977779ee422bdbaadc7e4fbfc314d5910bf512fd85f99f74af02a5d330fb5cddf2d36f0c9585a4faf4fe09dd63d9b136465536ba66ef
   languageName: node
   linkType: hard
 
@@ -2950,7 +3188,20 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyterlab/ui-components@npm:^4.0.4, @jupyterlab/ui-components@npm:^4.0.5":
+"@jupyterlab/translation@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/translation@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/services": ^7.0.6
+    "@jupyterlab/statedb": ^4.0.6
+    "@lumino/coreutils": ^2.1.2
+  checksum: 490243a26898bbdcc1909e8e1649be90580c6d4502417590fd1b3ca24db5aeff2323e567dbfb1d528c56df89ed2e7717753ece784134f9e409d14df92bf25682
+  languageName: node
+  linkType: hard
+
+"@jupyterlab/ui-components@npm:^4.0.5":
   version: 4.0.5
   resolution: "@jupyterlab/ui-components@npm:4.0.5"
   dependencies:
@@ -2979,6 +3230,35 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jupyterlab/ui-components@npm:^4.0.6":
+  version: 4.0.6
+  resolution: "@jupyterlab/ui-components@npm:4.0.6"
+  dependencies:
+    "@jupyterlab/coreutils": ^6.0.6
+    "@jupyterlab/observables": ^5.0.6
+    "@jupyterlab/rendermime-interfaces": ^3.8.6
+    "@jupyterlab/translation": ^4.0.6
+    "@lumino/algorithm": ^2.0.1
+    "@lumino/commands": ^2.1.3
+    "@lumino/coreutils": ^2.1.2
+    "@lumino/disposable": ^2.1.2
+    "@lumino/messaging": ^2.0.1
+    "@lumino/polling": ^2.1.2
+    "@lumino/properties": ^2.0.1
+    "@lumino/signaling": ^2.1.2
+    "@lumino/virtualdom": ^2.0.1
+    "@lumino/widgets": ^2.3.0
+    "@rjsf/core": ^5.1.0
+    "@rjsf/utils": ^5.1.0
+    react: ^18.2.0
+    react-dom: ^18.2.0
+    typestyle: ^2.0.4
+  peerDependencies:
+    react: ^18.2.0
+  checksum: 02997c3c35c15c0eda6a0d49fe9cfd12a3c5194c026b2ae8d8e53c7af80b769ba1598e7c24283450bacae7b8fab838d18f6c614d686c98d3d43e68f1f00b2528
+  languageName: node
+  linkType: hard
+
 "@lerna/child-process@npm:6.5.1":
   version: 6.5.1
   resolution: "@lerna/child-process@npm:6.5.1"
@@ -3159,7 +3439,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/algorithm@npm:*, @lumino/algorithm@npm:^2.0.0, @lumino/algorithm@npm:^2.0.1":
+"@lumino/algorithm@npm:*, @lumino/algorithm@npm:^2.0.1":
   version: 2.0.1
   resolution: "@lumino/algorithm@npm:2.0.1"
   checksum: cbf7fcf6ee6b785ea502cdfddc53d61f9d353dcb9659343511d5cd4b4030be2ff2ca4c08daec42f84417ab0318a3d9972a17319fa5231693e109ab112dcf8000
@@ -3201,7 +3481,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/coreutils@npm:^1.11.0 || ^2.0.0, @lumino/coreutils@npm:^1.11.0 || ^2.1.2, @lumino/coreutils@npm:^2.1.1, @lumino/coreutils@npm:^2.1.2":
+"@lumino/coreutils@npm:^1.11.0 || ^2.0.0, @lumino/coreutils@npm:^1.11.0 || ^2.1.2, @lumino/coreutils@npm:^2.1.2":
   version: 2.1.2
   resolution: "@lumino/coreutils@npm:2.1.2"
   checksum: 7865317ac0676b448d108eb57ab5d8b2a17c101995c0f7a7106662d9fe6c859570104525f83ee3cda12ae2e326803372206d6f4c1f415a5b59e4158a7b81066f
@@ -3241,7 +3521,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/messaging@npm:^2.0.0, @lumino/messaging@npm:^2.0.1":
+"@lumino/messaging@npm:^2.0.1":
   version: 2.0.1
   resolution: "@lumino/messaging@npm:2.0.1"
   dependencies:
@@ -3251,7 +3531,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/polling@npm:^2.1.1, @lumino/polling@npm:^2.1.2":
+"@lumino/polling@npm:^2.1.2":
   version: 2.1.2
   resolution: "@lumino/polling@npm:2.1.2"
   dependencies:
@@ -3269,7 +3549,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/signaling@npm:^1.10.0 || ^2.0.0, @lumino/signaling@npm:^2.1.1, @lumino/signaling@npm:^2.1.2":
+"@lumino/signaling@npm:^1.10.0 || ^2.0.0, @lumino/signaling@npm:^2.1.2":
   version: 2.1.2
   resolution: "@lumino/signaling@npm:2.1.2"
   dependencies:
@@ -3288,7 +3568,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lumino/widgets@npm:^1.37.2 || ^2.3.0, @lumino/widgets@npm:^2.1.1, @lumino/widgets@npm:^2.3.0":
+"@lumino/widgets@npm:^1.37.2 || ^2.3.0, @lumino/widgets@npm:^2.3.0":
   version: 2.3.0
   resolution: "@lumino/widgets@npm:2.3.0"
   dependencies:

From 151ee8e0e9c0cc67f340db59ed5cef8b67b10bab Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Thu, 14 Sep 2023 23:32:57 +0100
Subject: [PATCH 35/50] Ignore mypy false positive on traitlet

---
 python_packages/jupyter_lsp/jupyter_lsp/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python_packages/jupyter_lsp/jupyter_lsp/manager.py b/python_packages/jupyter_lsp/jupyter_lsp/manager.py
index 2d9597e0c..c847e8c4d 100644
--- a/python_packages/jupyter_lsp/jupyter_lsp/manager.py
+++ b/python_packages/jupyter_lsp/jupyter_lsp/manager.py
@@ -61,7 +61,7 @@ class LanguageServerManager(LanguageServerManagerAPI):
         config=True
     )  # type: KeyedLanguageServerSpecs
 
-    autodetect: bool = Bool(
+    autodetect: bool = Bool(  # type:ignore[assignment]
         True, help=_("try to find known language servers in sys.prefix (and elsewhere)")
     ).tag(
         config=True

From 2a8f7d99265602ac07815790720d61631b0f4942 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 15 Sep 2023 00:25:46 +0100
Subject: [PATCH 36/50] Fix width heuristic followup upstream fix

---
 packages/jupyterlab-lsp/src/features/completion/renderer.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/renderer.ts b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
index 204be004e..1bd306606 100644
--- a/packages/jupyterlab-lsp/src/features/completion/renderer.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
@@ -223,7 +223,7 @@ export class LSPCompletionRenderer
   }
 
   itemWidthHeuristic(item: CompletionItem): number {
-    const labelSize = item.label.replace(/<\?mark>/g, '').length;
+    const labelSize = item.label.replace(/<(\/)?mark>/g, '').length;
     const extraTextSize = this.getExtraInfo(item).length;
     if (this.options.settings.composite.layout === 'side-by-side') {
       // in 'side-by-side' take the sum

From f173870605534076867ea376f134e1f4d674065a Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 15 Sep 2023 00:26:07 +0100
Subject: [PATCH 37/50] Fix selector in tests

---
 atest/05_Features/Completion.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atest/05_Features/Completion.robot b/atest/05_Features/Completion.robot
index 3ae35b818..7b5788e33 100644
--- a/atest/05_Features/Completion.robot
+++ b/atest/05_Features/Completion.robot
@@ -470,4 +470,4 @@ Should Complete While Kernel Is Busy
     Page Should Contain Element    ${KERNEL_BUSY_INDICATOR}
 
 Wait For Our Completer To Initialize
-    Wait Until Page Contains Element    css:.body[data-lsp-completer-layout]
+    Wait Until Page Contains Element    css:body[data-lsp-completer-layout]

From 313349e161836905b65d429de591c9f40c77e1bd Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 15 Sep 2023 21:21:07 +0100
Subject: [PATCH 38/50] Re-enable connection check, fix token offset

---
 .../jupyterlab-lsp/src/features/completion/provider.ts   | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index dbd136fcc..b4c3209ce 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -213,9 +213,9 @@ export class CompletionProvider implements ICompletionProvider<CompletionItem> {
       editor.getPositionAt(request.offset)!
     ) as IEditorPosition;
     const token = editor.getTokenAt(request.offset);
-    const positionInToken = token.offset - request.offset;
+    const positionInToken = request.offset - token.offset;
     // TODO: (typedCharacter can serve as a proxy for triggerCharacter)
-    const typedCharacter = token.value[positionInToken + 1];
+    const typedCharacter = token.value[positionInToken - 1];
 
     // TODO: direct mapping
     // because we need editorAccessor, not the editor itself we perform this rather sad dance:
@@ -296,17 +296,12 @@ export class CompletionProvider implements ICompletionProvider<CompletionItem> {
     if (this.options.settings.composite.disable) {
       return false;
     }
-    /*
-    // Disabled due to the result being effectively cached until user changes
-    // cells which can lead to bad UX; upstream issue:
-    // https://github.com/jupyterlab/jupyterlab/issues/15016
     const manager = this.options.connectionManager;
     const widget = context.widget as IDocumentWidget;
     const adapter = manager.adapters.get(widget.context.path);
     if (!adapter) {
       return false;
     }
-    */
     return true;
   }
 }

From 7c0ea07cb6e81f166f50fe4243aa1d3295891c61 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 15 Sep 2023 21:37:22 +0100
Subject: [PATCH 39/50] Workaround spread syntax upstream

https://github.com/jupyterlab/jupyterlab/issues/15125
---
 .../src/features/completion/item.ts           | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/packages/jupyterlab-lsp/src/features/completion/item.ts b/packages/jupyterlab-lsp/src/features/completion/item.ts
index 0de5fde1b..c431eecda 100644
--- a/packages/jupyterlab-lsp/src/features/completion/item.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/item.ts
@@ -67,6 +67,13 @@ export class CompletionItem implements IExtendedCompletionItem {
     this.self = this;
     this.source = options.source;
     this.icon = options.icon ? options.icon : undefined;
+
+    // Upstream is sometimes using spread operator to copy the object (in reconciliator),
+    // which does not copy getters because these are not enumerable; we should use
+    // `Object.assign` upstream, but them, but for now marking relevant properties as enumerable is enough
+    // Ideally this would be fixed and tested e2e in JupyterLab 4.0.7.
+    // https://github.com/jupyterlab/jupyterlab/issues/15125
+    makeGetterEnumerable(this, 'insertText');
   }
 
   get type() {
@@ -185,3 +192,23 @@ export class CompletionItem implements IExtendedCompletionItem {
     );
   }
 }
+
+function makeGetterEnumerable(instance: object, name: string) {
+  const generatedDescriptor = findDescriptor(instance, name);
+  Object.defineProperty(instance, name, {
+    enumerable: true,
+    get: generatedDescriptor.get,
+    set: generatedDescriptor.set
+  });
+}
+
+function findDescriptor(instance: object, name: string) {
+  while (instance) {
+    const desc = Object.getOwnPropertyDescriptor(instance, name);
+    if (desc) {
+      return desc;
+    }
+    instance = Object.getPrototypeOf(instance);
+  }
+  throw Error(`No ${name} descriptor found.`);
+}

From 640453b1e3912d95033c83e431ae6e30316970c0 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Fri, 15 Sep 2023 22:41:50 +0100
Subject: [PATCH 40/50] Fallback to default renderer for non-LSP items

---
 packages/jupyterlab-lsp/src/features/completion/renderer.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/jupyterlab-lsp/src/features/completion/renderer.ts b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
index 1bd306606..b100eb2fc 100644
--- a/packages/jupyterlab-lsp/src/features/completion/renderer.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
@@ -213,6 +213,9 @@ export class LSPCompletionRenderer
         })
         .catch(this.options.console.warn);
       return this.options.markdownRenderer.node;
+    } else if (item.source != 'LSP') {
+      // fallback to default implementation for non-LSP completions
+      return super.createDocumentationNode(item);
     } else {
       let node = document.createElement('pre');
       if (item.documentation) {

From 6a15ba150925d1d94427c4d1ea21f8b8749e77f6 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 16 Sep 2023 00:12:31 +0100
Subject: [PATCH 41/50] Fix highlights, jump to and hover in foreign documents

Prior to this commit the positions would be always resolved with respect
to the root document because virtual document was being taken from
adapter using `virtualPositionToRootPosition` func which is error
prone as the assumption of virtual document == root document is not
obvious; this function was removed. To make the fixes work, a
workaround for foreign document created in inherited classes always having
the usptream class was added; the issue was reported in
https://github.com/jupyterlab/jupyterlab/issues/15126
---
 packages/jupyterlab-lsp/src/converter.ts      | 26 +++++++----------
 .../jupyterlab-lsp/src/features/highlights.ts | 11 +++----
 packages/jupyterlab-lsp/src/features/hover.ts | 13 +++++----
 .../jupyterlab-lsp/src/features/jump_to.ts    | 29 ++++++++++++-------
 .../jupyterlab-lsp/src/virtual/document.ts    |  5 ++--
 5 files changed, 45 insertions(+), 39 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/converter.ts b/packages/jupyterlab-lsp/src/converter.ts
index 7149114fd..d357582ab 100644
--- a/packages/jupyterlab-lsp/src/converter.ts
+++ b/packages/jupyterlab-lsp/src/converter.ts
@@ -10,7 +10,7 @@ import type {
 } from '@jupyterlab/lsp';
 import type * as lsProtocol from 'vscode-languageserver-protocol';
 
-import type { VirtualDocument } from './virtual/document';
+import { VirtualDocument } from './virtual/document';
 
 export class PositionConverter {
   static lsp_to_cm(position: lsProtocol.Position): IPosition {
@@ -86,18 +86,6 @@ export function rootPositionToVirtualPosition(
   return adapter.virtualDocument!.virtualPositionAtDocument(rootAsSource);
 }
 
-export function virtualPositionToRootPosition(
-  adapter: WidgetLSPAdapter<any>,
-  position: IVirtualPosition
-): IRootPosition | null {
-  if (!adapter.virtualDocument) {
-    throw Error('Virtual document of adapter disposed!');
-  }
-  return (adapter.virtualDocument as VirtualDocument).transformVirtualToRoot(
-    position
-  );
-}
-
 export function rootPositionToEditorPosition(
   adapter: WidgetLSPAdapter<any>,
   position: IRootPosition
@@ -126,12 +114,20 @@ export function editorPositionToRootPosition(
 export function rangeToEditorRange(
   adapter: WidgetLSPAdapter<any>,
   range: lsProtocol.Range,
-  editor: CodeEditor.IEditor | null
+  editor: CodeEditor.IEditor | null,
+  virtualDocument: VirtualDocument
 ): IEditorRange {
   let start = PositionConverter.lsp_to_cm(range.start) as IVirtualPosition;
   let end = PositionConverter.lsp_to_cm(range.end) as IVirtualPosition;
 
-  let startInRoot = virtualPositionToRootPosition(adapter, start);
+  // because `openForeign()` does not use new this.constructor, we need to workaround it for now:
+  // const startInRoot = virtualDocument.transformVirtualToRoot(start);
+  // https://github.com/jupyterlab/jupyterlab/issues/15126
+  const startInRoot = VirtualDocument.prototype.transformVirtualToRoot.call(
+    virtualDocument,
+    start
+  );
+
   if (!startInRoot) {
     throw Error('Could not determine position in root');
   }
diff --git a/packages/jupyterlab-lsp/src/features/highlights.ts b/packages/jupyterlab-lsp/src/features/highlights.ts
index 558a083fc..f22dc418f 100644
--- a/packages/jupyterlab-lsp/src/features/highlights.ts
+++ b/packages/jupyterlab-lsp/src/features/highlights.ts
@@ -14,8 +14,7 @@ import {
   ILSPFeatureManager,
   IEditorPosition,
   ILSPDocumentConnectionManager,
-  WidgetLSPAdapter,
-  VirtualDocument
+  WidgetLSPAdapter
 } from '@jupyterlab/lsp';
 import { ISettingRegistry } from '@jupyterlab/settingregistry';
 import { LabIcon } from '@jupyterlab/ui-components';
@@ -37,6 +36,7 @@ import { DocumentHighlightKind } from '../lsp';
 import { createMarkManager, ISimpleMarkManager } from '../marks';
 import { PLUGIN_ID } from '../tokens';
 import { BrowserConsole } from '../virtual/console';
+import { VirtualDocument } from '../virtual/document';
 
 export const highlightIcon = new LabIcon({
   name: 'lsp:highlight',
@@ -160,7 +160,8 @@ export class HighlightsFeature extends Feature {
 
   protected handleHighlight(
     items: lsProtocol.DocumentHighlight[] | null,
-    adapter: WidgetLSPAdapter<any>
+    adapter: WidgetLSPAdapter<any>,
+    document: VirtualDocument
   ) {
     this.markManager.clearAllMarks();
 
@@ -174,7 +175,7 @@ export class HighlightsFeature extends Feature {
     >();
 
     for (let item of items) {
-      let range = rangeToEditorRange(adapter, item.range, null);
+      let range = rangeToEditorRange(adapter, item.range, null, document);
       const editor = range.editor;
 
       let optionsList = highlightsByEditor.get(editor);
@@ -350,7 +351,7 @@ export class HighlightsFeature extends Feature {
         return;
       }
 
-      this.handleHighlight(highlights, adapter);
+      this.handleHighlight(highlights, adapter, document);
       this._lastToken = token;
     } catch (e) {
       this.console.warn('Could not get highlights:', e);
diff --git a/packages/jupyterlab-lsp/src/features/hover.ts b/packages/jupyterlab-lsp/src/features/hover.ts
index 86f205d52..bc223688d 100644
--- a/packages/jupyterlab-lsp/src/features/hover.ts
+++ b/packages/jupyterlab-lsp/src/features/hover.ts
@@ -18,7 +18,6 @@ import {
   isEqual,
   ILSPDocumentConnectionManager,
   WidgetLSPAdapter,
-  VirtualDocument,
   Document
 } from '@jupyterlab/lsp';
 import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
@@ -45,6 +44,7 @@ import { createMarkManager, ISimpleMarkManager } from '../marks';
 import { PLUGIN_ID } from '../tokens';
 import { getModifierState } from '../utils';
 import { BrowserConsole } from '../virtual/console';
+import { VirtualDocument } from '../virtual/document';
 
 export const hoverIcon = new LabIcon({
   name: 'lsp:hover',
@@ -365,7 +365,8 @@ export class HoverFeature extends Feature {
       context.adapter,
       response,
       context.token,
-      context.editor
+      context.editor,
+      virtualDocument
     );
     return this._addRange(
       context.adapter,
@@ -619,7 +620,8 @@ export class HoverFeature extends Feature {
             adapter,
             response,
             token,
-            editor
+            editor,
+            document
           );
           responseData = {
             response: response,
@@ -689,10 +691,11 @@ export class HoverFeature extends Feature {
     adapter: WidgetLSPAdapter<any>,
     response: lsProtocol.Hover,
     token: CodeEditor.IToken,
-    editor: CodeEditor.IEditor
+    editor: CodeEditor.IEditor,
+    document: VirtualDocument
   ): IEditorRange {
     if (typeof response.range !== 'undefined') {
-      return rangeToEditorRange(adapter, response.range, editor);
+      return rangeToEditorRange(adapter, response.range, editor, document);
     }
 
     const startInEditor = editor.getPositionAt(token.offset);
diff --git a/packages/jupyterlab-lsp/src/features/jump_to.ts b/packages/jupyterlab-lsp/src/features/jump_to.ts
index 492713485..2d9748de8 100644
--- a/packages/jupyterlab-lsp/src/features/jump_to.ts
+++ b/packages/jupyterlab-lsp/src/features/jump_to.ts
@@ -48,13 +48,13 @@ import {
   documentAtRootPosition,
   editorAtRootPosition,
   rootPositionToVirtualPosition,
-  rootPositionToEditorPosition,
-  virtualPositionToRootPosition
+  rootPositionToEditorPosition
 } from '../converter';
 import { FeatureSettings, Feature } from '../feature';
 import { PLUGIN_ID } from '../tokens';
 import { getModifierState, uriToContentsPath, urisEqual } from '../utils';
 import { BrowserConsole } from '../virtual/console';
+import { VirtualDocument } from '../virtual/document';
 
 export const jumpToIcon = new LabIcon({
   name: 'lsp:jump-to',
@@ -245,7 +245,7 @@ export class NavigationFeature extends Feature {
     connection.clientRequests['textDocument/definition']
       .request(positionParams)
       .then(targets => {
-        this.handleJump(targets, positionParams, adapter)
+        this.handleJump(targets, positionParams, adapter, document)
           .then((result: JumpResult | undefined) => {
             if (
               result === JumpResult.NoTargetsFound ||
@@ -259,7 +259,7 @@ export class NavigationFeature extends Feature {
                 })
                 .then(targets =>
                   // TODO: explain that we are now presenting references?
-                  this.handleJump(targets, positionParams, adapter)
+                  this.handleJump(targets, positionParams, adapter, document)
                 )
                 .catch(this.console.warn);
             }
@@ -361,7 +361,8 @@ export class NavigationFeature extends Feature {
   async handleJump(
     locationData: AnyLocation,
     positionParams: lsp.TextDocumentPositionParams,
-    adapter: WidgetLSPAdapter<any>
+    adapter: WidgetLSPAdapter<any>,
+    document: VirtualDocument
   ) {
     const locations = this._harmonizeLocations(locationData);
     const targetInfo = await this._chooseTarget(locations);
@@ -382,10 +383,16 @@ export class NavigationFeature extends Feature {
 
     if (urisEqual(uri, positionParams.textDocument.uri)) {
       // if in current file, transform from the position within virtual document to the editor position:
-      const rootPosition = virtualPositionToRootPosition(
-        adapter,
-        virtualPosition
-      );
+
+      // because `openForeign()` does not use new this.constructor, we need to workaround it for now:
+      // const rootPosition = document.transformVirtualToRoot(virtualPosition);
+      // https://github.com/jupyterlab/jupyterlab/issues/15126
+      const rootPosition =
+        VirtualDocument.prototype.transformVirtualToRoot.call(
+          document,
+          virtualPosition
+        );
+
       if (rootPosition === null) {
         this.console.warn(
           'Could not jump: conversion from virtual position to editor position failed',
@@ -581,7 +588,7 @@ export const JUMP_PLUGIN: JupyterFrontEndPlugin<void> = {
         const targets = await connection.clientRequests[
           'textDocument/definition'
         ].request(positionParams);
-        await feature.handleJump(targets, positionParams, adapter);
+        await feature.handleJump(targets, positionParams, adapter, document);
       },
       label: trans.__('Jump to definition'),
       icon: jumpToIcon,
@@ -627,7 +634,7 @@ export const JUMP_PLUGIN: JupyterFrontEndPlugin<void> = {
           ...positionParams,
           context: { includeDeclaration: false }
         });
-        await feature.handleJump(targets, positionParams, adapter);
+        await feature.handleJump(targets, positionParams, adapter, document);
       },
       label: trans.__('Jump to references'),
       icon: jumpToIcon,
diff --git a/packages/jupyterlab-lsp/src/virtual/document.ts b/packages/jupyterlab-lsp/src/virtual/document.ts
index 72447ab29..29f8e643d 100644
--- a/packages/jupyterlab-lsp/src/virtual/document.ts
+++ b/packages/jupyterlab-lsp/src/virtual/document.ts
@@ -184,11 +184,10 @@ export class VirtualDocument extends VirtualDocumentBase {
    * @experimental
    */
   transformVirtualToRoot(position: IVirtualPosition): IRootPosition | null {
-    // a method which was previously implemented in CodeMirrorIntegration
-    // but probably should have been in VirtualDocument all along
     let editor = this.virtualLines.get(position.line)!.editor;
     let editorPosition = this.transformVirtualToEditor(position);
-    return this.transformFromEditorToRoot(editor, editorPosition!);
+    // only root holds the full editor mapping
+    return this.root.transformFromEditorToRoot(editor, editorPosition!);
   }
 
   /**

From 6eacd1048569dbee060196ec581c42f7a506125b Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:30:18 +0100
Subject: [PATCH 42/50] Fix/improve completion sorting

---
 .../src/features/completion/item.ts           | 79 +++++++++++--------
 .../src/features/completion/model.spec.ts     | 67 +++++++++++++++-
 .../src/features/completion/model.ts          | 42 +++++++---
 .../src/features/completion/overrides.ts      | 24 +++---
 .../src/features/completion/provider.ts       |  4 +-
 5 files changed, 152 insertions(+), 64 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/item.ts b/packages/jupyterlab-lsp/src/features/completion/item.ts
index c431eecda..a68757b46 100644
--- a/packages/jupyterlab-lsp/src/features/completion/item.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/item.ts
@@ -30,10 +30,6 @@ namespace CompletionItem {
 }
 
 export class CompletionItem implements IExtendedCompletionItem {
-  private _detail: string | undefined;
-  private _documentation: string | undefined;
-  private _isDocumentationMarkdown: boolean;
-  private _resolved: boolean;
   /**
    * Self-reference to make sure that the instance for will remain accessible
    * after any copy operation (whether via spread syntax or Object.assign)
@@ -42,7 +38,6 @@ export class CompletionItem implements IExtendedCompletionItem {
   public self: CompletionItem;
   public element: HTMLLIElement;
   public source: string;
-  private _currentInsertText: string;
 
   get isDocumentationMarkdown(): boolean {
     return this._isDocumentationMarkdown;
@@ -55,7 +50,6 @@ export class CompletionItem implements IExtendedCompletionItem {
   public label: string;
 
   icon: LabIcon | undefined;
-  private match: lsProtocol.CompletionItem;
 
   constructor(protected options: CompletionItem.IOptions) {
     const match = options.match;
@@ -63,7 +57,7 @@ export class CompletionItem implements IExtendedCompletionItem {
     this._setDocumentation(match.documentation);
     this._resolved = false;
     this._detail = match.detail;
-    this.match = match;
+    this._match = match;
     this.self = this;
     this.source = options.source;
     this.icon = options.icon ? options.icon : undefined;
@@ -74,48 +68,38 @@ export class CompletionItem implements IExtendedCompletionItem {
     // Ideally this would be fixed and tested e2e in JupyterLab 4.0.7.
     // https://github.com/jupyterlab/jupyterlab/issues/15125
     makeGetterEnumerable(this, 'insertText');
+    makeGetterEnumerable(this, 'sortText');
+    makeGetterEnumerable(this, 'filterText');
   }
 
   get type() {
     return this.options.type;
   }
 
-  private _setDocumentation(
-    documentation: string | lsProtocol.MarkupContent | undefined
-  ) {
-    if (lsProtocol.MarkupContent.is(documentation)) {
-      this._documentation = documentation.value;
-      this._isDocumentationMarkdown = documentation.kind === 'markdown';
-    } else {
-      this._documentation = documentation;
-      this._isDocumentationMarkdown = false;
-    }
-  }
-
   /**
    * Completion to be inserted.
    */
   get insertText(): string {
-    return this._currentInsertText || this.match.insertText || this.match.label;
+    return (
+      this._currentInsertText || this._match.insertText || this._match.label
+    );
   }
-
   set insertText(text: string) {
     this._currentInsertText = text;
   }
 
   get sortText(): string {
-    return this.match.sortText || this.match.label;
+    return this._currentSortText || this._match.sortText || this._match.label;
+  }
+  set sortText(text: string) {
+    this._currentSortText = text;
   }
 
   get filterText(): string | undefined {
-    return this.match.filterText;
+    return this._match.filterText;
   }
-
-  private _supportsResolution(): boolean {
-    const connection = this.options.connection;
-    return (
-      connection.serverCapabilities.completionProvider?.resolveProvider ?? false
-    );
+  set filterText(text: string | undefined) {
+    this._match.filterText = text;
   }
 
   get detail(): string | undefined {
@@ -153,7 +137,7 @@ export class CompletionItem implements IExtendedCompletionItem {
 
     const resolvedCompletionItem = await connection.clientRequests[
       'completionItem/resolve'
-    ].request(this.match);
+    ].request(this._match);
 
     if (resolvedCompletionItem === null) {
       return this;
@@ -181,16 +165,43 @@ export class CompletionItem implements IExtendedCompletionItem {
    * Indicates if the item is deprecated.
    */
   get deprecated(): boolean {
-    if (this.match.deprecated) {
-      return this.match.deprecated;
+    if (this._match.deprecated) {
+      return this._match.deprecated;
     }
     return (
-      this.match.tags != null &&
-      this.match.tags.some(
+      this._match.tags != null &&
+      this._match.tags.some(
         tag => tag == lsProtocol.CompletionItemTag.Deprecated
       )
     );
   }
+
+  private _setDocumentation(
+    documentation: string | lsProtocol.MarkupContent | undefined
+  ) {
+    if (lsProtocol.MarkupContent.is(documentation)) {
+      this._documentation = documentation.value;
+      this._isDocumentationMarkdown = documentation.kind === 'markdown';
+    } else {
+      this._documentation = documentation;
+      this._isDocumentationMarkdown = false;
+    }
+  }
+
+  private _supportsResolution(): boolean {
+    const connection = this.options.connection;
+    return (
+      connection.serverCapabilities.completionProvider?.resolveProvider ?? false
+    );
+  }
+
+  private _detail: string | undefined;
+  private _documentation: string | undefined;
+  private _isDocumentationMarkdown: boolean;
+  private _resolved: boolean;
+  private _currentInsertText: string;
+  private _currentSortText: string;
+  private _match: lsProtocol.CompletionItem;
 }
 
 function makeGetterEnumerable(instance: object, name: string) {
diff --git a/packages/jupyterlab-lsp/src/features/completion/model.spec.ts b/packages/jupyterlab-lsp/src/features/completion/model.spec.ts
index 3cf2ae073..19e4189f0 100644
--- a/packages/jupyterlab-lsp/src/features/completion/model.spec.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/model.spec.ts
@@ -8,14 +8,15 @@ describe('LSPCompleterModel', () => {
 
   function createDummyItem(
     match: lsProtocol.CompletionItem,
-    type: string = 'dummy'
+    type: string = 'dummy',
+    source: string = 'lsp'
   ) {
     return new CompletionItem({
       type,
       icon: null as any,
       match,
       connection: null as any,
-      source: 'lsp'
+      source: source
     });
   }
 
@@ -79,6 +80,68 @@ describe('LSPCompleterModel', () => {
     expect(sortedItems.map(item => item.sortText)).toEqual(['a', 'b', 'c']);
   });
 
+  describe('sorting by source', () => {
+    const testCompletionA = createDummyItem(
+      {
+        label: 'test'
+      },
+      'a',
+      'LSP'
+    );
+    const testCompletionB = createDummyItem(
+      {
+        label: 'test'
+      },
+      'b',
+      'kernel'
+    );
+    const testCompletionC = createDummyItem(
+      {
+        label: 'test'
+      },
+      'c',
+      'context'
+    );
+    const testCompletionD = createDummyItem(
+      {
+        label: 'test'
+      },
+      'd',
+      'unknown'
+    );
+    const completionsFromDifferentSources = [
+      testCompletionA,
+      testCompletionC,
+      testCompletionB
+    ];
+
+    it('completions are sorted by source', () => {
+      model.setCompletionItems(completionsFromDifferentSources);
+      model.query = 'test';
+      let sortedItems = model.completionItems();
+      expect(sortedItems.map(item => item.type)).toEqual(['a', 'b', 'c']);
+    });
+
+    it('kernel completions can be prioritised', () => {
+      model.setCompletionItems(completionsFromDifferentSources);
+      model.query = 'test';
+      model.settings.kernelCompletionsFirst = true;
+      let sortedItems = model.completionItems();
+      expect(sortedItems.map(item => item.type)).toEqual(['b', 'a', 'c']);
+    });
+
+    it('completions from unknown source land at the end', () => {
+      model.setCompletionItems([
+        testCompletionD,
+        ...completionsFromDifferentSources
+      ]);
+      model.query = 'test';
+      let sortedItems = model.completionItems();
+      const types = sortedItems.map(item => item.type);
+      expect(types[types.length - 1]).toEqual('d');
+    });
+  });
+
   it('ignores perfect matches when asked', () => {
     model = new LSPCompleterModel({
       includePerfectMatches: false
diff --git a/packages/jupyterlab-lsp/src/features/completion/model.ts b/packages/jupyterlab-lsp/src/features/completion/model.ts
index 90b485359..d69924a3b 100644
--- a/packages/jupyterlab-lsp/src/features/completion/model.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/model.ts
@@ -38,14 +38,16 @@ export class GenericCompleterModel<
 
   completionItems(): T[] {
     let query = this.query;
-    // TODO: make use of `processedItemsCache` when made available upstream,
-    // see https://github.com/jupyterlab/jupyterlab/pull/15025
     // (setting query is bad because it resets the cache; ideally we would
     // modify the sorting and filtering algorithm upstream).
+
+    // TODO processedItemsCache
     this.query = '';
+
     let unfilteredItems = (
       super.completionItems() as CompletionHandler.ICompletionItem[]
     ).map(this.harmoniseItem);
+
     this.query = query;
 
     // always want to sort
@@ -178,7 +180,7 @@ export class GenericCompleterModel<
       }
     }
 
-    results.sort(this.compareMatches);
+    results.sort(this.compareMatches.bind(this));
 
     return results.map(x => x.item);
   }
@@ -206,14 +208,19 @@ export namespace GenericCompleterModel {
      */
     includePerfectMatches?: boolean;
     /**
-     * Wheteher matches should be pre-filtered (default = true)
+     * Whether matches should be pre-filtered (default = true)
      */
     preFilterMatches?: boolean;
+    /**
+     * Whether kernel completions should be shown first.
+     */
+    kernelCompletionsFirst?: boolean;
   }
   export const defaultOptions: IOptions = {
     caseSensitive: true,
     includePerfectMatches: true,
-    preFilterMatches: true
+    preFilterMatches: true,
+    kernelCompletionsFirst: false
   };
 }
 
@@ -239,13 +246,22 @@ export class LSPCompleterModel extends GenericCompleterModel<MaybeCompletionItem
     a: ICompletionMatch<MaybeCompletionItem>,
     b: ICompletionMatch<MaybeCompletionItem>
   ): number {
-    const delta = a.score - b.score;
-    if (delta !== 0) {
-      return delta;
-    }
-    // solve ties using sortText
-
-    // note: locale compare is case-insensitive
-    return (a.item.sortText ?? 'z').localeCompare(b.item.sortText ?? 'z');
+    // TODO: take source order from provider ranks, upstream this code
+    const sourceOrder = {
+      LSP: 1,
+      kernel: this.settings.kernelCompletionsFirst ? 0 : 2,
+      context: 3
+    };
+    const aRank = a.item.source
+      ? sourceOrder[a.item.source as keyof typeof sourceOrder] ?? 4
+      : 4;
+    const bRank = b.item.source
+      ? sourceOrder[b.item.source as keyof typeof sourceOrder] ?? 4
+      : 4;
+    return (
+      aRank - bRank ||
+      (a.item.sortText ?? 'z').localeCompare(b.item.sortText ?? 'z') ||
+      a.score - b.score
+    );
   }
 }
diff --git a/packages/jupyterlab-lsp/src/features/completion/overrides.ts b/packages/jupyterlab-lsp/src/features/completion/overrides.ts
index 42fe44ad9..d1a267c42 100644
--- a/packages/jupyterlab-lsp/src/features/completion/overrides.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/overrides.ts
@@ -36,21 +36,17 @@ export class EnhancedContextCompleterProvider extends ContextCompleterProvider {
     super();
   }
 
-  private get _kernelCompletionsFirst(): boolean {
-    return this.options.settings.composite.kernelCompletionsFirst;
-  }
-
   async fetch(
     request: CompletionHandler.IRequest,
     context: ICompletionContext
   ): Promise<CompletionHandler.ICompletionItemsReply> {
     const result = await super.fetch(request, context);
-    result.items = result.items.map(i => {
+    result.items = result.items.map((item, order) => {
       return {
-        ...i,
-        icon: this.iconFor(i.type ?? 'Text') ?? this.iconFor('Text'),
-        type: i.type === '<unknown>' ? undefined : (i.type as string),
-        sortText: this._kernelCompletionsFirst ? 'a' : 'z',
+        ...item,
+        icon: this.iconFor(item.type ?? 'Text') ?? this.iconFor('Text'),
+        type: item.type === '<unknown>' ? undefined : (item.type as string),
+        sortText: String.fromCharCode(order),
         source: this.label
       };
     });
@@ -75,11 +71,11 @@ export class EnhancedKernelCompleterProvider extends KernelCompleterProvider {
     context: ICompletionContext
   ): Promise<CompletionHandler.ICompletionItemsReply> {
     const result = await super.fetch(request, context);
-    result.items = result.items.map(i => {
+    result.items = result.items.map((item, order) => {
       return {
-        ...i,
-        icon: this.iconFor(i.type ?? KernelKind) ?? this.iconFor(KernelKind),
-        sortText: 'z',
+        ...item,
+        icon: this.iconFor(item.type ?? KernelKind) ?? this.iconFor(KernelKind),
+        sortText: String.fromCharCode(order),
         source: this.label
       };
     });
@@ -88,7 +84,7 @@ export class EnhancedKernelCompleterProvider extends KernelCompleterProvider {
 
   async isApplicable(context: ICompletionContext): Promise<boolean> {
     // Note: this method logs errors instead of throwing to ensure we do not ever
-    // break the upstream kernel completer, even if there is an error elsehwere.
+    // break the upstream kernel completer, even if there is an error elsewhere.
     const upstream = await super.isApplicable(context);
 
     if (upstream === false) {
diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index b4c3209ce..0776433f2 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -65,13 +65,15 @@ export class CompletionProvider implements ICompletionProvider<CompletionItem> {
     const model = new LSPCompleterModel({
       caseSensitive: composite.caseSensitive,
       preFilterMatches: composite.preFilterMatches,
-      includePerfectMatches: composite.includePerfectMatches
+      includePerfectMatches: composite.includePerfectMatches,
+      kernelCompletionsFirst: composite.kernelCompletionsFirst
     });
     this.options.settings.changed.connect(() => {
       const composite = this.options.settings.composite;
       model.settings.caseSensitive = composite.caseSensitive;
       model.settings.preFilterMatches = composite.preFilterMatches;
       model.settings.includePerfectMatches = composite.includePerfectMatches;
+      model.settings.kernelCompletionsFirst = composite.kernelCompletionsFirst;
     });
     return model;
   };

From b693d56530c745b48954c22be4ea11c28b223f4f Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 16 Sep 2023 22:15:26 +0100
Subject: [PATCH 43/50] Fix path completions and eliding in path rendering

---
 .../src/features/completion/model.ts          | 23 +++++++++++--
 .../src/features/completion/provider.ts       | 33 +++++++++++++------
 .../src/features/completion/renderer.ts       |  8 ++---
 3 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/model.ts b/packages/jupyterlab-lsp/src/features/completion/model.ts
index d69924a3b..f709451f1 100644
--- a/packages/jupyterlab-lsp/src/features/completion/model.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/model.ts
@@ -62,19 +62,24 @@ export class GenericCompleterModel<
   setCompletionItems(newValue: T[]) {
     super.setCompletionItems(newValue);
 
-    if (this.settings.preFilterMatches && this.current && this.cursor) {
+    if (this.current && this.cursor) {
       // set initial query to pre-filter items; in future we should use:
       // https://github.com/jupyterlab/jupyterlab/issues/9763#issuecomment-1001603348
       const { start, end } = this.cursor;
       let query = this.current.text.substring(start, end).trim();
+      let trimmedQuotes = false;
       // special case for "Completes Paths In Strings" test case
       if (query.startsWith('"') || query.startsWith("'")) {
         query = query.substring(1);
+        trimmedQuotes = true;
       }
       if (query.endsWith('"') || query.endsWith("'")) {
         query = query.substring(0, -1);
+        trimmedQuotes = true;
+      }
+      if (this.settings.preFilterMatches || trimmedQuotes) {
+        this.query = query;
       }
-      this.query = query;
     }
   }
 
@@ -99,6 +104,20 @@ export class GenericCompleterModel<
     return index > -1 ? item.label.substring(0, index) : item.label;
   }
 
+  createPatch(patch: string) {
+    if (this.subsetMatch) {
+      // Prevent insertion code path when auto-populating subset on tab, to avoid problems with
+      // prefix which is a subset of token incorrectly replacing a string with file system path.
+      // - Q: Which code path is being blocked?
+      //   A: The code path (b) discussed in https://github.com/jupyterlab/jupyterlab/issues/15130.
+      // - Q: Why are we short- circuiting here?
+      //   A: we want to prevent `onCompletionSelected()` from proceeding with text insertion,
+      //      but direct extension of Completer handler is difficult.
+      return undefined;
+    }
+    return super.createPatch(patch);
+  }
+
   private _sortAndFilter(query: string, items: T[]): T[] {
     let results: ICompletionMatch<T>[] = [];
 
diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index 0776433f2..bdc6addef 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -307,6 +307,13 @@ export class CompletionProvider implements ICompletionProvider<CompletionItem> {
     return true;
   }
 }
+
+function stripQuotes(path: string): string {
+  return path.slice(
+    path.startsWith("'") || path.startsWith('"') ? 1 : 0,
+    path.endsWith("'") || path.endsWith('"') ? -1 : path.length
+  );
+}
 export function transformLSPCompletions<T>(
   token: CodeEditor.IToken,
   positionInToken: number,
@@ -317,6 +324,7 @@ export function transformLSPCompletions<T>(
   let prefix = token.value.slice(0, positionInToken + 1);
   let allNonPrefixed = true;
   let items: T[] = [];
+  let prefixLength = prefix.length;
   lspCompletionItems.forEach(match => {
     let kind = match.kind ? CompletionItemKind[match.kind] : '';
 
@@ -338,22 +346,27 @@ export function transformLSPCompletions<T>(
       }
     }
     // add prefix if needed
-    else if (token.type === 'string' && prefix.includes('/')) {
+    else if (token.type === 'String' && prefix.includes('/')) {
       // special case for path completion in strings, ensuring that:
       //     '/Com<tab> → '/Completion.ipynb
       // when the returned insert text is `Completion.ipynb` (the token here is `'/Com`)
       // developed against pyls and pylsp server, may not work well in other cases
-      const parts = prefix.split('/');
+
+      const parts = stripQuotes(prefix).split('/');
       if (
         text.toLowerCase().startsWith(parts[parts.length - 1].toLowerCase())
       ) {
         let pathPrefix = parts.slice(0, -1).join('/') + '/';
-        match.insertText = pathPrefix + text;
-        // for label removing the prefix quote if present
-        if (pathPrefix.startsWith("'") || pathPrefix.startsWith('"')) {
-          pathPrefix = pathPrefix.substr(1);
-        }
+        const withStartingQuote =
+          (prefix.startsWith("'") || prefix.startsWith('"') ? prefix[0] : '') +
+          pathPrefix +
+          text;
+        match.insertText = withStartingQuote;
+        // for label without quotes
         match.label = pathPrefix + match.label;
+        if (prefix.endsWith("'") || prefix.endsWith('"')) {
+          prefixLength = prefix.length - 1;
+        }
         allNonPrefixed = false;
       }
     }
@@ -369,8 +382,8 @@ export function transformLSPCompletions<T>(
   // completion of dictionaries for Python with jedi-language-server was
   // causing an issue for dic['<tab>'] case; to avoid this let's make
   // sure that prefix.length >= prefix.offset
-  if (allNonPrefixed && prefixOffset > prefix.length) {
-    prefixOffset = prefix.length;
+  if (allNonPrefixed && prefixOffset > prefixLength) {
+    prefixOffset = prefixLength;
   }
 
   let response = {
@@ -384,7 +397,7 @@ export function transformLSPCompletions<T>(
     // text = token.value + text;
     // but it did not work for "from statistics <tab>" and lead to "from statisticsimport" (no space)
     start: token.offset + (allNonPrefixed ? prefixOffset : 0),
-    end: token.offset + prefix.length,
+    end: token.offset + prefixLength,
     items: items,
     source: 'LSP'
   };
diff --git a/packages/jupyterlab-lsp/src/features/completion/renderer.ts b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
index b100eb2fc..61fc8bd50 100644
--- a/packages/jupyterlab-lsp/src/features/completion/renderer.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
@@ -161,14 +161,14 @@ export class LSPCompletionRenderer
     const originalHTMLLabel = labelElement.childNodes;
     let hasMark = false;
     for (const node of originalHTMLLabel) {
-      if (node.nodeType === Node.ELEMENT_NODE) {
-        const element = node as Element;
+      if (node instanceof HTMLElement) {
+        const element = node as HTMLElement;
         const text = element.textContent;
-        if (element.tagName === 'MARK' && text) {
+        if (element.tagName === 'MARK' && text && text.length > 3) {
           const elidableElement = document.createElement('bdo');
           elidableElement.setAttribute('dir', 'ltr');
           elidableElement.textContent = text;
-          elidableElement.title = text;
+          element.title = text;
           element.replaceChildren(elidableElement);
           element.classList.add('lsp-elide');
           hasMark = true;

From 730da0c929d9a4cde6335e0bc9692d17c905d3e6 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sat, 16 Sep 2023 22:42:55 +0100
Subject: [PATCH 44/50] Add rank to provider

---
 packages/jupyterlab-lsp/src/features/completion/provider.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index bdc6addef..e710d58e8 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -42,6 +42,7 @@ interface IOptions {
 export class CompletionProvider implements ICompletionProvider<CompletionItem> {
   readonly identifier = 'lsp';
   readonly label = 'LSP';
+  readonly rank = 1000;
   protected console = new BrowserConsole().scope('Completion provider');
 
   constructor(protected options: IOptions) {

From b7f04a3663da00baf6e2f88c656c14c94b81b921 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 11:54:27 +0100
Subject: [PATCH 45/50] Robustify handling of special cases in completion
 provider

Notably, we no longer adjust start/end position based on individual
items because that was not predictable; instead we revert the logic
and populate insert text as needed to match start/end. We only change
end to equal start if there are absolutely no prefix matches, which
is required because otherwise everything would be filtered out.
---
 atest/05_Features/Completion.robot            |  2 +-
 .../src/features/completion/model.ts          |  5 +-
 .../src/features/completion/provider.ts       | 95 ++++++++++---------
 3 files changed, 53 insertions(+), 49 deletions(-)

diff --git a/atest/05_Features/Completion.robot b/atest/05_Features/Completion.robot
index 7b5788e33..2c7936ed3 100644
--- a/atest/05_Features/Completion.robot
+++ b/atest/05_Features/Completion.robot
@@ -85,7 +85,7 @@ Invalidates On Focus Loss
     Completer Should Not Suggest    test
     Enter Cell Editor    1    line=2
 
-Uses LSP Completions When Kernel Resoponse Times Out
+Uses LSP Completions When Kernel Response Times Out
     [Tags]    requires:busy-indicator
     Configure JupyterLab Plugin    {"kernelResponseTimeout": 1, "waitForBusyKernel": true}
     ...    plugin id=${COMPLETION PLUGIN ID}
diff --git a/packages/jupyterlab-lsp/src/features/completion/model.ts b/packages/jupyterlab-lsp/src/features/completion/model.ts
index f709451f1..7c1caa9a0 100644
--- a/packages/jupyterlab-lsp/src/features/completion/model.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/model.ts
@@ -256,7 +256,10 @@ export class LSPCompleterModel extends GenericCompleterModel<MaybeCompletionItem
 
   protected harmoniseItem(item: CompletionHandler.ICompletionItem) {
     if ((item as any).self) {
-      return (item as any).self;
+      const self = (item as any).self;
+      // reflect any changes made on copy
+      self.insertText = item.insertText;
+      return self;
     }
     return super.harmoniseItem(item);
   }
diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index e710d58e8..9eb7d81c5 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -322,53 +322,66 @@ export function transformLSPCompletions<T>(
   createCompletionItem: (kind: string, match: lsProtocol.CompletionItem) => T,
   console: ILSPLogConsole
 ) {
-  let prefix = token.value.slice(0, positionInToken + 1);
-  let allNonPrefixed = true;
+  let prefix = token.value.slice(0, positionInToken);
+  let suffix = token.value.slice(positionInToken, token.value.length);
   let items: T[] = [];
-  let prefixLength = prefix.length;
+  // If there are no prefixes, we will just insert the text without replacing the token,
+  // which is the case for example in R for `stats::<tab>` which returns module members
+  // without `::` prefix.
+  // If there are prefixes, we will replace the token so we may need to prepend/append to,
+  // or otherwise modify the insert text of individual completion items.
+  let anyPrefixed = false;
+
   lspCompletionItems.forEach(match => {
     let kind = match.kind ? CompletionItemKind[match.kind] : '';
 
-    // Update prefix values
     let text = match.insertText ? match.insertText : match.label;
+    let intendedText = match.insertText ? match.insertText : match.label;
 
-    // declare prefix presence if needed and update it
-    if (text.toLowerCase().startsWith(prefix.toLowerCase())) {
-      allNonPrefixed = false;
-      if (prefix !== token.value) {
-        if (text.toLowerCase().startsWith(token.value.toLowerCase())) {
-          // given a completion insert text "display_table" and two test cases:
-          // disp<tab>data →  display_table<cursor>data
-          // disp<tab>lay  →  display_table<cursor>
-          // we have to adjust the prefix for the latter (otherwise we would get display_table<cursor>lay),
-          // as we are constrained NOT to replace after the prefix (which would be "disp" otherwise)
-          prefix = token.value;
-        }
+    if (intendedText.toLowerCase().startsWith(prefix.toLowerCase())) {
+      anyPrefixed = true;
+    }
+
+    // Add overlap with token prefix
+    if (intendedText.toLowerCase().startsWith(token.value.toLowerCase())) {
+      anyPrefixed = true;
+      // remove overlap with prefix before expanding it
+      if (intendedText.toLowerCase().startsWith(prefix.toLowerCase())) {
+        text = text.substring(prefix.length, text.length);
+        match.insertText = text;
       }
+      text = token.value + text;
+      match.insertText = text;
     }
-    // add prefix if needed
-    else if (token.type === 'String' && prefix.includes('/')) {
-      // special case for path completion in strings, ensuring that:
-      //     '/Com<tab> → '/Completion.ipynb
-      // when the returned insert text is `Completion.ipynb` (the token here is `'/Com`)
-      // developed against pyls and pylsp server, may not work well in other cases
 
+    // special handling for paths
+    if (token.type === 'String' && prefix.includes('/')) {
       const parts = stripQuotes(prefix).split('/');
       if (
         text.toLowerCase().startsWith(parts[parts.length - 1].toLowerCase())
       ) {
         let pathPrefix = parts.slice(0, -1).join('/') + '/';
-        const withStartingQuote =
+        text =
           (prefix.startsWith("'") || prefix.startsWith('"') ? prefix[0] : '') +
           pathPrefix +
           text;
-        match.insertText = withStartingQuote;
+        match.insertText = text;
         // for label without quotes
         match.label = pathPrefix + match.label;
-        if (prefix.endsWith("'") || prefix.endsWith('"')) {
-          prefixLength = prefix.length - 1;
+        anyPrefixed = true;
+      }
+    } else {
+      // harmonise end to token
+      if (text.toLowerCase().endsWith(suffix.toLowerCase())) {
+        text = text.substring(0, text.length - suffix.length);
+        match.insertText = text;
+      } else if (token.type === 'String') {
+        // special case for completion in strings to preserve the closing quote;
+        // there is an issue that this gives opposing results in Notebook vs File editor
+        // probably due to reconciliator logic
+        if (suffix.startsWith("'") || suffix.startsWith('"')) {
+          match.insertText = text + suffix[0];
         }
-        allNonPrefixed = false;
       }
     }
 
@@ -377,29 +390,17 @@ export function transformLSPCompletions<T>(
     items.push(completionItem);
   });
   console.debug('Transformed');
-  // required to make the repetitive trigger characters like :: or ::: work for R with R languageserver,
-  // see https://github.com/jupyter-lsp/jupyterlab-lsp/issues/436
-  let prefixOffset = token.value.length;
-  // completion of dictionaries for Python with jedi-language-server was
-  // causing an issue for dic['<tab>'] case; to avoid this let's make
-  // sure that prefix.length >= prefix.offset
-  if (allNonPrefixed && prefixOffset > prefixLength) {
-    prefixOffset = prefixLength;
+
+  let start = token.offset;
+  let end = token.offset + token.value.length;
+  if (!anyPrefixed) {
+    start = end;
   }
 
   let response = {
-    // note in the ContextCompleter it was:
-    // start: token.offset,
-    // end: token.offset + token.value.length,
-    // which does not work with "from statistics import <tab>" as the last token ends at "t" of "import",
-    // so the completer would append "mean" as "from statistics importmean" (without space!);
-    // (in such a case the typedCharacters is undefined as we are out of range)
-    // a different workaround would be to prepend the token.value prefix:
-    // text = token.value + text;
-    // but it did not work for "from statistics <tab>" and lead to "from statisticsimport" (no space)
-    start: token.offset + (allNonPrefixed ? prefixOffset : 0),
-    end: token.offset + prefixLength,
-    items: items,
+    start,
+    end,
+    items,
     source: 'LSP'
   };
   if (response.start > response.end) {

From d90e67144faa76b35f0c14bd0511e86c7fc5b8c0 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 12:01:39 +0100
Subject: [PATCH 46/50] Bump version of other packages

---
 packages/code-jumpers/package.json     |  2 +-
 packages/completion-theme/package.json |  2 +-
 packages/jupyterlab-lsp/package.json   |  8 ++++----
 packages/theme-material/package.json   |  4 ++--
 packages/theme-vscode/package.json     |  4 ++--
 yarn.lock                              | 20 ++++++++++----------
 6 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/packages/code-jumpers/package.json b/packages/code-jumpers/package.json
index df2aefca0..67181e195 100644
--- a/packages/code-jumpers/package.json
+++ b/packages/code-jumpers/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/code-jumpers",
-  "version": "1.2.0",
+  "version": "2.0.0-rc.0",
   "description": "Implementation underlying the jump to definition functionality in JupyterLab-LSP",
   "keywords": [
     "jupyter",
diff --git a/packages/completion-theme/package.json b/packages/completion-theme/package.json
index 9e1e380e6..b679d7352 100644
--- a/packages/completion-theme/package.json
+++ b/packages/completion-theme/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/completion-theme",
-  "version": "3.3.0",
+  "version": "4.0.0-rc.0",
   "description": "Completion theme manager for JupyterLab-LSP",
   "keywords": [
     "jupyter",
diff --git a/packages/jupyterlab-lsp/package.json b/packages/jupyterlab-lsp/package.json
index 962e391d2..4b1306814 100644
--- a/packages/jupyterlab-lsp/package.json
+++ b/packages/jupyterlab-lsp/package.json
@@ -60,10 +60,10 @@
     "watch:src": "tsc -w"
   },
   "dependencies": {
-    "@jupyter-lsp/code-jumpers": "~1.2.0",
-    "@jupyter-lsp/completion-theme": "~3.3.0",
-    "@jupyter-lsp/theme-material": "~2.1.1",
-    "@jupyter-lsp/theme-vscode": "~2.1.1",
+    "@jupyter-lsp/code-jumpers": "~2.0.0-rc.0",
+    "@jupyter-lsp/completion-theme": "~4.0.0-rc.0",
+    "@jupyter-lsp/theme-material": "~3.0.0-rc.0",
+    "@jupyter-lsp/theme-vscode": "~3.0.0-rc.0",
     "@jupyterlab/lsp": "^4.0.6",
     "@rjsf/validator-ajv8": "^5.12.1",
     "lodash.mergewith": "^4.6.1"
diff --git a/packages/theme-material/package.json b/packages/theme-material/package.json
index cc448da90..15413edee 100644
--- a/packages/theme-material/package.json
+++ b/packages/theme-material/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/theme-material",
-  "version": "2.1.1",
+  "version": "3.0.0-rc.0",
   "description": "Material theme for JupyterLab-LSP",
   "keywords": [
     "jupyter",
@@ -34,7 +34,7 @@
     "clean": "rimraf lib"
   },
   "dependencies": {
-    "@jupyter-lsp/completion-theme": "^3.0.0"
+    "@jupyter-lsp/completion-theme": "^4.0.0-rc.0"
   },
   "jupyterlab": {
     "extension": true,
diff --git a/packages/theme-vscode/package.json b/packages/theme-vscode/package.json
index 613f48437..f12d713a2 100644
--- a/packages/theme-vscode/package.json
+++ b/packages/theme-vscode/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupyter-lsp/theme-vscode",
-  "version": "2.1.1",
+  "version": "3.0.0-rc.0",
   "description": "VSCode theme for JupyterLab-LSP",
   "keywords": [
     "jupyter",
@@ -34,7 +34,7 @@
     "clean": "rimraf lib"
   },
   "dependencies": {
-    "@jupyter-lsp/completion-theme": "^3.0.0"
+    "@jupyter-lsp/completion-theme": "^4.0.0-rc.0"
   },
   "jupyterlab": {
     "extension": true,
diff --git a/yarn.lock b/yarn.lock
index b82f3bd22..89faf36b0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2090,7 +2090,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@jupyter-lsp/code-jumpers@workspace:*, @jupyter-lsp/code-jumpers@workspace:packages/code-jumpers, @jupyter-lsp/code-jumpers@~1.2.0":
+"@jupyter-lsp/code-jumpers@workspace:*, @jupyter-lsp/code-jumpers@workspace:packages/code-jumpers, @jupyter-lsp/code-jumpers@~2.0.0-rc.0":
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/code-jumpers@workspace:packages/code-jumpers"
   dependencies:
@@ -2119,7 +2119,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@jupyter-lsp/completion-theme@^3.0.0, @jupyter-lsp/completion-theme@workspace:*, @jupyter-lsp/completion-theme@workspace:packages/completion-theme, @jupyter-lsp/completion-theme@~3.3.0":
+"@jupyter-lsp/completion-theme@^4.0.0-rc.0, @jupyter-lsp/completion-theme@workspace:*, @jupyter-lsp/completion-theme@workspace:packages/completion-theme, @jupyter-lsp/completion-theme@~4.0.0-rc.0":
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/completion-theme@workspace:packages/completion-theme"
   dependencies:
@@ -2176,10 +2176,10 @@ __metadata:
   resolution: "@jupyter-lsp/jupyterlab-lsp@workspace:packages/jupyterlab-lsp"
   dependencies:
     "@codemirror/lint": ^6.4.0
-    "@jupyter-lsp/code-jumpers": ~1.2.0
-    "@jupyter-lsp/completion-theme": ~3.3.0
-    "@jupyter-lsp/theme-material": ~2.1.1
-    "@jupyter-lsp/theme-vscode": ~2.1.1
+    "@jupyter-lsp/code-jumpers": ~2.0.0-rc.0
+    "@jupyter-lsp/completion-theme": ~4.0.0-rc.0
+    "@jupyter-lsp/theme-material": ~3.0.0-rc.0
+    "@jupyter-lsp/theme-vscode": ~3.0.0-rc.0
     "@jupyter-notebook/application": ^7.0.3
     "@jupyterlab/application": ^4.0.6
     "@jupyterlab/apputils": ^4.1.6
@@ -2241,19 +2241,19 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@jupyter-lsp/theme-material@workspace:*, @jupyter-lsp/theme-material@workspace:packages/theme-material, @jupyter-lsp/theme-material@~2.1.1":
+"@jupyter-lsp/theme-material@workspace:*, @jupyter-lsp/theme-material@workspace:packages/theme-material, @jupyter-lsp/theme-material@~3.0.0-rc.0":
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/theme-material@workspace:packages/theme-material"
   dependencies:
-    "@jupyter-lsp/completion-theme": ^3.0.0
+    "@jupyter-lsp/completion-theme": ^4.0.0-rc.0
   languageName: unknown
   linkType: soft
 
-"@jupyter-lsp/theme-vscode@workspace:*, @jupyter-lsp/theme-vscode@workspace:packages/theme-vscode, @jupyter-lsp/theme-vscode@~2.1.1":
+"@jupyter-lsp/theme-vscode@workspace:*, @jupyter-lsp/theme-vscode@workspace:packages/theme-vscode, @jupyter-lsp/theme-vscode@~3.0.0-rc.0":
   version: 0.0.0-use.local
   resolution: "@jupyter-lsp/theme-vscode@workspace:packages/theme-vscode"
   dependencies:
-    "@jupyter-lsp/completion-theme": ^3.0.0
+    "@jupyter-lsp/completion-theme": ^4.0.0-rc.0
   languageName: unknown
   linkType: soft
 

From aacc79805a86b8ec2e2e4f7182af0456e4ba6d9c Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 16:53:25 +0100
Subject: [PATCH 47/50] Fix completer tests (make token substitution case
 sensitive, otherwise

selecting `ValueError` on `va` would produce `valueError`.
---
 atest/05_Features/Completion.robot            |  4 +++-
 .../src/features/completion/model.ts          | 22 ++++++++++++++++++-
 .../src/features/completion/provider.ts       |  7 +++---
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/atest/05_Features/Completion.robot b/atest/05_Features/Completion.robot
index 2c7936ed3..0081daa02 100644
--- a/atest/05_Features/Completion.robot
+++ b/atest/05_Features/Completion.robot
@@ -150,7 +150,7 @@ Continuous Hinting Works
     [Setup]    Prepare File for Editing    Python    completion    completion.py
     Configure JupyterLab Plugin    {"continuousHinting": true}    plugin id=${COMPLETION PLUGIN ID}
     # TODO: remove once we resolve https://github.com/jupyterlab/jupyterlab/issues/15022
-    Configure JupyterLab Plugin    {"autoCompletion": true}    plugin id=${MANAGER PLUGIN ID}
+    Configure JupyterLab Plugin    {"autoCompletion": true, "providerTimeout": 2500}    plugin id=${MANAGER PLUGIN ID}
     Place Cursor In File Editor At    9    2
     Wait For Ready State
     Press Keys    None    d
@@ -373,6 +373,8 @@ Completes Paths In Strings
 *** Keywords ***
 Setup Completion Test
     Setup Notebook    Python    Completion.ipynb
+    # TODO: this should be per-provider (upstream issue)
+    Configure JupyterLab Plugin    {"providerTimeout": 2500}    plugin id=${MANAGER PLUGIN ID}
 
 Get Cell Editor Content
     [Arguments]    ${cell_nr}
diff --git a/packages/jupyterlab-lsp/src/features/completion/model.ts b/packages/jupyterlab-lsp/src/features/completion/model.ts
index 7c1caa9a0..b2c0981ff 100644
--- a/packages/jupyterlab-lsp/src/features/completion/model.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/model.ts
@@ -65,8 +65,28 @@ export class GenericCompleterModel<
     if (this.current && this.cursor) {
       // set initial query to pre-filter items; in future we should use:
       // https://github.com/jupyterlab/jupyterlab/issues/9763#issuecomment-1001603348
+
+      // note: start/end from cursor are not ideal because these get populated from fetch
+      // reply which will vary depending on what providers decide to return; we want the
+      // actual position in token, the same as passed in request to fetch. We can get it
+      // by searching for longest common prefix as seen below (or by counting characters).
+      // Maybe upstream should expose it directly?
       const { start, end } = this.cursor;
-      let query = this.current.text.substring(start, end).trim();
+      const { text, line, column } = this.original!;
+
+      const queryRange = text.substring(start, end).trim();
+      const linePrefix = text.split('\n')[line].substring(0, column).trim();
+      let query = '';
+      for (let i = queryRange.length; i > 0; i--) {
+        if (queryRange.slice(0, i) == linePrefix.slice(-i)) {
+          query = linePrefix.slice(-i);
+          break;
+        }
+      }
+      if (!query) {
+        return;
+      }
+
       let trimmedQuotes = false;
       // special case for "Completes Paths In Strings" test case
       if (query.startsWith('"') || query.startsWith("'")) {
diff --git a/packages/jupyterlab-lsp/src/features/completion/provider.ts b/packages/jupyterlab-lsp/src/features/completion/provider.ts
index 9eb7d81c5..6ecc90863 100644
--- a/packages/jupyterlab-lsp/src/features/completion/provider.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/provider.ts
@@ -343,10 +343,10 @@ export function transformLSPCompletions<T>(
     }
 
     // Add overlap with token prefix
-    if (intendedText.toLowerCase().startsWith(token.value.toLowerCase())) {
+    if (intendedText.startsWith(token.value)) {
       anyPrefixed = true;
       // remove overlap with prefix before expanding it
-      if (intendedText.toLowerCase().startsWith(prefix.toLowerCase())) {
+      if (intendedText.startsWith(prefix)) {
         text = text.substring(prefix.length, text.length);
         match.insertText = text;
       }
@@ -364,7 +364,8 @@ export function transformLSPCompletions<T>(
         text =
           (prefix.startsWith("'") || prefix.startsWith('"') ? prefix[0] : '') +
           pathPrefix +
-          text;
+          text +
+          (suffix.startsWith("'") || suffix.startsWith('"') ? suffix[0] : '');
         match.insertText = text;
         // for label without quotes
         match.label = pathPrefix + match.label;

From cfac3fce59434b6da2f7791a1ec8068576e69bce Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 17:27:15 +0100
Subject: [PATCH 48/50] Restore pre-fetching resolve(), remove unused
 `activityObserver`

---
 .../src/features/completion/renderer.ts       | 39 +------------------
 1 file changed, 1 insertion(+), 38 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/renderer.ts b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
index 61fc8bd50..0c814fa1e 100644
--- a/packages/jupyterlab-lsp/src/features/completion/renderer.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
@@ -21,14 +21,11 @@ export class LSPCompletionRenderer
   implements Completer.IRenderer
 {
   // signals
-  public activeChanged: Signal<LSPCompletionRenderer, ICompletionData>;
   public itemShown: Signal<LSPCompletionRenderer, ICompletionData>;
   // observers
   private visibilityObserver: IntersectionObserver;
-  private activityObserver: MutationObserver;
   // element data maps (with weak references for better GC)
   private elementToItem: WeakMap<HTMLLIElement, CompletionItem>;
-  private wasActivated: WeakMap<HTMLLIElement, boolean>;
 
   protected ITEM_PLACEHOLDER_CLASS = 'lsp-detail-placeholder';
   protected EXTRA_INFO_CLASS = 'jp-Completer-typeExtended';
@@ -36,10 +33,8 @@ export class LSPCompletionRenderer
 
   constructor(protected options: LSPCompletionRenderer.IOptions) {
     super();
-    this.activeChanged = new Signal(this);
     this.itemShown = new Signal(this);
     this.elementToItem = new WeakMap();
-    this.wasActivated = new WeakMap();
 
     this.visibilityObserver = new IntersectionObserver(
       entries => {
@@ -49,41 +44,13 @@ export class LSPCompletionRenderer
           }
           let li = entry.target as HTMLLIElement;
           let item = this.elementToItem.get(li)!;
-          this.itemShown.emit({
-            item: item,
-            element: li
-          });
+          item.resolve().catch(console.error);
         });
       },
       {
         threshold: 0.25
       }
     );
-
-    // note: there should be no need to unobserve deleted elements as per:
-    // https://stackoverflow.com/a/51106262/6646912
-    this.activityObserver = new MutationObserver(mutations => {
-      mutations.forEach(mutation => {
-        let li = mutation.target;
-        if (!(li instanceof HTMLLIElement)) {
-          return;
-        }
-        let inactive = !this.wasActivated.get(li);
-
-        if (li.classList.contains('jp-mod-active')) {
-          if (inactive) {
-            this.wasActivated.set(li, true);
-            let item = this.elementToItem.get(li)!;
-            this.activeChanged.emit({
-              item: item,
-              element: li
-            });
-          }
-        } else {
-          this.wasActivated.set(li, false);
-        }
-      });
-    });
   }
 
   protected getExtraInfo(item: CompletionItem): string {
@@ -132,10 +99,6 @@ export class LSPCompletionRenderer
     if (lspItem) {
       lspItem.element = li;
       this.elementToItem.set(li, lspItem);
-      this.activityObserver.observe(li, {
-        attributes: true,
-        attributeFilter: ['class']
-      });
       this.visibilityObserver.observe(li);
       // TODO: build custom li from ground up
       this.updateExtraInfo(lspItem, li);

From 14046c490bea11eb95f7512c4f630fac8e0aa6a3 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 17:54:00 +0100
Subject: [PATCH 49/50] Fix highlights not updating when switching documents

---
 .../jupyterlab-lsp/src/features/highlights.ts    | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/highlights.ts b/packages/jupyterlab-lsp/src/features/highlights.ts
index f22dc418f..66abc37df 100644
--- a/packages/jupyterlab-lsp/src/features/highlights.ts
+++ b/packages/jupyterlab-lsp/src/features/highlights.ts
@@ -70,7 +70,10 @@ export class HighlightsFeature extends Feature {
   >;
   private _virtualPosition: IVirtualPosition;
   private _versionSent: number;
-  private _lastToken: CodeEditor.IToken | null = null;
+  private _lastToken: {
+    token: CodeEditor.IToken;
+    adapter: WidgetLSPAdapter<any>;
+  } | null = null;
 
   constructor(options: HighlightsFeature.IOptions) {
     super(options);
@@ -297,10 +300,12 @@ export class HighlightsFeature extends Feature {
     const token = editor.getTokenAt(offset);
 
     // if token has not changed, no need to update highlight, unless it is an empty token
-    // which would indicate that the cursor is at the first character
+    // which would indicate that the cursor is at the first character; we also need to check
+    // adapter in case if user switched between documents/notebooks.
     if (
       this._lastToken &&
-      token.value === this._lastToken.value &&
+      token.value === this._lastToken.token.value &&
+      adapter === this._lastToken.adapter &&
       token.value !== ''
     ) {
       this.console.log(
@@ -352,7 +357,10 @@ export class HighlightsFeature extends Feature {
       }
 
       this.handleHighlight(highlights, adapter, document);
-      this._lastToken = token;
+      this._lastToken = {
+        token,
+        adapter
+      };
     } catch (e) {
       this.console.warn('Could not get highlights:', e);
     }

From 382cad3c2c7258d715d68f66be8735c17c49dfef Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 17 Sep 2023 17:59:29 +0100
Subject: [PATCH 50/50] Remove unused signal

---
 packages/jupyterlab-lsp/src/features/completion/renderer.ts | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/jupyterlab-lsp/src/features/completion/renderer.ts b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
index 0c814fa1e..358e41407 100644
--- a/packages/jupyterlab-lsp/src/features/completion/renderer.ts
+++ b/packages/jupyterlab-lsp/src/features/completion/renderer.ts
@@ -3,7 +3,6 @@
 
 import { Completer } from '@jupyterlab/completer';
 import { IRenderMime } from '@jupyterlab/rendermime';
-import { Signal } from '@lumino/signaling';
 
 import { CodeCompletion as LSPCompletionSettings } from '../../_completion';
 import { FeatureSettings } from '../../feature';
@@ -20,8 +19,6 @@ export class LSPCompletionRenderer
   extends Completer.Renderer
   implements Completer.IRenderer
 {
-  // signals
-  public itemShown: Signal<LSPCompletionRenderer, ICompletionData>;
   // observers
   private visibilityObserver: IntersectionObserver;
   // element data maps (with weak references for better GC)
@@ -33,7 +30,6 @@ export class LSPCompletionRenderer
 
   constructor(protected options: LSPCompletionRenderer.IOptions) {
     super();
-    this.itemShown = new Signal(this);
     this.elementToItem = new WeakMap();
 
     this.visibilityObserver = new IntersectionObserver(