DB properties storage refactoring

.clj-kondo/config.edn

@@ -7,7 +7,7 @@
  ;; false positive with match/match and _
  frontend.handler.paste {:linters {:used-underscored-binding {:level :off}}}
  frontend.db {:linters {:aliased-namespace-symbol
- {:exclude [frontend.db.conn frontend.db.react logseq.db.frontend.default]}}}}
+ {:exclude [frontend.db.conn frontend.db.react]}}}}
 
  :linters
  {:path-invalid-construct/string-join {:level :info}
@@ -139,9 +139,12 @@
  logseq.db ldb
  logseq.db.frontend.content db-content
  logseq.db.frontend.inputs db-inputs
+ logseq.db.frontend.order db-order
  logseq.db.frontend.property db-property
+ logseq.db.frontend.property.build db-property-build
  logseq.db.frontend.property.type db-property-type
  logseq.db.frontend.property.util db-property-util
+ logseq.db.frontend.entity-plus entity-plus
  logseq.db.frontend.rules rules
  logseq.db.frontend.schema db-schema
  logseq.db.frontend.validate db-validate

.gitignore

@@ -63,3 +63,4 @@ deps/shui/.lsp
 deps/shui/.lsp-cache
 deps/shui/.clj-kondo
 deps/shui/shui-graph/logseq/bak
+tx-log*

bb.edn

@@ -1,7 +1,7 @@
 {:paths ["scripts/src" "src/main" "src/resources"]
  :deps
  {metosin/malli
- {:mvn/version "0.10.0"}
+ {:mvn/version "0.16.1"}
  logseq/bb-tasks
  #_{:local/root "../bb-tasks"}
  {:git/url "https://github.com/logseq/bb-tasks"

deps.edn

@@ -4,7 +4,7 @@
  rum/rum {:mvn/version "0.12.9"}
 
  datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
- :sha "fc3645fa4a6b8160e087e2ad1ce1c242675680cb"}
+ :sha "aa2f963be5d507b55d05c4d9a6ee3ce9454c4415"}
 
  datascript-transit/datascript-transit {:mvn/version "0.3.0"}
  borkdude/rewrite-edn {:mvn/version "0.4.7"}
@@ -16,6 +16,7 @@
  org.clojure/core.match {:mvn/version "1.0.0"}
  com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
  :sha "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
+ ;; TODO: delete cljs-drag-n-drop and use dnd-kit
  cljs-drag-n-drop/cljs-drag-n-drop {:mvn/version "0.1.0"}
  cljs-http/cljs-http {:mvn/version "0.1.46"}
  org.babashka/sci {:mvn/version "0.3.2"}
@@ -36,20 +37,20 @@
  logseq/outliner {:local/root "deps/outliner"}
  logseq/publishing {:local/root "deps/publishing"}
  logseq/shui {:local/root "deps/shui"}
- metosin/malli {:mvn/version "0.10.0"}
+ metosin/malli {:mvn/version "0.16.1"}
  com.cognitect/transit-cljs {:mvn/version "0.8.280"}
  missionary/missionary {:mvn/version "b.39"}}
 
  :aliases {:cljs {:extra-paths ["src/dev-cljs/" "src/test/" "src/electron/"]
- :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.54"}
+ :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.132"}
  org.clojure/tools.namespace {:mvn/version "0.2.11"}
  cider/cider-nrepl {:mvn/version "0.47.0"}
  org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}
  tortue/spy {:mvn/version "2.14.0"}}
  :main-opts ["-m" "shadow.cljs.devtools.cli"]}
 
  :test {:extra-paths ["src/test/"]
- :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.54"}
+ :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.132"}
  org.clojure/test.check {:mvn/version "1.1.1"}
  pjstadig/humane-test-output {:mvn/version "0.11.0"}
  org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}

deps/common/.carve/config.edn

@@ -1,5 +1,7 @@
 {:paths ["src"]
  :api-namespaces [logseq.common.path
+ logseq.common.fractional-index
+ logseq.common.missionary-util
  logseq.common.util.page-ref
  logseq.common.util.block-ref
  logseq.common.util

deps/common/deps.edn

@@ -5,7 +5,7 @@
  :aliases
  {:test {:extra-paths ["test"]
  :extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}
- org.clojure/clojurescript {:mvn/version "1.11.54"}}
+ org.clojure/clojurescript {:mvn/version "1.11.132"}}
  :main-opts ["-m" "cljs-test-runner.main"]}
  :clj-kondo
  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}

deps/common/package.json

@@ -3,7 +3,7 @@
  "version": "1.0.0",
  "private": true,
  "devDependencies": {
- "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v10"
+ "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v12"
  },
  "scripts": {
  "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/resources/templates/config.edn

@@ -215,7 +215,6 @@
  [?h :block/marker ?marker]
  [(contains? #{"NOW" "DOING"} ?marker)]
  [?h :block/page ?p]
- [?p :block/journal? true]
  [?p :block/journal-day ?d]
  [(>= ?d ?start)]
  [(<= ?d ?today)]]
@@ -232,7 +231,6 @@
  [?h :block/marker ?marker]
  [(contains? #{"NOW" "LATER" "TODO"} ?marker)]
  [?h :block/page ?p]
- [?p :block/journal? true]
  [?p :block/journal-day ?d]
  [(> ?d ?start)]
  [(< ?d ?next)]]

deps/common/src/logseq/common/fractional_index.cljs

@@ -0,0 +1,163 @@
+(ns logseq.common.fractional-index
+ "Fractional indexing to create an ordering that can be used for Realtime Editing of Ordered Sequences")
+
+;; Original code from https://github.com/rocicorp/fractional-indexing,
+;; It's converted to cljs by using AI.
+
+(def base-62-digits
+ "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
+
+(defn char->int
+ [^js c]
+ (.charCodeAt c))
+
+(defn get-integer-length
+ [head]
+ (let [head-char (char->int head)]
+ (cond
+ (and (>= head-char (char->int \a)) (<= head-char (char->int \z)))
+ (+ (- head-char (char->int \a)) 2)
+
+ (and (>= head-char (char->int \A)) (<= head-char (char->int \Z)))
+ (+ (- (char->int \Z) head-char) 2)
+
+ :else
+ (throw (js/Error. (str "invalid order key head: " head))))))
+
+(defn validate-integer
+ [int]
+ (when-not (= (count int) (get-integer-length (first int)))
+ (throw (js/Error. (str "invalid integer part of order key: " int)))))
+
+(defn get-integer-part
+ [key]
+ (let [integer-part-length (get-integer-length (first key))]
+ (when (> integer-part-length (count key))
+ (throw (js/Error. (str "invalid order key: " key))))
+ (subs key 0 integer-part-length)))
+
+(defn validate-order-key
+ [key digits]
+ (when (= key (str "A" (repeat 26 (first digits))))
+ (throw (js/Error. (str "invalid order key: " key))))
+ (let [i (get-integer-part key)
+ f (subs key (count i))]
+ (when (= (last f) (first digits))
+ (throw (js/Error. (str "invalid order key: " key))))))
+
+(defn increment-integer
+ [x digits]
+ (validate-integer x)
+ (let [[head & digs] (seq x)
+ [carry? diff] (reduce
+ (fn [[carry? digs] dig]
+ (if carry?
+ (let [d (inc (.indexOf digits dig))]
+ (if (= d (count digits))
+ [true (conj digs (first digits))]
+ [false (conj digs (nth digits d))]))
+ [carry? digs]))
+ [true []]
+ (reverse digs))
+ digs (into (subvec (vec digs) 0 (- (count digs)
+ (count diff)))
+ (reverse diff))]
+ (if carry?
+ (cond
+ (= head \Z) (str "a" (first digits))
+ (= head \z) nil
+ :else (let [h (char (inc (.charCodeAt head 0)))
+ digs (if (> (compare h \a) 0)
+ (conj digs (first digits))
+ (pop digs))]
+ (str h (apply str digs))))
+ (str head (apply str digs)))))
+
+(defn decrement-integer
+ [x digits]
+ (validate-integer x)
+ (let [[head & digs] (seq x)
+ [borrow digs] (reduce
+ (fn [[_ digs] dig]
+ (let [d (dec (.indexOf digits dig))]
+ (if (= d -1)
+ [true (conj digs (last digits))]
+ [false (conj digs (nth digits d))])))
+ [true []]
+ (reverse digs))]
+ (if borrow
+ (cond
+ (= head \a) (str "Z" (last digits))
+ (= head \A) nil
+ :else (let [h (char (- (.charCodeAt head 0) 1))
+ digs (if (< (compare h \Z) 0)
+ (conj digs (last digits))
+ (pop digs))]
+ (str h (apply str digs))))
+ (str head (apply str digs)))))
+
+(defn midpoint
+ [a b digits]
+ (let [zero (first digits)]
+ (when (and b (or (>= (compare a b) 0) (= (last a) zero) (= (last b) zero)))
+ (throw (js/Error. (str a " >= " b " or trailing zero"))))
+ (let [n (when b
+ (first (keep-indexed (fn [i _c] (when-not (= (nth a i zero) (nth b i)) i)) b)))]
+ (if (and n (> n 0))
+ (str (subs b 0 n) (midpoint (subs a n) (subs b n) digits))
+ (let [digit-a (if (seq a) (.indexOf digits (first a)) 0)
+ digit-b (if (seq b) (.indexOf digits (first b)) (count digits))]
+ (if (> (- digit-b digit-a) 1)
+ (str (nth digits (Math/round (* 0.5 (+ digit-a digit-b)))))
+ (if (and (seq b) (> (count b) 1))
+ (subs b 0 1)
+ (str (nth digits digit-a) (midpoint (subs a 1) nil digits)))))))))
+
+(defn generate-key-between
+ [a b & {:keys [digits]
+ :or {digits base-62-digits}}]
+ (when a (validate-order-key a digits))
+ (when b (validate-order-key b digits))
+ (when (and a b (>= (compare a b) 0))
+ (throw (js/Error. (str a " >= " b))))
+ (cond
+ (nil? a) (if (nil? b)
+ (str "a" (first digits))
+ (let [ib (get-integer-part b)
+ fb (subs b (count ib))]
+ (if (= ib (str "A" (apply str (repeat 26 (first digits)))))
+ (str ib (midpoint "" fb digits))
+ (if (< (compare ib b) 0)
+ ib
+ (let [res (decrement-integer ib digits)]
+ (if (nil? res)
+ (throw (js/Error. "cannot decrement any more"))
+ res))))))
+ (nil? b) (let [ia (get-integer-part a)
+ fa (subs a (count ia))
+ i (increment-integer ia digits)]
+ (if (nil? i)
+ (str ia (midpoint fa nil digits))
+ i))
+ :else (let [ia (get-integer-part a)
+ fa (subs a (count ia))
+ ib (get-integer-part b)
+ fb (subs b (count ib))]
+ (if (= ia ib)
+ (str ia (midpoint fa fb digits))
+ (let [i (increment-integer ia digits)]
+ (if (nil? i)
+ (throw (js/Error. "cannot increment any more"))
+ (if (< (compare i b) 0) i (str ia (midpoint fa nil digits)))))))))
+
+(defn generate-n-keys-between
+ [a b n & {:keys [digits]
+ :or {digits base-62-digits}}]
+ (cond
+ (= n 0) []
+ (= n 1) [(generate-key-between a b digits)]
+ :else (let [c (generate-key-between a b digits)]
+ (concat
+ (generate-n-keys-between a c (Math/floor (/ n 2)) digits)
+ [c]
+ (generate-n-keys-between c b (- n (Math/floor (/ n 2)) 1) digits)))))

deps/common/test/logseq/common/fractional_index_test.cljs

@@ -0,0 +1,62 @@
+(ns logseq.common.fractional-index-test
+ (:require [clojure.test :refer [deftest are]]
+ [logseq.common.fractional-index :as index]))
+
+(deftest increment-integer-test
+ (are [x y]
+ (= (index/increment-integer x index/base-62-digits) y)
+ "a0" "a1"
+ "r3333333333333333zz" "r333333333333333400"))
+
+(deftest generate-key-between-test
+ (are [x y]
+ (= (index/generate-key-between x nil) y)
+ "a0" "a1"
+ "rzzzzzzzzzzzzzzzzzz" "s0000000000000000000"))
+
+(deftest generate-n-keys-between-test
+ (are [x y]
+ (= (index/generate-n-keys-between (first x) (second x) 20) y)
+ ["ZxV" "Zy7"]
+ ["ZxX"
+ "ZxZ"
+ "Zxd"
+ "Zxf"
+ "Zxh"
+ "Zxl"
+ "Zxn"
+ "Zxp"
+ "Zxt"
+ "Zxx"
+ "Zy"
+ "Zy0V"
+ "Zy1"
+ "Zy2"
+ "Zy3"
+ "Zy4"
+ "Zy4V"
+ "Zy5"
+ "Zy6"
+ "Zy6V"]
+
+ ["Zy7" "axV"]
+ ["ZyB"
+ "ZyE"
+ "ZyL"
+ "ZyP"
+ "ZyS"
+ "ZyZ"
+ "Zyd"
+ "Zyg"
+ "Zyn"
+ "Zyu"
+ "Zz"
+ "Zz8"
+ "ZzG"
+ "ZzV"
+ "Zzl"
+ "a0"
+ "a0G"
+ "a0V"
+ "a1"
+ "a2"]))

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v10":
- version "1.2.173-feat-db-v10"
- resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8a4bfaae55d85292476ac8f976516b86e6ae4036"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v12":
+ version "1.2.173-feat-db-v12"
+ resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/8274d8b40b7fec84532c54e16b5a8c6f78bf673b"
  dependencies:
  import-meta-resolve "^2.1.0"
 

deps/db/.carve/config.edn

@@ -5,8 +5,10 @@
  logseq.db.sqlite.util
  logseq.db.sqlite.cli
  logseq.db.frontend.property
+ logseq.db.frontend.property.build
  logseq.db.frontend.property.util
  logseq.db.frontend.content
+ logseq.db.frontend.order
  logseq.db.sqlite.create-graph
  logseq.db.frontend.malli-schema
  ;; Some fns are used by frontend but not worth moving over yet

deps/db/.carve/ignore

@@ -5,8 +5,6 @@ logseq.db.frontend.rules/db-query-dsl-rules
 ;; API
 logseq.db.frontend.rules/extract-rules
 ;; API
-logseq.db.frontend.property.type/type-or-closed-value?
-;; API
 logseq.db.frontend.property.type/infer-property-type-from-value
 ;; Internal API
 logseq.db.frontend.rules/rules

deps/db/deps.edn

@@ -1,13 +1,14 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
- :sha "fc3645fa4a6b8160e087e2ad1ce1c242675680cb"}
+ :sha "aa2f963be5d507b55d05c4d9a6ee3ce9454c4415"}
  datascript-transit/datascript-transit {:mvn/version "0.3.0"
  :exclusions [datascript/datascript]}
  cljs-bean/cljs-bean {:mvn/version "1.5.0"}
  com.cognitect/transit-cljs {:mvn/version "0.8.280"}
  logseq/common {:local/root "../common"}
- funcool/promesa {:mvn/version "4.0.2"}}
+ funcool/promesa {:mvn/version "4.0.2"}
+ org.flatland/ordered {:mvn/version "1.15.11"}}
 
  :aliases
  {:clj-kondo