Slightly improve editor memory usage for large projects (#8375)

* Use open addressing for WeakInterner hash map

* Added endpoint interner statistics to dev module

* Refactor WeakInterner in preparation for non-linear open addressing

* Added timings to WeakInterner stats

* Fixed WeakInterner tests to not use the global endpoint interner

* Use prime-sized hash table capacities in WeakInterner

* Add substeps for large WeakInterner capacity jumps

* Improved hash code calculation of Endpoints

* Use double hashing to resolve slot conflicts in WeakInterner

* Use non-boxed node-id in Endpoints
# Conflicts:
#	editor/src/clj/util/coll.clj

editor/src/clj/dev.clj

@@ -21,6 +21,7 @@
  [editor.console :as console]
  [editor.curve-view :as curve-view]
  [editor.defold-project :as project]
+ [editor.math :as math]
  [editor.outline-view :as outline-view]
  [editor.prefs :as prefs]
  [editor.properties-view :as properties-view]
@@ -29,8 +30,9 @@
  [internal.node :as in]
  [internal.system :as is]
  [internal.util :as util]
- [util.coll :refer [pair]])
- (:import [internal.graph.types Arc]
+ [util.coll :as coll :refer [pair]])
+ (:import [com.defold.util WeakInterner]
+ [internal.graph.types Arc]
  [java.beans BeanInfo Introspector MethodDescriptor PropertyDescriptor]
  [java.lang.reflect Modifier]
  [javafx.stage Window]))
@@ -610,4 +612,73 @@
  :output (name output)
  :node-count (node-type-freqs node-type)})
  dangling-outputs)))
- pprint/print-table)))
+ pprint/print-table)))
+
+(defn weak-interner-info [^WeakInterner weak-interner]
+ (into {}
+ (map (fn [[key value]]
+ (let [keyword-key (keyword key)]
+ (pair keyword-key
+ (case keyword-key
+ :hash-table (mapv (fn [entry-info]
+ (when entry-info
+ (into {}
+ (map (fn [[key value]]
+ (let [keyword-key (keyword key)]
+ (pair keyword-key
+ (case keyword-key
+ :status (keyword value)
+ value)))))
+ entry-info)))
+ value)
+ value)))))
+ (.getDebugInfo weak-interner)))
+
+(defn weak-interner-stats [^WeakInterner weak-interner]
+ (let [info (weak-interner-info weak-interner)
+ hash-table (:hash-table info)
+ entry-count (:count info)
+ capacity (count hash-table)
+ occupancy-factor (/ (double entry-count) (double capacity))
+
+ next-capacity
+ (util/first-where
+ #(< capacity %)
+ (:growth-sequence info))
+
+ attempt-frequencies
+ (->> hash-table
+ (keep :attempt)
+ (frequencies)
+ (into (sorted-map-by coll/descending-order)))
+
+ elapsed-nanosecond-values
+ (keep :elapsed hash-table)
+
+ median-elapsed-nanoseconds
+ (if (empty? elapsed-nanosecond-values)
+ 0
+ (->> elapsed-nanosecond-values
+ (math/median)
+ (long)))
+
+ max-elapsed-nanoseconds
+ (if (empty? elapsed-nanosecond-values)
+ 0
+ (->> elapsed-nanosecond-values
+ (reduce max Long/MIN_VALUE)))]
+
+ {:count entry-count
+ :capacity capacity
+ :next-capacity next-capacity
+ :growth-threshold (:growth-threshold info)
+ :occupancy-factor (math/round-with-precision occupancy-factor math/precision-general)
+ :median-elapsed-nanoseconds median-elapsed-nanoseconds
+ :max-elapsed-nanoseconds max-elapsed-nanoseconds
+ :attempt-frequencies attempt-frequencies}))
+
+(defn endpoint-interner-stats []
+ ;; Trigger a GC and give it a moment to clear out unused weak references.
+ (System/gc)
+ (Thread/sleep 500)
+ (weak-interner-stats gt/endpoint-interner))

editor/src/clj/editor/gui.clj

@@ -51,7 +51,7 @@
  [internal.graph.types :as gt]
  [internal.util :as util]
  [schema.core :as s]
- [util.coll :refer [pair]])
+ [util.coll :as coll :refer [pair]])
  (:import [com.dynamo.gamesys.proto Gui$SceneDesc Gui$SceneDesc$AdjustReference Gui$NodeDesc Gui$NodeDesc$XAnchor Gui$NodeDesc$YAnchor
  Gui$NodeDesc$Pivot Gui$NodeDesc$AdjustMode Gui$NodeDesc$BlendMode Gui$NodeDesc$ClippingMode Gui$NodeDesc$PieBounds Gui$NodeDesc$SizeMode]
  [com.jogamp.opengl GL GL2]
@@ -3075,10 +3075,8 @@
  child-indices (g/node-value parent :child-indices)
  before? (partial > node-index)
  after? (partial < node-index)
- ascending-order #(compare %1 %2)
- descending-order #(compare %2 %1)
  neighbour (first (sort-by second
- (if (= offset -1) descending-order ascending-order)
+ (if (= offset -1) coll/descending-order coll/ascending-order)
  (filter (comp (if (= offset -1) before? after?) second)
  child-indices)))]
  (when neighbour

editor/src/clj/editor/math.clj

@@ -36,6 +36,19 @@
  ^double [^double rad]
  (* rad 180.0 recip-pi))
 
+(defn median [numbers]
+ (let [sorted-numbers (sort numbers)
+ count (count sorted-numbers)]
+ (if (zero? count)
+ (throw (ex-info "Cannot calculate the median of an empty sequence." {:numbers numbers}))
+ (let [middle-index (quot count 2)
+ middle-number (nth sorted-numbers middle-index)]
+ (if (odd? count)
+ middle-number
+ (-> middle-number
+ (+ (nth sorted-numbers (dec middle-index)))
+ (/ 2.0)))))))
+
 (defn round-with-precision
  "Slow but precise rounding to a specified precision. Use with UI elements that
  produce doubles, not for performance-sensitive code. The precision is expected

editor/src/clj/editor/welcome.clj

@@ -35,6 +35,7 @@
  [editor.ui.fuzzy-choices :as fuzzy-choices]
  [editor.ui.updater :as ui.updater]
  [schema.core :as s]
+ [util.coll :as coll]
  [util.net :as net]
  [util.time :as time])
  (:import [clojure.lang ExceptionInfo]
@@ -194,9 +195,6 @@
  :last-opened instant
  :title title})))))))
 
-(defn- descending-order [a b]
- (compare b a))
-
 (defn- recent-projects
  "Returns a sequence of recently opened projects. Project files that no longer
  exist will be filtered out. If the user has an older preference file that does
@@ -205,7 +203,7 @@
  the most recently opened project first."
  [prefs]
  (sort-by :last-opened
- descending-order
+ coll/descending-order
  (if-some [timestamps-by-path (prefs/get-prefs prefs recent-projects-prefs-key nil)]
  (into [] xform-timestamps-by-path->recent-projects timestamps-by-path)
  (if-some [paths (prefs/get-prefs prefs legacy-recent-project-paths-prefs-key nil)]

editor/src/clj/internal/graph/types.clj

@@ -13,11 +13,12 @@
 ;; specific language governing permissions and limitations under the License.
 
 (ns internal.graph.types
- (:import [clojure.lang IHashEq Keyword]
+ (:import [clojure.lang IHashEq Keyword Murmur3 Util]
  [com.defold.util WeakInterner]
  [java.io Writer]))
 
 (set! *warn-on-reflection* true)
+(set! *unchecked-math* :warn-on-boxed)
 
 (defrecord Arc [source-id source-label target-id target-label])
 
@@ -28,33 +29,34 @@
 (defn target-label [^Arc arc] (.target-label arc))
 (defn target [^Arc arc] [(.target-id arc) (.target-label arc)])
 
-(deftype Endpoint [^Long node-id ^Keyword label]
+(definline node-id-hash [node-id]
+ `(Murmur3/hashLong ~node-id))
+
+(deftype Endpoint [^long node-id ^Keyword label]
  Comparable
  (compareTo [_ that]
  (let [^Endpoint that that
- node-id-comparison (.compareTo node-id (.-node-id that))]
+ node-id-comparison (Long/compare node-id (.-node-id that))]
  (if (zero? node-id-comparison)
  (.compareTo label (.-label that))
  node-id-comparison)))
  IHashEq
  (hasheq [_]
- (unchecked-add-int
- (unchecked-multiply-int 31 (.hashCode node-id))
+ (Util/hashCombine
+ (node-id-hash node-id)
  (.hasheq label)))
  Object
  (toString [_]
  (str "#g/endpoint [" node-id " " label "]"))
  (hashCode [_]
- (unchecked-add-int
- (unchecked-multiply-int 31 (.hashCode node-id))
+ (Util/hashCombine
+ (node-id-hash node-id)
  (.hasheq label)))
  (equals [this that]
  (or (identical? this that)
  (and (instance? Endpoint that)
- (let [^Endpoint that that]
- (and
- (.equals (.-node-id that) node-id)
- (.equals (.-label that) label)))))))
+ (= node-id (.-node-id ^Endpoint that))
+ (identical? label (.-label ^Endpoint that))))))
 
 (defmethod print-method Endpoint [^Endpoint ep ^Writer writer]
  (.write writer "#g/endpoint [")

editor/src/clj/util/coll.clj

@@ -19,6 +19,18 @@
 (set! *warn-on-reflection* true)
 (set! *unchecked-math* :warn-on-boxed)
 
+(def empty-sorted-map (sorted-map))
+
+(defn ascending-order
+ "Comparator that orders items in ascending order."
+ [a b]
+ (compare a b))
+
+(defn descending-order
+ "Comparator that orders items in descending order."
+ [a b]
+ (compare b a))
+
 (defn supports-transient?
  "Returns true if the supplied persistent collection can be made into a
  transient collection."
@@ -31,6 +43,24 @@
  [a b]
  (MapEntry. a b))
 
+(defmacro pair-fn
+ "Returns a function that takes a value and returns a pair from it. The
+ supplied key-fn is expected to take a value and return the key to use for
+ that value. If supplied, the value-fn is applied to the value to produce the
+ value portion of the pair. Useful when transforming sequences into maps, and
+ can be a drop-in replacement for (juxt key-fn value-fn)."
+ ([key-fn]
+ `(let [key-fn# ~key-fn]
+ (fn ~'value->pair [~'value]
+ (pair (key-fn# ~'value)
+ ~'value))))
+ ([key-fn value-fn]
+ `(let [key-fn# ~key-fn
+ value-fn# ~value-fn]
+ (fn ~'value->pair [~'value]
+ (pair (key-fn# ~'value)
+ (value-fn# ~'value))))))
+
 (defn flipped-pair
  "Constructs a two-element collection that implements IPersistentVector from
  the reversed arguments."
@@ -85,6 +115,23 @@
  :else
  (not (seq coll))))
 
+(defn pair-map-by
+ "Returns a hash-map where the keys are the result of applying the supplied
+ key-fn to each item in the input sequence and the values are the items
+ themselves. Returns a stateless transducer if no input sequence is
+ provided. Optionally, a value-fn can be supplied to transform the values."
+ ([key-fn]
+ {:pre [(ifn? key-fn)]}
+ (map (pair-fn key-fn)))
+ ([key-fn coll]
+ (into {}
+ (map (pair-fn key-fn))
+ coll))
+ ([key-fn value-fn coll]
+ (into {}
+ (map (pair-fn key-fn value-fn))
+ coll)))
+
 (defn separate-by
  "Separates items in the supplied collection into two based on a predicate.
  Returns a pair of [true-items, false-items]. The resulting collections will
@@ -122,3 +169,44 @@
  (conj (val result) item))))
  (pair empty-coll empty-coll)
  coll))))
+
+(defn sorted-assoc-in
+ "Like core.assoc-in, but sorted maps will be created for any levels that do not exist."
+ [coll [key & remaining-keys] value]
+ (if remaining-keys
+ (assoc coll key (sorted-assoc-in (get coll key empty-sorted-map) remaining-keys value))
+ (assoc coll key value)))
+
+(def xform-nested-map->path-map
+ "Transducer that takes a nested map and returns a flat map of vector paths to
+ the innermost values."
+ (letfn [(path-entries [path [key value]]
+ (let [path (conj path key)]
+ (if (coll? value)
+ (eduction
+ (mapcat #(path-entries path %))
+ value)
+ [(pair path value)])))]
+ (mapcat #(path-entries [] %))))
+
+(defn nested-map->path-map
+ "Takes a nested map and returns a flat map of vector paths to the innermost
+ values."
+ [nested-map]
+ {:pre [(map? nested-map)]}
+ (into (empty nested-map)
+ xform-nested-map->path-map
+ nested-map))
+
+(defn path-map->nested-map
+ "Takes a flat map of vector paths to values and returns a nested map to the
+ same values."
+ [path-map]
+ {:pre [(map? path-map)]}
+ (reduce (if (sorted? path-map)
+ (fn [nested-map [path value]]
+ (sorted-assoc-in nested-map path value))
+ (fn [nested-map [path value]]
+ (assoc-in nested-map path value)))
+ (empty path-map)
+ path-map))