fix(upgrade): include timestamps for partner supervoxel parents

Author: akhileshh

pychunkedgraph/__init__.py

@@ -1 +1 @@
-__version__ = "3.0.6"
+__version__ = "3.0.7"

pychunkedgraph/ingest/upgrade/atomic_layer.py

@@ -12,7 +12,7 @@
 
 
 def update_cross_edges(
-    cg: ChunkedGraph, node, cx_edges_d, node_ts, timestamps, earliest_ts
+    cg: ChunkedGraph, node, cx_edges_d: dict, node_ts, timestamps: set, earliest_ts
 ) -> list:
     """
     Helper function to update a single L2 ID.
@@ -27,7 +27,11 @@ def update_cross_edges(
         assert not exists_as_parent(cg, node, edges[:, 0])
         return rows
 
-    for ts in timestamps:
+    partner_parent_ts_d = get_parent_timestamps(cg, edges[:, 1])
+    for v in partner_parent_ts_d.values():
+        timestamps.update(v)
+
+    for ts in sorted(timestamps):
         if ts < earliest_ts:
             ts = earliest_ts
         val_dict = {}

pychunkedgraph/ingest/upgrade/parent_layer.py

@@ -163,8 +163,10 @@ def update_chunk(
         tasks.append(args)
 
     with mp.Pool(min(mp.cpu_count(), len(tasks))) as pool:
-        tqdm(
-            pool.imap_unordered(_update_cross_edges_helper, tasks),
-            total=len(tasks),
+        _ = list(
+            tqdm(
+                pool.imap_unordered(_update_cross_edges_helper, tasks),
+                total=len(tasks),
+            )
         )
     print(f"total elaspsed time: {time.time() - start}")

pychunkedgraph/ingest/utils.py

@@ -2,6 +2,7 @@
 
 import logging
 import functools
+import math
 from os import environ
 from time import sleep
 from typing import Any, Generator, Tuple
@@ -163,7 +164,7 @@ def print_status(imanager: IngestionManager, redis, upgrade: bool = False):
     )
     print(header)
     for layer, done, count in zip(layers, completed, layer_counts):
-        print(f"{layer}\t| {done:9} / {count} \t| {done/count:6.1%}")
+        print(f"{layer}\t| {done:9} / {count} \t| {math.floor((done/count)*100):6}%")
 
     print("\n\nqueue status:")
     for layer, q, f, wb in zip(layers, queued, failed, worker_busy):