improve sync_dbs

Author: jmosbacher

xedocs/json_records.py

@@ -77,8 +77,10 @@ def read(self, schema=None) -> List[dict]:
                 docs =[]
         if schema is not None:
             docs = [schema(**doc).pandas_dict() for doc in docs]
-            df = pd.DataFrame(docs)
+            df = pd.DataFrame(docs, columns=list(schema.__fields__))
             idx_names = list(schema.get_index_fields())
+            if not all([n in df.columns for n in idx_names]):
+                return df
             if len(idx_names) == 1:
                 idx_names = idx_names[0]
             df = df.set_index(idx_names)

xedocs/xedocs.py

@@ -173,9 +173,15 @@ def sync_dbs(from_db, to_db, schemas=None):
     if isinstance(to_db, str):
         to_db = getattr(xedocs.databases, to_db)()
 
+    results = {}
     for name, accessor in tqdm(from_db.items(), desc="Syncing databases"):
         if schemas is None or name in schemas:
-            docs = accessor.find_docs()
+            sort=None
+            if "time" in accessor.schema.get_index_fields():
+                sort = "time"
+            docs = accessor.find_docs(sort=sort)
             if name not in to_db:
                 continue
-            to_db[name].insert(docs)
+            results[name] = to_db[name].insert(docs, raise_on_error=False)
+
+    return results