update

Author: stefan

Collector.py

@@ -1,4 +1,3 @@
-
 import os
 import constant
 import common
@@ -17,6 +16,7 @@ class Collector(QThread):
     def __init__(self, ui):
         QThread.__init__(self)
         self.map = {}
+        self.duplicate_db = DuplicateDB(ui)
         self.ui = ui
 
 
@@ -32,28 +32,21 @@ def clear(self):
         self.map = {}
 
 
-    def add_dir(self, path, recursive, cmd, duplicate_db, skipExisting):
+    def add_dir(self, path, recursive, cmd, skipExisting):
         self.argPath = path
         self.argRecursive = recursive
         self.argCmd = cmd
         self.argSkipExisting = skipExisting
-        self.arg_duplicate_db = duplicate_db
-        self.worker = self.add_dir_wrapper
-        if constant.USE_THREADS:
-            self.start()
-        else:
-            self.worker()
+        self.worker = self.add_dir_worker
+        self.exec()
 
 
     def find_extern_duplicates(self, srcDir, recursive, simulate):
         self.argSrcDir = srcDir
         self.argRecursive = recursive
         self.arg_simulate = simulate
-        self.worker = self.find_extern_duplicates_wrapper
-        if constant.USE_THREADS:
-            self.start()
-        else:
-            self.worker()
+        self.worker = self.find_extern_duplicates_worker
+        self.exec()
 
 
     def find_hash(self, hash):
@@ -83,19 +76,19 @@ def run(self):
         self.worker()
 
 
-    def skip_dir(self, path):
+    def is_skip_dir(self, path):
         return os.path.isfile(os.path.join(path, constant.NOHASHFILE))
 
 
-    def add_dir_wrapper(self):
-        self.add_dir_impl(self.argPath, self.argRecursive, self.argSkipExisting, self.argCmd, self.arg_duplicate_db)
+    def add_dir_worker(self):
+        self.ui.reset()
+        self.add_dir_impl(self.argPath, self.argRecursive, self.argSkipExisting, self.argCmd)
         self.ui.stats()
 
 
-
-    def add_dir_impl2(self, path, recursive, skipExisting, cmd, duplicate_db):
+    def add_dir_impl2(self, path, recursive, skipExisting, cmd):
         dir = os.path.normpath(path)
-        if self.skip_dir(dir):
+        if self.is_skip_dir(dir):
             self.ui.info("Skipping dir: %s" % dir)
             self.ui.inc_dir_skipped()
             return
@@ -106,7 +99,7 @@ def add_dir_impl2(self, path, recursive, skipExisting, cmd, duplicate_db):
             pass
 
         if cmd is CollectorCmd.scan:
-            db.scan(duplicate_db, skipExisting)
+            db.scan(self.duplicate_db, skipExisting)
             self.ui.inc_dir_scanned()
             db.save()
         elif cmd is CollectorCmd.verify:
@@ -116,17 +109,17 @@ def add_dir_impl2(self, path, recursive, skipExisting, cmd, duplicate_db):
             dirList = []
             dirList.extend(common.get_dir_list_absolute(path, False))
             for dir in dirList:
-                self.add_dir_impl2(dir, recursive, skipExisting, cmd, duplicate_db)
+                self.add_dir_impl2(dir, recursive, skipExisting, cmd)
                 if self.ui.is_abort():
                     return
 
 
-    def add_dir_impl(self, path, recursive, skipExisting, cmd, duplicate_db):
-        duplicate_db.reset()
+    def add_dir_impl(self, path, recursive, skipExisting, cmd):
+        self.duplicate_db.reset()
         self.ui.info("Loading HashDB %sfrom: %s" % ('recursively ' if recursive else '', path))
-        self.add_dir_impl2(path, recursive, skipExisting, cmd, duplicate_db)
+        self.add_dir_impl2(path, recursive, skipExisting, cmd)
         self.ui.debug("Finished loading %d HashDB." % (len(self.map)))
-        duplicate_db.show_duplicates()
+        self.duplicate_db.show_duplicates()
 
 
     def remove_hash(self, path, hash):
@@ -137,6 +130,15 @@ def remove_hash(self, path, hash):
             db.remove(hash)
 
 
+    def remove_file(self, filepath):
+        path = os.path.dirname(filepath)
+        filename = os.path.basename(filepath)
+        db = self.map.get(path)
+        if db:
+            db.remove_filename(filename)
+        else:
+            self.ui.debug("remove_file: HashDB not found: %s" % path)
+
     def save_hashes(self, forceSave = False):
         self.ui.info("Start saving HashDB")
         for path, db in self.map.items():
@@ -145,23 +147,18 @@ def save_hashes(self, forceSave = False):
         pass
 
 
-    def find_extern_duplicates_wrapper(self):
+    def find_extern_duplicates_worker(self):
+        self.ui.reset()
         self.find_extern_duplicates_impl(self.argSrcDir, self.argRecursive, self.arg_simulate)
+        self.ui.stats()
 
 
     def find_extern_duplicates_impl(self, srcDir, recursive, simulate):
-
-        if None == srcDir:
-            self.ui.error("No src dir set")
-            return
-
         self.ui.info("Duplicates found in %s:" % srcDir)
         srcDirList = [srcDir]
         if recursive:
             srcDirList.extend(common.get_dir_list_absolute(srcDir, recursive))
 
-        cntDuplicates = 0
-
         for curSrcDir in srcDirList:
             fileList = common.get_file_list(curSrcDir)
 
@@ -170,7 +167,92 @@ def find_extern_duplicates_impl(self, srcDir, recursive, simulate):
                 hash = common.get_hash_from_file(srcFilepath, self.ui)
                 found_file = self.find_hash(hash)
                 if None != found_file:
-                    cntDuplicates += 1
-                    self.ui.info(srcFilepath)
+                    self.ui.inc_file_duplicates()
+                    self.ui.file(srcFilepath)
+
+        #self.ui.info("Finished finding duplicates. %d files" % (cntDuplicates))
+
+
+
+    def find_duplicates_in_hashDB_impl(self):
+        self.ui.info("Start finding duplicates in HashDB...")
+        self.duplicate_db.reset()
+        for path, db in self.map.items():
+            for hash, name in db.map.items():
+                filepath = os.path.normpath(os.path.join(db.path, name))
+                self.duplicate_db.add_hash(hash, filepath)
+                self.ui.inc_file_processed()
+        self.ui.debug("Finished finding duplicates")
+
+
+    def find_and_show_duplicates_in_hashDB_worker(self):
+        self.find_duplicates_in_hashDB_impl()
+        self.duplicate_db.show_duplicates(self.arg_path)
+        self.ui.stats()
+
+
+    def find_and_show_duplicates_in_hashDB(self, path):
+        self.arg_path = path
+        self.worker = self.find_and_show_duplicates_in_hashDB_worker
+        self.exec()
+
+
+    def exec(self):
+        if constant.USE_THREADS:
+            self.start()
+        else:
+            self.worker()
+
+
+    def move_duplicates_with_master_dir(self, master_path, dest_dir, move_flat, simulate):
+        self.arg_master_path = master_path
+        self.arg_dest_dir = dest_dir
+        self.arg_move_flat = move_flat
+        self.arg_simulate = simulate
+        self.worker = self.move_duplicates_with_master_dir_worker
+        self.exec()
+
+
+    def move_duplicates_with_master_dir_worker(self):
+        filenames = self.duplicate_db.get_list_with_files_to_move_keep_master_path(self.arg_master_path)
+        self.move_files(filenames, self.arg_move_flat, self.arg_dest_dir, self.arg_simulate)
+
+
+    def move_files(self, filenames, move_flat, dest_dir, is_simulation):
+        self.ui.reset()
+        for filename in filenames:
+            path = common.create_duplicate_dest_path(filename, dest_dir, move_flat)
+            common.move_file(filename, path, False, is_simulation, self.ui)
+            if not is_simulation:
+                self.remove_file(filename)
+        if not is_simulation:
+            self.save_hashes()
+        self.ui.stats()
+
+
+    def find_and_show_duplicates_in_folder(self, path):
+        self.arg_path = path
+        self.worker = self.find_and_show_duplicates_in_folder_worker
+        self.exec()
+
+
+    def find_and_show_duplicates_in_folder_worker(self):
+        self.ui.reset()
+        self.find_and_show_duplicates_in_folder_impl(self.arg_path)
+        self.duplicate_db.show_duplicates(None)
+        self.ui.stats()
+
+
+    def find_and_show_duplicates_in_folder_impl(self, path):
+        self.duplicate_db.reset()
+        files = common.get_file_list(path)
+        self.ui.info("Scannning %d files for duplicates in %s" % (len(files), path))
+        for item in files:
+            if self.ui.is_abort():
+                return
+            self.ui.info("Hashing: %s" % item)
+            filepath = os.path.normpath(os.path.join(path, item))
+            hash = common.get_hash_from_file(filepath, self.ui)
+            self.duplicate_db.add_hash(hash, filepath)
+        self.ui.info("Finished finding duplicates.")
 
-        self.ui.info("Finished finding duplicates. %d files" % (cntDuplicates))

DuplicateDB.py

@@ -58,34 +58,17 @@ def show_duplicates(self, path = None):
         #    self.ui.info("No duplicates found.")
 
 
-    def move_duplicates_with_master_dir_impl(self, master_path, duplicate_path, simulate, collector):
+    def get_list_with_files_to_move_keep_master_path(self, master_path):
         self.ui.info("Start moving duplicates...")
+        all_files_to_move = []
         for hash, files in self.map.items():
             if len(files) > 1:
-                files_to_move = []
-                master_file = None
                 for filename in files:
                     p = os.path.dirname(filename)
-                    if p == master_path:
-                        #self.ui.info("Found master: %s" % filename)
-                        master_file = filename
+                    if p == master_path: # found master file to keep
                         files_to_move = list(files)
                         files_to_move.remove(filename)
+                        all_files_to_move.extend(files_to_move)
                         break
-
-                for src_path in files_to_move:
-                    try:
-                        path = "." + os.path.splitdrive(src_path)[1]
-                        path = os.path.normpath(path)
-                        dest_path = os.path.join(duplicate_path, path)
-                        self.ui.debug("Move %s to %s - Master: %s" % (src_path, dest_path, master_file))
-                        common.move_file(src_path, dest_path, False, simulate, self.ui)
-                        if not simulate:
-                            dir_name = os.path.dirname(src_path)
-                            collector.remove_hash(dir_name, hash)
-                    except Exception as e:
-                        self.ui.error("Error moving: %s, %s" % (str(e), src_path))
-                        pass
-
-        collector.save_hashes()                            
-        #self.ui.info("Finished moving %d duplicates. Errors: %d" % (cnt_moved, cnt_error))
+        return all_files_to_move
+        

HashDb.py

@@ -148,3 +148,9 @@ def find_filename(self, filename):
     def find_hash(self, hash):
         value = self.map.get(hash)
         return value
+
+
+    def remove_filename(self, filename):
+        hash = self.find_filename(filename)
+        if hash:
+            self.remove(hash)

Logger.py

@@ -57,6 +57,7 @@ def reset(self):
         self.cnt_hash_duplicates = 0
         self.cnt_file_duplicates = 0
         self.cnt_missing_file = 0
+        self.cnt_file_processed = 0
 
     def m(self, msg, cnt):
         if cnt > 0:
@@ -77,9 +78,13 @@ def stats(self):
         str += self.m("Skipped dir", self.cnt_dir_skipped)
         str += self.m("Hash duplicates", self.cnt_hash_duplicates)
         str += self.m("File duplicates", self.cnt_file_duplicates)
-        str += self.m("File missing", self.cnt_missing_file)
+        str += self.m("Files missing", self.cnt_missing_file)
+        str += self.m("Files processed", self.cnt_file_processed)
         self.info(str)
 
+    def inc_file_processed(self):
+        self.cnt_file_processed += 1
+
     def inc_missing_file(self):
         self.cnt_missing_file += 1
 

common.py

@@ -62,34 +62,34 @@ def create_new_filename(filepath):
     return newfilepath
 
 
-def create_duplicate_dest_path(srcPath, duplicateDir, flat):
+def create_duplicate_dest_path(src_path, dest_dir, flat):
     if flat:
-        destPath = os.path.join(duplicateDir, os.path.basename(srcPath))
+        dest_path = os.path.join(dest_dir, os.path.basename(src_path))
     else:
-        destPath = '.' + os.path.splitdrive(srcPath)[1]
-        destPath = os.path.join(duplicateDir, destPath)
-    return os.path.normpath(destPath)
+        dest_path = '.' + os.path.splitdrive(src_path)[1]
+        dest_path = os.path.join(dest_dir, dest_path)
+    return os.path.normpath(dest_path)
 
 
-def move_file(srcPath, destPath, overwrite, simulate, ui):
-    ui.info("Move %s   ==>   %s" % (srcPath, destPath))
+def move_file(src_path, dest_path, overwrite, simulate, ui):
+    ui.info("Move %s   ==>   %s" % (src_path, dest_path))
     if not simulate:
         try:
-            destDir = os.path.dirname(destPath)
-            if not os.path.exists(destDir):
-                os.makedirs(destDir, exist_ok=True)
-
-            if not overwrite and os.path.isfile(destPath):
-                destPath = create_new_filename(destPath)
-                ui.info("Renamed moved file to %s" % destPath)
-                shutil.move(srcPath, destPath)
+            dest_path = os.path.dirname(dest_path)
+            if not os.path.exists(dest_path):
+                os.makedirs(dest_path, exist_ok=True)
+
+            if not overwrite and os.path.isfile(dest_path):
+                dest_path = create_new_filename(dest_path)
+                ui.info("Renamed moved file to %s" % dest_path)
+                shutil.move(src_path, dest_path)
                 ui.inc_moved_renamed()
             else:
-                shutil.move(srcPath, destPath)
+                shutil.move(src_path, dest_path)
                 ui.inc_moved()
         except:
             ui.error("Failed to move %s to %s" %
-                (srcPath, destPath))
+                (src_path, dest_path))
             ui.inc_error()
             pass
 

constant.py

@@ -1,3 +1,3 @@
 HASHFILE = "__hashes.txt"
 NOHASHFILE = "__nohashes.txt"
-USE_THREADS = True
+USE_THREADS = False