Dedupe pos/equality deletes to preserve previous behavior, prevent any duplicates in metadata

Author: amogh-jahagirdar

core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java

@@ -1081,9 +1081,11 @@ private List<ManifestFile> newDeleteFilesAsManifests() {
                   tableName,
                   ops().current().specsById(),
                   ThreadPools.getDeleteWorkerPool());
-
+      // Prevent commiting duplicate V2 deletes by deduping them
       Map<Integer, List<DeleteFile>> newDeleteFilesBySpec =
-          Streams.stream(Iterables.concat(mergedDVs, validDVs, positionAndEqualityDeletes))
+          Streams.stream(
+                  Iterables.concat(
+                      mergedDVs, validDVs, DeleteFileSet.of(positionAndEqualityDeletes)))
               .map(file -> Delegates.pendingDeleteFile(file, file.dataSequenceNumber()))
               .collect(Collectors.groupingBy(ContentFile::specId));
       newDeleteFilesBySpec.forEach(