Removed non-common tokens from GST

jplag · Jan 19, 2025 · 256364a · 256364a
1 parent 88e2120
commit 256364a
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/core/src/main/java/de/jplag/GreedyStringTiling.java b/core/src/main/java/de/jplag/GreedyStringTiling.java
@@ -106,8 +106,8 @@ private JPlagComparison compareInternal(Submission leftSubmission, Submission ri
         int[] leftValues = tokenValueListFromSubmission(leftSubmission, commonContexts);
         int[] rightValues = tokenValueListFromSubmission(rightSubmission, commonContexts);
 
-        boolean[] leftMarked = calculateInitiallyMarked(leftSubmission);
-        boolean[] rightMarked = calculateInitiallyMarked(rightSubmission);
+        boolean[] leftMarked = calculateInitiallyMarked(leftSubmission, commonContexts);
+        boolean[] rightMarked = calculateInitiallyMarked(rightSubmission, commonContexts);
 
         SubsequenceHashLookupTable leftLookupTable = subsequenceHashLookupTableForSubmission(leftSubmission, leftMarked, commonContexts);
         SubsequenceHashLookupTable rightLookupTable = subsequenceHashLookupTableForSubmission(rightSubmission, rightMarked, commonContexts);
@@ -200,9 +200,9 @@ private void addMatchIfNotOverlapping(List<Match> matches, Match match) {
         matches.add(match);
     }
 
-    private boolean[] calculateInitiallyMarked(Submission submission) {
+    private boolean[] calculateInitiallyMarked(Submission submission, Set<Object> contexts) {
         Set<Token> baseCodeTokens = baseCodeMarkings.get(submission);
-        List<Token> tokens = submission.getTokenList();
+        List<Token> tokens = submission.getTokenList(contexts);
         boolean[] result = new boolean[tokens.size()];
         for (int i = 0; i < result.length; i++) {
             result[i] = tokens.get(i).getType().isExcludedFromMatching() || (baseCodeTokens != null && baseCodeTokens.contains(tokens.get(i)));
@@ -222,7 +222,7 @@ private SubsequenceHashLookupTable subsequenceHashLookupTableForSubmission(Submi
      */
     private int[] tokenValueListFromSubmission(Submission submission, Set<Object> contexts) {
         return cachedTokenValueLists.computeIfAbsent(submission, (key -> {
-            List<Token> tokens = key.getTokenList();
+            List<Token> tokens = key.getTokenList(contexts);
             int[] tokenValueList = new int[tokens.size()];
             for (int i = 0; i < tokens.size(); i++) {
                 TokenType type = tokens.get(i).getType().constrained(contexts);

diff --git a/core/src/main/java/de/jplag/Submission.java b/core/src/main/java/de/jplag/Submission.java
@@ -141,6 +141,10 @@ public List<Token> getTokenList() {
         return tokenList;
     }
 
+    public List<Token> getTokenList(Set<Object> contexts) {
+        return tokenList.stream().filter(it -> !it.getType().constrained(contexts).getAttributes().isEmpty()).toList();
+    }
+
     /**
      * @return Whether a comparison between the submission and the base code is available.
      */