Fix JavaDoc

Add many comments and put a file in the 'passes' package. When JavaDoc cannot find a Java file in there, it quits.

core/src/main/java/de/jplag/Submission.java

@@ -65,6 +65,8 @@ public class Submission implements Comparable<Submission> {
 
     private Map<File, Integer> fileTokenCount;
 
+    private long parsingTimeInMillis;
+
     /**
      * Creates a submission.
      * @param name Identification of the submission (directory or filename).
@@ -97,6 +99,10 @@ public boolean equals(Object obj) {
         return otherSubmission.getName().equals(name);
     }
 
+    public long getParsingTime() {
+        return parsingTimeInMillis;
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(name);
@@ -252,7 +258,9 @@ private static File createErrorDirectory(String... subdirectoryNames) {
         }
 
         try {
+            long timeBeforeParsing = System.currentTimeMillis();
             tokenList = language.parse(new HashSet<>(files), normalize);
+            parsingTimeInMillis = System.currentTimeMillis() - timeBeforeParsing;
             if (logger.isDebugEnabled()) {
                 for (Token token : tokenList) {
                     logger.debug(String.join(" | ", token.getType().toString(), Integer.toString(token.getLine()), token.getSemantics().toString()));

core/src/main/java/de/jplag/SubmissionSet.java

@@ -51,6 +51,10 @@ public SubmissionSet(List<Submission> submissions, Submission baseCode, JPlagOpt
         invalidSubmissions = filterInvalidSubmissions();
     }
 
+    public long getTotalParsingTime() {
+        return submissions.stream().mapToLong(Submission::getParsingTime).sum();
+    }
+
     /**
      * @return Whether a basecode is available for this collection.
      */

languages/java-cpg/src/main/java/de/jplag/java_cpg/CpgAdapter.java

@@ -14,6 +14,7 @@
 import de.jplag.Token;
 import de.jplag.java_cpg.passes.*;
 import de.jplag.java_cpg.transformation.GraphTransformation;
+import de.jplag.java_cpg.transformation.GraphTransformation.ExecutionPhase;
 
 import kotlin.jvm.JvmClassMappingKt;
 import kotlin.reflect.KClass;
@@ -26,6 +27,10 @@ public class CpgAdapter {
     private List<Token> tokenList;
     private boolean reorderingEnabled = true;
 
+    /**
+     * Constructor for CpgAdapter.
+     * @param transformations a list of {@link GraphTransformation}s
+     */
     public CpgAdapter(GraphTransformation... transformations) {
         addTransformations(transformations);
     }
@@ -89,6 +94,10 @@ public void addTransformations(GraphTransformation[] transformations) {
         Arrays.stream(transformations).forEach(this::addTransformation);
     }
 
+    /**
+     * Adds a transformation at the end of its respective ATransformationPass.
+     * @param transformation a {@link GraphTransformation}
+     */
     public void addTransformation(GraphTransformation transformation) {
         switch (transformation.getPhase()) {
             case OBLIGATORY -> PrepareTransformationPass.registerTransformation(transformation);
@@ -97,11 +106,18 @@ public void addTransformation(GraphTransformation transformation) {
         }
     }
 
+    /**
+     * Clears all non-{@link ExecutionPhase#OBLIGATORY} transformations from the pipeline.
+     */
     public void clearTransformations() {
         AstTransformationPass.clearTransformations();
         CpgTransformationPass.clearTransformations();
     }
 
+    /**
+     * Sets <code>reorderingEnabled</code>. If true, statements may be reordered.
+     * @param enabled value for reorderingEnabled.
+     */
     public void setReorderingEnabled(boolean enabled) {
         this.reorderingEnabled = enabled;
     }

languages/java-cpg/src/main/java/de/jplag/java_cpg/JavaCpgLanguage.java

@@ -18,12 +18,15 @@
  */
 @MetaInfServices(de.jplag.Language.class)
 public class JavaCpgLanguage implements Language {
-    public static final int DEFAULT_MINIMUM_TOKEN_MATCH = 9;
-    public static final String[] FILE_EXTENSIONS = {".java"};
-    public static final String NAME = "Java Code Property Graph module";
+    private static final int DEFAULT_MINIMUM_TOKEN_MATCH = 9;
+    private static final String[] FILE_EXTENSIONS = {".java"};
+    private static final String NAME = "Java Code Property Graph module";
     private static final String IDENTIFIER = "java-cpg";
     private final CpgAdapter cpgAdapter;
 
+    /**
+     * Creates a new {@link JavaCpgLanguage}.
+     */
     public JavaCpgLanguage() {
         this.cpgAdapter = new CpgAdapter(allTransformations());
     }
@@ -69,6 +72,9 @@ public boolean requiresCoreNormalization() {
         return false;
     }
 
+    /**
+     * Resets the set of transformations to the obligatory transformations only.
+     */
     public void resetTransformations() {
         this.cpgAdapter.clearTransformations();
         this.cpgAdapter.addTransformations(this.obligatoryTransformations());
@@ -87,12 +93,12 @@ private GraphTransformation[] obligatoryTransformations() {
      * @return the array of recommended transformations
      */
     public GraphTransformation[] standardTransformations() {
-        return new GraphTransformation[] {removeOptionalOfCall,               // 3
-                removeOptionalGetCall,              // 4
-                moveConstantToOnlyUsingClass,       // 6
-                inlineSingleUseVariable,            // 8
-                removeLibraryRecord,                // 11
-                removeEmptyRecord,                  // 16
+        return new GraphTransformation[] {removeOptionalOfCall,               // 1
+                removeOptionalGetCall,              // 2
+                moveConstantToOnlyUsingClass,       // 5
+                inlineSingleUseVariable,            // 7
+                removeLibraryRecord,                // 10
+                removeEmptyRecord,                  // 15
         };
     }
 
@@ -101,22 +107,22 @@ public GraphTransformation[] standardTransformations() {
      * @return the array of all transformations
      */
     public GraphTransformation[] allTransformations() {
-        return new GraphTransformation[] {ifWithNegatedConditionResolution,   // 1
-                forStatementToWhileStatement,       // 2
-                removeOptionalOfCall,               // 3
-                removeOptionalGetCall,              // 4
-                removeGetterMethod,                 // 5
-                moveConstantToOnlyUsingClass,       // 6
-                inlineSingleUseConstant,            // 7
-                inlineSingleUseVariable,            // 8
-                removeEmptyDeclarationStatement,    // 9
-                removeImplicitStandardConstructor,  // 10
-                removeLibraryRecord,                // 11
-                removeLibraryField,                 // 12
-                removeEmptyConstructor,             // 13
-                removeUnsupportedConstructor,       // 14
-                removeUnsupportedMethod,            // 15
-                removeEmptyRecord,                  // 16
+        return new GraphTransformation[] {ifWithNegatedConditionResolution,   // 0
+                forStatementToWhileStatement,       // 1
+                removeOptionalOfCall,               // 2
+                removeOptionalGetCall,              // 3
+                removeGetterMethod,                 // 4
+                moveConstantToOnlyUsingClass,       // 5
+                inlineSingleUseConstant,            // 6
+                inlineSingleUseVariable,            // 7
+                removeEmptyDeclarationStatement,    // 8
+                removeImplicitStandardConstructor,  // 9
+                removeLibraryRecord,                // 10
+                removeLibraryField,                 // 11
+                removeEmptyConstructor,             // 12
+                removeUnsupportedConstructor,       // 13
+                removeUnsupportedMethod,            // 14
+                removeEmptyRecord,                  // 15
         };
     }
 

languages/java-cpg/src/main/java/de/jplag/java_cpg/passes/JTokenizationPass.java

@@ -0,0 +1,76 @@
+package de.jplag.java_cpg.passes;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.fraunhofer.aisec.cpg.TranslationContext;
+import de.fraunhofer.aisec.cpg.TranslationResult;
+import de.fraunhofer.aisec.cpg.graph.Name;
+import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker;
+import de.fraunhofer.aisec.cpg.passes.TranslationResultPass;
+import de.jplag.Token;
+import de.jplag.TokenType;
+import de.jplag.java_cpg.token.CpgNodeListener;
+import de.jplag.java_cpg.token.CpgToken;
+import de.jplag.java_cpg.token.CpgTokenConsumer;
+import de.jplag.java_cpg.visitorStrategy.NodeOrderStrategy;
+
+/**
+ * This pass tokenizes the {@link de.fraunhofer.aisec.cpg.TranslationResult}. It is a duplicate of
+ * de.jplag.java_cpg.passes.TokenizationPass implemented in Java as a workaround to make JavaDoc work. If the package
+ * 'passes' contains only Kotlin files, JavaDoc fails.
+ */
+public class JTokenizationPass extends TranslationResultPass {
+
+    private static final Logger logger = LoggerFactory.getLogger(TokenizationPass.class);
+    private final ArrayList<Token> tokenList = new ArrayList<>();
+    private final CpgTokenConsumer consumer = new ConcreteCpgTokenConsumer();
+    private final NodeOrderStrategy strategy = new NodeOrderStrategy();
+    Consumer<List<Token>> callback = null;
+
+    /**
+     * <p>
+     * Constructor for JTokenizationPass.
+     * </p>
+     * @param ctx a {@link de.fraunhofer.aisec.cpg.TranslationContext} object
+     */
+    public JTokenizationPass(@NotNull TranslationContext ctx) {
+        super(ctx);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void accept(TranslationResult translationResult) {
+        tokenList.clear();
+        CpgNodeListener listener = new CpgNodeListener(consumer);
+        SubgraphWalker.IterativeGraphWalker walker = new SubgraphWalker.IterativeGraphWalker();
+        walker.setStrategy(strategy::getIterator);
+        walker.registerOnNodeVisit(listener::visit);
+        walker.registerOnNodeExit(listener::exit);
+        walker.iterate(translationResult);
+        callback.accept(tokenList);
+    }
+
+    /**
+     * <p>
+     * cleanup.
+     * </p>
+     */
+    public void cleanup() {
+        logger.info("Found %d tokens".formatted(tokenList.size()));
+    }
+
+    private class ConcreteCpgTokenConsumer extends CpgTokenConsumer {
+        public void addToken(TokenType type, File file, int rowBegin, int colBegin, int length, Name name) {
+            CpgToken token = new CpgToken(type, file, rowBegin, colBegin, length, name);
+            tokenList.add(token);
+        }
+    }
+
+}

languages/java-cpg/src/main/java/de/jplag/java_cpg/token/ACpgNodeListener.java

@@ -6,10 +6,16 @@
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.*;
 
 /**
- * This class provides empty dummy implementations for {@link CpgNodeListener}s.
+ * This class provides empty dummy implementations for {@link de.jplag.java_cpg.token.CpgNodeListener}s.
  */
 public abstract class ACpgNodeListener extends IVisitorExitor<Node> {
 
+    /**
+     * Creates a new {@link ACpgNodeListener}.
+     */
+    public ACpgNodeListener() {
+    }
+
     void exit(AssertStatement assertStatement) {
     }
 

languages/java-cpg/src/main/java/de/jplag/java_cpg/token/CpgNodeListener.java

@@ -25,12 +25,21 @@ public class CpgNodeListener extends ACpgNodeListener {
     private final LinkedList<TokenType> openBlocks;
     private final LinkedList<BlockTokens> expectedBlocks;
 
+    /**
+     * Creates a new {@link CpgNodeListener}.
+     * @param consumer the {@link CpgTokenConsumer} that receives the tokens.
+     */
     public CpgNodeListener(CpgTokenConsumer consumer) {
         this.tokenConsumer = consumer;
         this.expectedBlocks = new LinkedList<>();
         this.openBlocks = new LinkedList<>();
     }
 
+    /**
+     * This is used to iterate over {@link Token} list, comparing them one {@link Token} at a time.
+     * @param nodes an iterator for {@link Node}s
+     * @return an iterator for {@link Token}s
+     */
     public static Iterator<Token> tokenIterator(Iterator<Node> nodes) {
         return new Iterator<>() {
 
@@ -388,6 +397,10 @@ public void visit(MemberCallExpression memberCallExpression) {
         tokenConsumer.addToken(METHOD_CALL, memberCallExpression, false);
     }
 
+    /**
+     * Visits a {@link Node}.
+     * @param node the node
+     */
     public void visit(Node node) {
         super.visit(node);
     }

languages/java-cpg/src/main/java/de/jplag/java_cpg/token/CpgToken.java

@@ -13,6 +13,15 @@
 public class CpgToken extends Token {
     private final Name name;
 
+    /**
+     * Creates a new {@link CpgToken}.
+     * @param tokenType the {@link de.jplag.TokenType}
+     * @param file the {@link java.io.File} that contains the represented piece of code
+     * @param startLine the starting line of the represented code
+     * @param startColumn the starting column of the represented code
+     * @param length the length of the represented code
+     * @param name the name of the represented CPG node
+     */
     public CpgToken(TokenType tokenType, File file, int startLine, int startColumn, int length, Name name) {
         super(tokenType, file, startLine, startColumn, length);
         this.name = name;

languages/java-cpg/src/main/java/de/jplag/java_cpg/token/CpgTokenConsumer.java

@@ -17,7 +17,7 @@
  */
 public abstract class CpgTokenConsumer implements TokenConsumer {
 
-    public static final Logger logger = LoggerFactory.getLogger(CpgTokenConsumer.class);
+    private static final Logger logger = LoggerFactory.getLogger(CpgTokenConsumer.class);
     /**
      * This is used as the Token's length if the token spans multiple lines. The value is supposed to be greater than the
      * length of any sensible line of code.
@@ -32,6 +32,12 @@ private static int calculateLength(Region region) {
             return MULTILINE_TOKEN_LENGTH;
     }
 
+    /**
+     * Adds a new {@link de.jplag.Token} for the given {@link TokenType} and {@link Node}.
+     * @param type the {@link de.jplag.TokenType}
+     * @param node the represented {@link de.fraunhofer.aisec.cpg.graph.Node}
+     * @param isEndToken true iff the token represents the end of a block
+     */
     public void addToken(TokenType type, Node node, boolean isEndToken) {
         logger.debug(type.toString() + "/" + node.toString());
         PhysicalLocation location = node.getLocation();

languages/java-cpg/src/main/java/de/jplag/java_cpg/token/IVisitorExitor.java

@@ -3,26 +3,32 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
+import de.fraunhofer.aisec.cpg.graph.Node;
 import de.fraunhofer.aisec.cpg.processing.IVisitable;
 import de.fraunhofer.aisec.cpg.processing.IVisitor;
 
 /**
  * This class adds exit methods to the {@link IVisitor} class. This implementation is supposed to be a close replication
  * of {@link IVisitor}.
+ * @param <V> the object type to visit and exit
  */
 public abstract class IVisitorExitor<V extends IVisitable<V>> extends IVisitor<V> {
 
-    public static final String EXIT_METHOD_IDENTIFIER = "exit";
+    private static final String EXIT_METHOD_IDENTIFIER = "exit";
 
-    public void exit(V t) {
+    /**
+     * Calls the most specific implementation of exit for the given {@link Node}.
+     * @param node the node
+     */
+    public void exit(V node) {
         try {
-            Method mostSpecificExit = this.getClass().getMethod(EXIT_METHOD_IDENTIFIER, t.getClass());
+            Method mostSpecificExit = this.getClass().getMethod(EXIT_METHOD_IDENTIFIER, node.getClass());
             mostSpecificExit.setAccessible(true);
-            mostSpecificExit.invoke(this, t);
+            mostSpecificExit.invoke(this, node);
         } catch (NoSuchMethodException e) {
             // This method is not implemented, that is okay
         } catch (InvocationTargetException | IllegalAccessException e) {
         }
     }
 
-}
+}