Use test entrypoints (#330)

* Initial test entrypoints.

* Progress.

edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF

@@ -24,7 +24,7 @@ Import-Package: com.google.common.collect,
  com.ibm.wala.cast.ir.ssa,
  com.ibm.wala.cast.loader,
  com.ibm.wala.cast.python.client;version="0.1.0",
- com.ibm.wala.cast.python.ipa.callgraph;version="0.24.0",
+ com.ibm.wala.cast.python.ipa.callgraph;version="0.25.0",
  com.ibm.wala.cast.python.loader;version="0.1.0",
  com.ibm.wala.cast.python.ml.analysis;version="0.16.0",
  com.ibm.wala.cast.python.ml.client;version="0.23.0",

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java

@@ -3,6 +3,7 @@
 import static org.eclipse.core.runtime.Platform.getLog;
 
 import java.io.File;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
@@ -284,4 +285,11 @@ public static boolean isContainerType(TypeReference reference) {
 		return reference.equals(PythonTypes.dict) || reference.equals(PythonTypes.enumerate) || reference.equals(PythonTypes.list)
 				|| reference.equals(PythonTypes.set) || reference.equals(PythonTypes.tuple);
 	}
+
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	public static void addEntryPoints(Collection target, Iterable source) {
+		for (Object entryPoint : source)
+			if (target.add(entryPoint))
+				LOG.info("Adding entrypoint: " + entryPoint);
+	}
 }

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java

@@ -1,10 +1,13 @@
 package edu.cuny.hunter.hybridize.core.refactorings;
 
+import static com.google.common.collect.Iterables.concat;
 import static java.lang.Boolean.TRUE;
 import static org.eclipse.core.runtime.Platform.getLog;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
@@ -30,12 +33,16 @@
 import org.python.pydev.core.preferences.InterpreterGeneralPreferences;
 
 import com.ibm.wala.cast.ipa.callgraph.CAstCallGraphUtil;
+import com.ibm.wala.cast.python.ipa.callgraph.PytestEntrypointBuilder;
+import com.ibm.wala.cast.python.ipa.callgraph.PytesttEntrypoint;
 import com.ibm.wala.cast.python.ipa.callgraph.PythonSSAPropagationCallGraphBuilder;
 import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis;
 import com.ibm.wala.ide.util.ProgressMonitorDelegate;
 import com.ibm.wala.ipa.callgraph.AnalysisOptions;
 import com.ibm.wala.ipa.callgraph.CallGraph;
 import com.ibm.wala.ipa.callgraph.CallGraphBuilderCancelException;
+import com.ibm.wala.ipa.callgraph.Entrypoint;
+import com.ibm.wala.ipa.cha.IClassHierarchy;
 import com.ibm.wala.util.CancelException;
 
 import edu.cuny.citytech.refactoring.common.core.RefactoringProcessor;
@@ -47,6 +54,7 @@
 import edu.cuny.hunter.hybridize.core.analysis.FunctionDefinition;
 import edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure;
 import edu.cuny.hunter.hybridize.core.analysis.UndeterminablePythonSideEffectsException;
+import edu.cuny.hunter.hybridize.core.analysis.Util;
 import edu.cuny.hunter.hybridize.core.descriptors.HybridizeFunctionRefactoringDescriptor;
 import edu.cuny.hunter.hybridize.core.messages.Messages;
 import edu.cuny.hunter.hybridize.core.wala.ml.EclipsePythonProjectTensorAnalysisEngine;
@@ -110,6 +118,11 @@ private static RefactoringStatus checkParameters(Function func) {
 
 	private boolean processFunctionsInParallel;
 
+	/**
+	 * True iff entry points corresponding to tests should be used in the {@link CallGraph} construction.
+	 */
+	private boolean useTestEntryPoints;
+
 	public HybridizeFunctionRefactoringProcessor() {
 		// Force the use of typeshed. It's an experimental feature of PyDev.
 		InterpreterGeneralPreferences.FORCE_USE_TYPESHED = TRUE;
@@ -140,6 +153,12 @@ public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffect
 		this.ignoreBooleansInLiteralCheck = ignoreBooleansInLiteralCheck;
 	}
 
+	public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel,
+			boolean alwaysCheckRecusion, boolean ignoreBooleansInLiteralCheck, boolean useTestEntryPoints) {
+		this(alwaysCheckPythonSideEffects, processFunctionsInParallel, alwaysCheckRecusion, ignoreBooleansInLiteralCheck);
+		this.useTestEntryPoints = useTestEntryPoints;
+	}
+
 	public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDefinitionSet)
 			throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ {
 		this();
@@ -176,6 +195,13 @@ public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDef
 		this.alwaysCheckRecursion = alwaysCheckRecursion;
 	}
 
+	public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDefinitionSet, boolean alwaysCheckPythonSideEffects,
+			boolean processFunctionsInParallel, boolean alwaysCheckRecursion, boolean useTestEntryPoints)
+			throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ {
+		this(functionDefinitionSet, alwaysCheckPythonSideEffects, processFunctionsInParallel, alwaysCheckRecursion);
+		this.useTestEntryPoints = useTestEntryPoints;
+	}
+
 	@Override
 	public RefactoringStatus checkFinalConditions(IProgressMonitor pm, CheckConditionsContext context)
 			throws CoreException, OperationCanceledException {
@@ -353,6 +379,25 @@ private CallGraph computeCallGraph(IProject project, PythonSSAPropagationCallGra
 		if (!projectToCallGraph.containsKey(project)) {
 			ProgressMonitorDelegate monitorDelegate = ProgressMonitorDelegate.createProgressMonitorDelegate(monitor);
 			AnalysisOptions options = builder.getOptions();
+
+			if (this.shouldUseTestEntryPoints()) {
+				// Get the current entrypoints.
+				Iterable<? extends Entrypoint> defaultEntrypoints = builder.getOptions().getEntrypoints();
+
+				// Get the pytest entrypoints.
+				Iterable<Entrypoint> pytestEntrypoints = new PytestEntrypointBuilder().createEntrypoints(builder.getClassHierarchy());
+
+				// Add the pytest entrypoints.
+				Iterable<Entrypoint> entrypoints = concat(defaultEntrypoints, pytestEntrypoints);
+
+				// Set the new entrypoints.
+				builder.getOptions().setEntrypoints(entrypoints);
+
+				for (Entrypoint ep : builder.getOptions().getEntrypoints())
+					if (ep instanceof PytesttEntrypoint)
+						LOG.info("Using test entrypoint: " + ep.getMethod().getDeclaringClass().getName() + ".");
+			}
+
 			CallGraph callGraph = builder.makeCallGraph(options, monitorDelegate);
 			projectToCallGraph.put(project, callGraph);
 		}
@@ -461,4 +506,13 @@ public Map<IProject, TensorTypeAnalysis> getProjectToTensorTypeAnalysis() {
 	private boolean getProcessFunctionsInParallel() {
 		return this.processFunctionsInParallel;
 	}
+
+	/**
+	 * True iff we should implicitly consider test cases as entry points in the {@link CallGraph} construction.
+	 *
+	 * @return True iff entry points from tests are considered.
+	 */
+	protected boolean shouldUseTestEntryPoints() {
+		return useTestEntryPoints;
+	}
 }

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java

@@ -49,11 +49,11 @@ public static Refactoring createRefactoring(Set<FunctionDefinition> functionDefi
 	}
 
 	public static HybridizeFunctionRefactoringProcessor createHybridizeFunctionRefactoring(IProject[] projects,
-			boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel, boolean alwaysCheckRecusion)
-			throws ExecutionException, CoreException, IOException {
+			boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel, boolean alwaysCheckRecusion,
+			boolean useTestEntryPoints) throws ExecutionException, CoreException, IOException {
 		Set<FunctionDefinition> functionDefinitions = getFunctionDefinitions(Arrays.asList(projects));
 		return new HybridizeFunctionRefactoringProcessor(functionDefinitions, alwaysCheckPythonSideEffects, processFunctionsInParallel,
-				alwaysCheckRecusion);
+				alwaysCheckRecusion, useTestEntryPoints);
 	}
 
 	public static Set<FunctionDefinition> getFunctionDefinitions(Iterable<?> iterable)

edu.cuny.hunter.hybridize.eval/Evaluate Hybridize Functions.launch

@@ -23,7 +23,7 @@
     <stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17"/>
     <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-os ${target.os} -ws ${target.ws} -arch ${target.arch} -nl ${target.nl} -consoleLog -clean"/>
     <stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>
-    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Declipse.p2.max.threads=10 -Doomph.update.url=https://download.eclipse.org/oomph/updates/milestone/latest -Doomph.redirection.index.redirection=index:/-&gt;http://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/ -Dosgi.requiredJavaVersion=17 [email protected]/eclipse-workspace -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true -Dsun.java.command=Eclipse -XX:+UseG1GC -XX:+UseStringDeduplication --add-modules=ALL-SYSTEM -Dosgi.requiredJavaVersion=11 -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true --add-modules=ALL-SYSTEM -ea:edu.cuny.hunter.hybridize... -Dedu.cuny.hunter.hybridize.eval.alwaysCheckPythonSideEffects=false -Dedu.cuny.hunter.hybridize.eval.alwaysCheckRecursion=false -Dedu.cuny.hunter.hybridize.eval.processFunctionsInParallel=false -Dedu.cuny.hunter.hybridize.dumpCallGraph=false -Xms1024m -Xmx10240m -Djava.util.logging.config.file=${resource_loc:/edu.cuny.hunter.hybridize/logging.properties}"/>
+    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Declipse.p2.max.threads=10 -Doomph.update.url=https://download.eclipse.org/oomph/updates/milestone/latest -Doomph.redirection.index.redirection=index:/-&gt;http://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/ -Dosgi.requiredJavaVersion=17 [email protected]/eclipse-workspace -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true -Dsun.java.command=Eclipse -XX:+UseG1GC -XX:+UseStringDeduplication --add-modules=ALL-SYSTEM -Dosgi.requiredJavaVersion=11 -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true --add-modules=ALL-SYSTEM -ea:edu.cuny.hunter.hybridize... -Dedu.cuny.hunter.hybridize.eval.alwaysCheckPythonSideEffects=false -Dedu.cuny.hunter.hybridize.eval.alwaysCheckRecursion=false -Dedu.cuny.hunter.hybridize.eval.processFunctionsInParallel=false -Dedu.cuny.hunter.hybridize.dumpCallGraph=false -Xms1024m -Xmx10240m -Djava.util.logging.config.file=${resource_loc:/edu.cuny.hunter.hybridize/logging.properties} -Dedu.cuny.hunter.hybridize.eval.useTestEntrypoints=true"/>
     <stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/edu.cuny.hunter.hybridize}/eval"/>
     <booleanAttribute key="pde.generated.config" value="false"/>
     <stringAttribute key="pde.version" value="3.3"/>

edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java

@@ -78,6 +78,8 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto
 
 	private static final String PROCESS_FUNCTIONS_IN_PARALLEL_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.processFunctionsInParallel";
 
+	private static final String USE_TEST_ENTRYPOINTS_KEY = "edu.cuny.hunter.hybridize.eval.useTestEntrypoints";
+
 	private static String[] buildAttributeColumnNames(String... additionalColumnNames) {
 		String[] primaryColumns = new String[] { "subject", "function", "module", "relative path" };
 		List<String> ret = new ArrayList<>(Arrays.asList(primaryColumns));
@@ -101,6 +103,8 @@ private static Object[] buildAttributeColumnValues(Function function, Object...
 
 	private boolean processFunctionsInParallel = Boolean.getBoolean(PROCESS_FUNCTIONS_IN_PARALLEL_PROPERTY_KEY);
 
+	private boolean useTestEntrypoints = Boolean.getBoolean(USE_TEST_ENTRYPOINTS_KEY);
+
 	@Override
 	public Object execute(ExecutionEvent event) throws ExecutionException {
 		Job.create("Evaluating Hybridize Functions refactoring...", monitor -> {
@@ -144,7 +148,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
 
 					resultsTimeCollector.start();
 					processor = createHybridizeFunctionRefactoring(new IProject[] { project }, this.getAlwaysCheckPythonSideEffects(),
-							this.getProcessFunctionsInParallel(), this.getAlwaysCheckRecusion());
+							this.getProcessFunctionsInParallel(), this.getAlwaysCheckRecusion(), this.getUseTestEntrypoints());
 					resultsTimeCollector.stop();
 
 					// run the precondition checking.
@@ -375,4 +379,8 @@ private boolean getAlwaysCheckRecusion() {
 	private boolean getProcessFunctionsInParallel() {
 		return this.processFunctionsInParallel;
 	}
+
+	private boolean getUseTestEntrypoints() {
+		return this.useTestEntrypoints;
+	}
 }

edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java

@@ -162,6 +162,8 @@ public int getInterpreterType() throws CoreException {
 
 	private static final boolean ALWAYS_CHECK_RECURSION = true;
 
+	private static final boolean USE_TEST_ENTRYPOINTS = false;
+
 	/**
 	 * Whether we should run the function processing in parallel. Running in parallel makes the logs difficult to read and doesn't offer
 	 * much in way of speedup since each test has only a few {@link Function}s.
@@ -584,7 +586,7 @@ private Set<Function> getFunctions(String fileNameWithoutExtension) throws Excep
 				.map(f -> new FunctionDefinition(f, fileNameWithoutExtension, inputTestFile, document, nature)).collect(Collectors.toSet());
 
 		HybridizeFunctionRefactoringProcessor processor = new HybridizeFunctionRefactoringProcessor(inputFunctionDefinitions,
-				ALWAYS_CHECK_PYTHON_SIDE_EFFECTS, PROCESS_FUNCTIONS_IN_PARALLEL, ALWAYS_CHECK_RECURSION);
+				ALWAYS_CHECK_PYTHON_SIDE_EFFECTS, PROCESS_FUNCTIONS_IN_PARALLEL, ALWAYS_CHECK_RECURSION, USE_TEST_ENTRYPOINTS);
 
 		ProcessorBasedRefactoring refactoring = new ProcessorBasedRefactoring(processor);
 

hybridize.target

@@ -29,7 +29,7 @@
 				<dependency>
 					<groupId>com.ibm.wala</groupId>
 					<artifactId>com.ibm.wala.cast.python.ml</artifactId>
-					<version>0.25.0-SNAPSHOT</version>
+					<version>0.26.0-SNAPSHOT</version>
 					<type>jar</type>
 				</dependency>
 			</dependencies>