Merge branch 'main' into dependabot/maven/com.puppycrawl.tools-checks…

…tyle-10.14.0

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java

@@ -1,5 +1,6 @@
 package edu.cuny.hunter.hybridize.core.analysis;
 
+import static edu.cuny.hunter.hybridize.core.analysis.Information.TYPE_INFERENCING;
 import static edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure.HAS_PRIMITIVE_PARAMETERS;
 import static edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS;
 import static edu.cuny.hunter.hybridize.core.analysis.PreconditionSuccess.P1;
@@ -386,11 +387,14 @@ public boolean getReduceRetracingParamExists() {
 
 	private boolean ignoreBooleans;
 
+	private boolean alwaysFollowTypeHints;
+
 	private static Map<MethodReference, Map<InstanceKey, Map<CallGraph, Boolean>>> creationsCache = Maps.newHashMap();
 
-	public Function(FunctionDefinition fd, boolean ignoreBooleans) {
+	public Function(FunctionDefinition fd, boolean ignoreBooleans, boolean alwaysFollowTypeHints) {
 		this.functionDefinition = fd;
 		this.ignoreBooleans = ignoreBooleans;
+		this.alwaysFollowTypeHints = alwaysFollowTypeHints;
 	}
 
 	public void computeRecursion(CallGraph callGraph) throws CantComputeRecursionException {
@@ -765,8 +769,12 @@ public TypeReference getDeclaringClass() {
 		return TypeReference.findOrCreate(PythonTypes.pythonLoader, typeName);
 	}
 
+	/**
+	 * Infer which parameters are likely tensor parameters.
+	 */
 	public void inferTensorTensorParameters(TensorTypeAnalysis analysis, CallGraph callGraph, IProgressMonitor monitor) throws Exception {
 		monitor.beginTask("Analyzing whether function has a tensor parameter.", IProgressMonitor.UNKNOWN);
+
 		// TODO: Use cast/assert statements?
 		FunctionDef functionDef = this.getFunctionDefinition().getFunctionDef();
 		argumentsType params = functionDef.args;
@@ -784,29 +792,30 @@ public void inferTensorTensorParameters(TensorTypeAnalysis analysis, CallGraph c
 						continue; // next parameter.
 
 					// check a special case where we consider type hints.
-
-					// if hybridization parameters are specified.
-					if (this.getHybridizationParameters() != null) {
-						// if we are considering type hints.
-						// TODO: Actually get the value here (#111).
-						if (this.getHybridizationParameters().getExperimentalFollowTypeHintsParamExists()) {
-							LOG.info("Following type hints for: " + this + " and parameter: " + paramName + ".");
-
-							// try to get its type from the AST.
-							TypeInfo argTypeInfo = NodeUtils.getTypeForParameterFromAST(paramName, functionDef);
-
-							if (argTypeInfo != null) {
-								LOG.info("Found type for parameter " + paramName + " in " + this + ": " + argTypeInfo.getActTok() + ".");
-
-								exprType node = argTypeInfo.getNode();
-								Set<Attribute> allAttributes = getAllAttributes(node);
-
-								if (attributesHaveTensorTypeHints(allAttributes, monitor.slice(IProgressMonitor.UNKNOWN))) {
-									this.likelyHasTensorParameter = Boolean.TRUE;
-									LOG.info(this + " likely has a tensor parameter: " + paramName + " due to a type hint.");
-									monitor.worked(1);
-									continue; // next parameter.
-								}
+					boolean followTypeHints = this.getAlwaysFollowTypeHints() || this.getHybridizationParameters() != null
+							// TODO: Actually get the value here (#111).
+							&& this.getHybridizationParameters().getExperimentalFollowTypeHintsParamExists();
+
+					// if we are considering type hints.
+					if (followTypeHints) {
+						LOG.info("Following type hints for: " + this + " and parameter: " + paramName + ".");
+
+						// try to get its type from the AST.
+						TypeInfo argTypeInfo = NodeUtils.getTypeForParameterFromAST(paramName, functionDef);
+
+						if (argTypeInfo != null) {
+							LOG.info("Found type for parameter " + paramName + " in " + this + ": " + argTypeInfo.getActTok() + ".");
+
+							exprType node = argTypeInfo.getNode();
+							Set<Attribute> allAttributes = getAllAttributes(node);
+
+							if (attributesHaveTensorTypeHints(allAttributes, monitor.slice(IProgressMonitor.UNKNOWN))) {
+								this.likelyHasTensorParameter = Boolean.TRUE;
+								LOG.info(this + " likely has a tensor parameter: " + paramName + " due to a type hint.");
+								monitor.worked(1);
+								this.addInfo(TYPE_INFERENCING, "Used a type hint to infer tensor type for parameter: " + paramName
+										+ " in function: " + this + ".");
+								continue; // next parameter.
 							}
 						}
 					}
@@ -1203,7 +1212,15 @@ public void addWarning(String message) {
 	}
 
 	public void addInfo(String message) {
-		addStatus(RefactoringStatus.INFO, message, RefactoringStatusEntry.NO_CODE);
+		this.addInfo(message, RefactoringStatusEntry.NO_CODE);
+	}
+
+	public void addInfo(Information information, String message) {
+		this.addInfo(message, information.getCode());
+	}
+
+	private void addInfo(String message, int code) {
+		addStatus(RefactoringStatus.INFO, message, code);
 	}
 
 	private void addStatus(int status, String message, int code) {
@@ -1568,4 +1585,13 @@ public boolean isMethod() {
 	protected boolean getIgnoreBooleans() {
 		return ignoreBooleans;
 	}
+
+	/**
+	 * Returns true iff we should use type hints regardless of a hybridization parameter.
+	 *
+	 * @return Whether we should use type hints regardless of what is specified in any hybridization parameters.
+	 */
+	public boolean getAlwaysFollowTypeHints() {
+		return alwaysFollowTypeHints;
+	}
 }

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Information.java

@@ -0,0 +1,28 @@
+package edu.cuny.hunter.hybridize.core.analysis;
+
+import java.util.Arrays;
+
+public enum Information {
+
+	/**
+	 * Information related to (tensor) type inferencing.
+	 */
+	TYPE_INFERENCING(1);
+
+	static {
+		// check that the codes are unique.
+		assert Arrays.stream(Information.values()).map(Information::getCode).distinct()
+				.count() == Information.values().length : "Codes must be unique.";
+	}
+
+	private int code;
+
+	private Information(int code) {
+		this.code = code;
+	}
+
+	public int getCode() {
+		return this.code;
+	}
+
+}

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java

@@ -119,6 +119,11 @@ private static RefactoringStatus checkParameters(Function func) {
 	 */
 	private boolean useTestEntryPoints;
 
+	/**
+	 * True iff we should use type hints regardless of any hybridization arguments.
+	 */
+	private boolean alwaysFollowTypeHints;
+
 	public HybridizeFunctionRefactoringProcessor() {
 		// Force the use of typeshed. It's an experimental feature of PyDev.
 		InterpreterGeneralPreferences.FORCE_USE_TYPESHED = TRUE;
@@ -155,6 +160,13 @@ public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffect
 		this.useTestEntryPoints = useTestEntryPoints;
 	}
 
+	public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel,
+			boolean alwaysCheckRecusion, boolean ignoreBooleansInLiteralCheck, boolean useTestEntryPoints, boolean alwaysFollowTypeHints) {
+		this(alwaysCheckPythonSideEffects, processFunctionsInParallel, alwaysCheckRecusion, ignoreBooleansInLiteralCheck,
+				useTestEntryPoints);
+		this.alwaysFollowTypeHints = alwaysFollowTypeHints;
+	}
+
 	public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDefinitionSet)
 			throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ {
 		this();
@@ -164,7 +176,7 @@ public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDef
 			Set<Function> functionSet = this.getFunctions();
 
 			for (FunctionDefinition fd : functionDefinitionSet) {
-				Function function = new Function(fd, this.ignoreBooleansInLiteralCheck);
+				Function function = new Function(fd, this.ignoreBooleansInLiteralCheck, this.alwaysFollowTypeHints);
 
 				// Add the Function to the Function set.
 				functionSet.add(function);
@@ -198,6 +210,31 @@ public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDef
 		this.useTestEntryPoints = useTestEntryPoints;
 	}
 
+	public HybridizeFunctionRefactoringProcessor(Set<FunctionDefinition> functionDefinitionSet, boolean alwaysCheckPythonSideEffects,
+			boolean processFunctionsInParallel, boolean alwaysCheckRecursion, boolean useTestEntryPoints, boolean alwaysFollowTypeHints)
+			throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ {
+		this();
+
+		this.alwaysCheckPythonSideEffects = alwaysCheckPythonSideEffects;
+		this.alwaysCheckRecursion = alwaysCheckRecursion;
+		this.processFunctionsInParallel = processFunctionsInParallel;
+		this.useTestEntryPoints = useTestEntryPoints;
+		this.alwaysFollowTypeHints = alwaysFollowTypeHints;
+
+		// Convert the FunctionDefs to Functions.
+		if (functions != null) {
+			Set<Function> functionSet = this.getFunctions();
+
+			for (FunctionDefinition fd : functionDefinitionSet) {
+				Function function = new Function(fd, this.ignoreBooleansInLiteralCheck, this.alwaysFollowTypeHints);
+
+				// Add the Function to the Function set.
+				functionSet.add(function);
+			}
+		}
+
+	}
+
 	@Override
 	public RefactoringStatus checkFinalConditions(IProgressMonitor pm, CheckConditionsContext context)
 			throws CoreException, OperationCanceledException {
@@ -511,4 +548,13 @@ private boolean getProcessFunctionsInParallel() {
 	protected boolean shouldUseTestEntryPoints() {
 		return useTestEntryPoints;
 	}
+
+	/**
+	 * Returns true iff we should follow type hints regardless of any hybridization arguments.
+	 *
+	 * @return True iff we should follow type hints regardless of any hybridization arguments.
+	 */
+	public boolean getAlwaysFollowTypeHints() {
+		return alwaysFollowTypeHints;
+	}
 }

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java

@@ -50,10 +50,10 @@ public static Refactoring createRefactoring(Set<FunctionDefinition> functionDefi
 
 	public static HybridizeFunctionRefactoringProcessor createHybridizeFunctionRefactoring(IProject[] projects,
 			boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel, boolean alwaysCheckRecusion,
-			boolean useTestEntryPoints) throws ExecutionException, CoreException, IOException {
+			boolean useTestEntryPoints, boolean alwaysFollowTypeHints) throws ExecutionException, CoreException, IOException {
 		Set<FunctionDefinition> functionDefinitions = getFunctionDefinitions(Arrays.asList(projects));
 		return new HybridizeFunctionRefactoringProcessor(functionDefinitions, alwaysCheckPythonSideEffects, processFunctionsInParallel,
-				alwaysCheckRecusion, useTestEntryPoints);
+				alwaysCheckRecusion, useTestEntryPoints, alwaysFollowTypeHints);
 	}
 
 	public static Set<FunctionDefinition> getFunctionDefinitions(Iterable<?> iterable)

edu.cuny.hunter.hybridize.core/tensorflow.xml

@@ -45,6 +45,8 @@
         <new def="layers" class="Lobject" />
         <putfield class="LRoot" field="layers" fieldType="LRoot" ref="x" value="layers" />
         <putfield class="LRoot" field="layers" fieldType="LRoot" ref="keras" value="layers" />
+        <new def="models" class="Lobject" />
+        <putfield class="LRoot" field="models" fieldType="LRoot" ref="keras" value="models" />
         <new def="app" class="Lobject" />
         <putfield class="LRoot" field="app" fieldType="LRoot" ref="x" value="app" />
         <new def="run" class="Ltensorflow/app/run" />
@@ -146,6 +148,9 @@
         <putfield class="LRoot" field="Input" fieldType="LRoot" ref="layers" value="Input" />
         <new def="Dense" class="Ltensorflow/keras/layers/Dense" />
         <putfield class="LRoot" field="Dense" fieldType="LRoot" ref="layers" value="Dense" />
+        <new def="Model" class="Ltensorflow/keras/models/Model" />
+        <putfield class="LRoot" field="Model" fieldType="LRoot" ref="keras" value="Model" />
+        <putfield class="LRoot" field="Model" fieldType="LRoot" ref="models" value="Model" />
         <new def="Variable" class="Ltensorflow/functions/Variable" />
         <putfield class="LRoot" field="Variable" fieldType="LRoot" ref="x" value="Variable" />
         <putfield class="LRoot" field="Variable" fieldType="LRoot" ref="variables" value="Variable" />
@@ -729,6 +734,42 @@
         </method>
       </class>
     </package>
+    <package name="tensorflow/keras/models">
+      <class name="Model" allocatable="true">
+        <method name="read_data" descriptor="()LRoot;">
+          <new def="x" class="Ltensorflow/keras/Model/attribute" />
+          <return value="x" />
+        </method>
+        <!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/keras/models/Model -->
+        <method name="do" descriptor="()LRoot;" numArgs="4" paramNames="self inputs outputs name">
+          <new def="__call__" class="Ltensorflow/keras/models/__call__" />
+          <putfield class="LRoot" field="__call__" fieldType="LRoot" ref="arg0" value="__call__" />
+          <new def="call" class="Ltensorflow/keras/models/call" />
+          <putfield class="LRoot" field="call" fieldType="LRoot" ref="arg0" value="call" />
+          <new def="x" class="Llist" />
+          <call class="LRoot" name="read_data" descriptor="()LRoot;" type="virtual" arg0="arg0" def="xx" />
+          <putfield class="LRoot" field="0" fieldType="LRoot" ref="x" value="xx" />
+          <!-- https://www.tensorflow.org/guide/keras/transfer_learning#freezing_layers_understanding_the_trainable_attribute -->
+          <putfield class="LRoot" field="trainable_weights" fieldType="LRoot" ref="arg0" value="x" />
+          <putfield class="LRoot" field="weights" fieldType="LRoot" ref="arg0" value="x" />
+          <putfield class="LRoot" field="non_trainable_weights" fieldType="LRoot" ref="arg0" value="x" />
+          <return value="arg0" />
+        </method>
+      </class>
+      <class name="__call__" allocatable="true">
+        <!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/keras/Model#call -->
+        <method name="do" descriptor="()LRoot;" numArgs="4" paramNames="self inputs training mask">
+          <return value="inputs" />
+        </method>
+      </class>
+      <!-- FIXME: Workaround for https://github.com/wala/ML/issues/106. -->
+      <class name="call" allocatable="true">
+        <!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/keras/Model#call -->
+        <method name="do" descriptor="()LRoot;" numArgs="4" paramNames="self inputs training mask">
+          <return value="inputs" />
+        </method>
+      </class>
+    </package>
     <package name="tensorflow/keras/layers">
       <class name="Dense" allocatable="true">
         <!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/keras/layers/Dense -->

edu.cuny.hunter.hybridize.eval/Evaluate Hybridize Functions.launch

@@ -23,7 +23,7 @@
     <stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17"/>
     <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-os ${target.os} -ws ${target.ws} -arch ${target.arch} -nl ${target.nl} -consoleLog -clean"/>
     <stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>
-    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Declipse.p2.max.threads=10 -Doomph.update.url=https://download.eclipse.org/oomph/updates/milestone/latest -Doomph.redirection.index.redirection=index:/-&gt;http://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/ -Dosgi.requiredJavaVersion=17 [email protected]/eclipse-workspace -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true -Dsun.java.command=Eclipse -XX:+UseG1GC -XX:+UseStringDeduplication --add-modules=ALL-SYSTEM -Dosgi.requiredJavaVersion=11 -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true --add-modules=ALL-SYSTEM -ea:edu.cuny.hunter.hybridize... -Dedu.cuny.hunter.hybridize.eval.alwaysCheckPythonSideEffects=false -Dedu.cuny.hunter.hybridize.eval.alwaysCheckRecursion=false -Dedu.cuny.hunter.hybridize.eval.processFunctionsInParallel=false -Dedu.cuny.hunter.hybridize.dumpCallGraph=false -Xms1024m -Xmx10240m -Djava.util.logging.config.file=${resource_loc:/edu.cuny.hunter.hybridize/logging.properties} -Dedu.cuny.hunter.hybridize.eval.useTestEntrypoints=true"/>
+    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Declipse.p2.max.threads=10 -Doomph.update.url=https://download.eclipse.org/oomph/updates/milestone/latest -Doomph.redirection.index.redirection=index:/-&gt;http://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/ -Dosgi.requiredJavaVersion=17 [email protected]/eclipse-workspace -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true -Dsun.java.command=Eclipse -XX:+UseG1GC -XX:+UseStringDeduplication --add-modules=ALL-SYSTEM -Dosgi.requiredJavaVersion=11 -Dosgi.dataAreaRequiresExplicitInit=true -Dorg.eclipse.swt.graphics.Resource.reportNonDisposed=true --add-modules=ALL-SYSTEM -ea:edu.cuny.hunter.hybridize... -Dedu.cuny.hunter.hybridize.eval.alwaysCheckPythonSideEffects=false -Dedu.cuny.hunter.hybridize.eval.alwaysCheckRecursion=false -Dedu.cuny.hunter.hybridize.eval.processFunctionsInParallel=false -Dedu.cuny.hunter.hybridize.dumpCallGraph=false -Xms1024m -Xmx10240m -Djava.util.logging.config.file=${resource_loc:/edu.cuny.hunter.hybridize/logging.properties} -Dedu.cuny.hunter.hybridize.eval.useTestEntrypoints=true -Dedu.cuny.hunter.hybridize.eval.alwaysFollowTypeHints=true"/>
     <stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/edu.cuny.hunter.hybridize}/eval"/>
     <booleanAttribute key="pde.generated.config" value="false"/>
     <stringAttribute key="pde.version" value="3.3"/>

edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java

@@ -80,6 +80,8 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto
 
 	private static final String USE_TEST_ENTRYPOINTS_KEY = "edu.cuny.hunter.hybridize.eval.useTestEntrypoints";
 
+	private static final String ALWAYS_FOLLOW_TYPE_HINTS_KEY = "edu.cuny.hunter.hybridize.eval.alwaysFollowTypeHints";
+
 	private static String[] buildAttributeColumnNames(String... additionalColumnNames) {
 		String[] primaryColumns = new String[] { "subject", "function", "module", "relative path" };
 		List<String> ret = new ArrayList<>(Arrays.asList(primaryColumns));
@@ -105,6 +107,8 @@ private static Object[] buildAttributeColumnValues(Function function, Object...
 
 	private boolean useTestEntrypoints = Boolean.getBoolean(USE_TEST_ENTRYPOINTS_KEY);
 
+	private boolean alwaysFollowTypeHints = Boolean.getBoolean(ALWAYS_FOLLOW_TYPE_HINTS_KEY);
+
 	@Override
 	public Object execute(ExecutionEvent event) throws ExecutionException {
 		Job.create("Evaluating Hybridize Functions refactoring...", monitor -> {
@@ -148,7 +152,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
 
 					resultsTimeCollector.start();
 					processor = createHybridizeFunctionRefactoring(new IProject[] { project }, this.getAlwaysCheckPythonSideEffects(),
-							this.getProcessFunctionsInParallel(), this.getAlwaysCheckRecusion(), this.getUseTestEntrypoints());
+							this.getProcessFunctionsInParallel(), this.getAlwaysCheckRecusion(), this.getUseTestEntrypoints(), this.getAlwaysFollowTypeHints());
 					resultsTimeCollector.stop();
 
 					// run the precondition checking.
@@ -383,4 +387,8 @@ private boolean getProcessFunctionsInParallel() {
 	private boolean getUseTestEntrypoints() {
 		return this.useTestEntrypoints;
 	}
+
+	public boolean getAlwaysFollowTypeHints() {
+		return alwaysFollowTypeHints;
+	}
 }

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testHasLikelyTensorParameter158/in/A.py

@@ -0,0 +1,13 @@
+# From https://www.tensorflow.org/guide/function#usage.
+
+import tensorflow as tf
+
+
+@tf.function
+def add(t: tuple[tf.Tensor, tf.Tensor]):
+  return t[0] + t[1]
+
+
+arg = (tf.constant(2), tf.constant(2))
+assert type(arg) == tuple
+result = add(arg)

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testHasLikelyTensorParameter158/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel10/in/A.py

@@ -0,0 +1,17 @@
+# From https://github.com/tensorflow/tensorflow/issues/14359#issue-272179775
+
+import tensorflow as tf
+
+
+def f(a):
+    pass
+
+
+inputs = tf.keras.Input(shape=(3,))
+x = tf.keras.layers.Dense(4, activation=tf.nn.relu)(inputs)
+outputs = tf.keras.layers.Dense(5, activation=tf.nn.softmax)(x)
+model = tf.keras.Model(inputs=inputs, outputs=outputs)
+
+# From https://www.tensorflow.org/guide/keras/transfer_learning#freezing_layers_understanding_the_trainable_attribute
+for i in model.weights:
+    f(i)

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel10/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3