From 0672e7bc36d768f9f5b6ddce761304619850a3db Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 13 Oct 2023 16:50:17 -0400 Subject: [PATCH 001/172] Add Python side-effects test. --- .../hybridize/core/analysis/Function.java | 6 ++++++ .../HybridizeFunctionRefactoringProcessor.java | 2 ++ .../testPythonSideEffects/in/A.py | 13 +++++++++++++ .../testPythonSideEffects/in/requirements.txt | 1 + .../tests/HybridizeFunctionRefactoringTest.java | 17 +++++++++++++++++ 5 files changed, 39 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index d84f0fa65..bef76886d 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -292,6 +292,8 @@ public boolean getReduceRetracingParamExists() { */ private Boolean likelyHasTensorParameter; + private boolean hasPythonSideEffects; + /** * TODO: Populate. */ @@ -771,4 +773,8 @@ public Refactoring getRefactoring() { public void setRefactoring(Refactoring refactoring) { this.refactoring = refactoring; } + + public boolean getHasPythonSideEffects() { + return this.hasPythonSideEffects; + } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index d04e676b2..79b4815ba 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -194,6 +194,8 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat throw new CoreException(Status.error("Could not infer tensor parameters for: : " + func, e)); } + // TODO: Check Python side-effects. + // check the function preconditions. func.check(); status.merge(func.getStatus()); diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/A.py new file mode 100644 index 000000000..d8a954964 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/A.py @@ -0,0 +1,13 @@ +# From https://www.tensorflow.org/guide/function#executing_python_side_effects. + +import tensorflow as tf + + +def f(x): + print("Traced with", x) + tf.print("Executed with", x) + + +f(1) +f(1) +f(2) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 4d0d07f8e..e4fa2fc1f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4556,4 +4556,21 @@ public void testPreconditionChecking2() throws Exception { // it's not hybrid but it has a tensor parameter. Let's make it hybrid. testPreconditionCheckingHelper(false, true, Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, Transformation.CONVERT_TO_HYBRID, P1); } + + @Test + public void testPythonSideEffects() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. + assertTrue(function.getHasPythonSideEffects()); + } + + private Function getSingleFunction() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + return function; + } } From 89f55a82b12d58a6eb8fa1fcb21a3a3fd60f189f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 10:40:08 -0400 Subject: [PATCH 002/172] More tests for side-effects. --- .../testPythonSideEffects2/in/A.py | 13 +++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects3/in/A.py | 15 ++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects4/in/A.py | 10 +++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 29 ++++++++++++++++++- 7 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/A.py new file mode 100644 index 000000000..5a6b05645 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/A.py @@ -0,0 +1,13 @@ +# From https://www.tensorflow.org/guide/function#executing_python_side_effects. + +import tensorflow as tf + + +def f(x): + # print("Traced with", x) # This is a Python side-effect. + tf.print("Executed with", x) # THis isn't. + + +f(1) +f(1) +f(2) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects2/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/A.py new file mode 100644 index 000000000..e743bea73 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/A.py @@ -0,0 +1,15 @@ +# From https://www.tensorflow.org/guide/function#executing_python_side_effects. + +import tensorflow as tf + + +def f(x): + # Assigning x to a." + a = x + print("Traced with", a) # This is a transitive Python side-effect. + tf.print("Executed with", x) # This isn't. + + +f(1) +f(1) +f(2) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects3/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/A.py new file mode 100644 index 000000000..2bb9174ee --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/A.py @@ -0,0 +1,10 @@ +# From https://www.tensorflow.org/guide/function#executing_python_side_effects. + + +def f(x): + x = 5 # no side-effect as `x` is a local variable.. + + +f(1) +f(1) +f(2) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects4/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e4fa2fc1f..f3532b222 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4562,7 +4562,34 @@ public void testPythonSideEffects() throws Exception { Function function = getSingleFunction(); assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. - assertTrue(function.getHasPythonSideEffects()); + assertTrue("Expecting a Python side-effect.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects2() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. + // there's a call to a TF operation. So, no "Python" side-effects. + assertFalse("TF operations shouldn't be considered Python side-effects.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects3() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. + // there's a transitive Python side-effect. + assertTrue("Expecting a Python side-effect from a transitive local variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects4() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. + // there's a Python statement but no side-effect. + assertFalse("This Python statement only modifies a local variable, so no side-effects.", function.getHasPythonSideEffects()); } private Function getSingleFunction() throws Exception { From 8790bfc52a921e5ec2e24cb441a3a39f2e2b5e6e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 14:55:02 -0400 Subject: [PATCH 003/172] Add list manipulation test. --- .../HybridizeFunction/testPythonSideEffects5/in/A.py | 8 ++++++++ .../testPythonSideEffects5/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 9 +++++++++ 3 files changed, 17 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/A.py new file mode 100644 index 000000000..afe6aa25c --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/A.py @@ -0,0 +1,8 @@ +my_list = [10] + + +def f(): + my_list[0] = 1 + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects5/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index f3532b222..a27598435 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4592,6 +4592,15 @@ public void testPythonSideEffects4() throws Exception { assertFalse("This Python statement only modifies a local variable, so no side-effects.", function.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects5() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("This Python statement modifies a global variable, so it has side-effects.", function.getHasPythonSideEffects()); + } + private Function getSingleFunction() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); From 2787b11ce5783df1ff95bc81668e45c8069fc765 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 17:37:09 -0400 Subject: [PATCH 004/172] Progress. --- .../META-INF/MANIFEST.MF | 2 + .../hybridize/core/analysis/Function.java | 88 ++++++++++++++++++- ...HybridizeFunctionRefactoringProcessor.java | 7 +- .../META-INF/MANIFEST.MF | 1 + ...teHybridizeFunctionRefactoringHandler.java | 10 +-- .../testPythonSideEffects6/in/A.py | 9 ++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 9 ++ 8 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index ed0ee3cfd..519f6c9d9 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -27,6 +27,8 @@ Import-Package: com.ibm.wala.cast.ipa.callgraph, com.ibm.wala.cast.python.loader;version="0.1.0", com.ibm.wala.cast.python.ml.analysis;version="0.1.0", com.ibm.wala.cast.python.ml.client;version="0.1.0", + com.ibm.wala.cast.python.modref, + com.ibm.wala.cast.python.types, com.ibm.wala.cast.python.util, com.ibm.wala.cast.tree, com.ibm.wala.classLoader, diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index bef76886d..bb3bc3494 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -5,12 +5,16 @@ import static edu.cuny.hunter.hybridize.core.analysis.Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID; import static edu.cuny.hunter.hybridize.core.analysis.Refactoring.OPTIMIZE_HYBRID_FUNCTION; import static edu.cuny.hunter.hybridize.core.analysis.Transformation.CONVERT_TO_EAGER; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; import static org.eclipse.core.runtime.Platform.getLog; import java.io.File; import java.util.HashSet; +import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import org.eclipse.core.resources.IProject; import org.eclipse.core.runtime.ILog; @@ -36,12 +40,22 @@ import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.cast.python.ml.analysis.TensorVariable; +import com.ibm.wala.cast.python.modref.PythonModRef; +import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; +import com.ibm.wala.cast.types.AstMethodReference; import com.ibm.wala.classLoader.IMethod; import com.ibm.wala.ipa.callgraph.CGNode; +import com.ibm.wala.ipa.callgraph.CallGraph; +import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey; +import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; +import com.ibm.wala.ipa.modref.ModRef; +import com.ibm.wala.types.MethodReference; +import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.Pair; +import com.ibm.wala.util.intset.OrdinalSet; import edu.cuny.citytech.refactoring.common.core.RefactorableProgramEntity; import edu.cuny.hunter.hybridize.core.utils.RefactoringAvailabilityTester; @@ -292,7 +306,10 @@ public boolean getReduceRetracingParamExists() { */ private Boolean likelyHasTensorParameter; - private boolean hasPythonSideEffects; + /** + * True iff this {@link Function} has Python side-effects. + */ + private Boolean hasPythonSideEffects; /** * TODO: Populate. @@ -318,6 +335,73 @@ public Function(FunctionDefinition fd) { this.functionDefinition = fd; } + /** + * Infer the side-effects potentially produced by executing this {@link Function}. + * + * @param callGraph The system {@link CallGraph}. + * @param pointerAnalysis The system {@link PointerAnalysis}. + * @throws IllegalArgumentException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. + */ + public void inferSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws IllegalArgumentException { + ModRef modRef = new PythonModRef(); + Map> mod = modRef.computeMod(callGraph, pointerAnalysis); + + // Get the nodes corresponding to this function. + Set nodes = getCallGraphNodes(callGraph); + + // for each node. + for (CGNode cgNode : nodes) { + // Get the locations (pointers) modified by this function. + OrdinalSet modSet = mod.get(cgNode); + LOG.info("Found " + modSet.size() + " modified location(s)."); + + if (!modSet.isEmpty()) { + modSet.forEach(pk -> LOG.info("Modified location: " + pk + ".")); + this.hasPythonSideEffects = TRUE; + return; + } + } + + this.hasPythonSideEffects = FALSE; + LOG.info(this + " does not have side-effects."); + } + + /** + * Get the {@link CallGraph} nodes corresponding to this {@link Function}. + * + * @param callGraph The {@link CallGraph} to search. + * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. + * @throws IllegalArgumentException If this {@link Function} can't be found in the given {@link CallGraph}. + * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. + */ + private Set getCallGraphNodes(CallGraph callGraph) throws IllegalArgumentException { + MethodReference methodReference = this.getMethodReference(); + Set nodes = callGraph.getNodes(methodReference); + + if (nodes.isEmpty()) { + LOG.error("Can't get call graph nodes for: " + this + "."); + LOG.info("Method reference is: " + methodReference + "."); + LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); + + throw new IllegalArgumentException("Can't find: " + methodReference + " in call graph."); + } + + LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); + return nodes; + } + + public MethodReference getMethodReference() { + File containingFile = this.getContainingFile(); + String filename = containingFile.getName(); + String functionName = this.getSimpleName(); + + TypeReference typeReference = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lscript " + filename + "/" + functionName); + + MethodReference methodReference = MethodReference.findOrCreate(typeReference, AstMethodReference.fnSelector); + return methodReference; + + } + public void inferTensorTensorParameters(TensorTypeAnalysis analysis, IProgressMonitor monitor) throws BadLocationException { monitor.beginTask("Analyzing whether function has a tensor parameter.", IProgressMonitor.UNKNOWN); // TODO: What if there are no current calls to the function? How will we determine its type? @@ -774,7 +858,7 @@ public void setRefactoring(Refactoring refactoring) { this.refactoring = refactoring; } - public boolean getHasPythonSideEffects() { + public Boolean getHasPythonSideEffects() { return this.hasPythonSideEffects; } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 79b4815ba..97f23f2a6 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -194,7 +194,12 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat throw new CoreException(Status.error("Could not infer tensor parameters for: : " + func, e)); } - // TODO: Check Python side-effects. + // Check Python side-effects. + try { + func.inferSideEffects(callGraph, builder.getPointerAnalysis()); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Can't infer side-effects of: " + this + ".", e); + } // check the function preconditions. func.check(); diff --git a/edu.cuny.hunter.hybridize.eval/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.eval/META-INF/MANIFEST.MF index d65007340..fdd920402 100644 --- a/edu.cuny.hunter.hybridize.eval/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.eval/META-INF/MANIFEST.MF @@ -7,6 +7,7 @@ Bundle-Version: 1.0.0.qualifier Export-Package: edu.cuny.hunter.hybridize.eval.handlers, edu.cuny.hunter.hybridize.eval.ui.plugins Import-Package: com.google.common.collect, + com.ibm.wala.types, org.python.pydev.parser.jython.ast, org.python.pydev.plugin.nature Require-Bundle: org.eclipse.ui, diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index dbdee1135..cf551e3b5 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -72,19 +72,19 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto private static final String PERFORM_CHANGE_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.performChange"; private static String[] buildAttributeColumnNames(String... additionalColumnNames) { - String[] primaryColumns = new String[] { "subject", "function", "module", "relative path" }; + String[] primaryColumns = new String[] { "subject", "function", "module", "relative path", "method reference" }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnNames)); return ret.toArray(String[]::new); } - private static Object[] buildAttributeColumnValues(Function function, Object... additioanlColumnValues) { + private static Object[] buildAttributeColumnValues(Function function, Object... additionalColumnValues) { IProject project = function.getProject(); Path relativePath = project.getLocation().toFile().toPath().relativize(function.getContainingFile().toPath()); - String[] primaryColumns = new String[] { project.getName(), function.getIdentifer(), function.getContainingModuleName(), - relativePath.toString() }; + Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifer(), function.getContainingModuleName(), + relativePath, function.getMethodReference() }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); - ret.addAll(Arrays.asList(additioanlColumnValues)); + ret.addAll(Arrays.asList(additionalColumnValues)); return ret.toArray(Object[]::new); } diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/A.py new file mode 100644 index 000000000..61ad64c04 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/A.py @@ -0,0 +1,9 @@ +my_list = [10] + + +def f(): + my_list[0] = 1 + + +f() +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects6/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index a27598435..190556b66 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4601,6 +4601,15 @@ public void testPythonSideEffects5() throws Exception { assertTrue("This Python statement modifies a global variable, so it has side-effects.", function.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects6() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. Multiple calls to the function. + assertTrue("This Python statement modifies a global variable, so it has side-effects.", function.getHasPythonSideEffects()); + } + private Function getSingleFunction() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); From 4e38b13dbfd002b9e55e344976b8dc5d4dfd9e00 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 17:37:24 -0400 Subject: [PATCH 005/172] Fix progress monitors. --- .../refactorings/HybridizeFunctionRefactoringProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 97f23f2a6..94ff6eb39 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -181,7 +181,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // Find out if it's hybrid via the tf.function decorator. try { - func.computeHybridization(monitor); + func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); } catch (BadLocationException e) { throw new CoreException(Status.error("Could not compute hybridization for: : " + func, e)); } @@ -189,7 +189,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // TODO: Whether a function has a tensor argument should probably be an initial // condition: functions w/o such arguments should not be candidates. try { - func.inferTensorTensorParameters(analysis, monitor); + func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); } catch (BadLocationException e) { throw new CoreException(Status.error("Could not infer tensor parameters for: : " + func, e)); } From 399bb7c2c82fb6975c11f67734002240940d03cf Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 17:37:43 -0400 Subject: [PATCH 006/172] Fix logs. --- .../wala/ml/EclipsePythonProjectTensorAnalysisEngine.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/EclipsePythonProjectTensorAnalysisEngine.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/EclipsePythonProjectTensorAnalysisEngine.java index 89a4557c4..d21e8159c 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/EclipsePythonProjectTensorAnalysisEngine.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/EclipsePythonProjectTensorAnalysisEngine.java @@ -52,13 +52,13 @@ public EclipsePythonProjectTensorAnalysisEngine(IProject project) { this.project = project; IPath projectPath = getPath(project); Module dirModule = new EclipseSourceDirectoryTreeModule(projectPath, null, ".py"); - LOG.info("Creating engine from: " + dirModule); + LOG.info("Creating engine from: " + dirModule + "."); this.setModuleFiles(Collections.singleton(dirModule)); for (Iterator entries = dirModule.getEntries(); entries.hasNext();) { ModuleEntry entry = entries.next(); - LOG.info("Found entry: " + entry); + LOG.info("Found entry: " + entry + "."); } } From a5c029bddb9d9fe71ab609e7d070c61b20ab699f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 16 Oct 2023 21:27:29 -0400 Subject: [PATCH 007/172] Fix checkstyle warning. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index cf551e3b5..e3378e931 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -82,7 +82,7 @@ private static Object[] buildAttributeColumnValues(Function function, Object... IProject project = function.getProject(); Path relativePath = project.getLocation().toFile().toPath().relativize(function.getContainingFile().toPath()); Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifer(), function.getContainingModuleName(), - relativePath, function.getMethodReference() }; + relativePath, function.getMethodReference() }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnValues)); return ret.toArray(Object[]::new); From 6a85b87facc68b91b27a396e378228bbecd59cca Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 10:27:28 -0400 Subject: [PATCH 008/172] Fix exception message. --- .../refactorings/HybridizeFunctionRefactoringProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 94ff6eb39..669340498 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -198,7 +198,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat try { func.inferSideEffects(callGraph, builder.getPointerAnalysis()); } catch (IllegalArgumentException e) { - throw new IllegalStateException("Can't infer side-effects of: " + this + ".", e); + throw new IllegalStateException("Can't infer side-effects of: " + func + ".", e); } // check the function preconditions. From 23fc5abe417a185c3a0742d515525f5c00343afd Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 10:46:12 -0400 Subject: [PATCH 009/172] Adjust indentation. --- checkstyle.xml | 2 +- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkstyle.xml b/checkstyle.xml index 77a54d114..2bb74ebef 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -262,7 +262,7 @@ - + diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index e3378e931..cf551e3b5 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -82,7 +82,7 @@ private static Object[] buildAttributeColumnValues(Function function, Object... IProject project = function.getProject(); Path relativePath = project.getLocation().toFile().toPath().relativize(function.getContainingFile().toPath()); Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifer(), function.getContainingModuleName(), - relativePath, function.getMethodReference() }; + relativePath, function.getMethodReference() }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnValues)); return ret.toArray(Object[]::new); From 8a8ea13ed8d9b197d70373fa244b5c12967578f8 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 10:53:23 -0400 Subject: [PATCH 010/172] Fix spelling. --- .../hybridize/core/analysis/Function.java | 4 +-- ...teHybridizeFunctionRefactoringHandler.java | 2 +- .../HybridizeFunctionRefactoringTest.java | 26 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index bb3bc3494..3b6bc66a9 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -763,7 +763,7 @@ protected FunctionDefinition getFunctionDefinition() { * @see PEP 3155 * @return This {@link Function}'s QN. */ - public String getIdentifer() { + public String getIdentifier() { FunctionDefinition functionDefinition = this.getFunctionDefinition(); FunctionDef functionDef = functionDefinition.getFunctionDef(); return Util.getQualifiedName(functionDef); @@ -790,7 +790,7 @@ public Boolean getLikelyHasTensorParameter() { @Override public String toString() { - return this.getIdentifer() + "()"; + return this.getIdentifier() + "()"; } @Override diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index cf551e3b5..99a736f24 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -81,7 +81,7 @@ private static String[] buildAttributeColumnNames(String... additionalColumnName private static Object[] buildAttributeColumnValues(Function function, Object... additionalColumnValues) { IProject project = function.getProject(); Path relativePath = project.getLocation().toFile().toPath().relativize(function.getContainingFile().toPath()); - Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifer(), function.getContainingModuleName(), + Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifier(), function.getContainingModuleName(), relativePath, function.getMethodReference() }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnValues)); diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 190556b66..a307fb1a4 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -898,7 +898,7 @@ public void testQN() throws Exception { LOG.info("Expected signature: " + expectedSignature); - String actualSignature = func.getIdentifer(); + String actualSignature = func.getIdentifier(); LOG.info("Actual signature: " + actualSignature); @@ -1180,7 +1180,7 @@ public void testSameFileSameName() throws Exception { for (Function func : functions) { assertNotNull(func); - functionNames.add(func.getIdentifer()); + functionNames.add(func.getIdentifier()); } assertEquals(2, functionNames.size()); @@ -1200,7 +1200,7 @@ public void testSameFileSameName2() throws Exception { for (Function func : functions) { assertNotNull(func); - functionNames.add(func.getIdentifer()); + functionNames.add(func.getIdentifier()); } // NOTE: Both of these functions have the same qualified name. @@ -1233,7 +1233,7 @@ private void testDifferentFileSameNameHelper(int expectedNumberOfFunctions, int for (Function func : functions) { assertNotNull(func); - functionNames.add(func.getIdentifer()); + functionNames.add(func.getIdentifier()); } assertEquals(expectedNumberOfFunctionNames, functionNames.size()); @@ -1262,7 +1262,7 @@ private void testDifferentFileSameNameHelper(Set functionsToT int foundCount = 0; for (FunctionUnderTest funcUnderTest : functionsToTest) { - if (funcUnderTest.getName().equals(func.getIdentifer()) + if (funcUnderTest.getName().equals(func.getIdentifier()) && (funcUnderTest.getModuleName() == null || funcUnderTest.getModuleName().equals(func.getContainingModuleName())) && funcUnderTest.getParameters().size() == func.getNumberOfParameters()) { // found it. @@ -1284,7 +1284,7 @@ private void testDifferentFileSameNameHelper(Set functionsToT for (Function func : functions) { assertNotNull(func); - functionNames.add(func.getIdentifer()); + functionNames.add(func.getIdentifier()); } assertEquals(expectedNumberOfFunctionNames, functionNames.size()); @@ -1408,7 +1408,7 @@ public void testFunctionEquality() throws Exception { for (Function func : functions) { assertNotNull(func); - String id = func.getIdentifer(); + String id = func.getIdentifier(); assertNotNull(id); assertTrue(id.equals("a") || id.equals("b")); } @@ -1420,7 +1420,7 @@ public void testFunctionEquality() throws Exception { Function func1 = iterator.next(); assertNotNull(func1); - String identifer1 = func1.getIdentifer(); + String identifer1 = func1.getIdentifier(); assertNotNull(identifer1); assertTrue(iterator.hasNext()); @@ -1428,7 +1428,7 @@ public void testFunctionEquality() throws Exception { Function func2 = iterator.next(); assertNotNull(func2); - String identifer2 = func2.getIdentifer(); + String identifer2 = func2.getIdentifier(); assertNotNull(identifer2); assertTrue(!identifer1.equals("a") || identifer2.equals("b")); @@ -1449,7 +1449,7 @@ public void testFunctionEquality2() throws Exception { for (Function func : functions) { assertNotNull(func); - String id = func.getIdentifer(); + String id = func.getIdentifier(); assertNotNull(id); assertTrue(id.equals("a")); } @@ -1461,7 +1461,7 @@ public void testFunctionEquality2() throws Exception { Function func1 = iterator.next(); assertNotNull(func1); - String identifer1 = func1.getIdentifer(); + String identifer1 = func1.getIdentifier(); assertNotNull(identifer1); assertTrue(iterator.hasNext()); @@ -1469,7 +1469,7 @@ public void testFunctionEquality2() throws Exception { Function func2 = iterator.next(); assertNotNull(func2); - String identifer2 = func2.getIdentifer(); + String identifer2 = func2.getIdentifier(); assertNotNull(identifer2); assertTrue(!func1.equals(func2)); @@ -1492,7 +1492,7 @@ public void testFunctionEquality3() throws Exception { Function func = iterator.next(); assertNotNull(func); - String id = func.getIdentifer(); + String id = func.getIdentifier(); assertNotNull(id); assertTrue(id.equals("a")); From 3d2dd879b7df0bf84296a468efb5f2014659f850 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 12:06:55 -0400 Subject: [PATCH 011/172] Progress. --- .../hunter/hybridize/core/analysis/Function.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 3b6bc66a9..10bc3f4fc 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -387,19 +387,23 @@ private Set getCallGraphNodes(CallGraph callGraph) throws IllegalArgumen } LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); + LOG.info("Nodes:\n" + nodes.stream().map(Objects::toString).collect(Collectors.joining("\n"))); + return nodes; } public MethodReference getMethodReference() { + TypeReference typeReference = getTypeReference(); + return MethodReference.findOrCreate(typeReference, AstMethodReference.fnSelector); + } + + public TypeReference getTypeReference() { File containingFile = this.getContainingFile(); String filename = containingFile.getName(); - String functionName = this.getSimpleName(); - - TypeReference typeReference = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lscript " + filename + "/" + functionName); - - MethodReference methodReference = MethodReference.findOrCreate(typeReference, AstMethodReference.fnSelector); - return methodReference; + String modifiedIdentifier = this.getIdentifier().replace('.', '/'); + String typeName = "Lscript " + filename + "/" + modifiedIdentifier; + return TypeReference.findOrCreate(PythonTypes.pythonLoader, typeName); } public void inferTensorTensorParameters(TensorTypeAnalysis analysis, IProgressMonitor monitor) throws BadLocationException { From 8714c340a65c343bccd3448c12e00e854a81ec32 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 12:45:51 -0400 Subject: [PATCH 012/172] Progress. --- ...valuateHybridizeFunctionRefactoringHandler.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 99a736f24..9c9954622 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -72,7 +72,7 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto private static final String PERFORM_CHANGE_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.performChange"; private static String[] buildAttributeColumnNames(String... additionalColumnNames) { - String[] primaryColumns = new String[] { "subject", "function", "module", "relative path", "method reference" }; + String[] primaryColumns = new String[] { "subject", "function", "module", "relative path" }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnNames)); return ret.toArray(String[]::new); @@ -82,7 +82,7 @@ private static Object[] buildAttributeColumnValues(Function function, Object... IProject project = function.getProject(); Path relativePath = project.getLocation().toFile().toPath().relativize(function.getContainingFile().toPath()); Object[] primaryColumns = new Object[] { project.getName(), function.getIdentifier(), function.getContainingModuleName(), - relativePath, function.getMethodReference() }; + relativePath }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); ret.addAll(Arrays.asList(additionalColumnValues)); return ret.toArray(Object[]::new); @@ -239,14 +239,14 @@ public Object execute(ExecutionEvent event) throws ExecutionException { } private static String[] buildFunctionAttributeColumnNames() { - return buildAttributeColumnNames("parameters", "tensor parameter", "hybrid", "autograph", "experimental_autograph_options", - "experimental_follow_type_hints", "experimental_implements", "func", "input_signature", "jit_compile", "reduce_retracing", - "refactoring", "passing precondition", "status"); + return buildAttributeColumnNames("method reference", "type reference", "parameters", "tensor parameter", "hybrid", "autograph", + "experimental_autograph_options", "experimental_follow_type_hints", "experimental_implements", "func", "input_signature", + "jit_compile", "reduce_retracing", "refactoring", "passing precondition", "status"); } private static void printFunction(CSVPrinter printer, Function function) throws IOException { - Object[] initialColumnValues = buildAttributeColumnValues(function, function.getNumberOfParameters(), - function.getLikelyHasTensorParameter(), function.isHybrid()); + Object[] initialColumnValues = buildAttributeColumnValues(function, function.getMethodReference(), function.getTypeReference(), + function.getNumberOfParameters(), function.getLikelyHasTensorParameter(), function.isHybrid()); for (Object columnValue : initialColumnValues) printer.print(columnValue); From 889fdd71c6261c321bee1f5c67772dd61c37d1b2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 12:49:59 -0400 Subject: [PATCH 013/172] Add (Python) side-effects to CSVs. --- .../EvaluateHybridizeFunctionRefactoringHandler.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 9c9954622..1ebc9cedd 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -239,14 +239,15 @@ public Object execute(ExecutionEvent event) throws ExecutionException { } private static String[] buildFunctionAttributeColumnNames() { - return buildAttributeColumnNames("method reference", "type reference", "parameters", "tensor parameter", "hybrid", "autograph", - "experimental_autograph_options", "experimental_follow_type_hints", "experimental_implements", "func", "input_signature", - "jit_compile", "reduce_retracing", "refactoring", "passing precondition", "status"); + return buildAttributeColumnNames("method reference", "type reference", "parameters", "tensor parameter", "hybrid", "side-effects", + "autograph", "experimental_autograph_options", "experimental_follow_type_hints", "experimental_implements", "func", + "input_signature", "jit_compile", "reduce_retracing", "refactoring", "passing precondition", "status"); } private static void printFunction(CSVPrinter printer, Function function) throws IOException { Object[] initialColumnValues = buildAttributeColumnValues(function, function.getMethodReference(), function.getTypeReference(), - function.getNumberOfParameters(), function.getLikelyHasTensorParameter(), function.isHybrid()); + function.getNumberOfParameters(), function.getLikelyHasTensorParameter(), function.isHybrid(), + function.getHasPythonSideEffects()); for (Object columnValue : initialColumnValues) printer.print(columnValue); From e5af9aa2697204e35e419bf8251d3f820314ae4a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 17 Oct 2023 17:59:14 -0400 Subject: [PATCH 014/172] Progress. --- checkstyle.xml | 1 - .../hybridize/core/analysis/Function.java | 22 +++---- .../core/analysis/PreconditionFailure.java | 2 +- ...HybridizeFunctionRefactoringProcessor.java | 26 ++++++++- .../HybridizeFunctionRefactoringTest.java | 57 ++++++++++++++++--- 5 files changed, 87 insertions(+), 21 deletions(-) diff --git a/checkstyle.xml b/checkstyle.xml index 2bb74ebef..ab4bd739e 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -141,7 +141,6 @@ - diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 10bc3f4fc..1666d5d59 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -68,6 +68,15 @@ */ public class Function extends RefactorableProgramEntity { + public static final String PLUGIN_ID = FrameworkUtil.getBundle(Function.class).getSymbolicName(); + + private final class FunctionStatusContext extends RefactoringStatusContext { + @Override + public Object getCorrespondingElement() { + return Function.this; + } + } + /** * Parameters that may be passed to a tf.fuction decorator. Parameter descriptions found at: * https://tensorflow.org/versions/r2.9/api_docs/python/tf/function Note: We are also parsing the deprecated parameters specified in the @@ -670,17 +679,10 @@ private static boolean isHybrid(decoratorsType decorator, String containingModul return false; } - private void addStatusEntry(PreconditionFailure failure, String message) { - RefactoringStatusContext context = new RefactoringStatusContext() { - - @Override - public Object getCorrespondingElement() { - return Function.this.getFunctionDefinition().getFunctionDef(); - } - }; + public void addStatusEntry(PreconditionFailure failure, String message) { + RefactoringStatusContext context = new FunctionStatusContext(); - this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, FrameworkUtil.getBundle(Function.class).getSymbolicName(), - failure.getCode(), this); + this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, PLUGIN_ID, failure.getCode(), this); } /** diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 2fe5424cc..7c25b3ef3 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -3,7 +3,7 @@ import java.util.Arrays; public enum PreconditionFailure { - CURRENTLY_NOT_HANDLED(1), OPTIMIZATION_NOT_AVAILABLE(2); + CURRENTLY_NOT_HANDLED(1), OPTIMIZATION_NOT_AVAILABLE(2), UNDETERMINABLE_SIDE_EFFECTS(3); static { // check that the codes are unique. diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 669340498..7f8bc1498 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -20,6 +20,8 @@ import org.eclipse.ltk.core.refactoring.Change; import org.eclipse.ltk.core.refactoring.NullChange; import org.eclipse.ltk.core.refactoring.RefactoringStatus; +import org.eclipse.ltk.core.refactoring.RefactoringStatusContext; +import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.eclipse.ltk.core.refactoring.participants.CheckConditionsContext; import org.eclipse.ltk.core.refactoring.participants.RefactoringParticipant; import org.eclipse.ltk.core.refactoring.participants.SharableParticipants; @@ -39,6 +41,7 @@ import edu.cuny.citytech.refactoring.common.core.TimeCollector; import edu.cuny.hunter.hybridize.core.analysis.Function; import edu.cuny.hunter.hybridize.core.analysis.FunctionDefinition; +import edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure; import edu.cuny.hunter.hybridize.core.descriptors.HybridizeFunctionRefactoringDescriptor; import edu.cuny.hunter.hybridize.core.messages.Messages; import edu.cuny.hunter.hybridize.core.wala.ml.EclipsePythonProjectTensorAnalysisEngine; @@ -46,6 +49,19 @@ @SuppressWarnings("unused") public class HybridizeFunctionRefactoringProcessor extends RefactoringProcessor { + private final class FunctionStatusContext extends RefactoringStatusContext { + private final Function func; + + private FunctionStatusContext(Function func) { + this.func = func; + } + + @Override + public Object getCorrespondingElement() { + return func; + } + } + private static final ILog LOG = getLog(HybridizeFunctionRefactoringProcessor.class); private static RefactoringStatus checkDecorators(Function func) { @@ -198,12 +214,17 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat try { func.inferSideEffects(callGraph, builder.getPointerAnalysis()); } catch (IllegalArgumentException e) { - throw new IllegalStateException("Can't infer side-effects of: " + func + ".", e); + LOG.warn("Unable to infer side-effects of: " + func + ".", e); + func.addStatusEntry(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, + "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); + // next function. + status.merge(func.getStatus()); + subMonitor.worked(1); + continue; } // check the function preconditions. func.check(); - status.merge(func.getStatus()); status.merge(checkParameters(func)); subMonitor.checkCanceled(); @@ -211,6 +232,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat status.merge(checkDecorators(func)); subMonitor.checkCanceled(); + status.merge(func.getStatus()); subMonitor.worked(1); } } diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index a307fb1a4..91db72f34 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -40,6 +40,7 @@ import org.eclipse.jface.text.IDocument; import org.eclipse.ltk.core.refactoring.RefactoringCore; import org.eclipse.ltk.core.refactoring.RefactoringStatus; +import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.eclipse.ltk.core.refactoring.participants.ProcessorBasedRefactoring; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -96,6 +97,7 @@ import edu.cuny.hunter.hybridize.core.analysis.Function; import edu.cuny.hunter.hybridize.core.analysis.FunctionDefinition; import edu.cuny.hunter.hybridize.core.analysis.FunctionExtractor; +import edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure; import edu.cuny.hunter.hybridize.core.analysis.PreconditionSuccess; import edu.cuny.hunter.hybridize.core.analysis.Refactoring; import edu.cuny.hunter.hybridize.core.analysis.Transformation; @@ -607,6 +609,13 @@ public void testAmbiguousDefinition() throws Exception { assertNotNull(function); assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); + + switch (function.getIdentifier()) { + case "Test.value": + case "Test.name": + checkSideEffectStatus(function); + break; + } } } @@ -819,6 +828,7 @@ public void testComputeParameters10() throws Exception { // to return False. assertNull(args); + checkSideEffectStatus(function); } /** @@ -843,6 +853,17 @@ public void testComputeParameters11() throws Exception { assertTrue(!args.getFuncParamExists() && !args.getInputSignatureParamExists() & args.getAutoGraphParamExists() && !args.getJitCompileParamExists() && !args.getReduceRetracingParamExists() && !args.getExperimentalImplementsParamExists() && !args.getExperimentalAutographOptParamExists() && !args.getExperimentalFollowTypeHintsParamExists()); + + checkSideEffectStatus(function); + } + + private void checkSideEffectStatus(Function function) { + RefactoringStatus status = function.getStatus(); + assertTrue("Should fail due to a call graph issue, either a decorated function or missing function invocation.", status.hasError()); + assertNull(function.getHasPythonSideEffects()); + RefactoringStatusEntry entry = status.getEntryMatchingCode(Function.PLUGIN_ID, + PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()); + assertNotNull(entry); } /** @@ -996,9 +1017,14 @@ public void testIsHybrid3() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(2, functions.size()); // one function is for the decorator. - Function function = functions.iterator().next(); - assertNotNull(function); - assertFalse(function.isHybrid()); + + functions.stream().forEach(f -> { + assertNotNull(f); + assertFalse(f.isHybrid()); + + if (f.getIdentifier().equals("func1")) + checkSideEffectStatus(f); + }); } /** @@ -1012,6 +1038,7 @@ public void testIsHybrid4() throws Exception { Function function = functions.iterator().next(); assertNotNull(function); assertFalse(function.isHybrid()); + checkSideEffectStatus(function); } /** @@ -1038,6 +1065,7 @@ public void testIsHybrid6() throws Exception { Function function = functions.iterator().next(); assertNotNull(function); assertFalse(function.isHybrid()); + checkSideEffectStatus(function); } /** @@ -1064,6 +1092,7 @@ public void testIsHybrid8() throws Exception { Function function = functions.iterator().next(); assertNotNull(function); assertTrue(function.isHybrid()); + checkSideEffectStatus(function); } /** @@ -1091,6 +1120,9 @@ public void testIsHybridFalse() throws Exception { for (Function func : functions) { assertNotNull(func); assertFalse(func.isHybrid()); + + if (func.getIdentifier().equals("dummy_func2")) + checkSideEffectStatus(func); } } @@ -1106,6 +1138,7 @@ public void testIsHybridMultipleAttributes() throws Exception { for (Function func : functions) { assertNotNull(func); assertFalse(func.isHybrid()); + checkSideEffectStatus(func); } } @@ -1121,6 +1154,7 @@ public void testIsHybridMultipleDecorators() throws Exception { for (Function func : functions) { assertNotNull(func); assertTrue(func.isHybrid()); + checkSideEffectStatus(func); } } @@ -1164,6 +1198,7 @@ public void testProcessDecorator() throws Exception { Function function = functions.iterator().next(); assertNotNull(function); assertTrue(function.isHybrid()); + checkSideEffectStatus(function); } /** @@ -4356,8 +4391,10 @@ public void testModel() throws Exception { assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); break; case "__call__": - // TODO: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. + // NOTE: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + // NOTE: Should be error-free once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + checkSideEffectStatus(f); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); @@ -4388,8 +4425,10 @@ public void testModel2() throws Exception { assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); break; case "call": - // TODO: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. + // NOTE: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + // NOTE: Remove once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + checkSideEffectStatus(f); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); @@ -4479,8 +4518,10 @@ public void testModel5() throws Exception { assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); break; case "call": - // TODO: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. + // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. assertFalse("Expecting " + simpleName + " not to have a tensor param.", f.getLikelyHasTensorParameter()); + // NOTE: Remove once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + checkSideEffectStatus(f); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); @@ -4510,8 +4551,10 @@ public void testModel6() throws Exception { assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); break; case "__call__": - // TODO: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. + // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. assertFalse("Expecting " + simpleName + " not to have a tensor param.", f.getLikelyHasTensorParameter()); + // No invocation, so we won't be able to infer side-effects. + checkSideEffectStatus(f); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); From 99272cb0c0653dcba3d27a11ec20b8b298375447 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 09:51:06 -0400 Subject: [PATCH 015/172] Remove unused import. --- .../core/refactorings/HybridizeFunctionRefactoringProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 7f8bc1498..189f676cd 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -21,7 +21,6 @@ import org.eclipse.ltk.core.refactoring.NullChange; import org.eclipse.ltk.core.refactoring.RefactoringStatus; import org.eclipse.ltk.core.refactoring.RefactoringStatusContext; -import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.eclipse.ltk.core.refactoring.participants.CheckConditionsContext; import org.eclipse.ltk.core.refactoring.participants.RefactoringParticipant; import org.eclipse.ltk.core.refactoring.participants.SharableParticipants; From 081b13208bc398ce293caf879ca1acbf1c14eed9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 09:55:47 -0400 Subject: [PATCH 016/172] Suppress warning. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 1 + 1 file changed, 1 insertion(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 1666d5d59..400affdf3 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -68,6 +68,7 @@ */ public class Function extends RefactorableProgramEntity { + @SuppressWarnings("hiding") public static final String PLUGIN_ID = FrameworkUtil.getBundle(Function.class).getSymbolicName(); private final class FunctionStatusContext extends RefactoringStatusContext { From 25f766a9407ba83e21f4e947f1c1c8fef6b92818 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:08:13 -0400 Subject: [PATCH 017/172] Change field name. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 400affdf3..2ff966b57 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -68,8 +68,7 @@ */ public class Function extends RefactorableProgramEntity { - @SuppressWarnings("hiding") - public static final String PLUGIN_ID = FrameworkUtil.getBundle(Function.class).getSymbolicName(); + private static final String BUNDLE_SYMBOLIC_NAME = FrameworkUtil.getBundle(Function.class).getSymbolicName(); private final class FunctionStatusContext extends RefactoringStatusContext { @Override @@ -683,7 +682,7 @@ private static boolean isHybrid(decoratorsType decorator, String containingModul public void addStatusEntry(PreconditionFailure failure, String message) { RefactoringStatusContext context = new FunctionStatusContext(); - this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, PLUGIN_ID, failure.getCode(), this); + this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, BUNDLE_SYMBOLIC_NAME, failure.getCode(), this); } /** From c676527c4fcdfff32d22f77292e27ba4c367fed9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:22:50 -0400 Subject: [PATCH 018/172] Fix compilation error. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 2ff966b57..7e85ff50f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -68,7 +68,7 @@ */ public class Function extends RefactorableProgramEntity { - private static final String BUNDLE_SYMBOLIC_NAME = FrameworkUtil.getBundle(Function.class).getSymbolicName(); + public static final String BUNDLE_SYMBOLIC_NAME = FrameworkUtil.getBundle(Function.class).getSymbolicName(); private final class FunctionStatusContext extends RefactoringStatusContext { @Override diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 91db72f34..b10a9a27d 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -861,7 +861,7 @@ private void checkSideEffectStatus(Function function) { RefactoringStatus status = function.getStatus(); assertTrue("Should fail due to a call graph issue, either a decorated function or missing function invocation.", status.hasError()); assertNull(function.getHasPythonSideEffects()); - RefactoringStatusEntry entry = status.getEntryMatchingCode(Function.PLUGIN_ID, + RefactoringStatusEntry entry = status.getEntryMatchingCode(Function.BUNDLE_SYMBOLIC_NAME, PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()); assertNotNull(entry); } From 18ebc57e47de00edb986e3a01fd898ad624c6d72 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:40:23 -0400 Subject: [PATCH 019/172] Add transitive test. --- .../testPythonSideEffects7/in/A.py | 12 ++++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 24 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/A.py new file mode 100644 index 000000000..b8c40d625 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/A.py @@ -0,0 +1,12 @@ +my_list = [10] + + +def g(): + my_list[0] = 1 + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects7/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index b10a9a27d..71e97b162 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -104,6 +104,7 @@ import edu.cuny.hunter.hybridize.core.analysis.Util; import edu.cuny.hunter.hybridize.core.refactorings.HybridizeFunctionRefactoringProcessor; import edu.cuny.hunter.hybridize.core.utils.RefactoringAvailabilityTester; +import junit.framework.Assert; @SuppressWarnings("restriction") public class HybridizeFunctionRefactoringTest extends RefactoringTest { @@ -4653,6 +4654,29 @@ public void testPythonSideEffects6() throws Exception { assertTrue("This Python statement modifies a global variable, so it has side-effects.", function.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects7() throws Exception { + Set functionSet = getFunctions(); + assertEquals(2, functionSet.size()); + + functionSet.forEach(f -> { + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + + switch (f.getIdentifier()) { + case "f": + case "g": + // there's a Python statement with (transitive) side-effects. + assertTrue("This Python statement modifies a global variable, so it has side-effects.", f.getHasPythonSideEffects()); + break; + + default: + fail("Not expecting: " + f.getIdentifier() + "."); + break; + } + }); + } + private Function getSingleFunction() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); From 3c57a7617952587b8d401c8bbf1f501a0f3582cf Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:46:01 -0400 Subject: [PATCH 020/172] Not having a switch default is OK. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 05706bf56..ae9da2a8e 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,7 @@ -err:-discouraged -err:-emptyBlock -err:-boxing + -err:-switchdefault From a78e995796e40b3a17a483ee8fcbec6a8e0ca104 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:46:19 -0400 Subject: [PATCH 021/172] Add doc. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 71e97b162..616f3a3dd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4677,6 +4677,11 @@ public void testPythonSideEffects7() throws Exception { }); } + /** + * Returns the only function defined in the test file. + * + * @return The only function defined in the test file. + */ private Function getSingleFunction() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); From 99b3d6b771f153f79c6e4e17c672454e4af329e1 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 10:46:59 -0400 Subject: [PATCH 022/172] Make static. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 616f3a3dd..79836edb3 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -858,7 +858,7 @@ public void testComputeParameters11() throws Exception { checkSideEffectStatus(function); } - private void checkSideEffectStatus(Function function) { + private static void checkSideEffectStatus(Function function) { RefactoringStatus status = function.getStatus(); assertTrue("Should fail due to a call graph issue, either a decorated function or missing function invocation.", status.hasError()); assertNull(function.getHasPythonSideEffects()); From 0c1f532658612019bab3937b251f2b07d3eed1cf Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:07:58 -0400 Subject: [PATCH 023/172] Remove unused import. --- .../hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 79836edb3..5f679a59e 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -104,7 +104,6 @@ import edu.cuny.hunter.hybridize.core.analysis.Util; import edu.cuny.hunter.hybridize.core.refactorings.HybridizeFunctionRefactoringProcessor; import edu.cuny.hunter.hybridize.core.utils.RefactoringAvailabilityTester; -import junit.framework.Assert; @SuppressWarnings("restriction") public class HybridizeFunctionRefactoringTest extends RefactoringTest { From 460aa317fd9e90553840269d93f80b9c98768d9d Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:22:48 -0400 Subject: [PATCH 024/172] Add doc. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 5f679a59e..8e8bdfbb2 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4653,6 +4653,9 @@ public void testPythonSideEffects6() throws Exception { assertTrue("This Python statement modifies a global variable, so it has side-effects.", function.getHasPythonSideEffects()); } + /** + * Test transitive side-effects in the same file. + */ @Test public void testPythonSideEffects7() throws Exception { Set functionSet = getFunctions(); From 75ff03aab7e4e66c6a156f8d7da3ad66f0dc7a44 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:23:49 -0400 Subject: [PATCH 025/172] Add helper method. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 8e8bdfbb2..516a22afd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4660,7 +4660,10 @@ public void testPythonSideEffects6() throws Exception { public void testPythonSideEffects7() throws Exception { Set functionSet = getFunctions(); assertEquals(2, functionSet.size()); + testTransitivePythonSideEffects(functionSet); + } + private void testTransitivePythonSideEffects(Set functionSet) { functionSet.forEach(f -> { assertFalse(f.isHybrid()); assertFalse(f.getLikelyHasTensorParameter()); From 757094c355940209aef01e5ab8eda36d0a93ac78 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:33:02 -0400 Subject: [PATCH 026/172] Fix doc. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 516a22afd..1a2a419be 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4683,9 +4683,9 @@ private void testTransitivePythonSideEffects(Set functionSet) { } /** - * Returns the only function defined in the test file. + * Returns the only function defined in the default test file. * - * @return The only function defined in the test file. + * @return The only function defined in the default test file. */ private Function getSingleFunction() throws Exception { Set functions = this.getFunctions(); From 40cf6a92e903ba5051ed102c149c0caa4961bd89 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:33:55 -0400 Subject: [PATCH 027/172] Add helper methods. --- .../HybridizeFunctionRefactoringTest.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 1a2a419be..849163870 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4688,7 +4688,26 @@ private void testTransitivePythonSideEffects(Set functionSet) { * @return The only function defined in the default test file. */ private Function getSingleFunction() throws Exception { - Set functions = this.getFunctions(); + return this.getSingleFunction(this.getFunctions()); + } + + /** + * Returns the only function defined in the given test file. + * + * @param fileNameWithoutExtension The name of the file declaring the function without a file extension. + * @return The only function defined in the test file. + */ + private Function getSingleFunction(String fileNameWithoutExtension) throws Exception { + return getSingleFunction(this.getFunctions(fileNameWithoutExtension)); + } + + /** + * Returns the only function contained in the given set of functions. + * + * @param functions The set of functions containing only one function. + * @return The sole function contained in the given set of functions. + */ + private Function getSingleFunction(Set functions) { assertNotNull(functions); assertEquals(1, functions.size()); Function function = functions.iterator().next(); From bc31996dca69db9b2485814a02b6130b748c3594 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:34:21 -0400 Subject: [PATCH 028/172] Test transitive side-effects in different files. --- .../testPythonSideEffects8/in/A.py | 7 +++++++ .../testPythonSideEffects8/in/B.py | 5 +++++ .../testPythonSideEffects8/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 15 +++++++++++++++ 4 files changed, 27 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/B.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/A.py new file mode 100644 index 000000000..b370344f0 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/A.py @@ -0,0 +1,7 @@ +from B import g + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/B.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/B.py new file mode 100644 index 000000000..e83bf9547 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/B.py @@ -0,0 +1,5 @@ +my_list = [10] + + +def g(): + my_list[0] = 1 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects8/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 849163870..088d7ea45 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4714,4 +4714,19 @@ private Function getSingleFunction(Set functions) { assertNotNull(function); return function; } + + /** + * Test transitive side-effects in different files. + */ + @Test + public void testPythonSideEffects8() throws Exception { + Function functionFromA = this.getSingleFunction("A"); + assertEquals("f", functionFromA.getIdentifier()); + + Function functionFromB = this.getSingleFunction("B"); + assertEquals("g", functionFromB.getIdentifier()); + + Set functionSet = new HashSet<>(Arrays.asList(functionFromA, functionFromB)); + testTransitivePythonSideEffects(functionSet); + } } From e3cb664587bdda759d60fbd69b25afcdffdd8be4 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:37:17 -0400 Subject: [PATCH 029/172] Revert "Not having a switch default is OK." This reverts commit 3c57a7617952587b8d401c8bbf1f501a0f3582cf. I am still getting the warning. --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae9da2a8e..05706bf56 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,6 @@ -err:-discouraged -err:-emptyBlock -err:-boxing - -err:-switchdefault From 175b7a65cde2324c69d167bb49133fe49550a48c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:40:34 -0400 Subject: [PATCH 030/172] Squelch warning about missing default case. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 088d7ea45..52611c21c 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -615,6 +615,8 @@ public void testAmbiguousDefinition() throws Exception { case "Test.name": checkSideEffectStatus(function); break; + default: + break; } } } From 19467d9e8984f201a4b8f9ae991b4382f20e97d6 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:51:58 -0400 Subject: [PATCH 031/172] Make methods static. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 52611c21c..0fdb87eb6 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4665,7 +4665,7 @@ public void testPythonSideEffects7() throws Exception { testTransitivePythonSideEffects(functionSet); } - private void testTransitivePythonSideEffects(Set functionSet) { + private static void testTransitivePythonSideEffects(Set functionSet) { functionSet.forEach(f -> { assertFalse(f.isHybrid()); assertFalse(f.getLikelyHasTensorParameter()); @@ -4709,7 +4709,7 @@ private Function getSingleFunction(String fileNameWithoutExtension) throws Excep * @param functions The set of functions containing only one function. * @return The sole function contained in the given set of functions. */ - private Function getSingleFunction(Set functions) { + private static Function getSingleFunction(Set functions) { assertNotNull(functions); assertEquals(1, functions.size()); Function function = functions.iterator().next(); From fc4048223019127f879b071156489bf748c437e9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 11:54:13 -0400 Subject: [PATCH 032/172] Fix warning. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 0fdb87eb6..766972e14 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4690,7 +4690,7 @@ private static void testTransitivePythonSideEffects(Set functionSet) { * @return The only function defined in the default test file. */ private Function getSingleFunction() throws Exception { - return this.getSingleFunction(this.getFunctions()); + return getSingleFunction(this.getFunctions()); } /** From 8c871c150a25356ec7a233d2555768d2ac79890c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 12:32:54 -0400 Subject: [PATCH 033/172] Add extra checks. --- .../tests/HybridizeFunctionRefactoringTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 766972e14..ac5480f19 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4391,6 +4391,7 @@ public void testModel() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "__call__": // NOTE: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. @@ -4425,6 +4426,7 @@ public void testModel2() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "call": // NOTE: Change to assertTrue when https://github.com/wala/ML/issues/24 is fixed. @@ -4458,6 +4460,7 @@ public void testModel3() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "call": assertTrue("Expecting " + simpleName + " to have a tensor param.", f.getLikelyHasTensorParameter()); @@ -4468,6 +4471,13 @@ public void testModel3() throws Exception { }); } + private static void checkOptimizationNotAvailableStatus(Function f) { + RefactoringStatus status = f.getStatus(); + assertTrue("Should not be available for optimization.", status.hasError()); + Object entry = status.getEntryMatchingCode(Function.BUNDLE_SYMBOLIC_NAME, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()); + assertNotNull(entry); + } + /** * Test a model. No tf.function in this one. Explicit call method. */ @@ -4488,6 +4498,7 @@ public void testModel4() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "__call__": assertTrue("Expecting " + simpleName + " to have a tensor param.", f.getLikelyHasTensorParameter()); @@ -4518,6 +4529,7 @@ public void testModel5() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "call": // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. @@ -4551,6 +4563,7 @@ public void testModel6() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); break; case "__call__": // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. From e24a5e44959d40ca8e4eafd2738aff8c6e0f49a0 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 12:34:58 -0400 Subject: [PATCH 034/172] Add side-effects checks to model tests. These have explicits calls, so we should be able to infer them successfully. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index ac5480f19..b7fea285a 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4464,6 +4464,8 @@ public void testModel3() throws Exception { break; case "call": assertTrue("Expecting " + simpleName + " to have a tensor param.", f.getLikelyHasTensorParameter()); + assertTrue("Should pass preconditions.", f.getStatus().isOK()); + assertFalse("No Python side-effects.", f.getHasPythonSideEffects()); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); @@ -4502,6 +4504,8 @@ public void testModel4() throws Exception { break; case "__call__": assertTrue("Expecting " + simpleName + " to have a tensor param.", f.getLikelyHasTensorParameter()); + assertTrue("Should pass preconditions.", f.getStatus().isOK()); + assertFalse("No Python side-effects.", f.getHasPythonSideEffects()); break; default: throw new IllegalStateException("Not expecting function: " + simpleName + "."); From 7bc23b04981b202ab55dea76c499bddc34870e42 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 12:35:34 -0400 Subject: [PATCH 035/172] Fix comment. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index b7fea285a..c98f046f8 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4538,7 +4538,7 @@ public void testModel5() throws Exception { case "call": // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/229 is fixed. assertFalse("Expecting " + simpleName + " not to have a tensor param.", f.getLikelyHasTensorParameter()); - // NOTE: Remove once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + // Can't infer side-effects here because there's no invocation of this method. checkSideEffectStatus(f); break; default: From 1800dbffa1ebb392f1a39e147a5f6344341df228 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 14:22:24 -0400 Subject: [PATCH 036/172] Update comment. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 7e85ff50f..6d498c946 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -355,7 +355,8 @@ public void inferSideEffects(CallGraph callGraph, PointerAnalysis p ModRef modRef = new PythonModRef(); Map> mod = modRef.computeMod(callGraph, pointerAnalysis); - // Get the nodes corresponding to this function. + // Get the nodes corresponding to this function's declaration. NOTE: There can be multiple nodes for a function declaration under + // the current representation. It seems that there is a declaration node for each call to the function. Set nodes = getCallGraphNodes(callGraph); // for each node. From dd5d366ed92651f961f6758d362b93bbae95f306 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 15:26:15 -0400 Subject: [PATCH 037/172] Add a model test with Python side-effects. --- .../HybridizeFunction/testModel7/in/A.py | 53 +++++++++++++++++++ .../testModel7/in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 36 +++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/A.py new file mode 100644 index 000000000..0146f58c7 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/A.py @@ -0,0 +1,53 @@ +import tensorflow as tf + +# Create an override model to classify pictures + + +class SequentialModel(tf.keras.Model): + + def __init__(self, **kwargs): + super(SequentialModel, self).__init__(**kwargs) + + self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28)) + + # Add a lot of small layers + num_layers = 100 + self.my_layers = [tf.keras.layers.Dense(64, activation="relu") + for n in range(num_layers)] + + self.dropout = tf.keras.layers.Dropout(0.2) + self.dense_2 = tf.keras.layers.Dense(10) + + self._stuff = 5 + + def call(self, x): + x = self.flatten(x) + + for layer in self.my_layers: + x = layer(x) + + x = self.dropout(x) + x = self.dense_2(x) + + self._stuff = 6 + + return x + + def get_stuff(self): + return self._stuff + + +input_data = tf.random.uniform([20, 28, 28]) +print("Input:") +print(type(input_data)) +print(input_data) + +model = SequentialModel() +print(model.get_stuff()) + +result = model.call(input_data) +print(model.get_stuff()) + +print("Output:") +print(type(input_data)) +print(result) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testModel7/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index c98f046f8..3cfb443bd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4581,6 +4581,42 @@ public void testModel6() throws Exception { }); } + /** + * Test a model. No tf.function in this one. Explicit call method. Unlike testModel3, there are Python side-effects in + * SequentialModel.__init__() and SequentialModel.call(). + * + * @see testModel3. + */ + @Test + public void testModel7() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + + LOG.info("Found functions: " + functions.size()); + assertEquals("Expecting two functions.", 3, functions.size()); + + // no hybrids. + assertTrue(functions.stream().map(Function::isHybrid).allMatch(b -> b == false)); + + // check function parameters. + functions.forEach(f -> { + String simpleName = f.getSimpleName(); + switch (simpleName) { + case "__init__": + case "get_stuff": + assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + checkOptimizationNotAvailableStatus(f); + break; + case "call": + assertTrue("Expecting " + simpleName + " to have a tensor param.", f.getLikelyHasTensorParameter()); + assertTrue("Should have python side-effects.", f.getHasPythonSideEffects()); + break; + default: + throw new IllegalStateException("Not expecting function: " + simpleName + "."); + } + }); + } + // TODO: Test models that have tf.functions. private void testPreconditionCheckingHelper(boolean expectedHybrid, boolean expectedTensorParameter, Refactoring expectedRefactoring, From 8a3cdc81ae628670eaf82d18510961968463d9f9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 18 Oct 2023 17:36:34 -0400 Subject: [PATCH 038/172] Fix javadoc comment. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 3cfb443bd..690244a3b 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4585,7 +4585,7 @@ public void testModel6() throws Exception { * Test a model. No tf.function in this one. Explicit call method. Unlike testModel3, there are Python side-effects in * SequentialModel.__init__() and SequentialModel.call(). * - * @see testModel3. + * @see HybridizeFunctionRefactoringTest#testModel3 */ @Test public void testModel7() throws Exception { From 6ebbad62f00affe6e1ff2d4d6643b605d676bc44 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 19 Oct 2023 09:52:05 -0400 Subject: [PATCH 039/172] Simplify test. --- .../testPythonSideEffects9/in/A.py | 11 +++++++++++ .../testPythonSideEffects9/in/requirements.txt | 1 + .../tests/HybridizeFunctionRefactoringTest.java | 14 ++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py new file mode 100644 index 000000000..45c396517 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py @@ -0,0 +1,11 @@ +# From https://www.tensorflow.org/guide/function#executing_python_side_effects. + +import tensorflow as tf + + +def f(x): + print("Traced with", x) + tf.print("Executed with", x) + + +f(1) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 690244a3b..df6dc8360 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4784,4 +4784,18 @@ public void testPythonSideEffects8() throws Exception { Set functionSet = new HashSet<>(Arrays.asList(functionFromA, functionFromB)); testTransitivePythonSideEffects(functionSet); } + + /** + * Like testPythonSideEffects but only a single call. Simplifies the call graph since there seems to be a node for each call to a + * function. + * + * @see HybridizeFunctionRefactoringTest#testPythonSideEffects + */ + @Test + public void testPythonSideEffects9() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. + assertTrue("Expecting a Python side-effect.", function.getHasPythonSideEffects()); + } } From 05261e7df650c25497261c8bbb5846fc8875080e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 19 Oct 2023 17:15:57 -0400 Subject: [PATCH 040/172] Use our own ModRef. --- .../META-INF/MANIFEST.MF | 1 + .../hybridize/core/analysis/Function.java | 4 +- .../ml/PythonModRefWithBuiltinFunctions.java | 72 +++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index 519f6c9d9..b76bde0ad 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -28,6 +28,7 @@ Import-Package: com.ibm.wala.cast.ipa.callgraph, com.ibm.wala.cast.python.ml.analysis;version="0.1.0", com.ibm.wala.cast.python.ml.client;version="0.1.0", com.ibm.wala.cast.python.modref, + com.ibm.wala.cast.python.ssa, com.ibm.wala.cast.python.types, com.ibm.wala.cast.python.util, com.ibm.wala.cast.tree, diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 6d498c946..af4901338 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -40,7 +40,6 @@ import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.cast.python.ml.analysis.TensorVariable; -import com.ibm.wala.cast.python.modref.PythonModRef; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; import com.ibm.wala.cast.types.AstMethodReference; @@ -59,6 +58,7 @@ import edu.cuny.citytech.refactoring.common.core.RefactorableProgramEntity; import edu.cuny.hunter.hybridize.core.utils.RefactoringAvailabilityTester; +import edu.cuny.hunter.hybridize.core.wala.ml.PythonModRefWithBuiltinFunctions; /** * A representation of a Python function. @@ -352,7 +352,7 @@ public Function(FunctionDefinition fd) { * @throws IllegalArgumentException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. */ public void inferSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws IllegalArgumentException { - ModRef modRef = new PythonModRef(); + ModRef modRef = new PythonModRefWithBuiltinFunctions(); Map> mod = modRef.computeMod(callGraph, pointerAnalysis); // Get the nodes corresponding to this function's declaration. NOTE: There can be multiple nodes for a function declaration under diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java new file mode 100644 index 000000000..511ab59d9 --- /dev/null +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -0,0 +1,72 @@ +package edu.cuny.hunter.hybridize.core.wala.ml; + +import java.util.Collection; + +import com.ibm.wala.cast.ir.ssa.AstGlobalWrite; +import com.ibm.wala.cast.ir.ssa.AstLexicalAccess.Access; +import com.ibm.wala.cast.ir.ssa.AstLexicalRead; +import com.ibm.wala.cast.ir.ssa.AstLexicalWrite; +import com.ibm.wala.cast.ir.ssa.AstPropertyWrite; +import com.ibm.wala.cast.python.modref.PythonModRef; +import com.ibm.wala.cast.python.modref.PythonModRef.PythonModVisitor; +import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; +import com.ibm.wala.cast.python.types.PythonTypes; +import com.ibm.wala.classLoader.IClass; +import com.ibm.wala.ipa.callgraph.CGNode; +import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; +import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; +import com.ibm.wala.ipa.callgraph.propagation.PointerKey; +import com.ibm.wala.ipa.modref.ExtendedHeapModel; +import com.ibm.wala.shrike.shrikeBT.IInvokeInstruction.IDispatch; +import com.ibm.wala.ssa.SSAInstruction; +import com.ibm.wala.ssa.SSAInvokeInstruction; +import com.ibm.wala.types.MethodReference; +import com.ibm.wala.types.TypeReference; +import com.ibm.wala.util.intset.OrdinalSet; + +public class PythonModRefWithBuiltinFunctions extends PythonModRef { + + public static class PythonModVisitorWithBuiltinFunctions extends PythonModVisitor { + + public PythonModVisitorWithBuiltinFunctions(CGNode n, Collection result, ExtendedHeapModel h, PointerAnalysis pa, + boolean ignoreAllocHeapDefs) { + super(n, result, h, pa, ignoreAllocHeapDefs); + } + + @Override + public void visitPythonInvoke(PythonInvokeInstruction inst) { + int use = inst.getUse(0); + SSAInstruction def = this.n.getDU().getDef(use); + + if (def instanceof AstLexicalRead) { + AstLexicalRead read = (AstLexicalRead) def; + Access[] accesses = read.getAccesses(); + + if (accesses.length > 0 && accesses[0].variableName.equals("str")) { + PointerKey pk = this.h.getPointerKeyForLocal(this.n, use); + OrdinalSet pointsToSet = this.pa.getPointsToSet(pk); + + pointsToSet.forEach(o -> { + if (o instanceof InstanceKey) { + InstanceKey ik = (InstanceKey) o; + IClass concreteType = ik.getConcreteType(); + TypeReference reference = concreteType.getReference(); + + TypeReference strFunction = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lwala/builtin/str"); + System.out.println(reference.equals(strFunction)); + } + + }); + } + } + + super.visitPythonInvoke(inst); + } + } + + @Override + protected ModVisitor makeModVisitor(CGNode n, Collection result, + PointerAnalysis pa, ExtendedHeapModel h, boolean ignoreAllocHeapDefs) { + return new PythonModVisitorWithBuiltinFunctions<>(n, result, h, pa, ignoreAllocHeapDefs); + } +} From 675aa045ebbc908c39ee3cfdb58897f1945f6e8a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 19 Oct 2023 17:17:24 -0400 Subject: [PATCH 041/172] Simplify test. --- .../HybridizeFunction/testPythonSideEffects9/in/A.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py index 45c396517..d355ff552 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py @@ -1,11 +1,5 @@ -# From https://www.tensorflow.org/guide/function#executing_python_side_effects. - -import tensorflow as tf - - def f(x): print("Traced with", x) - tf.print("Executed with", x) f(1) From cbff639b379ce0355803fc24144761216e8d273b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 19 Oct 2023 17:18:39 -0400 Subject: [PATCH 042/172] Use built-in function. Exercise https://github.com/wala/ML/issues/93. --- .../resources/HybridizeFunction/testPythonSideEffects9/in/A.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py index d355ff552..a2200f8ed 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/A.py @@ -1,5 +1,5 @@ def f(x): - print("Traced with", x) + print("Traced with: " + str(x)) f(1) From fe8510672c0c2f6608ba40eff6761dbb95b6c3d4 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 09:21:14 -0400 Subject: [PATCH 043/172] Remove unnecessarily cast. --- .../wala/ml/PythonModRefWithBuiltinFunctions.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 511ab59d9..bb61c1672 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -46,15 +46,12 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { PointerKey pk = this.h.getPointerKeyForLocal(this.n, use); OrdinalSet pointsToSet = this.pa.getPointsToSet(pk); - pointsToSet.forEach(o -> { - if (o instanceof InstanceKey) { - InstanceKey ik = (InstanceKey) o; - IClass concreteType = ik.getConcreteType(); - TypeReference reference = concreteType.getReference(); + pointsToSet.forEach(ik -> { + IClass concreteType = ik.getConcreteType(); + TypeReference reference = concreteType.getReference(); - TypeReference strFunction = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lwala/builtin/str"); - System.out.println(reference.equals(strFunction)); - } + TypeReference strFunction = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lwala/builtin/str"); + System.out.println(reference.equals(strFunction)); }); } From ea95aa6c3a0905cc9e732e3e70f26c8647e6649e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 15:37:56 -0400 Subject: [PATCH 044/172] Handle side-effects from print(). --- .../ml/PythonModRefWithBuiltinFunctions.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index bb61c1672..54db689a7 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -2,13 +2,10 @@ import java.util.Collection; -import com.ibm.wala.cast.ir.ssa.AstGlobalWrite; +import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; import com.ibm.wala.cast.ir.ssa.AstLexicalAccess.Access; import com.ibm.wala.cast.ir.ssa.AstLexicalRead; -import com.ibm.wala.cast.ir.ssa.AstLexicalWrite; -import com.ibm.wala.cast.ir.ssa.AstPropertyWrite; import com.ibm.wala.cast.python.modref.PythonModRef; -import com.ibm.wala.cast.python.modref.PythonModRef.PythonModVisitor; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.classLoader.IClass; @@ -17,10 +14,7 @@ import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; import com.ibm.wala.ipa.modref.ExtendedHeapModel; -import com.ibm.wala.shrike.shrikeBT.IInvokeInstruction.IDispatch; import com.ibm.wala.ssa.SSAInstruction; -import com.ibm.wala.ssa.SSAInvokeInstruction; -import com.ibm.wala.types.MethodReference; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.intset.OrdinalSet; @@ -28,6 +22,14 @@ public class PythonModRefWithBuiltinFunctions extends PythonModRef { public static class PythonModVisitorWithBuiltinFunctions extends PythonModVisitor { + private static final AstGlobalPointerKey GLOBAL_OUTPUT_STREAM_POINTER_KEY = new AstGlobalPointerKey( + PythonModVisitorWithBuiltinFunctions.class.getPackageName().replace('.', '/') + "/OUT"); + + private static final String PRINT_FUNCTION_VARIABLE_NAME = "print"; + + private static final TypeReference PRINT_FUNCTION_TYPE_REFERENCE = TypeReference.findOrCreate(PythonTypes.pythonLoader, + "Lwala/builtin/" + PRINT_FUNCTION_VARIABLE_NAME); + public PythonModVisitorWithBuiltinFunctions(CGNode n, Collection result, ExtendedHeapModel h, PointerAnalysis pa, boolean ignoreAllocHeapDefs) { super(n, result, h, pa, ignoreAllocHeapDefs); @@ -42,17 +44,19 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { AstLexicalRead read = (AstLexicalRead) def; Access[] accesses = read.getAccesses(); - if (accesses.length > 0 && accesses[0].variableName.equals("str")) { + if (accesses.length > 0 && accesses[0].variableName.equals(PRINT_FUNCTION_VARIABLE_NAME)) { PointerKey pk = this.h.getPointerKeyForLocal(this.n, use); OrdinalSet pointsToSet = this.pa.getPointsToSet(pk); pointsToSet.forEach(ik -> { IClass concreteType = ik.getConcreteType(); - TypeReference reference = concreteType.getReference(); - - TypeReference strFunction = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lwala/builtin/str"); - System.out.println(reference.equals(strFunction)); + TypeReference typeReference = concreteType.getReference(); + if (typeReference.equals(PRINT_FUNCTION_TYPE_REFERENCE)) { + // found a call to the built-in print function, which has side effects. + // add a pointer to a fake global variable representing a modification to the output stream. + this.result.add(GLOBAL_OUTPUT_STREAM_POINTER_KEY); + } }); } } From 6de19055c20bc6ea776e4a65047d33f0f7b4689d Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 15:59:37 -0400 Subject: [PATCH 045/172] We don't need tensorflow for this test. --- .../HybridizeFunction/testPythonSideEffects9/in/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt index b154f958f..e69de29bb 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects9/in/requirements.txt @@ -1 +0,0 @@ -tensorflow==2.9.3 From 1e678208ab769e2f263eb64a390fa9021a2b51de Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 16:28:59 -0400 Subject: [PATCH 046/172] Add write() tests. --- edu.cuny.hunter.hybridize.tests/.gitignore | 2 ++ .../testPythonSideEffects10/in/.gitignore | 1 + .../testPythonSideEffects10/in/A.py | 9 +++++++ .../in/requirements.txt | 0 .../testPythonSideEffects11/in/.gitignore | 1 + .../testPythonSideEffects11/in/A.py | 6 +++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 24 +++++++++++++++++++ 8 files changed, 43 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/.gitignore create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/.gitignore create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/.gitignore b/edu.cuny.hunter.hybridize.tests/.gitignore index b83d22266..9132ec700 100644 --- a/edu.cuny.hunter.hybridize.tests/.gitignore +++ b/edu.cuny.hunter.hybridize.tests/.gitignore @@ -1 +1,3 @@ /target/ +/file.txt +/spam.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/.gitignore b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/.gitignore new file mode 100644 index 000000000..37d479370 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/.gitignore @@ -0,0 +1 @@ +/spam.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/A.py new file mode 100644 index 000000000..6f1e7481c --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/A.py @@ -0,0 +1,9 @@ +# From https://docs.python.org/3/library/io.html#io.IOBase. + + +def f(): + with open('spam.txt', 'w') as file: + file.write('Spam and eggs!') + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects10/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/.gitignore b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/.gitignore new file mode 100644 index 000000000..37d479370 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/.gitignore @@ -0,0 +1 @@ +/spam.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/A.py new file mode 100644 index 000000000..8641cdb79 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/A.py @@ -0,0 +1,6 @@ +def f(): + with open('file.txt', 'w') as f: + f.writelines(['line1\n', 'line2\n']) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects11/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index df6dc8360..e37300085 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4798,4 +4798,28 @@ public void testPythonSideEffects9() throws Exception { assertFalse(function.getLikelyHasTensorParameter()); // the example uses a primitive type. assertTrue("Expecting a Python side-effect.", function.getHasPythonSideEffects()); } + + /** + * Test write(). + */ + @Test + public void testPythonSideEffects10() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // NOTE: Switch to asserTrue when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/273 is fixed. + assertFalse("Not expecting a Python side-effect.", function.getHasPythonSideEffects()); + } + + /** + * Test writelines(). + */ + @Test + public void testPythonSideEffects11() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // NOTE: This is passing, but it doesn't look like it's for the correct reason. I'm unsure why lists are being modeled as fields. + assertTrue("Expecting a Python side-effect.", function.getHasPythonSideEffects()); + } } From 1e6e6d4aaff9e8069a7068502045cfc167f59b5e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 17:36:06 -0400 Subject: [PATCH 047/172] Update to ML 0.10.0-SNAPSHOT. --- hybridize.target | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hybridize.target b/hybridize.target index 1169ab0bc..0faccce1b 100644 --- a/hybridize.target +++ b/hybridize.target @@ -29,7 +29,7 @@ com.ibm.wala com.ibm.wala.cast.python.ml - 0.9.0-SNAPSHOT + 0.10.0-SNAPSHOT jar From 2f0352e6121f14a5360a93ec79918ff7f117cff1 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 20 Oct 2023 17:36:19 -0400 Subject: [PATCH 048/172] Require at least ML 0.9.0 for call graphs. --- edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index b76bde0ad..64ab74931 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -23,7 +23,7 @@ Import-Package: com.ibm.wala.cast.ipa.callgraph, com.ibm.wala.cast.ir.ssa, com.ibm.wala.cast.loader, com.ibm.wala.cast.python.client;version="0.1.0", - com.ibm.wala.cast.python.ipa.callgraph, + com.ibm.wala.cast.python.ipa.callgraph;version="0.9.0", com.ibm.wala.cast.python.loader;version="0.1.0", com.ibm.wala.cast.python.ml.analysis;version="0.1.0", com.ibm.wala.cast.python.ml.client;version="0.1.0", From 878a9c5df6301a723525820640b2bcd275e63b83 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 23 Oct 2023 15:01:45 -0400 Subject: [PATCH 049/172] Fix method name. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- .../refactorings/HybridizeFunctionRefactoringProcessor.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index af4901338..4d2d9f63e 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -345,13 +345,13 @@ public Function(FunctionDefinition fd) { } /** - * Infer the side-effects potentially produced by executing this {@link Function}. + * Infer Python side-effects potentially produced by executing this {@link Function}. * * @param callGraph The system {@link CallGraph}. * @param pointerAnalysis The system {@link PointerAnalysis}. * @throws IllegalArgumentException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. */ - public void inferSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws IllegalArgumentException { + public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws IllegalArgumentException { ModRef modRef = new PythonModRefWithBuiltinFunctions(); Map> mod = modRef.computeMod(callGraph, pointerAnalysis); diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 189f676cd..425695427 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -211,7 +211,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // Check Python side-effects. try { - func.inferSideEffects(callGraph, builder.getPointerAnalysis()); + func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); } catch (IllegalArgumentException e) { LOG.warn("Unable to infer side-effects of: " + func + ".", e); func.addStatusEntry(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, From 5c599fd914a0ec39193ba70431d8aaa9c50bdc19 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 23 Oct 2023 15:07:02 -0400 Subject: [PATCH 050/172] Filter out metaprogramming locations. --- .../hybridize/core/analysis/Function.java | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 4d2d9f63e..bddfb790d 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -37,21 +37,29 @@ import org.python.pydev.parser.visitors.NodeUtils; import org.python.pydev.parser.visitors.TypeInfo; +import com.ibm.wala.cast.ipa.callgraph.ReflectedFieldPointerKey; import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.cast.python.ml.analysis.TensorVariable; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; import com.ibm.wala.cast.types.AstMethodReference; +import com.ibm.wala.classLoader.IClass; +import com.ibm.wala.classLoader.IField; import com.ibm.wala.classLoader.IMethod; +import com.ibm.wala.classLoader.NewSiteReference; +import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.CallGraph; +import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey; +import com.ibm.wala.ipa.callgraph.propagation.NormalAllocationInNode; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; import com.ibm.wala.ipa.modref.ModRef; import com.ibm.wala.types.MethodReference; +import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.Pair; import com.ibm.wala.util.intset.OrdinalSet; @@ -363,10 +371,15 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis modSet = mod.get(cgNode); - LOG.info("Found " + modSet.size() + " modified location(s)."); + LOG.info("Found " + modSet.size() + " original modified location(s)."); + modSet.forEach(pk -> LOG.info("Original modified location: " + pk + ".")); - if (!modSet.isEmpty()) { - modSet.forEach(pk -> LOG.info("Modified location: " + pk + ".")); + // filter out the modified locations. + Set filteredModSet = filterSideEffects(modSet, pointerAnalysis); + LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); + filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); + + if (!filteredModSet.isEmpty()) { this.hasPythonSideEffects = TRUE; return; } @@ -376,6 +389,45 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet, PointerAnalysis pa) { + Set ret = new HashSet<>(); + + for (PointerKey pointerKey : modSet) { + if (pointerKey instanceof InstanceFieldKey) { + InstanceFieldKey ifk = (InstanceFieldKey) pointerKey; + IField manipulatedField = ifk.getField(); + Atom manipulatedFieldName = manipulatedField.getName(); + + // let's assume that metaprogramming is done by the system and does not represent a Python side-effect. + if (manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$class")) + || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$self"))) + continue; + } else if (pointerKey instanceof ReflectedFieldPointerKey) { + ReflectedFieldPointerKey rfp = (ReflectedFieldPointerKey) pointerKey; + InstanceKey pointerInstanceKey = rfp.getInstanceKey(); + IClass pointerConcreteType = pointerInstanceKey.getConcreteType(); + TypeName pointerConcreteTypeName = pointerConcreteType.getName(); + + // Is it an allocation of the super class object? + if (pointerConcreteTypeName.equals(TypeName.findOrCreate("Lsuperfun")) + && pointerInstanceKey instanceof NormalAllocationInNode) { + NormalAllocationInNode alloc = (NormalAllocationInNode) pointerInstanceKey; + NewSiteReference allocSite = alloc.getSite(); + TypeReference allocDeclaredType = allocSite.getDeclaredType(); + int allocPC = allocSite.getProgramCounter(); + + if (allocDeclaredType.equals(TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lsuperfun")) && allocPC == 0) { + continue; + } + } + } + + ret.add(pointerKey); + } + + return ret; + } + /** * Get the {@link CallGraph} nodes corresponding to this {@link Function}. * From d8774cbee30e733ad4f5462eeb82aab7232b9dec Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 23 Oct 2023 15:36:41 -0400 Subject: [PATCH 051/172] Fix errors. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index bddfb790d..184efdad1 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -375,7 +375,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Original modified location: " + pk + ".")); // filter out the modified locations. - Set filteredModSet = filterSideEffects(modSet, pointerAnalysis); + Set filteredModSet = filterSideEffects(modSet); LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); @@ -389,7 +389,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet, PointerAnalysis pa) { + private static Set filterSideEffects(Iterable modSet) { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { From 9077d4eabb3f81e8eeb21e4c782a793c5fa98373 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 09:43:06 -0400 Subject: [PATCH 052/172] Don't print so much. --- .../hunter/hybridize/core/analysis/Function.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 184efdad1..ddfa77784 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -303,6 +303,11 @@ public boolean getReduceRetracingParamExists() { private static final ILog LOG = getLog(Function.class); + /** + * True iff verbose output is desired. + */ + private static final boolean VERBOSE = false; + /** * This {@link Function}'s associated hybridization parameters. */ @@ -442,8 +447,11 @@ private Set getCallGraphNodes(CallGraph callGraph) throws IllegalArgumen if (nodes.isEmpty()) { LOG.error("Can't get call graph nodes for: " + this + "."); - LOG.info("Method reference is: " + methodReference + "."); - LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); + + if (VERBOSE) { + LOG.info("Method reference is: " + methodReference + "."); + LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); + } throw new IllegalArgumentException("Can't find: " + methodReference + " in call graph."); } From 3793d638119f3c4a4418d427775291dcdffba23b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 10:44:35 -0400 Subject: [PATCH 053/172] Use defaults. --- edu.cuny.hunter.hybridize.tests/pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/pom.xml b/edu.cuny.hunter.hybridize.tests/pom.xml index a8113386a..70e0a932c 100644 --- a/edu.cuny.hunter.hybridize.tests/pom.xml +++ b/edu.cuny.hunter.hybridize.tests/pom.xml @@ -51,8 +51,6 @@ false true true - false - false false org.junit From 2eba547a122b1e2374565e6c5b577ae9e6ea97ba Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 11:10:30 -0400 Subject: [PATCH 054/172] Enable assertions for our project. --- edu.cuny.hunter.hybridize.tests/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/pom.xml b/edu.cuny.hunter.hybridize.tests/pom.xml index 70e0a932c..ca847dfcd 100644 --- a/edu.cuny.hunter.hybridize.tests/pom.xml +++ b/edu.cuny.hunter.hybridize.tests/pom.xml @@ -12,6 +12,10 @@ https://github.com/ponder-lab/Hybridize-Functions-Refactoring/tree/main/edu.cuny.hunter.hybridize.tests + + + -ea:edu.cuny.hunter.hybridize... + macosx From dc632e3accdba7092f2a9b6cec8d4b7ff6fb05e2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 11:22:56 -0400 Subject: [PATCH 055/172] Add sanity check for side-effects. If we find side-effects, they are determinable. --- .../hybridize/core/analysis/Function.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index ddfa77784..5444a6807 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -23,6 +23,7 @@ import org.eclipse.jface.text.IDocument; import org.eclipse.ltk.core.refactoring.RefactoringStatus; import org.eclipse.ltk.core.refactoring.RefactoringStatusContext; +import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.osgi.framework.FrameworkUtil; import org.python.pydev.core.IPythonNature; import org.python.pydev.core.docutils.PySelection; @@ -385,12 +386,12 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Filtered modified location: " + pk + ".")); if (!filteredModSet.isEmpty()) { - this.hasPythonSideEffects = TRUE; + this.setHasPythonSideEffects(TRUE); return; } } - this.hasPythonSideEffects = FALSE; + this.setHasPythonSideEffects(FALSE); LOG.info(this + " does not have side-effects."); } @@ -741,8 +742,11 @@ private static boolean isHybrid(decoratorsType decorator, String containingModul } public void addStatusEntry(PreconditionFailure failure, String message) { - RefactoringStatusContext context = new FunctionStatusContext(); + // If is side-effects is filled, we can't set a precondition failure that we can't determine them. + assert this.getHasPythonSideEffects() == null + || failure != PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS : "Can't both have side-effects filled and have tem undterminable."; + RefactoringStatusContext context = new FunctionStatusContext(); this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, BUNDLE_SYMBOLIC_NAME, failure.getCode(), this); } @@ -928,4 +932,11 @@ public void setRefactoring(Refactoring refactoring) { public Boolean getHasPythonSideEffects() { return this.hasPythonSideEffects; } + + protected void setHasPythonSideEffects(Boolean hasPythonSideEffects) { + assert hasPythonSideEffects == null || this.getStatus().getEntryMatchingCode(BUNDLE_SYMBOLIC_NAME, + PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()) == null : "Can't set side-effects if they are undeterminable."; + + this.hasPythonSideEffects = hasPythonSideEffects; + } } From f57432b3653a4d26d469f3e11e0258c873c3ee47 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:05:13 -0400 Subject: [PATCH 056/172] Use custom exception. --- .../hunter/hybridize/core/analysis/Function.java | 10 +++++----- .../UndeterminablePythonSideEffectsException.java | 13 +++++++++++++ .../HybridizeFunctionRefactoringProcessor.java | 3 ++- 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 5444a6807..11a20d539 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -363,9 +363,9 @@ public Function(FunctionDefinition fd) { * * @param callGraph The system {@link CallGraph}. * @param pointerAnalysis The system {@link PointerAnalysis}. - * @throws IllegalArgumentException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. + * @throws UndeterminablePythonSideEffectsException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. */ - public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws IllegalArgumentException { + public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws UndeterminablePythonSideEffectsException { ModRef modRef = new PythonModRefWithBuiltinFunctions(); Map> mod = modRef.computeMod(callGraph, pointerAnalysis); @@ -439,10 +439,10 @@ private static Set filterSideEffects(Iterable modSet) { * * @param callGraph The {@link CallGraph} to search. * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. - * @throws IllegalArgumentException If this {@link Function} can't be found in the given {@link CallGraph}. + * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ - private Set getCallGraphNodes(CallGraph callGraph) throws IllegalArgumentException { + private Set getCallGraphNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { MethodReference methodReference = this.getMethodReference(); Set nodes = callGraph.getNodes(methodReference); @@ -454,7 +454,7 @@ private Set getCallGraphNodes(CallGraph callGraph) throws IllegalArgumen LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); } - throw new IllegalArgumentException("Can't find: " + methodReference + " in call graph."); + throw new UndeterminablePythonSideEffectsException(methodReference, callGraph); } LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java new file mode 100644 index 000000000..9cd76e03a --- /dev/null +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java @@ -0,0 +1,13 @@ +package edu.cuny.hunter.hybridize.core.analysis; + +import com.ibm.wala.ipa.callgraph.CallGraph; +import com.ibm.wala.types.MethodReference; + +public class UndeterminablePythonSideEffectsException extends Exception { + + private static final long serialVersionUID = -1229657254725226075L; + + public UndeterminablePythonSideEffectsException(MethodReference methodReference, CallGraph callGraph) { + super("Can't find: " + methodReference + " in: " + callGraph + " ."); + } +} diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 425695427..63be290a5 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -41,6 +41,7 @@ import edu.cuny.hunter.hybridize.core.analysis.Function; import edu.cuny.hunter.hybridize.core.analysis.FunctionDefinition; import edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure; +import edu.cuny.hunter.hybridize.core.analysis.UndeterminablePythonSideEffectsException; import edu.cuny.hunter.hybridize.core.descriptors.HybridizeFunctionRefactoringDescriptor; import edu.cuny.hunter.hybridize.core.messages.Messages; import edu.cuny.hunter.hybridize.core.wala.ml.EclipsePythonProjectTensorAnalysisEngine; @@ -212,7 +213,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // Check Python side-effects. try { func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); - } catch (IllegalArgumentException e) { + } catch (UndeterminablePythonSideEffectsException e) { LOG.warn("Unable to infer side-effects of: " + func + ".", e); func.addStatusEntry(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); From 15b0afd88b6c85a2ee0739dc2d96a4a2cb6aaf5d Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:05:28 -0400 Subject: [PATCH 057/172] Use precomputed symbolic bundle name. --- .../EvaluateHybridizeFunctionRefactoringHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 1ebc9cedd..99496f083 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -1,5 +1,6 @@ package edu.cuny.hunter.hybridize.eval.handlers; +import static edu.cuny.hunter.hybridize.core.analysis.Function.BUNDLE_SYMBOLIC_NAME; import static edu.cuny.hunter.hybridize.core.utils.Util.createHybridizeFunctionRefactoring; import static org.eclipse.core.runtime.Platform.getLog; import static org.python.pydev.plugin.nature.PythonNature.PYTHON_NATURE_ID; @@ -146,9 +147,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // optimization available functions. These are the "filtered" functions. We consider functions to be candidates iff they // have a tensor-like parameter or are currently hybrid. - Set candidates = functions.stream().filter(Function::isHybridizationAvailable) - .filter(f -> f.getStatus().getEntryMatchingCode(FrameworkUtil.getBundle(Function.class).getSymbolicName(), - PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()) == null) + Set candidates = functions.stream().filter(Function::isHybridizationAvailable).filter(f -> f.getStatus() + .getEntryMatchingCode(BUNDLE_SYMBOLIC_NAME, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()) == null) .collect(Collectors.toSet()); resultsPrinter.print(candidates.size()); // number. From c09728d565f76588b9eebc2038f3a2310e505ea9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:06:04 -0400 Subject: [PATCH 058/172] Remove unused import. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 11a20d539..906ec7c88 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -23,7 +23,6 @@ import org.eclipse.jface.text.IDocument; import org.eclipse.ltk.core.refactoring.RefactoringStatus; import org.eclipse.ltk.core.refactoring.RefactoringStatusContext; -import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.osgi.framework.FrameworkUtil; import org.python.pydev.core.IPythonNature; import org.python.pydev.core.docutils.PySelection; From 1fa5be9ab4a96c335d8bc8cc21b013ade1144bc4 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:17:08 -0400 Subject: [PATCH 059/172] Format. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 906ec7c88..890092915 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -362,9 +362,11 @@ public Function(FunctionDefinition fd) { * * @param callGraph The system {@link CallGraph}. * @param pointerAnalysis The system {@link PointerAnalysis}. - * @throws UndeterminablePythonSideEffectsException If this {@link Function}'s representation isn't found in the given {@link CallGraph}. + * @throws UndeterminablePythonSideEffectsException If this {@link Function}'s representation isn't found in the given + * {@link CallGraph}. */ - public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) throws UndeterminablePythonSideEffectsException { + public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis pointerAnalysis) + throws UndeterminablePythonSideEffectsException { ModRef modRef = new PythonModRefWithBuiltinFunctions(); Map> mod = modRef.computeMod(callGraph, pointerAnalysis); From bc017a034468188aff1be6559e006c9ea2a79f3c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:17:23 -0400 Subject: [PATCH 060/172] Shorten log. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- .../analysis/UndeterminablePythonSideEffectsException.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 890092915..46341313a 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -455,7 +455,7 @@ private Set getCallGraphNodes(CallGraph callGraph) throws Undeterminable LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); } - throw new UndeterminablePythonSideEffectsException(methodReference, callGraph); + throw new UndeterminablePythonSideEffectsException(methodReference); } LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java index 9cd76e03a..8f247c60b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/UndeterminablePythonSideEffectsException.java @@ -1,13 +1,12 @@ package edu.cuny.hunter.hybridize.core.analysis; -import com.ibm.wala.ipa.callgraph.CallGraph; import com.ibm.wala.types.MethodReference; public class UndeterminablePythonSideEffectsException extends Exception { private static final long serialVersionUID = -1229657254725226075L; - public UndeterminablePythonSideEffectsException(MethodReference methodReference, CallGraph callGraph) { - super("Can't find: " + methodReference + " in: " + callGraph + " ."); + public UndeterminablePythonSideEffectsException(MethodReference methodReference) { + super("Can't find: " + methodReference + " in call graph."); } } From 0b0a0e75e968c1a0c6d5bd350482a002042c2021 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:17:50 -0400 Subject: [PATCH 061/172] Remove unused import. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 99496f083..4fcde824c 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -36,7 +36,6 @@ import org.eclipse.ltk.core.refactoring.participants.ProcessorBasedRefactoring; import org.eclipse.ui.ISources; import org.eclipse.ui.handlers.HandlerUtil; -import org.osgi.framework.FrameworkUtil; import org.python.pydev.navigator.elements.PythonSourceFolder; import com.google.common.collect.Sets; From eb3f92142efd19130da6552857fbd7a3aa7c64ab Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 12:37:20 -0400 Subject: [PATCH 062/172] Additional filtering. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 46341313a..d643da079 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -407,8 +407,12 @@ private static Set filterSideEffects(Iterable modSet) { // let's assume that metaprogramming is done by the system and does not represent a Python side-effect. if (manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$class")) - || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$self"))) + || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$self")) + || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$function"))) { + LOG.info("Filtering out manipulated metaprogramming field: " + manipulatedFieldName + "."); continue; + } else if (manipulatedFieldName.startsWith(Atom.findOrCreateAsciiAtom("$"))) + LOG.warn("Encountered unfiltered manipulated potential metaprogramming field: " + manipulatedFieldName + "."); } else if (pointerKey instanceof ReflectedFieldPointerKey) { ReflectedFieldPointerKey rfp = (ReflectedFieldPointerKey) pointerKey; InstanceKey pointerInstanceKey = rfp.getInstanceKey(); From 14c2424d122604befeb7330926852f02792874a4 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 13:19:44 -0400 Subject: [PATCH 063/172] Log removed locations. --- edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF | 3 ++- .../cuny/hunter/hybridize/core/analysis/Function.java | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index 64ab74931..964efe5de 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -19,7 +19,8 @@ Require-Bundle: org.eclipse.ltk.core.refactoring;bundle-version="3.12.200", org.eclipse.core.resources;bundle-version="3.18.0", org.eclipse.core.runtime;bundle-version="3.26.0", com.ibm.wala.ide;bundle-version="1.6.2" -Import-Package: com.ibm.wala.cast.ipa.callgraph, +Import-Package: com.google.common.collect, + com.ibm.wala.cast.ipa.callgraph, com.ibm.wala.cast.ir.ssa, com.ibm.wala.cast.loader, com.ibm.wala.cast.python.client;version="0.1.0", diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index d643da079..87c92105b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -37,6 +37,8 @@ import org.python.pydev.parser.visitors.NodeUtils; import org.python.pydev.parser.visitors.TypeInfo; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; import com.ibm.wala.cast.ipa.callgraph.ReflectedFieldPointerKey; import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; @@ -381,11 +383,16 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Original modified location: " + pk + ".")); - // filter out the modified locations. + // Filter out the modified locations. Set filteredModSet = filterSideEffects(modSet); LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); + // Log the locations we are removing. + SetView removed = Sets.difference(Sets.newHashSet(modSet), filteredModSet); + LOG.info("Removed " + removed.size() + " locations."); + removed.forEach(pk -> LOG.info("Removed modified location: " + pk + ".")); + if (!filteredModSet.isEmpty()) { this.setHasPythonSideEffects(TRUE); return; From 97e75a59883db8166fb2edba5513ba293858681c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 13:20:02 -0400 Subject: [PATCH 064/172] Only set side-effects once. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 1 + 1 file changed, 1 insertion(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 87c92105b..da70c2d64 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -946,6 +946,7 @@ public Boolean getHasPythonSideEffects() { } protected void setHasPythonSideEffects(Boolean hasPythonSideEffects) { + assert this.hasPythonSideEffects == null : "Can only set side-effects once."; assert hasPythonSideEffects == null || this.getStatus().getEntryMatchingCode(BUNDLE_SYMBOLIC_NAME, PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()) == null : "Can't set side-effects if they are undeterminable."; From 9f983b9715a1c10a1520324c4eb83e5ca6f2f1c2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 14:56:57 -0400 Subject: [PATCH 065/172] Change method name. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index da70c2d64..e092c3ddb 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -384,7 +384,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Original modified location: " + pk + ".")); // Filter out the modified locations. - Set filteredModSet = filterSideEffects(modSet); + Set filteredModSet = filterPointerKeys(modSet); LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); @@ -403,7 +403,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet) { + private static Set filterPointerKeys(Iterable modSet) { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { From d42f263a8ec23bdb92c6972289d1d25df5590f72 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 16:46:54 -0400 Subject: [PATCH 066/172] Fix side-effects. --- .../hybridize/core/analysis/Function.java | 57 ++++++++----------- .../testPythonSideEffects12/in/A.py | 8 +++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 13 ++++- 4 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index e092c3ddb..96f75ad13 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -11,6 +11,7 @@ import java.io.File; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -54,6 +55,7 @@ import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.CallGraph; import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldKey; +import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldPointerKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey; import com.ibm.wala.ipa.callgraph.propagation.NormalAllocationInNode; @@ -384,7 +386,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Original modified location: " + pk + ".")); // Filter out the modified locations. - Set filteredModSet = filterPointerKeys(modSet); + Set filteredModSet = this.filterSideEffects(modSet, callGraph); LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); @@ -403,41 +405,16 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterPointerKeys(Iterable modSet) { + private Set filterSideEffects(Iterable modSet, CallGraph callGraph) { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { - if (pointerKey instanceof InstanceFieldKey) { - InstanceFieldKey ifk = (InstanceFieldKey) pointerKey; - IField manipulatedField = ifk.getField(); - Atom manipulatedFieldName = manipulatedField.getName(); - - // let's assume that metaprogramming is done by the system and does not represent a Python side-effect. - if (manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$class")) - || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$self")) - || manipulatedFieldName.equals(Atom.findOrCreateAsciiAtom("$function"))) { - LOG.info("Filtering out manipulated metaprogramming field: " + manipulatedFieldName + "."); - continue; - } else if (manipulatedFieldName.startsWith(Atom.findOrCreateAsciiAtom("$"))) - LOG.warn("Encountered unfiltered manipulated potential metaprogramming field: " + manipulatedFieldName + "."); - } else if (pointerKey instanceof ReflectedFieldPointerKey) { - ReflectedFieldPointerKey rfp = (ReflectedFieldPointerKey) pointerKey; - InstanceKey pointerInstanceKey = rfp.getInstanceKey(); - IClass pointerConcreteType = pointerInstanceKey.getConcreteType(); - TypeName pointerConcreteTypeName = pointerConcreteType.getName(); - - // Is it an allocation of the super class object? - if (pointerConcreteTypeName.equals(TypeName.findOrCreate("Lsuperfun")) - && pointerInstanceKey instanceof NormalAllocationInNode) { - NormalAllocationInNode alloc = (NormalAllocationInNode) pointerInstanceKey; - NewSiteReference allocSite = alloc.getSite(); - TypeReference allocDeclaredType = allocSite.getDeclaredType(); - int allocPC = allocSite.getProgramCounter(); - - if (allocDeclaredType.equals(TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lsuperfun")) && allocPC == 0) { - continue; - } - } + if (pointerKey instanceof InstanceFieldPointerKey) { + InstanceFieldPointerKey fieldPointerKey = (InstanceFieldPointerKey) pointerKey; + InstanceKey instanceKey = fieldPointerKey.getInstanceKey(); + + if (allCreationsWithinThisFunction(instanceKey, callGraph)) + continue; // next pointer. } ret.add(pointerKey); @@ -446,6 +423,20 @@ private static Set filterPointerKeys(Iterable modSet) { return ret; } + private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGraph callGraph) { + // for each creation site of the given instance. + for (Iterator> it = instanceKey.getCreationSites(callGraph); it.hasNext();) { + Pair creationSite = it.next(); + CGNode creationNode = creationSite.fst; + + // is this instance being created outside this function? + if (!this.getMethodReference().equals(creationNode.getMethod().getReference())) + return false; + } + + return true; + } + /** * Get the {@link CallGraph} nodes corresponding to this {@link Function}. * diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py new file mode 100644 index 000000000..b00a20621 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py @@ -0,0 +1,8 @@ +def f(): + my_list = [10] + my_list[0] = 1 + return my_list + + +ret = f() +print(ret) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e37300085..78230ec04 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4819,7 +4819,16 @@ public void testPythonSideEffects11() throws Exception { Function function = getSingleFunction(); assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); - // NOTE: This is passing, but it doesn't look like it's for the correct reason. I'm unsure why lists are being modeled as fields. - assertTrue("Expecting a Python side-effect.", function.getHasPythonSideEffects()); + // NOTE: Switch to asserTrue when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/273 is fixed. + assertFalse("Not expecting a Python side-effect.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects12() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with no side-effects. + assertFalse("This Python statement modifies a local variable.", function.getHasPythonSideEffects()); } } From 7225a23e084d63731f4692c92dfc57792678bf40 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 16:47:02 -0400 Subject: [PATCH 067/172] Add log. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 1 + 1 file changed, 1 insertion(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 96f75ad13..c4eeffcc8 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -397,6 +397,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis Date: Tue, 24 Oct 2023 17:05:44 -0400 Subject: [PATCH 068/172] Cleanup test file. --- .../HybridizeFunction/testPythonSideEffects12/in/A.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py index b00a20621..279cd9e79 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects12/in/A.py @@ -1,8 +1,6 @@ def f(): my_list = [10] my_list[0] = 1 - return my_list -ret = f() -print(ret) +f() From 48dfe80d1da042c289ce32711d542645ef4f3184 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 17:06:14 -0400 Subject: [PATCH 069/172] Remove unused imports. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c4eeffcc8..d3761e071 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -40,30 +40,23 @@ import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; -import com.ibm.wala.cast.ipa.callgraph.ReflectedFieldPointerKey; import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.cast.python.ml.analysis.TensorVariable; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; import com.ibm.wala.cast.types.AstMethodReference; -import com.ibm.wala.classLoader.IClass; -import com.ibm.wala.classLoader.IField; import com.ibm.wala.classLoader.IMethod; import com.ibm.wala.classLoader.NewSiteReference; -import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.CallGraph; -import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldPointerKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey; -import com.ibm.wala.ipa.callgraph.propagation.NormalAllocationInNode; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; import com.ibm.wala.ipa.modref.ModRef; import com.ibm.wala.types.MethodReference; -import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.Pair; import com.ibm.wala.util.intset.OrdinalSet; From ef83ecbc9b46384f66a525525eb458a24179f1a7 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 24 Oct 2023 17:14:32 -0400 Subject: [PATCH 070/172] List comprehensions aren't working. --- .../testPythonSideEffects13/in/A.py | 8 ++++++ .../in/requirements.txt | 0 .../testPythonSideEffects14/in/A.py | 8 ++++++ .../in/requirements.txt | 0 .../testPythonSideEffects15/in/A.py | 10 +++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 27 +++++++++++++++++++ 7 files changed, 53 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/A.py new file mode 100644 index 000000000..3dfb5fb41 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/A.py @@ -0,0 +1,8 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def f(): + squares = [x ** 2 for x in range(10)] + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects13/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/A.py new file mode 100644 index 000000000..1e3cae621 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/A.py @@ -0,0 +1,8 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def f(): + squares = list(map(lambda x: x**2, range(10))) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects14/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/A.py new file mode 100644 index 000000000..04b407573 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/A.py @@ -0,0 +1,10 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def f(): + squares = [] + for x in range(10): + squares.append(x ** 2) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects15/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 78230ec04..00e826a0d 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4831,4 +4831,31 @@ public void testPythonSideEffects12() throws Exception { // there's a Python statement with no side-effects. assertFalse("This Python statement modifies a local variable.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects13() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with no side-effects. + assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects14() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with no side-effects. + assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects15() throws Exception { + Function function = getSingleFunction(); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with no side-effects. + assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); + } } From 31739613e969071e20c1da5e838b6fd191120015 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 10:39:30 -0400 Subject: [PATCH 071/172] Fix list comphrensions. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index d3761e071..b3b895a83 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -422,9 +422,11 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap for (Iterator> it = instanceKey.getCreationSites(callGraph); it.hasNext();) { Pair creationSite = it.next(); CGNode creationNode = creationSite.fst; + NewSiteReference newSiteReference = creationSite.snd; // is this instance being created outside this function? - if (!this.getMethodReference().equals(creationNode.getMethod().getReference())) + if (!this.getMethodReference().equals(creationNode.getMethod().getReference()) + && !this.getTypeReference().equals(newSiteReference.getDeclaredType())) return false; } From dfbf202b919973da82bc0633434515696238d534 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 10:52:58 -0400 Subject: [PATCH 072/172] Simplify boolean. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index b3b895a83..f98e0d2bd 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -425,8 +425,8 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap NewSiteReference newSiteReference = creationSite.snd; // is this instance being created outside this function? - if (!this.getMethodReference().equals(creationNode.getMethod().getReference()) - && !this.getTypeReference().equals(newSiteReference.getDeclaredType())) + if (!(this.getMethodReference().equals(creationNode.getMethod().getReference()) + || this.getTypeReference().equals(newSiteReference.getDeclaredType()))) return false; } From 53e701193247a9c0f39ed41a32e7bd04ef10c3c7 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 11:02:15 -0400 Subject: [PATCH 073/172] Switch expressions. More readable. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index f98e0d2bd..9d8e47cc7 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -425,8 +425,8 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap NewSiteReference newSiteReference = creationSite.snd; // is this instance being created outside this function? - if (!(this.getMethodReference().equals(creationNode.getMethod().getReference()) - || this.getTypeReference().equals(newSiteReference.getDeclaredType()))) + if (!(creationNode.getMethod().getReference().equals(this.getMethodReference()) + || newSiteReference.getDeclaredType().equals(this.getTypeReference()))) return false; } From aabbd847ff4b2aa9398f5ef826e8296140aa4f1d Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 11:51:12 -0400 Subject: [PATCH 074/172] Additional side-effects tests. --- .../testPythonSideEffects16/in/A.py | 15 ++++++++++ .../in/requirements.txt | 0 .../testPythonSideEffects17/in/A.py | 16 ++++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 29 +++++++++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/A.py new file mode 100644 index 000000000..2bfcb6846 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/A.py @@ -0,0 +1,15 @@ +# From https://stackoverflow.com/questions/5753597/is-it-pythonic-to-use-list-comprehensions-for-just-side-effects + +my_list = [10] + + +def fun_with_side_effects(y): + my_list[0] = 1 + return y ** 2 + + +def f(): + squares = [fun_with_side_effects(x) for x in range(10)] + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects16/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py new file mode 100644 index 000000000..b435ff480 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py @@ -0,0 +1,16 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + +my_list = [10] + + +def fun_with_side_effects(y): + my_list[0] = 1 + return y ** 2 + + +def f(): + squares = list(map(lambda x: fun_with_side_effects(x), range(10))) + print(squares) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 00e826a0d..c07a9acf6 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -572,6 +572,17 @@ private Set getFunctions() throws Exception { return getFunctions("A"); } + /** + * Returns the first {@link Function} in the default test file with the given identifier. + * + * @param functionIndentifier The {@link Function} to return. + * @return The first {@link Function} in the default test file with the given identifier. + */ + private Function getFunction(String functionIndentifier) throws Exception { + Set functions = this.getFunctions(); + return functions.stream().filter(f -> f.getIdentifier().equals(functionIndentifier)).findFirst().orElseThrow(); + } + /** * Return the {@link File} representing X.py, where X is fileNameWithoutExtension. * @@ -4858,4 +4869,22 @@ public void testPythonSideEffects15() throws Exception { // there's a Python statement with no side-effects. assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects16() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("This Python statement uses a list comprehension to modify a global variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects17() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("This Python statement uses a lambda to modify a global variable.", function.getHasPythonSideEffects()); + } } From 9fa43c1ca1eedc5b0f74283f596f87df334e6661 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 12:06:22 -0400 Subject: [PATCH 075/172] Add system property to dump call graphs. --- .../refactorings/HybridizeFunctionRefactoringProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 63be290a5..acc1ccf06 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -49,6 +49,8 @@ @SuppressWarnings("unused") public class HybridizeFunctionRefactoringProcessor extends RefactoringProcessor { + private static final String DUMP_CALL_GRAPH_PROPERTY_KEY = "edu.cuny.hunter.hybridize.dumpCallGraph"; + private final class FunctionStatusContext extends RefactoringStatusContext { private final Function func; @@ -93,7 +95,7 @@ private static RefactoringStatus checkParameters(Function func) { /** * True iff the {@link CallGraph} should be displayed. */ - private boolean dumpCallGraph; + private boolean dumpCallGraph = Boolean.getBoolean(DUMP_CALL_GRAPH_PROPERTY_KEY); public HybridizeFunctionRefactoringProcessor() { // Force the use of typeshed. It's an experimental feature of PyDev. From 73a9c1627efba9f632c99affaaf689be1c23e394 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 15:44:30 -0400 Subject: [PATCH 076/172] Add assertions. --- .../tests/HybridizeFunctionRefactoringTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index c07a9acf6..e9070a95d 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4402,6 +4402,8 @@ public void testModel() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "__call__": @@ -4437,6 +4439,8 @@ public void testModel2() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "call": @@ -4471,6 +4475,8 @@ public void testModel3() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "call": @@ -4511,6 +4517,8 @@ public void testModel4() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "__call__": @@ -4544,6 +4552,8 @@ public void testModel5() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "call": @@ -4578,6 +4588,8 @@ public void testModel6() throws Exception { switch (simpleName) { case "__init__": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); + assertTrue(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "__call__": @@ -4614,8 +4626,10 @@ public void testModel7() throws Exception { String simpleName = f.getSimpleName(); switch (simpleName) { case "__init__": + assertTrue(f.getHasPythonSideEffects()); case "get_stuff": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); checkOptimizationNotAvailableStatus(f); break; case "call": From a093c2ede3c80a2a4af203c38dc57deca0742cef Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 15:57:03 -0400 Subject: [PATCH 077/172] Suppress fall-through warning. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e9070a95d..0d8ce513f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4626,10 +4626,15 @@ public void testModel7() throws Exception { String simpleName = f.getSimpleName(); switch (simpleName) { case "__init__": + assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); + assertFalse(f.isHybrid()); assertTrue(f.getHasPythonSideEffects()); + checkOptimizationNotAvailableStatus(f); + break; case "get_stuff": assertFalse("Expecting " + simpleName + " to not have a tensor param.", f.getLikelyHasTensorParameter()); assertFalse(f.isHybrid()); + assertFalse(f.getHasPythonSideEffects()); checkOptimizationNotAvailableStatus(f); break; case "call": From 31e7ecf9bc67c4016b474e8d6f2843299dd7f8dc Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 25 Oct 2023 17:28:57 -0400 Subject: [PATCH 078/172] Tests for transitive side-effects. --- .../testPythonSideEffects18/in/A.py | 12 +++++ .../in/requirements.txt | 0 .../testPythonSideEffects19/in/A.py | 10 +++++ .../in/requirements.txt | 0 .../testPythonSideEffects20/in/A.py | 13 ++++++ .../in/requirements.txt | 0 .../testPythonSideEffects21/in/A.py | 10 +++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 44 +++++++++++++++++++ 9 files changed, 89 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/A.py new file mode 100644 index 000000000..b8c40d625 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/A.py @@ -0,0 +1,12 @@ +my_list = [10] + + +def g(): + my_list[0] = 1 + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects18/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/A.py new file mode 100644 index 000000000..00b2eb48a --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/A.py @@ -0,0 +1,10 @@ +def g(): + my_list = [10] + my_list[0] = 1 + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects19/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/A.py new file mode 100644 index 000000000..4104acfa9 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/A.py @@ -0,0 +1,13 @@ +a = 10 + + +def g(): + global a + a = a + 1 + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects20/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/A.py new file mode 100644 index 000000000..b0fa77da3 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/A.py @@ -0,0 +1,10 @@ +def g(): + a = 10 + a = a + 1 + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects21/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 0d8ce513f..1e4129cf9 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4906,4 +4906,48 @@ public void testPythonSideEffects17() throws Exception { // there's a Python statement with side-effects. assertTrue("This Python statement uses a lambda to modify a global variable.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects18() throws Exception { + Function f = getFunction("f"); + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + assertTrue(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertTrue(g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects19() throws Exception { + Function f = getFunction("f"); + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + assertFalse(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertFalse(g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects20() throws Exception { + Function f = getFunction("f"); + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + assertTrue(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertTrue(g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects21() throws Exception { + Function f = getFunction("f"); + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + assertFalse(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertFalse(g.getHasPythonSideEffects()); + } } From 83fce2c399525673c6585fc46a15192c50fb69ed Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 10:12:11 -0400 Subject: [PATCH 079/172] Add transitive list comphrension tests. --- .../testPythonSideEffects22/in/A.py | 12 +++++++ .../in/requirements.txt | 0 .../testPythonSideEffects23/in/A.py | 24 +++++++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 36 +++++++++++++++++++ 5 files changed, 72 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/A.py new file mode 100644 index 000000000..35f60132b --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/A.py @@ -0,0 +1,12 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def g(): + squares = [x ** 2 for x in range(10)] + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects22/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/A.py new file mode 100644 index 000000000..daaf7d09e --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/A.py @@ -0,0 +1,24 @@ +# From https://stackoverflow.com/questions/5753597/is-it-pythonic-to-use-list-comprehensions-for-just-side-effects + +my_list = [10] + + +def h(): + pass + + +def fun_with_side_effects(y): + my_list[0] = 1 + return y ** 2 + + +def g(): + squares = [fun_with_side_effects(x) for x in range(10)] + + +def f(): + g() + + +f() +h() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects23/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 1e4129cf9..3621f7a2d 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4950,4 +4950,40 @@ public void testPythonSideEffects21() throws Exception { Function g = getFunction("g"); assertFalse(g.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects22() throws Exception { + Set functionSet = getFunctions(); + + for (Function f : functionSet) { + assertFalse(f.isHybrid()); + assertFalse(f.getLikelyHasTensorParameter()); + assertFalse("This Python statement (transitively) uses a list comprehension to modify a local variable.", + f.getHasPythonSideEffects()); + } + } + + @Test + public void testPythonSideEffects23() throws Exception { + Set functionSet = getFunctions(); + + for (Function function : functionSet) { + switch (function.getIdentifier()) { + case "f": + case "g": + case "fun_with_side_effects": + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("This Python statement (transitively) uses a list comprehension to modify a global variable.", + function.getHasPythonSideEffects()); + break; + case "h": + assertFalse(function.getHasPythonSideEffects()); + break; + default: + throw new IllegalStateException("Unknown function: " + function + "."); + } + } + } } From eff997ca76d44ee974688dda27bec4658d9c1774 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 10:14:19 -0400 Subject: [PATCH 080/172] Fix assertion messages. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 3621f7a2d..262331e91 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4877,7 +4877,7 @@ public void testPythonSideEffects14() throws Exception { assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); // there's a Python statement with no side-effects. - assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); + assertFalse("This Python statement uses a lambda to modify a local variable.", function.getHasPythonSideEffects()); } @Test @@ -4886,7 +4886,7 @@ public void testPythonSideEffects15() throws Exception { assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); // there's a Python statement with no side-effects. - assertFalse("This Python statement uses a list comprehension to modify a local variable.", function.getHasPythonSideEffects()); + assertFalse("This Python statement uses a loop to modify a local variable.", function.getHasPythonSideEffects()); } @Test From f7feee108dd9e499ef80ffe41c5e002f821d9926 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 10:52:45 -0400 Subject: [PATCH 081/172] More tests. --- .../testPythonSideEffects24/in/A.py | 12 +++++++ .../in/requirements.txt | 0 .../testPythonSideEffects25/in/A.py | 14 ++++++++ .../in/requirements.txt | 0 .../testPythonSideEffects26/in/A.py | 11 ++++++ .../in/requirements.txt | 0 .../testPythonSideEffects27/in/A.py | 15 ++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 35 +++++++++++++++++++ 9 files changed, 87 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/A.py new file mode 100644 index 000000000..636ede0ed --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/A.py @@ -0,0 +1,12 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def g(): + squares = list(map(lambda x: x ** 2, range(10))) + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects24/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/A.py new file mode 100644 index 000000000..ae6140329 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/A.py @@ -0,0 +1,14 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + + +def g(): + squares = [] + for x in range(10): + squares.append(x ** 2) + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects25/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/A.py new file mode 100644 index 000000000..9f8daec6c --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/A.py @@ -0,0 +1,11 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + +squares = [] + + +def f(): + for x in range(10): + squares.append(x ** 2) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects26/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/A.py new file mode 100644 index 000000000..4064bc426 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/A.py @@ -0,0 +1,15 @@ +# From https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions + +squares = [] + + +def g(): + for x in range(10): + squares.append(x ** 2) + + +def f(): + g() + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects27/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 262331e91..9a5f3484f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4986,4 +4986,39 @@ public void testPythonSideEffects23() throws Exception { } } } + + @Test + public void testPythonSideEffects24() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertFalse("This Python statement (transitively) uses a lambda to modify a local variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects25() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with no side-effects. + assertFalse("This Python statement (transitively) uses a loop to modify a local variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects26() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("A loop to modifies a global variable.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects27() throws Exception { + Function function = getFunction("f"); + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // there's a Python statement with side-effects. + assertTrue("A loop to modifies a global variable.", function.getHasPythonSideEffects()); + } } From 359ae142f45e3882786bb7528a753dbbc9a167aa Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 10:52:53 -0400 Subject: [PATCH 082/172] Cleanup test file. --- .../resources/HybridizeFunction/testPythonSideEffects17/in/A.py | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py index b435ff480..51c6a080c 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py @@ -10,7 +10,6 @@ def fun_with_side_effects(y): def f(): squares = list(map(lambda x: fun_with_side_effects(x), range(10))) - print(squares) f() From da6c3aa5d8cdcce8f84ce0612ac448db41e9157f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 12:16:56 -0400 Subject: [PATCH 083/172] Add comments. --- .../core/wala/ml/PythonModRefWithBuiltinFunctions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 54db689a7..9b971181f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -37,8 +37,8 @@ public PythonModVisitorWithBuiltinFunctions(CGNode n, Collection res @Override public void visitPythonInvoke(PythonInvokeInstruction inst) { - int use = inst.getUse(0); - SSAInstruction def = this.n.getDU().getDef(use); + int use = inst.getUse(0); // a reference to the invoked function. + SSAInstruction def = this.n.getDU().getDef(use); // the definition of the invoked function. if (def instanceof AstLexicalRead) { AstLexicalRead read = (AstLexicalRead) def; From 66d5d23cad2e2b195c8a295605e2b632f7045de5 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 12:50:45 -0400 Subject: [PATCH 084/172] Use helper method. --- .../core/wala/ml/PythonModRefWithBuiltinFunctions.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 9b971181f..653052011 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -49,8 +49,7 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { OrdinalSet pointsToSet = this.pa.getPointsToSet(pk); pointsToSet.forEach(ik -> { - IClass concreteType = ik.getConcreteType(); - TypeReference typeReference = concreteType.getReference(); + TypeReference typeReference = getTypeReference(ik); if (typeReference.equals(PRINT_FUNCTION_TYPE_REFERENCE)) { // found a call to the built-in print function, which has side effects. @@ -63,6 +62,10 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { super.visitPythonInvoke(inst); } + + private TypeReference getTypeReference(T ik) { + return ik.getConcreteType().getReference(); + } } @Override From 59876325420c5492276aa8d524ade68191f681ee Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 12:50:56 -0400 Subject: [PATCH 085/172] Consider list.append as having side-effects. --- .../ml/PythonModRefWithBuiltinFunctions.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 653052011..9e273dc29 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -1,5 +1,8 @@ package edu.cuny.hunter.hybridize.core.wala.ml; +import static com.ibm.wala.cast.python.types.PythonTypes.list; +import static com.ibm.wala.cast.python.types.PythonTypes.string; + import java.util.Collection; import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; @@ -7,9 +10,11 @@ import com.ibm.wala.cast.ir.ssa.AstLexicalRead; import com.ibm.wala.cast.python.modref.PythonModRef; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; +import com.ibm.wala.cast.python.ssa.PythonPropertyRead; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.classLoader.IClass; import com.ibm.wala.ipa.callgraph.CGNode; +import com.ibm.wala.ipa.callgraph.propagation.ConstantKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; @@ -58,6 +63,33 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { } }); } + } else if (def instanceof PythonPropertyRead) { + PythonPropertyRead read = (PythonPropertyRead) def; + int memberRef = read.getMemberRef(); + PointerKey memberRefPK = this.h.getPointerKeyForLocal(this.n, memberRef); + OrdinalSet memberRefPointsToSet = this.pa.getPointsToSet(memberRefPK); + + memberRefPointsToSet.forEach(memberRefIK -> { + TypeReference typeReference = getTypeReference(memberRefIK); + + if (typeReference.equals(string) && memberRefIK instanceof ConstantKey) { + ConstantKey ck = (ConstantKey) memberRefIK; + Object value = ck.getValue(); + + if (value.equals("append")) { + // check that the receiver is of type list. + int objectRef = read.getObjectRef(); + PointerKey objectRefPK = this.h.getPointerKeyForLocal(this.n, objectRef); + OrdinalSet objectRefPointsToSet = this.pa.getPointsToSet(objectRefPK); + + objectRefPointsToSet.forEach(objectRefIK -> { + if (objectRefIK.getConcreteType().getReference().equals(list)) + // it's a list. Add the instance to the results. + this.result.add(objectRefPK); + }); + } + } + }); } super.visitPythonInvoke(inst); From cbe0381a4c9b47457ccae9ab099fce3cc3f2f879 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 13:02:03 -0400 Subject: [PATCH 086/172] Remove unused import. --- .../hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 9e273dc29..2ad8e1eb5 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -12,7 +12,6 @@ import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.ssa.PythonPropertyRead; import com.ibm.wala.cast.python.types.PythonTypes; -import com.ibm.wala.classLoader.IClass; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.propagation.ConstantKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; From bca5bb9805a7d54cf61f120a2173829b4ff6ed6a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 14:56:10 -0400 Subject: [PATCH 087/172] Handle local pointer keys. --- .../hunter/hybridize/core/analysis/Function.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 9d8e47cc7..9ba2e9010 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -379,7 +379,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis LOG.info("Original modified location: " + pk + ".")); // Filter out the modified locations. - Set filteredModSet = this.filterSideEffects(modSet, callGraph); + Set filteredModSet = this.filterSideEffects(modSet, callGraph, pointerAnalysis); LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); @@ -399,7 +399,8 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet, CallGraph callGraph) { + private Set filterSideEffects(Iterable modSet, CallGraph callGraph, + PointerAnalysis pointerAnalysis) { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { @@ -409,6 +410,17 @@ private Set filterSideEffects(Iterable modSet, CallGraph if (allCreationsWithinThisFunction(instanceKey, callGraph)) continue; // next pointer. + } else if (pointerKey instanceof LocalPointerKey) { + LocalPointerKey localPointerKey = (LocalPointerKey) pointerKey; + OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(localPointerKey); + + boolean skipPointerKey = true; + + for (InstanceKey ik : pointsToSet) + skipPointerKey &= allCreationsWithinThisFunction(ik, callGraph); + + if (skipPointerKey) + continue; // next pointer. } ret.add(pointerKey); From 752eb39a7072dfc847002d55b4a5e6268bc0c3b9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 15:15:08 -0400 Subject: [PATCH 088/172] Revert "Cleanup test file." This reverts commit 359ae142f45e3882786bb7528a753dbbc9a167aa. That's supposed to be there. --- .../resources/HybridizeFunction/testPythonSideEffects17/in/A.py | 1 + 1 file changed, 1 insertion(+) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py index 51c6a080c..b435ff480 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py @@ -10,6 +10,7 @@ def fun_with_side_effects(y): def f(): squares = list(map(lambda x: fun_with_side_effects(x), range(10))) + print(squares) f() From 93b101f16bfabade284c5e41a2fa2afbd5e8181f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 15:18:32 -0400 Subject: [PATCH 089/172] Revert "Revert "Cleanup test file."" This reverts commit 752eb39a7072dfc847002d55b4a5e6268bc0c3b9. Actually, it was added by mistake and is now causing a false negative. --- .../resources/HybridizeFunction/testPythonSideEffects17/in/A.py | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py index b435ff480..51c6a080c 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects17/in/A.py @@ -10,7 +10,6 @@ def fun_with_side_effects(y): def f(): squares = list(map(lambda x: fun_with_side_effects(x), range(10))) - print(squares) f() From 96237546e73e7e1a4e721794499a27e7121715b9 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 16:23:51 -0400 Subject: [PATCH 090/172] Work around https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/274. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 9a5f3484f..750e043cd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4903,8 +4903,8 @@ public void testPythonSideEffects17() throws Exception { Function function = getFunction("f"); assertFalse(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); - // there's a Python statement with side-effects. - assertTrue("This Python statement uses a lambda to modify a global variable.", function.getHasPythonSideEffects()); + // NOTE: Switch to assertTrue when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/274 is fixed. + assertFalse("This Python statement uses a lambda to modify a global variable.", function.getHasPythonSideEffects()); } @Test From 26af352605e8acb4fc1d497a682fe2d8db84eeea Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 17:15:20 -0400 Subject: [PATCH 091/172] Add parameter test case. --- .../testPythonSideEffects28/in/A.py | 10 ++++++++++ .../testPythonSideEffects28/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 12 ++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/A.py new file mode 100644 index 000000000..7454e88ac --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/A.py @@ -0,0 +1,10 @@ +def g(a_list): + a_list[0] = 1 + + +def f(): + my_list = [10] + g(my_list) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects28/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 750e043cd..dfa89084e 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5021,4 +5021,16 @@ public void testPythonSideEffects27() throws Exception { // there's a Python statement with side-effects. assertTrue("A loop to modifies a global variable.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects28() throws Exception { + Map> map = this.getFunctions().stream() + .collect(Collectors.groupingBy(Function::getIdentifier, Collectors.toSet())); + + assertEquals(2, map.size()); + assertEquals(2, map.keySet().size()); + + map.get("f").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertFalse(s)); + map.get("g").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertTrue(s)); + } } From 1e85172303f361543bda8bed71b5bf811868a5a3 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 17:24:59 -0400 Subject: [PATCH 092/172] Should suffice. --- .../hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index dfa89084e..d2401ed14 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5028,7 +5028,6 @@ public void testPythonSideEffects28() throws Exception { .collect(Collectors.groupingBy(Function::getIdentifier, Collectors.toSet())); assertEquals(2, map.size()); - assertEquals(2, map.keySet().size()); map.get("f").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertFalse(s)); map.get("g").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertTrue(s)); From b63e2ca4fbabe2e4a37ad2969c6de925e375ca7f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 17:32:52 -0400 Subject: [PATCH 093/172] Be more defensive. If we encounter a pointer key we don't know about, let's know about it. --- .../hybridize/core/analysis/Function.java | 22 ++++++++++++++----- .../ml/PythonModRefWithBuiltinFunctions.java | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 9ba2e9010..1a771e2e4 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -5,6 +5,7 @@ import static edu.cuny.hunter.hybridize.core.analysis.Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID; import static edu.cuny.hunter.hybridize.core.analysis.Refactoring.OPTIMIZE_HYBRID_FUNCTION; import static edu.cuny.hunter.hybridize.core.analysis.Transformation.CONVERT_TO_EAGER; +import static edu.cuny.hunter.hybridize.core.wala.ml.PythonModRefWithBuiltinFunctions.PythonModVisitorWithBuiltinFunctions.GLOBAL_OUTPUT_STREAM_POINTER_KEY; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; import static org.eclipse.core.runtime.Platform.getLog; @@ -40,6 +41,7 @@ import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; +import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; import com.ibm.wala.cast.loader.AstMethod; import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.cast.python.ml.analysis.TensorVariable; @@ -409,7 +411,9 @@ private Set filterSideEffects(Iterable modSet, CallGraph InstanceKey instanceKey = fieldPointerKey.getInstanceKey(); if (allCreationsWithinThisFunction(instanceKey, callGraph)) - continue; // next pointer. + continue; // filter this pointer out. + else + ret.add(fieldPointerKey); } else if (pointerKey instanceof LocalPointerKey) { LocalPointerKey localPointerKey = (LocalPointerKey) pointerKey; OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(localPointerKey); @@ -420,10 +424,18 @@ private Set filterSideEffects(Iterable modSet, CallGraph skipPointerKey &= allCreationsWithinThisFunction(ik, callGraph); if (skipPointerKey) - continue; // next pointer. - } - - ret.add(pointerKey); + continue; // filter this pointer out. + else + ret.add(localPointerKey); + } else if (pointerKey instanceof AstGlobalPointerKey) { + AstGlobalPointerKey globalPointerKey = (AstGlobalPointerKey) pointerKey; + + if (globalPointerKey.equals(GLOBAL_OUTPUT_STREAM_POINTER_KEY)) + ret.add(globalPointerKey); + else + throw new IllegalArgumentException("Not expecting global pointer key: " + globalPointerKey + "."); + } else + throw new IllegalArgumentException("Not expecting pointer key: " + pointerKey + " of type: " + pointerKey.getClass() + "."); } return ret; diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 2ad8e1eb5..3398491fa 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -26,7 +26,7 @@ public class PythonModRefWithBuiltinFunctions extends PythonModRef { public static class PythonModVisitorWithBuiltinFunctions extends PythonModVisitor { - private static final AstGlobalPointerKey GLOBAL_OUTPUT_STREAM_POINTER_KEY = new AstGlobalPointerKey( + public static final AstGlobalPointerKey GLOBAL_OUTPUT_STREAM_POINTER_KEY = new AstGlobalPointerKey( PythonModVisitorWithBuiltinFunctions.class.getPackageName().replace('.', '/') + "/OUT"); private static final String PRINT_FUNCTION_VARIABLE_NAME = "print"; From 0948ecaa84e0c89e64c9b69c1df4af2acbe7b8ac Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 17:40:44 -0400 Subject: [PATCH 094/172] Make API more consistent wih WALA. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 6 +++--- .../EvaluateHybridizeFunctionRefactoringHandler.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 1a771e2e4..a778e50b7 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -450,7 +450,7 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap // is this instance being created outside this function? if (!(creationNode.getMethod().getReference().equals(this.getMethodReference()) - || newSiteReference.getDeclaredType().equals(this.getTypeReference()))) + || newSiteReference.getDeclaredType().equals(this.getDeclaringClass()))) return false; } @@ -487,11 +487,11 @@ private Set getCallGraphNodes(CallGraph callGraph) throws Undeterminable } public MethodReference getMethodReference() { - TypeReference typeReference = getTypeReference(); + TypeReference typeReference = getDeclaringClass(); return MethodReference.findOrCreate(typeReference, AstMethodReference.fnSelector); } - public TypeReference getTypeReference() { + public TypeReference getDeclaringClass() { File containingFile = this.getContainingFile(); String filename = containingFile.getName(); String modifiedIdentifier = this.getIdentifier().replace('.', '/'); diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 4fcde824c..1a36e7ce0 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -244,7 +244,7 @@ private static String[] buildFunctionAttributeColumnNames() { } private static void printFunction(CSVPrinter printer, Function function) throws IOException { - Object[] initialColumnValues = buildAttributeColumnValues(function, function.getMethodReference(), function.getTypeReference(), + Object[] initialColumnValues = buildAttributeColumnValues(function, function.getMethodReference(), function.getDeclaringClass(), function.getNumberOfParameters(), function.getLikelyHasTensorParameter(), function.isHybrid(), function.getHasPythonSideEffects()); From 13a902270f99eaed5281297e70cd28e3d6c1dd5b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 18:15:54 -0400 Subject: [PATCH 095/172] Remove else clause. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index a778e50b7..124f15c97 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -412,8 +412,8 @@ private Set filterSideEffects(Iterable modSet, CallGraph if (allCreationsWithinThisFunction(instanceKey, callGraph)) continue; // filter this pointer out. - else - ret.add(fieldPointerKey); + + ret.add(fieldPointerKey); } else if (pointerKey instanceof LocalPointerKey) { LocalPointerKey localPointerKey = (LocalPointerKey) pointerKey; OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(localPointerKey); @@ -425,8 +425,8 @@ private Set filterSideEffects(Iterable modSet, CallGraph if (skipPointerKey) continue; // filter this pointer out. - else - ret.add(localPointerKey); + + ret.add(localPointerKey); } else if (pointerKey instanceof AstGlobalPointerKey) { AstGlobalPointerKey globalPointerKey = (AstGlobalPointerKey) pointerKey; From 0b9f25a3392a018263d56ddb185f726796f1b38a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 26 Oct 2023 18:38:34 -0400 Subject: [PATCH 096/172] Progress. --- .../hybridize/core/analysis/Function.java | 55 ++++++++++++++++--- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 124f15c97..73f40568b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -402,7 +402,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet, CallGraph callGraph, - PointerAnalysis pointerAnalysis) { + PointerAnalysis pointerAnalysis) throws UndeterminablePythonSideEffectsException { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { @@ -410,7 +410,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph InstanceFieldPointerKey fieldPointerKey = (InstanceFieldPointerKey) pointerKey; InstanceKey instanceKey = fieldPointerKey.getInstanceKey(); - if (allCreationsWithinThisFunction(instanceKey, callGraph)) + if (allCreationsWithinClosure(this.getMethodReference(), instanceKey, callGraph)) continue; // filter this pointer out. ret.add(fieldPointerKey); @@ -421,7 +421,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph boolean skipPointerKey = true; for (InstanceKey ik : pointsToSet) - skipPointerKey &= allCreationsWithinThisFunction(ik, callGraph); + skipPointerKey &= allCreationsWithinClosure(this.getMethodReference(), ik, callGraph); if (skipPointerKey) continue; // filter this pointer out. @@ -441,7 +441,29 @@ private Set filterSideEffects(Iterable modSet, CallGraph return ret; } - private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGraph callGraph) { + private static boolean allCreationsWithinClosure(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) + throws UndeterminablePythonSideEffectsException { + // check this function. + if (allCreationsWithin(methodReference, instanceKey, callGraph)) + return true; + + // otherwise, check its callees. + Set cgNodes = getCallGraphNodes(methodReference, callGraph); + + for (CGNode node : cgNodes) + // check the called functions. + for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { + CGNode next = succNodes.next(); + MethodReference reference = next.getMethod().getReference(); + + if (allCreationsWithinClosure(reference, instanceKey, callGraph)) + return true; + } + + return false; + } + + private static boolean allCreationsWithin(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) { // for each creation site of the given instance. for (Iterator> it = instanceKey.getCreationSites(callGraph); it.hasNext();) { Pair creationSite = it.next(); @@ -449,8 +471,8 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap NewSiteReference newSiteReference = creationSite.snd; // is this instance being created outside this function? - if (!(creationNode.getMethod().getReference().equals(this.getMethodReference()) - || newSiteReference.getDeclaredType().equals(this.getDeclaringClass()))) + if (!(creationNode.getMethod().getReference().equals(methodReference) + || newSiteReference.getDeclaredType().equals(methodReference.getDeclaringClass()))) return false; } @@ -462,15 +484,30 @@ private boolean allCreationsWithinThisFunction(InstanceKey instanceKey, CallGrap * * @param callGraph The {@link CallGraph} to search. * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. - * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. + * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This + * needs to be generic. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ private Set getCallGraphNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { - MethodReference methodReference = this.getMethodReference(); + return getCallGraphNodes(this.getMethodReference(), callGraph); + } + + /** + * Get the {@link CallGraph} nodes corresponding to the given {@link MethodReference}. + * + * @param methodReference The method to search for. + * @param callGraph The {@link CallGraph} to search. + * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. + * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This + * needs to be generic. + * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. + */ + private static Set getCallGraphNodes(MethodReference methodReference, CallGraph callGraph) + throws UndeterminablePythonSideEffectsException { Set nodes = callGraph.getNodes(methodReference); if (nodes.isEmpty()) { - LOG.error("Can't get call graph nodes for: " + this + "."); + LOG.error("Can't get call graph nodes for: " + methodReference + "."); if (VERBOSE) { LOG.info("Method reference is: " + methodReference + "."); From ac85cece1523bb600a2677a2c2f87f3522573a41 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 13:19:25 -0400 Subject: [PATCH 097/172] More global keyword testing. --- .../testPythonSideEffects29/in/A.py | 15 +++++++++ .../in/requirements.txt | 0 .../testPythonSideEffects30/in/A.py | 15 +++++++++ .../in/requirements.txt | 0 .../testPythonSideEffects31/in/A.py | 14 ++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 33 +++++++++++++++++-- 7 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/A.py new file mode 100644 index 000000000..a94124fdc --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/A.py @@ -0,0 +1,15 @@ +a = 10 + + +def g(): + global a + a = 1 + + +def f(): + g() + + +assert a == 10 +f() +assert a == 1, "Function f() calls g(), function g() modifies a global variable." diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects29/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/A.py new file mode 100644 index 000000000..4be27c3ec --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/A.py @@ -0,0 +1,15 @@ +a = 10 + + +def g(): + a = 1 + + +def f(): + g() + + +assert a == 10 +f() +assert a == 10 + diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects30/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/A.py new file mode 100644 index 000000000..4597b490c --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/A.py @@ -0,0 +1,14 @@ +a = [10] + + +def g(): + a[0] = 1 + + +def f(): + g() + + +assert a == [10] +f() +assert a == [1] diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects31/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index d2401ed14..3e2bb8f73 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4934,10 +4934,11 @@ public void testPythonSideEffects20() throws Exception { Function f = getFunction("f"); assertFalse(f.isHybrid()); assertFalse(f.getLikelyHasTensorParameter()); - assertTrue(f.getHasPythonSideEffects()); + assertTrue("Function f() calls g(), which has Python side-effets. Thus, f() also has Python side-effects.", + f.getHasPythonSideEffects()); Function g = getFunction("g"); - assertTrue(g.getHasPythonSideEffects()); + assertTrue("Function g() modifies a global variable through the global keyword.", g.getHasPythonSideEffects()); } @Test @@ -5032,4 +5033,32 @@ public void testPythonSideEffects28() throws Exception { map.get("f").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertFalse(s)); map.get("g").stream().map(Function::getHasPythonSideEffects).forEach(s -> assertTrue(s)); } + + @Test + public void testPythonSideEffects29() throws Exception { + Function f = getFunction("f"); + assertTrue("Function f() calls g(), which has Python side-effets. Thus, f() also has Python side-effects.", + f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertTrue("Function g() modifies a global variable through the global keyword.", g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects30() throws Exception { + Function f = getFunction("f"); + assertFalse("Removed the global keyword from g().", f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertFalse("Function g() modifies a lobal variable (removed the global keyword).", g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects31() throws Exception { + Function f = getFunction("f"); + assertTrue(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertTrue(g.getHasPythonSideEffects()); + } } From e2e273588bb3ad9578aa03d236db8732cb857d73 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 17:01:27 -0400 Subject: [PATCH 098/172] If thera are no creations, then I guess they're not in the function. --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 73f40568b..52f349ba8 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -464,6 +464,8 @@ private static boolean allCreationsWithinClosure(MethodReference methodReference } private static boolean allCreationsWithin(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) { + int numCreations = 0; + // for each creation site of the given instance. for (Iterator> it = instanceKey.getCreationSites(callGraph); it.hasNext();) { Pair creationSite = it.next(); @@ -474,8 +476,14 @@ private static boolean allCreationsWithin(MethodReference methodReference, Insta if (!(creationNode.getMethod().getReference().equals(methodReference) || newSiteReference.getDeclaredType().equals(methodReference.getDeclaringClass()))) return false; + + ++numCreations; } + if (numCreations == 0) // if there are no creations. + // then, they can't be within this method. + return false; + return true; } From 4e8256612a8a8be7ef9134b34afa364fde4c63ef Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 17:01:54 -0400 Subject: [PATCH 099/172] Fix comment. --- .../core/wala/ml/PythonModRefWithBuiltinFunctions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 3398491fa..1d7074c8b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -83,7 +83,7 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { objectRefPointsToSet.forEach(objectRefIK -> { if (objectRefIK.getConcreteType().getReference().equals(list)) - // it's a list. Add the instance to the results. + // it's a list. Add the pointer to the results. this.result.add(objectRefPK); }); } From 6bba96465eaad199179e98e1250d1a4a409e3d3f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 17:22:14 -0400 Subject: [PATCH 100/172] Handle globals. --- .../hybridize/core/analysis/Function.java | 8 +++++++ .../ml/PythonModRefWithBuiltinFunctions.java | 21 +++++++++++++++++++ .../testPythonSideEffects32/in/A.py | 21 +++++++++++++++++++ .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 12 +++++++++++ 5 files changed, 62 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 52f349ba8..c1281089d 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -57,6 +57,7 @@ import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; +import com.ibm.wala.ipa.callgraph.propagation.StaticFieldKey; import com.ibm.wala.ipa.modref.ModRef; import com.ibm.wala.types.MethodReference; import com.ibm.wala.types.TypeReference; @@ -434,6 +435,13 @@ private Set filterSideEffects(Iterable modSet, CallGraph ret.add(globalPointerKey); else throw new IllegalArgumentException("Not expecting global pointer key: " + globalPointerKey + "."); + } else if (pointerKey instanceof StaticFieldKey) { + StaticFieldKey staticFieldKey = (StaticFieldKey) pointerKey; + if (staticFieldKey.getField().getName().toString().startsWith("global")) + // it's a global variable. It's accessible from anywhere. + ret.add(staticFieldKey); + else + throw new IllegalArgumentException("Not expecting a non-global static field: " + staticFieldKey + "."); } else throw new IllegalArgumentException("Not expecting pointer key: " + pointerKey + " of type: " + pointerKey.getClass() + "."); } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 1d7074c8b..8f32a18f7 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -6,17 +6,21 @@ import java.util.Collection; import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; +import com.ibm.wala.cast.ir.ssa.AstGlobalWrite; import com.ibm.wala.cast.ir.ssa.AstLexicalAccess.Access; import com.ibm.wala.cast.ir.ssa.AstLexicalRead; import com.ibm.wala.cast.python.modref.PythonModRef; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.ssa.PythonPropertyRead; import com.ibm.wala.cast.python.types.PythonTypes; +import com.ibm.wala.classLoader.IField; +import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.propagation.ConstantKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; +import com.ibm.wala.ipa.callgraph.propagation.StaticFieldKey; import com.ibm.wala.ipa.modref.ExtendedHeapModel; import com.ibm.wala.ssa.SSAInstruction; import com.ibm.wala.types.TypeReference; @@ -97,6 +101,23 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { private TypeReference getTypeReference(T ik) { return ik.getConcreteType().getReference(); } + + @Override + public void visitAstGlobalWrite(AstGlobalWrite instruction) { + super.visitAstGlobalWrite(instruction); + String globalName = instruction.getGlobalName(); + + // find the pointer key corresponding to this global. + for (PointerKey pk : this.pa.getPointerKeys()) { + if (pk instanceof StaticFieldKey) { + StaticFieldKey staticFieldKey = (StaticFieldKey) pk; + IField field = staticFieldKey.getField(); + Atom fieldName = field.getName(); + if (fieldName.toString().equals(globalName)) + this.result.add(this.h.getPointerKeyForStaticField(field)); + } + } + } } @Override diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/A.py new file mode 100644 index 000000000..258af26a4 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/A.py @@ -0,0 +1,21 @@ +a = 10 + + +def h(): + global a + a = 1 + + +def g(): + pass + + +def f(): + g() + + +assert a == 10 +f() +assert a == 10 +h() +assert a == 1 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects32/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 3e2bb8f73..a8c58e585 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5061,4 +5061,16 @@ public void testPythonSideEffects31() throws Exception { Function g = getFunction("g"); assertTrue(g.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects32() throws Exception { + Function f = getFunction("f"); + assertFalse(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertFalse(g.getHasPythonSideEffects()); + + Function h = this.getFunction("h"); + assertTrue(h.getHasPythonSideEffects()); + } } From e84a3e3e792e858fafa81de3a4cddff848010d0b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 18:08:08 -0400 Subject: [PATCH 101/172] Add FIXMEs. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c1281089d..55551e040 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -424,7 +424,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph for (InstanceKey ik : pointsToSet) skipPointerKey &= allCreationsWithinClosure(this.getMethodReference(), ik, callGraph); - if (skipPointerKey) + if (skipPointerKey) // FIXME: What if the points-to set is empty? continue; // filter this pointer out. ret.add(localPointerKey); @@ -503,6 +503,7 @@ private static boolean allCreationsWithin(MethodReference methodReference, Insta * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This * needs to be generic. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. + * FIXME: Change to getNodes(). */ private Set getCallGraphNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { return getCallGraphNodes(this.getMethodReference(), callGraph); From 57e9cfb7db0b9fd8c76f893f9569cf392b58682c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 18:09:50 -0400 Subject: [PATCH 102/172] Fix globals. --- .../hybridize/core/analysis/Function.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 55551e040..a79af58c5 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -436,12 +436,19 @@ private Set filterSideEffects(Iterable modSet, CallGraph else throw new IllegalArgumentException("Not expecting global pointer key: " + globalPointerKey + "."); } else if (pointerKey instanceof StaticFieldKey) { + // FIXME: Looks too much like the LocalPointerKey case. StaticFieldKey staticFieldKey = (StaticFieldKey) pointerKey; - if (staticFieldKey.getField().getName().toString().startsWith("global")) - // it's a global variable. It's accessible from anywhere. - ret.add(staticFieldKey); - else - throw new IllegalArgumentException("Not expecting a non-global static field: " + staticFieldKey + "."); + OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(staticFieldKey); + + boolean skipPointerKey = true; + + for (InstanceKey ik : pointsToSet) + skipPointerKey &= allCreationsWithinClosure(this.getMethodReference(), ik, callGraph); + + if (skipPointerKey && !pointsToSet.isEmpty()) + continue; // filter this pointer out. + + ret.add(staticFieldKey); } else throw new IllegalArgumentException("Not expecting pointer key: " + pointerKey + " of type: " + pointerKey.getClass() + "."); } @@ -502,8 +509,7 @@ private static boolean allCreationsWithin(MethodReference methodReference, Insta * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This * needs to be generic. - * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. - * FIXME: Change to getNodes(). + * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. FIXME: Change to getNodes(). */ private Set getCallGraphNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { return getCallGraphNodes(this.getMethodReference(), callGraph); From 05a0af1a461758fc98d7e9569e89e682e6b8e811 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 27 Oct 2023 18:15:16 -0400 Subject: [PATCH 103/172] Handle empty points-to set case. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index a79af58c5..e4d8ebc24 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -424,7 +424,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph for (InstanceKey ik : pointsToSet) skipPointerKey &= allCreationsWithinClosure(this.getMethodReference(), ik, callGraph); - if (skipPointerKey) // FIXME: What if the points-to set is empty? + if (skipPointerKey && !pointsToSet.isEmpty()) continue; // filter this pointer out. ret.add(localPointerKey); From b446460a821c406b9fc0ddec4cfe9e2fa2cd7029 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 30 Oct 2023 12:27:22 -0400 Subject: [PATCH 104/172] Upgrade to Ariadne 0.11.0-SNAPSHOT. Use the globals ModRef analysis from there. --- .../META-INF/MANIFEST.MF | 2 +- .../ml/PythonModRefWithBuiltinFunctions.java | 21 ------------------- hybridize.target | 2 +- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index 964efe5de..d1f03f776 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -28,7 +28,7 @@ Import-Package: com.google.common.collect, com.ibm.wala.cast.python.loader;version="0.1.0", com.ibm.wala.cast.python.ml.analysis;version="0.1.0", com.ibm.wala.cast.python.ml.client;version="0.1.0", - com.ibm.wala.cast.python.modref, + com.ibm.wala.cast.python.modref;version="0.10.0", com.ibm.wala.cast.python.ssa, com.ibm.wala.cast.python.types, com.ibm.wala.cast.python.util, diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java index 8f32a18f7..1d7074c8b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/wala/ml/PythonModRefWithBuiltinFunctions.java @@ -6,21 +6,17 @@ import java.util.Collection; import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; -import com.ibm.wala.cast.ir.ssa.AstGlobalWrite; import com.ibm.wala.cast.ir.ssa.AstLexicalAccess.Access; import com.ibm.wala.cast.ir.ssa.AstLexicalRead; import com.ibm.wala.cast.python.modref.PythonModRef; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.ssa.PythonPropertyRead; import com.ibm.wala.cast.python.types.PythonTypes; -import com.ibm.wala.classLoader.IField; -import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.propagation.ConstantKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; import com.ibm.wala.ipa.callgraph.propagation.PointerKey; -import com.ibm.wala.ipa.callgraph.propagation.StaticFieldKey; import com.ibm.wala.ipa.modref.ExtendedHeapModel; import com.ibm.wala.ssa.SSAInstruction; import com.ibm.wala.types.TypeReference; @@ -101,23 +97,6 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { private TypeReference getTypeReference(T ik) { return ik.getConcreteType().getReference(); } - - @Override - public void visitAstGlobalWrite(AstGlobalWrite instruction) { - super.visitAstGlobalWrite(instruction); - String globalName = instruction.getGlobalName(); - - // find the pointer key corresponding to this global. - for (PointerKey pk : this.pa.getPointerKeys()) { - if (pk instanceof StaticFieldKey) { - StaticFieldKey staticFieldKey = (StaticFieldKey) pk; - IField field = staticFieldKey.getField(); - Atom fieldName = field.getName(); - if (fieldName.toString().equals(globalName)) - this.result.add(this.h.getPointerKeyForStaticField(field)); - } - } - } } @Override diff --git a/hybridize.target b/hybridize.target index 0faccce1b..1c412cc0a 100644 --- a/hybridize.target +++ b/hybridize.target @@ -29,7 +29,7 @@ com.ibm.wala com.ibm.wala.cast.python.ml - 0.10.0-SNAPSHOT + 0.11.0-SNAPSHOT jar From ceb822a0dbaaee4086a907e566586550b33cfdec Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 30 Oct 2023 13:27:39 -0400 Subject: [PATCH 105/172] Shorten method name. --- .../hunter/hybridize/core/analysis/Function.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index e4d8ebc24..c65f68e8e 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -372,7 +372,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis nodes = getCallGraphNodes(callGraph); + Set nodes = getNodes(callGraph); // for each node. for (CGNode cgNode : nodes) { @@ -463,7 +463,7 @@ private static boolean allCreationsWithinClosure(MethodReference methodReference return true; // otherwise, check its callees. - Set cgNodes = getCallGraphNodes(methodReference, callGraph); + Set cgNodes = getNodes(methodReference, callGraph); for (CGNode node : cgNodes) // check the called functions. @@ -509,10 +509,10 @@ private static boolean allCreationsWithin(MethodReference methodReference, Insta * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This * needs to be generic. - * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. FIXME: Change to getNodes(). + * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ - private Set getCallGraphNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { - return getCallGraphNodes(this.getMethodReference(), callGraph); + private Set getNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { + return getNodes(this.getMethodReference(), callGraph); } /** @@ -525,7 +525,7 @@ private Set getCallGraphNodes(CallGraph callGraph) throws Undeterminable * needs to be generic. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ - private static Set getCallGraphNodes(MethodReference methodReference, CallGraph callGraph) + private static Set getNodes(MethodReference methodReference, CallGraph callGraph) throws UndeterminablePythonSideEffectsException { Set nodes = callGraph.getNodes(methodReference); From 15daf3ad3494e94bf59b13468d9097e407f9278a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 30 Oct 2023 17:39:35 -0400 Subject: [PATCH 106/172] Fix exception handling. --- .../hybridize/core/analysis/Function.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c65f68e8e..7a78944e4 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -374,6 +374,9 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis nodes = getNodes(callGraph); + if (nodes.isEmpty()) + throw new UndeterminablePythonSideEffectsException(this.getMethodReference()); + // for each node. for (CGNode cgNode : nodes) { // Get the locations (pointers) modified by this function. @@ -403,7 +406,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis filterSideEffects(Iterable modSet, CallGraph callGraph, - PointerAnalysis pointerAnalysis) throws UndeterminablePythonSideEffectsException { + PointerAnalysis pointerAnalysis) { Set ret = new HashSet<>(); for (PointerKey pointerKey : modSet) { @@ -456,8 +459,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph return ret; } - private static boolean allCreationsWithinClosure(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) - throws UndeterminablePythonSideEffectsException { + private static boolean allCreationsWithinClosure(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) { // check this function. if (allCreationsWithin(methodReference, instanceKey, callGraph)) return true; @@ -465,6 +467,9 @@ private static boolean allCreationsWithinClosure(MethodReference methodReference // otherwise, check its callees. Set cgNodes = getNodes(methodReference, callGraph); + if (cgNodes.isEmpty()) + throw new IllegalArgumentException("Can't find call graph nodes corresponding to: " + methodReference + "."); + for (CGNode node : cgNodes) // check the called functions. for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { @@ -507,11 +512,9 @@ private static boolean allCreationsWithin(MethodReference methodReference, Insta * * @param callGraph The {@link CallGraph} to search. * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. - * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This - * needs to be generic. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ - private Set getNodes(CallGraph callGraph) throws UndeterminablePythonSideEffectsException { + private Set getNodes(CallGraph callGraph) { return getNodes(this.getMethodReference(), callGraph); } @@ -521,12 +524,9 @@ private Set getNodes(CallGraph callGraph) throws UndeterminablePythonSid * @param methodReference The method to search for. * @param callGraph The {@link CallGraph} to search. * @return The nodes in the {@link CallGraph} corresponding to this {@link Function}. - * @throws UndeterminablePythonSideEffectsException If this {@link Function} can't be found in the given {@link CallGraph}. FIXME: This - * needs to be generic. * @apiNote There can be multiple nodes for a single {@link Function} under the current representation. */ - private static Set getNodes(MethodReference methodReference, CallGraph callGraph) - throws UndeterminablePythonSideEffectsException { + private static Set getNodes(MethodReference methodReference, CallGraph callGraph) { Set nodes = callGraph.getNodes(methodReference); if (nodes.isEmpty()) { @@ -536,8 +536,6 @@ private static Set getNodes(MethodReference methodReference, CallGraph c LOG.info("Method reference is: " + methodReference + "."); LOG.info("Call graph nodes:\n" + callGraph.stream().map(Objects::toString).collect(Collectors.joining("\n"))); } - - throw new UndeterminablePythonSideEffectsException(methodReference); } LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); From e05287972a8dff1ec1c1e17b750d73ad28bf3016 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 30 Oct 2023 17:39:45 -0400 Subject: [PATCH 107/172] Use `this`.` --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 7a78944e4..343ba93ab 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -372,7 +372,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis nodes = getNodes(callGraph); + Set nodes = this.getNodes(callGraph); if (nodes.isEmpty()) throw new UndeterminablePythonSideEffectsException(this.getMethodReference()); From 2b34b298943cb01b516c4c89b3b544b9fc3f97a1 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 30 Oct 2023 17:46:19 -0400 Subject: [PATCH 108/172] Cleanup. --- .../hybridize/core/analysis/Function.java | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 343ba93ab..7d734aced 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -418,9 +418,8 @@ private Set filterSideEffects(Iterable modSet, CallGraph continue; // filter this pointer out. ret.add(fieldPointerKey); - } else if (pointerKey instanceof LocalPointerKey) { - LocalPointerKey localPointerKey = (LocalPointerKey) pointerKey; - OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(localPointerKey); + } else if (pointerKey instanceof LocalPointerKey || pointerKey instanceof StaticFieldKey) { + OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(pointerKey); boolean skipPointerKey = true; @@ -430,7 +429,7 @@ private Set filterSideEffects(Iterable modSet, CallGraph if (skipPointerKey && !pointsToSet.isEmpty()) continue; // filter this pointer out. - ret.add(localPointerKey); + ret.add(pointerKey); } else if (pointerKey instanceof AstGlobalPointerKey) { AstGlobalPointerKey globalPointerKey = (AstGlobalPointerKey) pointerKey; @@ -438,20 +437,6 @@ private Set filterSideEffects(Iterable modSet, CallGraph ret.add(globalPointerKey); else throw new IllegalArgumentException("Not expecting global pointer key: " + globalPointerKey + "."); - } else if (pointerKey instanceof StaticFieldKey) { - // FIXME: Looks too much like the LocalPointerKey case. - StaticFieldKey staticFieldKey = (StaticFieldKey) pointerKey; - OrdinalSet pointsToSet = pointerAnalysis.getPointsToSet(staticFieldKey); - - boolean skipPointerKey = true; - - for (InstanceKey ik : pointsToSet) - skipPointerKey &= allCreationsWithinClosure(this.getMethodReference(), ik, callGraph); - - if (skipPointerKey && !pointsToSet.isEmpty()) - continue; // filter this pointer out. - - ret.add(staticFieldKey); } else throw new IllegalArgumentException("Not expecting pointer key: " + pointerKey + " of type: " + pointerKey.getClass() + "."); } From 59137747c2beec754512b7379380e7edd66981f3 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 11:32:22 -0400 Subject: [PATCH 109/172] Add Eclipse version to CONTRIBUTING.md. There seems to be an M2E bug (or fix?) that is preventing the resolution of the depedencies. --- CONTRIBUTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f97a78ae3..7f2304705 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,15 @@ Please see our [wiki] for more information regarding development. +## Eclipse Environment + +The plug-ins are being developed on the following Eclipse versions. Currently, newer versions of Eclipse will not resolve M2E dependencies: + + Eclipse IDE for RCP and RAP Developers (includes Incubating components) + + Version: 2023-03 (4.27.0) + Build id: 20230309-1520 + ## Building The project includes a maven configuration file using the Tycho plug-in, which is part of the [maven eclipse plugin](http://www.eclipse.org/m2e). Running `mvn install` will install *most* dependencies. Note that if you are not using maven, this plugin depends on the [Common Eclipse Refactoring Framework], the **Eclipse SDK**, **Eclipse SDK tests**, the **Eclipse testing framework** (may also be called the **Eclipse Test Framework**), [Ariadne], [WALA], and [PyDev]. Some of these can be installed from the "Install New Software..." menu option under "Help" in Eclipse (choose to "work with" "The Eclipse Project Updates"). Others may need to be obtained from their respective update sites (see below). From 9ceca36f28227e993c25e6dcd17e8c65090a8815 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 11:58:50 -0400 Subject: [PATCH 110/172] Eclipse metadata updates. --- edu.cuny.hunter.hybridize.core/.classpath | 1 + .../build.properties | 1 + edu.cuny.hunter.hybridize.eval/.classpath | 8 +++-- .../.settings/org.eclipse.jdt.core.prefs | 1 + .../.settings/org.eclipse.pde.prefs | 35 +++++++++++++++++++ edu.cuny.hunter.hybridize.tests/.classpath | 1 + .../.settings/org.eclipse.pde.prefs | 35 +++++++++++++++++++ edu.cuny.hunter.hybridize.ui/.classpath | 1 + .../.settings/org.eclipse.pde.prefs | 35 +++++++++++++++++++ 9 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.pde.prefs create mode 100644 edu.cuny.hunter.hybridize.tests/.settings/org.eclipse.pde.prefs create mode 100644 edu.cuny.hunter.hybridize.ui/.settings/org.eclipse.pde.prefs diff --git a/edu.cuny.hunter.hybridize.core/.classpath b/edu.cuny.hunter.hybridize.core/.classpath index 685a6999c..c6fdc8fb8 100644 --- a/edu.cuny.hunter.hybridize.core/.classpath +++ b/edu.cuny.hunter.hybridize.core/.classpath @@ -3,6 +3,7 @@ + diff --git a/edu.cuny.hunter.hybridize.core/build.properties b/edu.cuny.hunter.hybridize.core/build.properties index 67f334176..9ce5e7704 100644 --- a/edu.cuny.hunter.hybridize.core/build.properties +++ b/edu.cuny.hunter.hybridize.core/build.properties @@ -8,3 +8,4 @@ bin.includes = META-INF/,\ pandas.xml,\ functools.xml,\ tensorflow.xml +jre.compilation.profile = JavaSE-11 diff --git a/edu.cuny.hunter.hybridize.eval/.classpath b/edu.cuny.hunter.hybridize.eval/.classpath index 1db08c6b4..1d45a8a9f 100644 --- a/edu.cuny.hunter.hybridize.eval/.classpath +++ b/edu.cuny.hunter.hybridize.eval/.classpath @@ -1,7 +1,11 @@ - + + + + + - + diff --git a/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.jdt.core.prefs b/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.jdt.core.prefs index c9545f06a..052ee6edb 100644 --- a/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.jdt.core.prefs +++ b/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.jdt.core.prefs @@ -4,6 +4,7 @@ org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning org.eclipse.jdt.core.compiler.release=enabled org.eclipse.jdt.core.compiler.source=11 diff --git a/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.pde.prefs b/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.pde.prefs new file mode 100644 index 000000000..a37b7af59 --- /dev/null +++ b/edu.cuny.hunter.hybridize.eval/.settings/org.eclipse.pde.prefs @@ -0,0 +1,35 @@ +compilers.f.unresolved-features=1 +compilers.f.unresolved-plugins=1 +compilers.incompatible-environment=1 +compilers.p.build=1 +compilers.p.build.bin.includes=1 +compilers.p.build.encodings=2 +compilers.p.build.java.compiler=2 +compilers.p.build.java.compliance=1 +compilers.p.build.missing.output=2 +compilers.p.build.output.library=1 +compilers.p.build.source.library=1 +compilers.p.build.src.includes=1 +compilers.p.deprecated=1 +compilers.p.discouraged-class=1 +compilers.p.exec-env-too-low=2 +compilers.p.internal=1 +compilers.p.missing-packages=2 +compilers.p.missing-version-export-package=2 +compilers.p.missing-version-import-package=2 +compilers.p.missing-version-require-bundle=2 +compilers.p.no-required-att=0 +compilers.p.no.automatic.module=1 +compilers.p.not-externalized-att=2 +compilers.p.service.component.without.lazyactivation=1 +compilers.p.unknown-attribute=1 +compilers.p.unknown-class=1 +compilers.p.unknown-element=1 +compilers.p.unknown-identifier=1 +compilers.p.unknown-resource=1 +compilers.p.unresolved-ex-points=0 +compilers.p.unresolved-import=0 +compilers.s.create-docs=false +compilers.s.doc-folder=doc +compilers.s.open-tags=1 +eclipse.preferences.version=1 diff --git a/edu.cuny.hunter.hybridize.tests/.classpath b/edu.cuny.hunter.hybridize.tests/.classpath index 3071286c1..20ff5e592 100644 --- a/edu.cuny.hunter.hybridize.tests/.classpath +++ b/edu.cuny.hunter.hybridize.tests/.classpath @@ -3,6 +3,7 @@ + diff --git a/edu.cuny.hunter.hybridize.tests/.settings/org.eclipse.pde.prefs b/edu.cuny.hunter.hybridize.tests/.settings/org.eclipse.pde.prefs new file mode 100644 index 000000000..a37b7af59 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/.settings/org.eclipse.pde.prefs @@ -0,0 +1,35 @@ +compilers.f.unresolved-features=1 +compilers.f.unresolved-plugins=1 +compilers.incompatible-environment=1 +compilers.p.build=1 +compilers.p.build.bin.includes=1 +compilers.p.build.encodings=2 +compilers.p.build.java.compiler=2 +compilers.p.build.java.compliance=1 +compilers.p.build.missing.output=2 +compilers.p.build.output.library=1 +compilers.p.build.source.library=1 +compilers.p.build.src.includes=1 +compilers.p.deprecated=1 +compilers.p.discouraged-class=1 +compilers.p.exec-env-too-low=2 +compilers.p.internal=1 +compilers.p.missing-packages=2 +compilers.p.missing-version-export-package=2 +compilers.p.missing-version-import-package=2 +compilers.p.missing-version-require-bundle=2 +compilers.p.no-required-att=0 +compilers.p.no.automatic.module=1 +compilers.p.not-externalized-att=2 +compilers.p.service.component.without.lazyactivation=1 +compilers.p.unknown-attribute=1 +compilers.p.unknown-class=1 +compilers.p.unknown-element=1 +compilers.p.unknown-identifier=1 +compilers.p.unknown-resource=1 +compilers.p.unresolved-ex-points=0 +compilers.p.unresolved-import=0 +compilers.s.create-docs=false +compilers.s.doc-folder=doc +compilers.s.open-tags=1 +eclipse.preferences.version=1 diff --git a/edu.cuny.hunter.hybridize.ui/.classpath b/edu.cuny.hunter.hybridize.ui/.classpath index 685a6999c..c6fdc8fb8 100644 --- a/edu.cuny.hunter.hybridize.ui/.classpath +++ b/edu.cuny.hunter.hybridize.ui/.classpath @@ -3,6 +3,7 @@ + diff --git a/edu.cuny.hunter.hybridize.ui/.settings/org.eclipse.pde.prefs b/edu.cuny.hunter.hybridize.ui/.settings/org.eclipse.pde.prefs new file mode 100644 index 000000000..a37b7af59 --- /dev/null +++ b/edu.cuny.hunter.hybridize.ui/.settings/org.eclipse.pde.prefs @@ -0,0 +1,35 @@ +compilers.f.unresolved-features=1 +compilers.f.unresolved-plugins=1 +compilers.incompatible-environment=1 +compilers.p.build=1 +compilers.p.build.bin.includes=1 +compilers.p.build.encodings=2 +compilers.p.build.java.compiler=2 +compilers.p.build.java.compliance=1 +compilers.p.build.missing.output=2 +compilers.p.build.output.library=1 +compilers.p.build.source.library=1 +compilers.p.build.src.includes=1 +compilers.p.deprecated=1 +compilers.p.discouraged-class=1 +compilers.p.exec-env-too-low=2 +compilers.p.internal=1 +compilers.p.missing-packages=2 +compilers.p.missing-version-export-package=2 +compilers.p.missing-version-import-package=2 +compilers.p.missing-version-require-bundle=2 +compilers.p.no-required-att=0 +compilers.p.no.automatic.module=1 +compilers.p.not-externalized-att=2 +compilers.p.service.component.without.lazyactivation=1 +compilers.p.unknown-attribute=1 +compilers.p.unknown-class=1 +compilers.p.unknown-element=1 +compilers.p.unknown-identifier=1 +compilers.p.unknown-resource=1 +compilers.p.unresolved-ex-points=0 +compilers.p.unresolved-import=0 +compilers.s.create-docs=false +compilers.s.doc-folder=doc +compilers.s.open-tags=1 +eclipse.preferences.version=1 From fc7824cc8c64a5babfb0409ab6af615c1503eca3 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 12:39:28 -0400 Subject: [PATCH 111/172] Use PLUGIN_ID. --- .../cuny/hunter/hybridize/core/analysis/Function.java | 9 ++++----- .../EvaluateHybridizeFunctionRefactoringHandler.java | 4 ++-- .../tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 7d734aced..9fb602041 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -64,7 +64,6 @@ import com.ibm.wala.util.collections.Pair; import com.ibm.wala.util.intset.OrdinalSet; -import edu.cuny.citytech.refactoring.common.core.RefactorableProgramEntity; import edu.cuny.hunter.hybridize.core.utils.RefactoringAvailabilityTester; import edu.cuny.hunter.hybridize.core.wala.ml.PythonModRefWithBuiltinFunctions; @@ -74,9 +73,9 @@ * @author Raffi Khatchadourian * @author Tatiana Castro Vélez */ -public class Function extends RefactorableProgramEntity { +public class Function { - public static final String BUNDLE_SYMBOLIC_NAME = FrameworkUtil.getBundle(Function.class).getSymbolicName(); + public static final String PLUGIN_ID = FrameworkUtil.getBundle(Function.class).getSymbolicName(); private final class FunctionStatusContext extends RefactoringStatusContext { @Override @@ -813,7 +812,7 @@ public void addStatusEntry(PreconditionFailure failure, String message) { || failure != PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS : "Can't both have side-effects filled and have tem undterminable."; RefactoringStatusContext context = new FunctionStatusContext(); - this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, BUNDLE_SYMBOLIC_NAME, failure.getCode(), this); + this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, PLUGIN_ID, failure.getCode(), this); } /** @@ -1001,7 +1000,7 @@ public Boolean getHasPythonSideEffects() { protected void setHasPythonSideEffects(Boolean hasPythonSideEffects) { assert this.hasPythonSideEffects == null : "Can only set side-effects once."; - assert hasPythonSideEffects == null || this.getStatus().getEntryMatchingCode(BUNDLE_SYMBOLIC_NAME, + assert hasPythonSideEffects == null || this.getStatus().getEntryMatchingCode(PLUGIN_ID, PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()) == null : "Can't set side-effects if they are undeterminable."; this.hasPythonSideEffects = hasPythonSideEffects; diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 1a36e7ce0..4329ad272 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -1,6 +1,6 @@ package edu.cuny.hunter.hybridize.eval.handlers; -import static edu.cuny.hunter.hybridize.core.analysis.Function.BUNDLE_SYMBOLIC_NAME; +import static edu.cuny.hunter.hybridize.core.analysis.Function.PLUGIN_ID; import static edu.cuny.hunter.hybridize.core.utils.Util.createHybridizeFunctionRefactoring; import static org.eclipse.core.runtime.Platform.getLog; import static org.python.pydev.plugin.nature.PythonNature.PYTHON_NATURE_ID; @@ -147,7 +147,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // optimization available functions. These are the "filtered" functions. We consider functions to be candidates iff they // have a tensor-like parameter or are currently hybrid. Set candidates = functions.stream().filter(Function::isHybridizationAvailable).filter(f -> f.getStatus() - .getEntryMatchingCode(BUNDLE_SYMBOLIC_NAME, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()) == null) + .getEntryMatchingCode(PLUGIN_ID, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()) == null) .collect(Collectors.toSet()); resultsPrinter.print(candidates.size()); // number. diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index a8c58e585..96586b3bd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -874,7 +874,7 @@ private static void checkSideEffectStatus(Function function) { RefactoringStatus status = function.getStatus(); assertTrue("Should fail due to a call graph issue, either a decorated function or missing function invocation.", status.hasError()); assertNull(function.getHasPythonSideEffects()); - RefactoringStatusEntry entry = status.getEntryMatchingCode(Function.BUNDLE_SYMBOLIC_NAME, + RefactoringStatusEntry entry = status.getEntryMatchingCode(Function.PLUGIN_ID, PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS.getCode()); assertNotNull(entry); } @@ -4493,7 +4493,7 @@ public void testModel3() throws Exception { private static void checkOptimizationNotAvailableStatus(Function f) { RefactoringStatus status = f.getStatus(); assertTrue("Should not be available for optimization.", status.hasError()); - Object entry = status.getEntryMatchingCode(Function.BUNDLE_SYMBOLIC_NAME, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()); + Object entry = status.getEntryMatchingCode(Function.PLUGIN_ID, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()); assertNotNull(entry); } From b5251ccff9b5148ccb22cfd759d3ec1b2a9cfe8f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 12:55:00 -0400 Subject: [PATCH 112/172] Additional parameter tests. --- .../testPythonSideEffects33/in/A.py | 12 ++++++++++++ .../testPythonSideEffects33/in/requirements.txt | 0 .../testPythonSideEffects34/in/A.py | 12 ++++++++++++ .../testPythonSideEffects34/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 15 +++++++++++++++ 5 files changed, 39 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/A.py new file mode 100644 index 000000000..80f93d473 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/A.py @@ -0,0 +1,12 @@ +def g(a_list): + return a_list[0] + + +def f(): + my_list = [10] + assert my_list[0] == 10 + g(my_list) + assert my_list[0] == 10 + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects33/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/A.py new file mode 100644 index 000000000..1f8fee694 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/A.py @@ -0,0 +1,12 @@ +def g(a): + a = 1 + + +def f(): + x = 10 + assert x == 10 + g(x) + assert x == 10 + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects34/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 96586b3bd..99d2a8caf 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5073,4 +5073,19 @@ public void testPythonSideEffects32() throws Exception { Function h = this.getFunction("h"); assertTrue(h.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects33() throws Exception { + Function g = getFunction("g"); + assertFalse("g() only returns the parameter.", g.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects34() throws Exception { + Function f = getFunction("f"); + assertFalse(f.getHasPythonSideEffects()); + + Function g = getFunction("g"); + assertFalse("g() modifies a copy of a parameter.", g.getHasPythonSideEffects()); + } } From b53b425f7d472e56e64e5f794dccc34ad9f43a72 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 13:26:18 -0400 Subject: [PATCH 113/172] Add side-effect test from docs. --- .../testPythonSideEffects35/in/A.py | 17 +++++++++++++++++ .../testPythonSideEffects35/in/requirements.txt | 1 + .../tests/HybridizeFunctionRefactoringTest.java | 12 +++++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/A.py new file mode 100644 index 000000000..1c25902ec --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/A.py @@ -0,0 +1,17 @@ +# From https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. +import tensorflow as tf + +external_list = [] + + +@tf.function +def side_effect(x): + tf.print('Python side effect') + external_list.append(x) + + +side_effect(1) +side_effect(1) +side_effect(1) +# The list append only happened once! +assert len(external_list) == 1 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects35/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 99d2a8caf..07f14b991 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -2133,9 +2133,6 @@ public void testHasLikelyTensorParameter17() throws Exception { } } - // TODO: Left off at https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. The model is not going to work - // because call() is called implicitly. See https://github.com/wala/ML/issues/24. - /** * Test for #2. From https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/function#features. Example with closures. */ @@ -5088,4 +5085,13 @@ public void testPythonSideEffects34() throws Exception { Function g = getFunction("g"); assertFalse("g() modifies a copy of a parameter.", g.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects35() throws Exception { + Function function = getFunction("side_effect"); + + assertTrue(function.isHybrid()); + assertFalse("side_effect() is passed an integer (from docs).", function.getLikelyHasTensorParameter()); + assertTrue("side_effect() modifies a global list.", function.getHasPythonSideEffects()); + } } From dc7d59c5d85053449904bde16b023478547f9590 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 14:40:38 -0400 Subject: [PATCH 114/172] More side-effect testing from docs. --- .../testPythonSideEffects36/in/A.py | 17 +++++++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects37/in/A.py | 17 +++++++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 18 ++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/A.py new file mode 100644 index 000000000..42d7e1929 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/A.py @@ -0,0 +1,17 @@ +# From https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. +# No printing in this one. +import tensorflow as tf + +external_list = [] + + +@tf.function +def side_effect(x): + external_list.append(x) + + +side_effect(1) +side_effect(1) +side_effect(1) +# The list append only happened once! +assert len(external_list) == 1 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects36/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/A.py new file mode 100644 index 000000000..ac0084f1d --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/A.py @@ -0,0 +1,17 @@ +# From https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. +import tensorflow as tf + +external_list = [1, 3, 5] + + +@tf.function +def no_side_effect(x): + tf.print('No python side effect') + return external_list[x] + +assert len(external_list) == 3 +no_side_effect(1) +no_side_effect(1) +no_side_effect(1) +# The list append only happened once! +assert len(external_list) == 3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects37/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 07f14b991..6e9bccd54 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5094,4 +5094,22 @@ public void testPythonSideEffects35() throws Exception { assertFalse("side_effect() is passed an integer (from docs).", function.getLikelyHasTensorParameter()); assertTrue("side_effect() modifies a global list.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects36() throws Exception { + Function function = getFunction("side_effect"); + + assertTrue(function.isHybrid()); + assertFalse("side_effect() is passed an integer (from docs).", function.getLikelyHasTensorParameter()); + assertTrue("side_effect() modifies a global list.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects37() throws Exception { + Function function = getFunction("no_side_effect"); + + assertTrue(function.isHybrid()); + assertFalse("no_side_effect() is passed an integer (from docs).", function.getLikelyHasTensorParameter()); + assertFalse("no_side_effect() doesn't modifies a global list.", function.getHasPythonSideEffects()); + } } From e604cdcdf768661c8f80ee6785f3430d19fa0afb Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 15:01:29 -0400 Subject: [PATCH 115/172] Add model side-effect tests. --- .../testPythonSideEffects38/in/A.py | 23 +++++++++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects39/in/A.py | 23 +++++++++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 21 +++++++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/A.py new file mode 100644 index 000000000..9b8666b64 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/A.py @@ -0,0 +1,23 @@ +# From https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. +import tensorflow as tf + + +class Model(tf.Module): + + def __init__(self): + self.v = tf.Variable(0) + self.counter = 0 + + @tf.function + def __call__(self): + if self.counter == 0: + # A python side-effect + self.counter += 1 + self.v.assign_add(1) + + return self.v + + +m = Model() +for n in range(3): + print(m().numpy()) # prints 1, 2, 3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects38/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/A.py new file mode 100644 index 000000000..a81c328c1 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/A.py @@ -0,0 +1,23 @@ +# From https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables. +import tensorflow as tf + + +class Model(tf.Module): + + def __init__(self): + self.v = tf.Variable(0) + self.counter = 0 + + @tf.function + def __call__(self): + if self.counter == 0: + # A python side-effect + self.counter += 1 + self.v.assign_add(1) + + return self.v + + +m = Model() +for n in range(3): + print(m.__call__().numpy()) # prints 1, 2, 3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects39/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 6e9bccd54..afce2ada9 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5112,4 +5112,25 @@ public void testPythonSideEffects37() throws Exception { assertFalse("no_side_effect() is passed an integer (from docs).", function.getLikelyHasTensorParameter()); assertFalse("no_side_effect() doesn't modifies a global list.", function.getHasPythonSideEffects()); } + + @Test + public void testPythonSideEffects38() throws Exception { + Function function = getFunction("Model.__call__"); + assertNotNull(function); + + assertTrue(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // Change to assertTrue() once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + assertNull(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects39() throws Exception { + Function function = getFunction("Model.__call__"); + assertNotNull(function); + + assertTrue(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + } } From 98e54d86611a63a8e5a8c17803f2ee9bd98a7f83 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 15:01:53 -0400 Subject: [PATCH 116/172] Add TODO. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index afce2ada9..ae062a456 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5133,4 +5133,6 @@ public void testPythonSideEffects39() throws Exception { assertFalse(function.getLikelyHasTensorParameter()); assertTrue(function.getHasPythonSideEffects()); } + + // TODO: Left off at https://www.tensorflow.org/guide/function#using_python_iterators_and_generators. } From 2e3f4f6d4025063581747d00158eb8b239c85d80 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 31 Oct 2023 23:02:38 -0400 Subject: [PATCH 117/172] More side-effects tests from the docs. --- .../testPythonSideEffects40/in/A.py | 16 +++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects41/in/A.py | 16 +++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 24 +++++++++++++++++-- 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/A.py new file mode 100644 index 000000000..4aee72880 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/A.py @@ -0,0 +1,16 @@ +# From https://www.tensorflow.org/guide/function#using_python_iterators_and_generators + +import tensorflow as tf + + +@tf.function +def buggy_consume_next(iterator): + tf.print("Value:", next(iterator)) + + +iterator = iter([1, 2, 3]) + +buggy_consume_next(iterator) +# This reuses the first value from the iterator, rather than consuming the next value. +buggy_consume_next(iterator) +buggy_consume_next(iterator) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects40/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/A.py new file mode 100644 index 000000000..d3f32ad38 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/A.py @@ -0,0 +1,16 @@ +# From https://www.tensorflow.org/guide/function#using_python_iterators_and_generators + +import tensorflow as tf + + +@tf.function +def good_consume_next(iterator): + # This is ok, iterator is a tf.data.Iterator + tf.print("Value:", next(iterator)) + + +ds = tf.data.Dataset.from_tensor_slices([1, 2, 3]) +iterator = iter(ds) +good_consume_next(iterator) +good_consume_next(iterator) +good_consume_next(iterator) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects41/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index ae062a456..bc91a1846 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5120,7 +5120,7 @@ public void testPythonSideEffects38() throws Exception { assertTrue(function.isHybrid()); assertFalse(function.getLikelyHasTensorParameter()); - // Change to assertTrue() once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed. + // Change to assertTrue() once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/271 is fixed: assertNull(function.getHasPythonSideEffects()); } @@ -5134,5 +5134,25 @@ public void testPythonSideEffects39() throws Exception { assertTrue(function.getHasPythonSideEffects()); } - // TODO: Left off at https://www.tensorflow.org/guide/function#using_python_iterators_and_generators. + @Test + public void testPythonSideEffects40() throws Exception { + Function function = getFunction("buggy_consume_next"); + + assertTrue(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + // TODO: Change to assertTrue() when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/278 is fixed: + assertFalse("next() moves the iterator's cursor, and the iterator is over a list.", function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects41() throws Exception { + Function function = getFunction("good_consume_next"); + + assertTrue(function.isHybrid()); + assertFalse("iterator still isn't a tensor. I wonder if you get speedup from that.", function.getLikelyHasTensorParameter()); + assertFalse("next() moves the iterator's cursor, but the iterator is over a dataset.", function.getHasPythonSideEffects()); + } + + // TODO: Left off at https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values. THese are still + // side-effects, I believe. } From 687802e4cb7ce941112035de3451c7c22a8a8f71 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 1 Nov 2023 10:45:28 -0400 Subject: [PATCH 118/172] Add tests for "All outputs of a tf.function must be return values." https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values --- .../testPythonSideEffects42/in/A.py | 21 ++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects43/in/A.py | 21 ++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects44/in/A.py | 21 ++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 33 +++++++++++++++++++ 7 files changed, 99 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/A.py new file mode 100644 index 000000000..61d032f7d --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +@tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects42/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/A.py new file mode 100644 index 000000000..97f300cfa --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +@tf.function +def leaky_function(a): + global x + # x = a + 1 # Bad - leaks local tensor + return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +# try: +# x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +# except AttributeError as expected: +# print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects43/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/A.py new file mode 100644 index 000000000..5603ac314 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +# @tf.function +def leaky_function(a): + global x + # x = a + 1 # Bad - leaks local tensor + return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +# try: +# x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +# except AttributeError as expected: +# print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects44/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index bc91a1846..ed846ec06 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -5153,6 +5154,38 @@ public void testPythonSideEffects41() throws Exception { assertFalse("next() moves the iterator's cursor, but the iterator is over a dataset.", function.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects42() throws Exception { + Function function = getFunction("leaky_function"); + + assertTrue(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects43() throws Exception { + Function function = getFunction("leaky_function"); + + assertTrue(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertFalse(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects44() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertFalse(function.getHasPythonSideEffects()); + + assertTrue(function.getStatus().isOK()); + assertTrue(function.getRefactoring() == Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID); + assertTrue(function.getPassingPrecondition() == PreconditionSuccess.P1); + assertEquals(Collections.singleton(Transformation.CONVERT_TO_HYBRID), function.getTransformations()); + } + // TODO: Left off at https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values. THese are still // side-effects, I believe. } From a866ed4fb63c1542e5795be3809b0f562d65a001 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 10:17:45 -0400 Subject: [PATCH 119/172] Add leaky test. --- .../testPythonSideEffects45/in/A.py | 21 +++++++++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 18 ++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/A.py new file mode 100644 index 000000000..177f430f7 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +@tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + # return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +# print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects45/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index ed846ec06..f286492e3 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5186,6 +5186,24 @@ public void testPythonSideEffects44() throws Exception { assertEquals(Collections.singleton(Transformation.CONVERT_TO_HYBRID), function.getTransformations()); } + @Test + public void testPythonSideEffects45() throws Exception { + Function function = getFunction("leaky_function"); + + assertTrue(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + + // We have a hybrid function with a tensor parameter and Python side-effects. Issue a warning. + RefactoringStatusEntry entry = function.getStatus().getEntryWithHighestSeverity(); + assertNotNull(entry); + assertEquals(RefactoringStatus.WARNING, entry.getSeverity()); + + assertNull(function.getRefactoring()); + assertNull(function.getPassingPrecondition()); + assertTrue(function.getTransformations().isEmpty()); + } + // TODO: Left off at https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values. THese are still // side-effects, I believe. } From a996a66b5eb913639ce9b8f4b25f075811ec8ce3 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 10:39:58 -0400 Subject: [PATCH 120/172] Add leaky test failure cases. --- .../core/analysis/PreconditionFailure.java | 17 +++++++- .../testPythonSideEffects46/in/A.py | 21 ++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects49/in/A.py | 21 ++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 42 +++++++++++++++++++ 6 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 7c25b3ef3..9764ba988 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -3,7 +3,22 @@ import java.util.Arrays; public enum PreconditionFailure { - CURRENTLY_NOT_HANDLED(1), OPTIMIZATION_NOT_AVAILABLE(2), UNDETERMINABLE_SIDE_EFFECTS(3); + CURRENTLY_NOT_HANDLED(1), + + /** + * Not a candidate. + */ + OPTIMIZATION_NOT_AVAILABLE(2), + + /** + * Either there is no call to the function, there is a call but don't handle it, or something about decorators? + */ + UNDETERMINABLE_SIDE_EFFECTS(3), + + /** + * P1 failure. + */ + HAS_SIDE_EFFECTS(4); static { // check that the codes are unique. diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/A.py new file mode 100644 index 000000000..e94607a9e --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +# @tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + # return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +# print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects46/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/A.py new file mode 100644 index 000000000..188ac7a96 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/A.py @@ -0,0 +1,21 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + +x = None + + +# @tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + return a + 2 + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects49/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index f286492e3..d9c0e7963 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5204,6 +5204,48 @@ public void testPythonSideEffects45() throws Exception { assertTrue(function.getTransformations().isEmpty()); } + @Test + public void testPythonSideEffects46() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + + RefactoringStatus status = function.getStatus(); + + // We have an eager function with a tensor parameter but Python side-effects. Should be a P1 failure. + assertFalse(status.isOK()); + assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); + assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); + assertNull(function.getPassingPrecondition()); + assertEquals(Collections.emptySet(), function.getTransformations()); + } + + @Test + public void testPythonSideEffects49() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertTrue(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + + // This is a P1 failure. + RefactoringStatus status = function.getStatus(); + assertFalse(status.isOK()); + assertTrue(status.hasError()); + assertFalse(status.hasFatalError()); + + RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); + assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); + assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, entry.getCode()); + assertEquals(function, entry.getData()); + + assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); + assertNull(function.getPassingPrecondition()); + assertTrue(function.getTransformations().isEmpty()); + } + // TODO: Left off at https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values. THese are still // side-effects, I believe. } From 230f2769df3823d6282b95b7c76314b0d17d652c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 11:10:24 -0400 Subject: [PATCH 121/172] Progress on testing. Some situations are already optimal. --- .../core/analysis/PreconditionFailure.java | 7 ++++++- .../HybridizeFunctionRefactoringTest.java | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 9764ba988..3ef90d3ac 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -18,7 +18,12 @@ public enum PreconditionFailure { /** * P1 failure. */ - HAS_SIDE_EFFECTS(4); + HAS_SIDE_EFFECTS(4), + + /** + * P2 "failure." + */ + ALREADY_OPTIMAL(5); static { // check that the codes are unique. diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index d9c0e7963..7363820e1 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5194,12 +5194,21 @@ public void testPythonSideEffects45() throws Exception { assertTrue(function.getLikelyHasTensorParameter()); assertTrue(function.getHasPythonSideEffects()); + RefactoringStatus status = function.getStatus(); + // We have a hybrid function with a tensor parameter and Python side-effects. Issue a warning. - RefactoringStatusEntry entry = function.getStatus().getEntryWithHighestSeverity(); - assertNotNull(entry); - assertEquals(RefactoringStatus.WARNING, entry.getSeverity()); + RefactoringStatusEntry warning = status.getEntryMatchingSeverity(RefactoringStatus.WARNING); + assertNotNull(warning); + assertEquals(RefactoringStatus.WARNING, warning.getSeverity()); + + // This situation is already "optimal," so we can't refactor it. + assertFalse(status.isOK()); + assertTrue(status.hasError()); + RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); + assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); + assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), entry.getCode()); - assertNull(function.getRefactoring()); + assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); assertNull(function.getPassingPrecondition()); assertTrue(function.getTransformations().isEmpty()); } @@ -5216,7 +5225,7 @@ public void testPythonSideEffects46() throws Exception { // We have an eager function with a tensor parameter but Python side-effects. Should be a P1 failure. assertFalse(status.isOK()); - assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); + assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS.getCode(), status.getEntryWithHighestSeverity().getCode()); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); assertNull(function.getPassingPrecondition()); assertEquals(Collections.emptySet(), function.getTransformations()); From eae5ac476b67ea9b8eed5c01e34d858100b5fa08 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 11:46:07 -0400 Subject: [PATCH 122/172] Add captured leaked tensor case. Hybrid case. See https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281. --- .../testPythonSideEffects47/in/A.py | 30 +++++++++++ .../in/requirements.txt | 2 + .../HybridizeFunctionRefactoringTest.java | 50 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/A.py new file mode 100644 index 000000000..75032e3d9 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/A.py @@ -0,0 +1,30 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf +from nose.tools import assert_raises + + +@tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + return x # Good - uses local tensor + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) + + +@tf.function +def captures_leaked_tensor(b): + b += x # Bad - `x` is leaked from `leaky_function` + return b + + +with assert_raises(TypeError): + captures_leaked_tensor(tf.constant(2)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/requirements.txt new file mode 100644 index 000000000..56020dd04 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects47/in/requirements.txt @@ -0,0 +1,2 @@ +tensorflow==2.9.3 +nose==1.3.7 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 7363820e1..454b50439 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5231,6 +5231,56 @@ public void testPythonSideEffects46() throws Exception { assertEquals(Collections.emptySet(), function.getTransformations()); } + @Test + public void testPythonSideEffects47() throws Exception { + Function leakyFunction = getFunction("leaky_function"); + + assertTrue(leakyFunction.isHybrid()); + assertTrue(leakyFunction.getLikelyHasTensorParameter()); + assertTrue(leakyFunction.getHasPythonSideEffects()); + + assertFalse("P2 \"failure.\"", leakyFunction.getStatus().isOK()); + assertEquals( + "Should have one warning and one error. The warning is for running a hybrid function that has side-effects. The error is that it is already \"optimal\".", + 2, leakyFunction.getStatus().getEntries().length); + assertEquals(RefactoringStatus.ERROR, leakyFunction.getStatus().getEntryWithHighestSeverity().getSeverity()); + assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), leakyFunction.getStatus().getEntryWithHighestSeverity().getCode()); + assertNotNull(leakyFunction.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING)); + + assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, leakyFunction.getRefactoring()); + assertNull(leakyFunction.getPassingPrecondition()); + assertTrue(leakyFunction.getTransformations().isEmpty()); + + Function capturesLeakedTensor = getFunction("captures_leaked_tensor"); + + assertTrue(capturesLeakedTensor.isHybrid()); + assertTrue(capturesLeakedTensor.getLikelyHasTensorParameter()); + + // NOTE: This function doesn't have Python side-effects, but it does capture a "leaky" tensor. See + // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281. + assertFalse(capturesLeakedTensor.getHasPythonSideEffects()); + + assertFalse(capturesLeakedTensor.getStatus().isOK()); + assertTrue(capturesLeakedTensor.getStatus().hasError()); + assertFalse(capturesLeakedTensor.getStatus().hasFatalError()); + RefactoringStatusEntry error = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.ERROR); + assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), error.getCode()); + + assertNotNull(capturesLeakedTensor.getRefactoring()); + assertEquals("P2 \"failure.\"", Refactoring.OPTIMIZE_HYBRID_FUNCTION, capturesLeakedTensor.getRefactoring()); + assertNull(capturesLeakedTensor.getPassingPrecondition()); + assertTrue(capturesLeakedTensor.getTransformations().isEmpty()); + + // NOTE: Change to assertTrue when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertFalse("We should warn about this.", capturesLeakedTensor.getStatus().hasWarning()); + + RefactoringStatusEntry warning = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING); + // NOTE: Change to assertNotNull when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + // NOTE: Add assertEquals(RefactoringStatus.WARNING, entry.getSeverity()) when + // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertNull("Warn about a hybrid function that leaks as a potential tensor.", warning); + } + @Test public void testPythonSideEffects49() throws Exception { Function function = getFunction("leaky_function"); From 699177be664b76d90a7681ce25090dccb0cfbd69 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 12:21:26 -0400 Subject: [PATCH 123/172] Add eager case for https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281. --- .../testPythonSideEffects48/in/A.py | 30 +++++++++++ .../in/requirements.txt | 2 + .../HybridizeFunctionRefactoringTest.java | 50 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/A.py new file mode 100644 index 000000000..e58c67b14 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/A.py @@ -0,0 +1,30 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf +from nose.tools import assert_raises + + +@tf.function +def leaky_function(a): + global x + x = a + 1 # Bad - leaks local tensor + return x # Good - uses local tensor + + +correct_a = leaky_function(tf.constant(1)) + +print(correct_a.numpy()) # Good - value obtained from function's returns +try: + x.numpy() # Bad - tensor leaked from inside the function, cannot be used here +except AttributeError as expected: + print(expected) + + +# @tf.function +def captures_leaked_tensor(b): + b += x # Bad - `x` is leaked from `leaky_function` + return b + + +with assert_raises(TypeError): + captures_leaked_tensor(tf.constant(2)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/requirements.txt new file mode 100644 index 000000000..56020dd04 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects48/in/requirements.txt @@ -0,0 +1,2 @@ +tensorflow==2.9.3 +nose==1.3.7 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 454b50439..3770d246f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5266,6 +5266,9 @@ public void testPythonSideEffects47() throws Exception { RefactoringStatusEntry error = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.ERROR); assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), error.getCode()); + // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertFalse("We should warn that the hybrid function is capturing leaked tensors.", capturesLeakedTensor.getStatus().hasWarning()); + assertNotNull(capturesLeakedTensor.getRefactoring()); assertEquals("P2 \"failure.\"", Refactoring.OPTIMIZE_HYBRID_FUNCTION, capturesLeakedTensor.getRefactoring()); assertNull(capturesLeakedTensor.getPassingPrecondition()); @@ -5281,6 +5284,53 @@ public void testPythonSideEffects47() throws Exception { assertNull("Warn about a hybrid function that leaks as a potential tensor.", warning); } + @Test + public void testPythonSideEffects48() throws Exception { + Function leakyFunction = getFunction("leaky_function"); + + assertTrue(leakyFunction.isHybrid()); + assertTrue(leakyFunction.getLikelyHasTensorParameter()); + assertTrue(leakyFunction.getHasPythonSideEffects()); + + // P2 "failure." + assertFalse(leakyFunction.getStatus().isOK()); + assertEquals( + "Should have one warning and one error. The warning is for running a hybrid function that has side-effects. The error is that it is already \"optimal\".", + 2, leakyFunction.getStatus().getEntries().length); + assertEquals(RefactoringStatus.ERROR, leakyFunction.getStatus().getEntryWithHighestSeverity().getSeverity()); + assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), leakyFunction.getStatus().getEntryWithHighestSeverity().getCode()); + assertNotNull(leakyFunction.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING)); + + Function capturesLeakedTensor = getFunction("captures_leaked_tensor"); + + assertFalse(capturesLeakedTensor.isHybrid()); + assertTrue(capturesLeakedTensor.getLikelyHasTensorParameter()); + + // NOTE: This function doesn't have Python side-effects, but it does capture a "leaky" tensor. See + // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281. + assertFalse(capturesLeakedTensor.getHasPythonSideEffects()); + + // NOTE: Change to assertFalse once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertTrue("Passes P1.", capturesLeakedTensor.getStatus().isOK()); + + assertFalse(capturesLeakedTensor.getStatus().hasWarning()); + // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertFalse(capturesLeakedTensor.getStatus().hasError()); + + assertNotNull(capturesLeakedTensor.getRefactoring()); + assertEquals("We shouldn't refactor this but we do currently. Nevertheless, the refactoring kind should remain intact.", + Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, capturesLeakedTensor.getRefactoring()); + + // NOTE: Change to assertNull once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertNotNull(capturesLeakedTensor.getPassingPrecondition()); + assertEquals("We really shouldn't refactor this.", capturesLeakedTensor.getPassingPrecondition(), PreconditionSuccess.P1); + + // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertFalse(capturesLeakedTensor.getTransformations().isEmpty()); + assertEquals("We really shouldn't transform this.", Collections.singleton(Transformation.CONVERT_TO_HYBRID), + capturesLeakedTensor.getTransformations()); + } + @Test public void testPythonSideEffects49() throws Exception { Function function = getFunction("leaky_function"); From 7c12e21a7516e9e6e4e405ebb208fb8b2ec0fe24 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 12:27:29 -0400 Subject: [PATCH 124/172] Add missing period. --- .../hunter/hybridize/core/analysis/PreconditionFailure.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 3ef90d3ac..15283797f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -11,7 +11,7 @@ public enum PreconditionFailure { OPTIMIZATION_NOT_AVAILABLE(2), /** - * Either there is no call to the function, there is a call but don't handle it, or something about decorators? + * Either there is no call to the function, there is a call but don't handle it, or something about decorators?. */ UNDETERMINABLE_SIDE_EFFECTS(3), From 98c40e38f871be84ed4ba185c1af573afb04307d Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 13:04:22 -0400 Subject: [PATCH 125/172] Add another leaky test. --- .../core/analysis/PreconditionFailure.java | 7 ++++- .../testPythonSideEffects50/in/A.py | 28 +++++++++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 24 ++++++++++++++-- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 15283797f..39918216a 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -23,7 +23,12 @@ public enum PreconditionFailure { /** * P2 "failure." */ - ALREADY_OPTIMAL(5); + ALREADY_OPTIMAL(5), + + /** + * P1 failure. + */ + HAS_NO_TENSOR_PARAMETERS(6); static { // check that the codes are unique. diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py new file mode 100644 index 000000000..401d7caf3 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py @@ -0,0 +1,28 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + + +class MyClass: + + def __init__(self): + self.field = None + + +external_list = [] +external_object = MyClass() + + +def leaky_function(): + a = tf.constant(1) + external_list.append(a) # Bad - leaks tensor + external_object.field = a # Bad - leaks tensor + +assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None +leaky_function() +assert len(external_list) == 1 +assert external_object is not None +assert external_object.field is not None +assert external_object.field == tf.constant(1) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 3770d246f..e00bfad96 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5355,6 +5355,26 @@ public void testPythonSideEffects49() throws Exception { assertTrue(function.getTransformations().isEmpty()); } - // TODO: Left off at https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values. THese are still - // side-effects, I believe. + @Test + public void testPythonSideEffects50() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + + RefactoringStatus status = function.getStatus(); + assertFalse("This is a P1 failure.", status.isOK()); + assertTrue(status.hasError()); + assertFalse(status.hasFatalError()); + + RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); + assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); + assertEquals(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode(), entry.getCode()); + assertEquals(function, entry.getData()); + + assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); + assertNull(function.getPassingPrecondition()); + assertTrue(function.getTransformations().isEmpty()); + } } From e4264ca7acc1dee1d95af647551e36eb31e951ec Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 14:21:28 -0400 Subject: [PATCH 126/172] Multiple errors on this test. --- .../tests/HybridizeFunctionRefactoringTest.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e00bfad96..18e91b086 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5368,10 +5368,19 @@ public void testPythonSideEffects50() throws Exception { assertTrue(status.hasError()); assertFalse(status.hasFatalError()); - RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); - assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); - assertEquals(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode(), entry.getCode()); - assertEquals(function, entry.getData()); + RefactoringStatusEntry[] statusEntries = status.getEntries(); + assertEquals(2, statusEntries.length); + + assertTrue(Arrays.stream(statusEntries).map(RefactoringStatusEntry::getSeverity) + .allMatch(s -> Objects.equals(s, RefactoringStatus.ERROR))); + + assertTrue(Arrays.stream(statusEntries).map(RefactoringStatusEntry::getData).allMatch(d -> Objects.equals(d, function))); + + Map> codeToEntry = Arrays.stream(statusEntries) + .collect(Collectors.groupingBy(RefactoringStatusEntry::getCode)); + + assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode())); + assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_SIDE_EFFECTS.getCode())); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); assertNull(function.getPassingPrecondition()); From 3c3420849edff79944d3ff1603506ae9cdc3f48a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 15:59:13 -0400 Subject: [PATCH 127/172] More tests. This last one here needs to fail. If there are no side-effects, then it could pass. Update table 2 in the paper. --- .../testPythonSideEffects50/in/A.py | 1 + .../testPythonSideEffects51/in/A.py | 28 +++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects52/in/A.py | 28 +++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects53/in/A.py | 28 +++++++++++++ .../in/requirements.txt | 1 + .../testPythonSideEffects54/in/A.py | 33 +++++++++++++++ .../in/requirements.txt | 2 + .../HybridizeFunctionRefactoringTest.java | 40 +++++++++++++++++++ 10 files changed, 163 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py index 401d7caf3..dcb2d4daa 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects50/in/A.py @@ -18,6 +18,7 @@ def leaky_function(): external_list.append(a) # Bad - leaks tensor external_object.field = a # Bad - leaks tensor + assert len(external_list) == 0 assert external_object is not None assert external_object.field is None diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/A.py new file mode 100644 index 000000000..8b17b3cb6 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/A.py @@ -0,0 +1,28 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + + +class MyClass: + + def __init__(self): + self.field = None + + +external_list = [] +# external_object = MyClass() + + +def leaky_function(): + a = tf.constant(1) + external_list.append(a) # Bad - leaks tensor + # external_object.field = a # Bad - leaks tensor + +assert len(external_list) == 0 +# assert external_object is not None +# assert external_object.field is None +leaky_function() +assert len(external_list) == 1 +# assert external_object is not None +# assert external_object.field is not None +# assert external_object.field == tf.constant(1) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects51/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/A.py new file mode 100644 index 000000000..2049545f4 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/A.py @@ -0,0 +1,28 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + + +class MyClass: + + def __init__(self): + self.field = None + + +# external_list = [] +external_object = MyClass() + + +def leaky_function(): + a = tf.constant(1) + # external_list.append(a) # Bad - leaks tensor + external_object.field = a # Bad - leaks tensor + +# assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None +leaky_function() +# assert len(external_list) == 1 +assert external_object is not None +assert external_object.field is not None +assert external_object.field == tf.constant(1) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects52/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/A.py new file mode 100644 index 000000000..34173b8e8 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/A.py @@ -0,0 +1,28 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + + +class MyClass: + + def __init__(self): + self.field = None + + +# external_list = [] +external_object = MyClass() + + +def not_leaky_function(): + a = tf.constant(1) + # external_list.append(a) # Bad - leaks tensor + return external_object.field + + +# assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None +not_leaky_function() +# assert len(external_list) == 1 +assert external_object is not None +assert external_object.field is None diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects53/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/A.py new file mode 100644 index 000000000..483cea0bc --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/A.py @@ -0,0 +1,33 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf +from nose.tools import assert_raises + + +class MyClass: + + def __init__(self): + self.field = None + + +external_list = [] +external_object = MyClass() + + +@tf.function +def leaky_function(): + a = tf.constant(1) + external_list.append(a) # Bad - leaks tensor + external_object.field = a # Bad - leaks tensor + + +assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None +leaky_function() +assert len(external_list) == 1 +assert external_object is not None +assert external_object.field is not None + +with assert_raises(TypeError): + assert external_object.field == tf.constant(1) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/requirements.txt new file mode 100644 index 000000000..56020dd04 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects54/in/requirements.txt @@ -0,0 +1,2 @@ +tensorflow==2.9.3 +nose==1.3.7 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 18e91b086..683faabef 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5386,4 +5386,44 @@ public void testPythonSideEffects50() throws Exception { assertNull(function.getPassingPrecondition()); assertTrue(function.getTransformations().isEmpty()); } + + @Test + public void testPythonSideEffects51() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects52() throws Exception { + Function function = getFunction("leaky_function"); + + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects53() throws Exception { + Function function = getFunction("not_leaky_function"); + + assertFalse(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertFalse(function.getHasPythonSideEffects()); + } + + @Test + public void testPythonSideEffects54() throws Exception { + Function function = getFunction("leaky_function"); + + assertTrue(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertTrue(function.getHasPythonSideEffects()); + + // TODO: We can't convert something to eager if it has side-effects because that will alter semantics. + } + + // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported } From cdc66c43404e18caeaf95b6a74320683a81bc4b2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 16:11:20 -0400 Subject: [PATCH 128/172] Fail the test. --- .../tests/HybridizeFunctionRefactoringTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 683faabef..e857c4477 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5422,7 +5422,14 @@ public void testPythonSideEffects54() throws Exception { assertFalse(function.getLikelyHasTensorParameter()); assertTrue(function.getHasPythonSideEffects()); - // TODO: We can't convert something to eager if it has side-effects because that will alter semantics. + RefactoringStatus status = function.getStatus(); + assertTrue("We can't convert something to eager if it has side-effects because that will alter semantics.", status.hasError()); + assertEquals(1, status.getEntries().length); + assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); + + assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); + assertNull(function.getPassingPrecondition()); + assertTrue(function.getTransformations().isEmpty()); } // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported From fae4c96efd6e3ad019a1fd1c2cc6cd1f605ddcbc Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 16:19:50 -0400 Subject: [PATCH 129/172] If there are no side-effects, then it could pass. --- .../testPythonSideEffects55/in/A.py | 30 +++++++++++++++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 20 +++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/A.py new file mode 100644 index 000000000..c97ab24ed --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/A.py @@ -0,0 +1,30 @@ +# From https://www.tensorflow.org/guide/function#all_outputs_of_a_tffunction_must_be_return_values + +import tensorflow as tf + + +class MyClass: + + def __init__(self): + self.field = None + + +external_list = [] +external_object = MyClass() + + +@tf.function +def leaky_function(): + a = tf.constant(1) + # external_list.append(a) # Bad - leaks tensor + # external_object.field = a # Bad - leaks tensor + return a + + +assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None +leaky_function() +assert len(external_list) == 0 +assert external_object is not None +assert external_object.field is None diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects55/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e857c4477..88b90a5f4 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5432,5 +5432,25 @@ public void testPythonSideEffects54() throws Exception { assertTrue(function.getTransformations().isEmpty()); } + @Test + public void testPythonSideEffects55() throws Exception { + Function function = getFunction("leaky_function"); + + assertTrue(function.isHybrid()); + assertFalse(function.getLikelyHasTensorParameter()); + assertFalse(function.getHasPythonSideEffects()); + + RefactoringStatus status = function.getStatus(); + assertFalse("We can convert something to eager if it does not have side-effects because that will not alter semantics.", + status.hasError()); + assertEquals(0, status.getEntries().length); + + assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); + assertNotNull(function.getPassingPrecondition()); + assertEquals(PreconditionSuccess.P2, function.getPassingPrecondition()); + assertFalse(function.getTransformations().isEmpty()); + assertEquals(Collections.singleton(Transformation.CONVERT_TO_EAGER), function.getTransformations()); + } + // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported } From c9459d9395ee3c743b3668c0d1013e0502183609 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 16:37:27 -0400 Subject: [PATCH 130/172] Add TODOs. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 2 +- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 4329ad272..a021dc3d9 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -112,7 +112,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException { buildAttributeColumnNames("transformation")); CSVPrinter optimizableFunctionPrinter = createCSVPrinter(OPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter nonOptimizableFunctionPrinter = createCSVPrinter(NONOPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); - CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, + CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, // TODO: Add a "warnings" file? Or non-zero? buildAttributeColumnNames("code", "message"));) { IProject[] pythonProjectsFromEvent = getSelectedPythonProjectsFromEvent(event); diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 88b90a5f4..7e263bff7 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5243,6 +5243,9 @@ public void testPythonSideEffects47() throws Exception { assertEquals( "Should have one warning and one error. The warning is for running a hybrid function that has side-effects. The error is that it is already \"optimal\".", 2, leakyFunction.getStatus().getEntries().length); + + // TODO: Actually, it has a tensor paramter but has side-effects. Isn't that the reason? + assertEquals(RefactoringStatus.ERROR, leakyFunction.getStatus().getEntryWithHighestSeverity().getSeverity()); assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), leakyFunction.getStatus().getEntryWithHighestSeverity().getCode()); assertNotNull(leakyFunction.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING)); From f4dad0b12280262b75da84f57005658269c31ed8 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 2 Nov 2023 22:34:12 -0400 Subject: [PATCH 131/172] Progress. --- .../hybridize/core/analysis/Function.java | 60 +++++++++++++------ .../core/analysis/PreconditionFailure.java | 12 ++-- ...HybridizeFunctionRefactoringProcessor.java | 2 +- .../HybridizeFunctionRefactoringTest.java | 8 +-- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 9fb602041..c5b11226c 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -25,6 +25,7 @@ import org.eclipse.jface.text.IDocument; import org.eclipse.ltk.core.refactoring.RefactoringStatus; import org.eclipse.ltk.core.refactoring.RefactoringStatusContext; +import org.eclipse.ltk.core.refactoring.RefactoringStatusEntry; import org.osgi.framework.FrameworkUtil; import org.python.pydev.core.IPythonNature; import org.python.pydev.core.docutils.PySelection; @@ -806,7 +807,7 @@ private static boolean isHybrid(decoratorsType decorator, String containingModul return false; } - public void addStatusEntry(PreconditionFailure failure, String message) { + public void addFailure(PreconditionFailure failure, String message) { // If is side-effects is filled, we can't set a precondition failure that we can't determine them. assert this.getHasPythonSideEffects() == null || failure != PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS : "Can't both have side-effects filled and have tem undterminable."; @@ -815,31 +816,52 @@ public void addStatusEntry(PreconditionFailure failure, String message) { this.getStatus().addEntry(RefactoringStatus.ERROR, message, context, PLUGIN_ID, failure.getCode(), this); } + public void addWarning(String message) { + RefactoringStatusContext context = new FunctionStatusContext(); + this.getStatus().addEntry(RefactoringStatus.WARNING, message, context, PLUGIN_ID, RefactoringStatusEntry.NO_CODE, this); + } + /** * Check refactoring preconditions. */ public void check() { - // we can't refactor it if either it doesn't have a tensor parameter or it's not currently hybrid. - if (!(this.getLikelyHasTensorParameter() || this.isHybrid())) - this.addStatusEntry(PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE, - "This function is not available for optimization. Either the function must have a tensor-like parameter or be currently hybrid."); - - // if this is not a hybrid function. - if (!this.isHybrid()) { - // but it likely has a tensor parameter. + if (!this.isHybrid()) { // Eager. Table 1. + this.setRefactoring(CONVERT_EAGER_FUNCTION_TO_HYBRID); + if (this.getLikelyHasTensorParameter()) { - // hybridize it. - this.setRefactoring(CONVERT_EAGER_FUNCTION_TO_HYBRID); - this.addTransformation(Transformation.CONVERT_TO_HYBRID); - this.setPassingPrecondition(P1); + if (!this.getHasPythonSideEffects()) { + this.addTransformation(Transformation.CONVERT_TO_HYBRID); + this.setPassingPrecondition(P1); + } else + this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "Can't hybridize a function with Python side-effects."); + } else { // no tensor parameters. + this.addFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS, + "This function has no tensor parameters and may not benefit from hybridization."); + + if (this.hasPythonSideEffects) + this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "Can't hybridize a function with Python side-effects."); } - } else { // this is a hybrid function. - // but it does not likely have a tensor parameter. + } else { // Hybrid. Use table 2. + this.setRefactoring(OPTIMIZE_HYBRID_FUNCTION); + if (!this.getLikelyHasTensorParameter()) { - // de-hybridize it. - this.setRefactoring(OPTIMIZE_HYBRID_FUNCTION); - this.addTransformation(CONVERT_TO_EAGER); - this.setPassingPrecondition(P2); + if (!this.getHasPythonSideEffects()) { + this.addTransformation(CONVERT_TO_EAGER); + this.setPassingPrecondition(P2); + } else { // it has side-effects. + this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, + "De-hybridizing a function with Python side-effects may alter semantics."); + } + } else { // it has a tensor parameter. + this.addFailure(PreconditionFailure.HAS_TENSOR_PARAMETERS, + "Functions with tensor parameters may benefit from hybreidization."); + + if (this.hasPythonSideEffects) { + this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, + "De-hybridizing a function with Python side-effects may alter semantics."); + + this.addWarning("This function is hybrid but potentially contains Python side-effects."); + } } } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 39918216a..3e43ecaec 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -15,20 +15,16 @@ public enum PreconditionFailure { */ UNDETERMINABLE_SIDE_EFFECTS(3), - /** - * P1 failure. - */ - HAS_SIDE_EFFECTS(4), + HAS_PYTHON_SIDE_EFFECTS(4), /** * P2 "failure." */ ALREADY_OPTIMAL(5), - /** - * P1 failure. - */ - HAS_NO_TENSOR_PARAMETERS(6); + HAS_NO_TENSOR_PARAMETERS(6), + + HAS_TENSOR_PARAMETERS(7); static { // check that the codes are unique. diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index acc1ccf06..7c9a21f06 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -217,7 +217,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); } catch (UndeterminablePythonSideEffectsException e) { LOG.warn("Unable to infer side-effects of: " + func + ".", e); - func.addStatusEntry(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, + func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); // next function. status.merge(func.getStatus()); diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 7e263bff7..a35e7b58f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5225,7 +5225,7 @@ public void testPythonSideEffects46() throws Exception { // We have an eager function with a tensor parameter but Python side-effects. Should be a P1 failure. assertFalse(status.isOK()); - assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS.getCode(), status.getEntryWithHighestSeverity().getCode()); + assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode(), status.getEntryWithHighestSeverity().getCode()); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); assertNull(function.getPassingPrecondition()); assertEquals(Collections.emptySet(), function.getTransformations()); @@ -5350,7 +5350,7 @@ public void testPythonSideEffects49() throws Exception { RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); - assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, entry.getCode()); + assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, entry.getCode()); assertEquals(function, entry.getData()); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); @@ -5383,7 +5383,7 @@ public void testPythonSideEffects50() throws Exception { .collect(Collectors.groupingBy(RefactoringStatusEntry::getCode)); assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode())); - assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_SIDE_EFFECTS.getCode())); + assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode())); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); assertNull(function.getPassingPrecondition()); @@ -5428,7 +5428,7 @@ public void testPythonSideEffects54() throws Exception { RefactoringStatus status = function.getStatus(); assertTrue("We can't convert something to eager if it has side-effects because that will alter semantics.", status.hasError()); assertEquals(1, status.getEntries().length); - assertEquals(PreconditionFailure.HAS_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); + assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); assertNull(function.getPassingPrecondition()); From 851e7738408895ef011376dcdf60dbfe0f9f921b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 11:51:08 -0400 Subject: [PATCH 132/172] Add count failures only once sanity check. --- .../hybridize/core/analysis/Function.java | 22 +++++++++++++++++++ ...HybridizeFunctionRefactoringProcessor.java | 2 ++ 2 files changed, 24 insertions(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c5b11226c..84e1e6d29 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -11,8 +11,10 @@ import static org.eclipse.core.runtime.Platform.getLog; import java.io.File; +import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -1027,4 +1029,24 @@ protected void setHasPythonSideEffects(Boolean hasPythonSideEffects) { this.hasPythonSideEffects = hasPythonSideEffects; } + + /** + * Returns true iff there is at most one {@link RefactoringStatusEntry} for a particular kind of failure. + * + * @apiNote This is to prevent counting a single kind of failure multiple times. Though that may be valid, I don't believe we have a + * situation like this currently. + * @return True iff there is at most one failure per failure kind. + */ + public boolean hasOnlyOneFailurePerKind() { + Map> failureCodeToEntries = Arrays.stream(this.getStatus().getEntries()) + .collect(Collectors.groupingBy(RefactoringStatusEntry::getCode)); + + for (Integer code : failureCodeToEntries.keySet()) { + List failuresForCode = failureCodeToEntries.get(code); + if (failuresForCode.size() > 1) + return false; + } + + return true; + } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 7c9a21f06..bc53534ec 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -236,6 +236,8 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat status.merge(func.getStatus()); subMonitor.worked(1); + + assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; } } From 50993862a387c320d396d157aec01b264a8ffba2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 11:51:25 -0400 Subject: [PATCH 133/172] Test code fixes. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index a35e7b58f..9a24495bd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5350,7 +5350,7 @@ public void testPythonSideEffects49() throws Exception { RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); - assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, entry.getCode()); + assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode(), entry.getCode()); assertEquals(function, entry.getData()); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); @@ -5382,8 +5382,8 @@ public void testPythonSideEffects50() throws Exception { Map> codeToEntry = Arrays.stream(statusEntries) .collect(Collectors.groupingBy(RefactoringStatusEntry::getCode)); - assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode())); - assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode())); + assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS.getCode()).size()); + assertEquals(1, codeToEntry.get(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode()).size()); assertEquals(Refactoring.CONVERT_EAGER_FUNCTION_TO_HYBRID, function.getRefactoring()); assertNull(function.getPassingPrecondition()); From 762ca9391b8bf975911c1f0cdf7817a01056eedc Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 13:38:17 -0400 Subject: [PATCH 134/172] More test fixes. --- .../HybridizeFunctionRefactoringTest.java | 74 +++++++++---------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 9a24495bd..e49389e4f 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.stream.Collectors; @@ -5191,28 +5192,37 @@ public void testPythonSideEffects45() throws Exception { Function function = getFunction("leaky_function"); assertTrue(function.isHybrid()); - assertTrue(function.getLikelyHasTensorParameter()); - assertTrue(function.getHasPythonSideEffects()); - - RefactoringStatus status = function.getStatus(); + // This is a hybrid function, so the refactoring should be OPTIMIZE_HYBRID_FUNCTION. + assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); - // We have a hybrid function with a tensor parameter and Python side-effects. Issue a warning. - RefactoringStatusEntry warning = status.getEntryMatchingSeverity(RefactoringStatus.WARNING); - assertNotNull(warning); - assertEquals(RefactoringStatus.WARNING, warning.getSeverity()); + assertTrue(function.getLikelyHasTensorParameter()); + // In table 2, we need it not to have a tensor parameter to de-hybridize, so this is a "failure." + assertTrue(getEntryMatchingFailure(function, PreconditionFailure.HAS_TENSOR_PARAMETERS).isError()); - // This situation is already "optimal," so we can't refactor it. - assertFalse(status.isOK()); - assertTrue(status.hasError()); - RefactoringStatusEntry entry = status.getEntryWithHighestSeverity(); - assertEquals(RefactoringStatus.ERROR, entry.getSeverity()); - assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), entry.getCode()); + assertTrue(function.getHasPythonSideEffects()); + // We also can't de-hybridize if it has Python side-effects. So, that's an error. + assertTrue(getEntryMatchingFailure(function, PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS).isError()); + // Also, we have a hybrid function with Python side-effects. Let's warn about that. + assertEquals(1, Arrays.stream(function.getStatus().getEntries()).map(RefactoringStatusEntry::getSeverity) + .filter(s -> s == RefactoringStatus.WARNING).count()); - assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); assertNull(function.getPassingPrecondition()); assertTrue(function.getTransformations().isEmpty()); } + /** + * Returns the first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in the given + * {@link Function}'s {@link RefactoringStatus}. + * + * @param function The {@link Function} being tested. + * @param failure The {@link PreconditionFailure} whose {@link RefactoringStatusEntry} to find. + * @return The first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in the given + * {@link Function}'s {@link RefactoringStatus}. + */ + private static RefactoringStatusEntry getEntryMatchingFailure(Function function, PreconditionFailure failure) { + return function.getStatus().getEntryMatchingCode(Function.PLUGIN_ID, failure.getCode()); + } + @Test public void testPythonSideEffects46() throws Exception { Function function = getFunction("leaky_function"); @@ -5239,21 +5249,6 @@ public void testPythonSideEffects47() throws Exception { assertTrue(leakyFunction.getLikelyHasTensorParameter()); assertTrue(leakyFunction.getHasPythonSideEffects()); - assertFalse("P2 \"failure.\"", leakyFunction.getStatus().isOK()); - assertEquals( - "Should have one warning and one error. The warning is for running a hybrid function that has side-effects. The error is that it is already \"optimal\".", - 2, leakyFunction.getStatus().getEntries().length); - - // TODO: Actually, it has a tensor paramter but has side-effects. Isn't that the reason? - - assertEquals(RefactoringStatus.ERROR, leakyFunction.getStatus().getEntryWithHighestSeverity().getSeverity()); - assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), leakyFunction.getStatus().getEntryWithHighestSeverity().getCode()); - assertNotNull(leakyFunction.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING)); - - assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, leakyFunction.getRefactoring()); - assertNull(leakyFunction.getPassingPrecondition()); - assertTrue(leakyFunction.getTransformations().isEmpty()); - Function capturesLeakedTensor = getFunction("captures_leaked_tensor"); assertTrue(capturesLeakedTensor.isHybrid()); @@ -5267,24 +5262,25 @@ public void testPythonSideEffects47() throws Exception { assertTrue(capturesLeakedTensor.getStatus().hasError()); assertFalse(capturesLeakedTensor.getStatus().hasFatalError()); RefactoringStatusEntry error = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.ERROR); - assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), error.getCode()); + assertEquals(PreconditionFailure.HAS_TENSOR_PARAMETERS.getCode(), error.getCode()); - // NOTE: Change to assertTrue once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertFalse("We should warn that the hybrid function is capturing leaked tensors.", capturesLeakedTensor.getStatus().hasWarning()); + // NOTE: Change to assertEquals(..., 1, ...) once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertEquals("We should warn that the hybrid function is capturing leaked tensors.", 0, + Arrays.stream(capturesLeakedTensor.getStatus().getEntries()).map(RefactoringStatusEntry::getSeverity) + .filter(s -> s == RefactoringStatus.WARNING).count()); assertNotNull(capturesLeakedTensor.getRefactoring()); assertEquals("P2 \"failure.\"", Refactoring.OPTIMIZE_HYBRID_FUNCTION, capturesLeakedTensor.getRefactoring()); assertNull(capturesLeakedTensor.getPassingPrecondition()); assertTrue(capturesLeakedTensor.getTransformations().isEmpty()); - // NOTE: Change to assertTrue when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertFalse("We should warn about this.", capturesLeakedTensor.getStatus().hasWarning()); + Optional warning = Arrays.stream(capturesLeakedTensor.getStatus().getEntries()) + .filter(e -> e.getSeverity() == RefactoringStatus.WARNING).findFirst(); - RefactoringStatusEntry warning = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING); - // NOTE: Change to assertNotNull when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + // NOTE: Change to assertFalse when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. // NOTE: Add assertEquals(RefactoringStatus.WARNING, entry.getSeverity()) when // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertNull("Warn about a hybrid function that leaks as a potential tensor.", warning); + assertTrue("Warn about a hybrid function that leaks as a potential tensor.", warning.isEmpty()); } @Test @@ -5428,7 +5424,7 @@ public void testPythonSideEffects54() throws Exception { RefactoringStatus status = function.getStatus(); assertTrue("We can't convert something to eager if it has side-effects because that will alter semantics.", status.hasError()); assertEquals(1, status.getEntries().length); - assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, status.getEntryWithHighestSeverity().getCode()); + assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode(), status.getEntryWithHighestSeverity().getCode()); assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); assertNull(function.getPassingPrecondition()); From 34580b6739c68598487b8dcd11449c93ceca1c65 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 13:40:57 -0400 Subject: [PATCH 135/172] Fix test. --- .../tests/HybridizeFunctionRefactoringTest.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e49389e4f..3ed0229ea 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5291,15 +5291,6 @@ public void testPythonSideEffects48() throws Exception { assertTrue(leakyFunction.getLikelyHasTensorParameter()); assertTrue(leakyFunction.getHasPythonSideEffects()); - // P2 "failure." - assertFalse(leakyFunction.getStatus().isOK()); - assertEquals( - "Should have one warning and one error. The warning is for running a hybrid function that has side-effects. The error is that it is already \"optimal\".", - 2, leakyFunction.getStatus().getEntries().length); - assertEquals(RefactoringStatus.ERROR, leakyFunction.getStatus().getEntryWithHighestSeverity().getSeverity()); - assertEquals(PreconditionFailure.ALREADY_OPTIMAL.getCode(), leakyFunction.getStatus().getEntryWithHighestSeverity().getCode()); - assertNotNull(leakyFunction.getStatus().getEntryMatchingSeverity(RefactoringStatus.WARNING)); - Function capturesLeakedTensor = getFunction("captures_leaked_tensor"); assertFalse(capturesLeakedTensor.isHybrid()); From 33ae3d7c3a8db85e4ab3694e398a3d12b832b814 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 14:19:01 -0400 Subject: [PATCH 136/172] Fix tests. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 3ed0229ea..16fc9fe1c 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4492,8 +4492,8 @@ public void testModel3() throws Exception { private static void checkOptimizationNotAvailableStatus(Function f) { RefactoringStatus status = f.getStatus(); assertTrue("Should not be available for optimization.", status.hasError()); - Object entry = status.getEntryMatchingCode(Function.PLUGIN_ID, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()); - assertNotNull(entry); + RefactoringStatusEntry noTensorsFailure = getEntryMatchingFailure(f, PreconditionFailure.HAS_NO_TENSOR_PARAMETERS); + assertTrue(!f.isHybrid() || (noTensorsFailure != null && noTensorsFailure.isError())); } /** From ef6863ea42ebd5eb0853f39a6fd985f15e52c777 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 14:43:07 -0400 Subject: [PATCH 137/172] Remove unused precondition failures. --- .../hybridize/core/analysis/Function.java | 12 ++++++++++++ .../core/analysis/PreconditionFailure.java | 10 ---------- ...teHybridizeFunctionRefactoringHandler.java | 5 ++--- .../HybridizeFunctionRefactoringTest.java | 19 +++---------------- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 84e1e6d29..00f4caa4d 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -1049,4 +1049,16 @@ public boolean hasOnlyOneFailurePerKind() { return true; } + + /** + * Returns the first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in this {@link Function}'s + * {@link RefactoringStatus}. + * + * @param failure The {@link PreconditionFailure} whose {@link RefactoringStatusEntry} to find. + * @return The first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in this {@link Function}'s + * {@link RefactoringStatus}. + */ + public RefactoringStatusEntry getEntryMatchingFailure(PreconditionFailure failure) { + return this.getStatus().getEntryMatchingCode(Function.PLUGIN_ID, failure.getCode()); + } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java index 3e43ecaec..cfe6eb72c 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/PreconditionFailure.java @@ -5,11 +5,6 @@ public enum PreconditionFailure { CURRENTLY_NOT_HANDLED(1), - /** - * Not a candidate. - */ - OPTIMIZATION_NOT_AVAILABLE(2), - /** * Either there is no call to the function, there is a call but don't handle it, or something about decorators?. */ @@ -17,11 +12,6 @@ public enum PreconditionFailure { HAS_PYTHON_SIDE_EFFECTS(4), - /** - * P2 "failure." - */ - ALREADY_OPTIMAL(5), - HAS_NO_TENSOR_PARAMETERS(6), HAS_TENSOR_PARAMETERS(7); diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index a021dc3d9..b73c3d025 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -1,6 +1,5 @@ package edu.cuny.hunter.hybridize.eval.handlers; -import static edu.cuny.hunter.hybridize.core.analysis.Function.PLUGIN_ID; import static edu.cuny.hunter.hybridize.core.utils.Util.createHybridizeFunctionRefactoring; import static org.eclipse.core.runtime.Platform.getLog; import static org.python.pydev.plugin.nature.PythonNature.PYTHON_NATURE_ID; @@ -146,8 +145,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // optimization available functions. These are the "filtered" functions. We consider functions to be candidates iff they // have a tensor-like parameter or are currently hybrid. - Set candidates = functions.stream().filter(Function::isHybridizationAvailable).filter(f -> f.getStatus() - .getEntryMatchingCode(PLUGIN_ID, PreconditionFailure.OPTIMIZATION_NOT_AVAILABLE.getCode()) == null) + Set candidates = functions.stream().filter(Function::isHybridizationAvailable) + .filter(f -> f.isHybrid() || f.getEntryMatchingFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS) == null) .collect(Collectors.toSet()); resultsPrinter.print(candidates.size()); // number. diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 16fc9fe1c..eee3590cd 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -4492,7 +4492,7 @@ public void testModel3() throws Exception { private static void checkOptimizationNotAvailableStatus(Function f) { RefactoringStatus status = f.getStatus(); assertTrue("Should not be available for optimization.", status.hasError()); - RefactoringStatusEntry noTensorsFailure = getEntryMatchingFailure(f, PreconditionFailure.HAS_NO_TENSOR_PARAMETERS); + RefactoringStatusEntry noTensorsFailure = f.getEntryMatchingFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS); assertTrue(!f.isHybrid() || (noTensorsFailure != null && noTensorsFailure.isError())); } @@ -5197,11 +5197,11 @@ public void testPythonSideEffects45() throws Exception { assertTrue(function.getLikelyHasTensorParameter()); // In table 2, we need it not to have a tensor parameter to de-hybridize, so this is a "failure." - assertTrue(getEntryMatchingFailure(function, PreconditionFailure.HAS_TENSOR_PARAMETERS).isError()); + assertTrue(function.getEntryMatchingFailure(PreconditionFailure.HAS_TENSOR_PARAMETERS).isError()); assertTrue(function.getHasPythonSideEffects()); // We also can't de-hybridize if it has Python side-effects. So, that's an error. - assertTrue(getEntryMatchingFailure(function, PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS).isError()); + assertTrue(function.getEntryMatchingFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS).isError()); // Also, we have a hybrid function with Python side-effects. Let's warn about that. assertEquals(1, Arrays.stream(function.getStatus().getEntries()).map(RefactoringStatusEntry::getSeverity) .filter(s -> s == RefactoringStatus.WARNING).count()); @@ -5210,19 +5210,6 @@ public void testPythonSideEffects45() throws Exception { assertTrue(function.getTransformations().isEmpty()); } - /** - * Returns the first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in the given - * {@link Function}'s {@link RefactoringStatus}. - * - * @param function The {@link Function} being tested. - * @param failure The {@link PreconditionFailure} whose {@link RefactoringStatusEntry} to find. - * @return The first {@link RefactoringStatusEntry} matching the given {@link PreconditionFailure}'s code in the given - * {@link Function}'s {@link RefactoringStatus}. - */ - private static RefactoringStatusEntry getEntryMatchingFailure(Function function, PreconditionFailure failure) { - return function.getStatus().getEntryMatchingCode(Function.PLUGIN_ID, failure.getCode()); - } - @Test public void testPythonSideEffects46() throws Exception { Function function = getFunction("leaky_function"); From 2cc58fc45c00270301f2caba18d0e3a68e092987 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 14:52:54 -0400 Subject: [PATCH 138/172] Get statistics only on candidates. --- .../EvaluateHybridizeFunctionRefactoringHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index b73c3d025..495df8fa9 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -196,17 +196,17 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // refactoring type counts. for (Refactoring refactoringKind : Refactoring.values()) - resultsPrinter.print(functions.parallelStream().map(Function::getRefactoring) + resultsPrinter.print(candidates.parallelStream().map(Function::getRefactoring) .filter(r -> Objects.equals(r, refactoringKind)).count()); // precondition success counts. for (PreconditionSuccess preconditionSuccess : PreconditionSuccess.values()) - resultsPrinter.print(functions.parallelStream().map(Function::getPassingPrecondition) + resultsPrinter.print(candidates.parallelStream().map(Function::getPassingPrecondition) .filter(pp -> Objects.equals(pp, preconditionSuccess)).count()); // transformation counts. for (Transformation transformation : Transformation.values()) - resultsPrinter.print(functions.parallelStream().map(Function::getTransformations).filter(Objects::nonNull) + resultsPrinter.print(candidates.parallelStream().map(Function::getTransformations).filter(Objects::nonNull) .flatMap(as -> as.parallelStream()).filter(a -> Objects.equals(a, transformation)).count()); // actually perform the refactoring if there are no fatal errors. From 62cbf483854ec0dccbdfd1ca70d3df5f4a8fd9fd Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 15:11:25 -0400 Subject: [PATCH 139/172] Add severity level to failures. They should be errors but just to verify. --- .../EvaluateHybridizeFunctionRefactoringHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 495df8fa9..07200a62b 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -112,7 +112,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException { CSVPrinter optimizableFunctionPrinter = createCSVPrinter(OPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter nonOptimizableFunctionPrinter = createCSVPrinter(NONOPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, // TODO: Add a "warnings" file? Or non-zero? - buildAttributeColumnNames("code", "message"));) { + buildAttributeColumnNames("severity", "code", "message"));) { IProject[] pythonProjectsFromEvent = getSelectedPythonProjectsFromEvent(event); monitor.beginTask("Analyzing projects...", pythonProjectsFromEvent.length); @@ -190,7 +190,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { Function failedFunction = (Function) correspondingElement; - errorPrinter.printRecord(buildAttributeColumnValues(failedFunction, entry.getCode(), entry.getMessage())); + errorPrinter.printRecord( + buildAttributeColumnValues(failedFunction, entry.getSeverity(), entry.getCode(), entry.getMessage())); } } From 7d1aa121dc1d63ccdd9e9fa4efddeffbcc57f63a Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 15:40:51 -0400 Subject: [PATCH 140/172] Print all statuses. --- ...teHybridizeFunctionRefactoringHandler.java | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 07200a62b..a8fa6d925 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.csv.CSVPrinter; @@ -68,6 +69,8 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto private static final String FAILED_PRECONDITIONS_CSV_FILENAME = "failed_preconditions.csv"; + private static final String STATUS_CSV_FILENAME = "statuses.csv"; + private static final String PERFORM_CHANGE_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.performChange"; private static String[] buildAttributeColumnNames(String... additionalColumnNames) { @@ -111,7 +114,9 @@ public Object execute(ExecutionEvent event) throws ExecutionException { buildAttributeColumnNames("transformation")); CSVPrinter optimizableFunctionPrinter = createCSVPrinter(OPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter nonOptimizableFunctionPrinter = createCSVPrinter(NONOPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); - CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, // TODO: Add a "warnings" file? Or non-zero? + CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, + buildAttributeColumnNames("severity", "code", "message")); + CSVPrinter statusPrinter = createCSVPrinter(STATUS_CSV_FILENAME, buildAttributeColumnNames("severity", "code", "message"));) { IProject[] pythonProjectsFromEvent = getSelectedPythonProjectsFromEvent(event); @@ -173,27 +178,16 @@ public Object execute(ExecutionEvent event) throws ExecutionException { nonOptimizableFunctionPrinter.printRecord(buildAttributeColumnValues(function)); // failed preconditions. - Collection errorEntries = failures.parallelStream().map(Function::getStatus) - .flatMap(s -> Arrays.stream(s.getEntries())).filter(RefactoringStatusEntry::isError) - .collect(Collectors.toSet()); + Collection errorEntries = getRefactoringStatusEntries(failures, + RefactoringStatusEntry::isError); resultsPrinter.print(errorEntries.size()); // number. - for (RefactoringStatusEntry entry : errorEntries) { - if (!entry.isFatalError()) { - Object correspondingElement = entry.getData(); + printStatusEntries(errorPrinter, errorEntries); - if (!(correspondingElement instanceof Function)) - throw new IllegalStateException("The element: " + correspondingElement - + " corresponding to a failed precondition is not a Function. Instead, it is a: " - + correspondingElement.getClass()); - - Function failedFunction = (Function) correspondingElement; - - errorPrinter.printRecord( - buildAttributeColumnValues(failedFunction, entry.getSeverity(), entry.getCode(), entry.getMessage())); - } - } + // general refactoring statuses. + Set generalEntries = getRefactoringStatusEntries(functions, x -> true); + printStatusEntries(statusPrinter, generalEntries); // refactoring type counts. for (Refactoring refactoringKind : Refactoring.values()) @@ -237,6 +231,28 @@ public Object execute(ExecutionEvent event) throws ExecutionException { return null; } + private static Set getRefactoringStatusEntries(Set functionSet, + Predicate predicate) { + return functionSet.parallelStream().map(Function::getStatus).flatMap(s -> Arrays.stream(s.getEntries())).filter(predicate) + .collect(Collectors.toSet()); + } + + private static void printStatusEntries(CSVPrinter printer, Collection entries) throws IOException { + for (RefactoringStatusEntry entry : entries) { + if (!entry.isFatalError()) { + Object correspondingElement = entry.getData(); + + if (!(correspondingElement instanceof Function)) + throw new IllegalStateException("The element: " + correspondingElement + " is not a Function. Instead, it is a: " + + correspondingElement.getClass()); + + Function function = (Function) correspondingElement; + + printer.printRecord(buildAttributeColumnValues(function, entry.getSeverity(), entry.getCode(), entry.getMessage())); + } + } + } + private static String[] buildFunctionAttributeColumnNames() { return buildAttributeColumnNames("method reference", "type reference", "parameters", "tensor parameter", "hybrid", "side-effects", "autograph", "experimental_autograph_options", "experimental_follow_type_hints", "experimental_implements", "func", From f4087d186f81311a6f21b345a58d239d000d59e1 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Fri, 3 Nov 2023 17:39:15 -0400 Subject: [PATCH 141/172] Add TODOs. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 00f4caa4d..1d364c70f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -379,7 +379,7 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis modSet = mod.get(cgNode); @@ -457,7 +457,7 @@ private static boolean allCreationsWithinClosure(MethodReference methodReference if (cgNodes.isEmpty()) throw new IllegalArgumentException("Can't find call graph nodes corresponding to: " + methodReference + "."); - for (CGNode node : cgNodes) + for (CGNode node : cgNodes) // FIXME: I don't think we need multiple nodes here. Also, do we need to consider synthetic nodes? // check the called functions. for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { CGNode next = succNodes.next(); From 1d7013dec1907788e5fe3d117558bdbc6cafe6a5 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 10:28:06 -0500 Subject: [PATCH 142/172] Don't revisit recursive calls. --- .../hunter/hybridize/core/analysis/Function.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 1d364c70f..2ba3f5068 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -447,6 +447,14 @@ private Set filterSideEffects(Iterable modSet, CallGraph } private static boolean allCreationsWithinClosure(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) { + Set seen = Sets.newHashSet(); + return allCreationsWithinClosureInteral(methodReference, instanceKey, callGraph, seen); + } + + private static boolean allCreationsWithinClosureInteral(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph, + Set seen) { + seen.add(methodReference); + // check this function. if (allCreationsWithin(methodReference, instanceKey, callGraph)) return true; @@ -463,8 +471,12 @@ private static boolean allCreationsWithinClosure(MethodReference methodReference CGNode next = succNodes.next(); MethodReference reference = next.getMethod().getReference(); - if (allCreationsWithinClosure(reference, instanceKey, callGraph)) - return true; + if (!seen.contains(reference)) { + seen.add(reference); + + if (allCreationsWithinClosureInteral(reference, instanceKey, callGraph, seen)) + return true; + } } return false; From 0f68c16f030f77c57ed6ba9911ab9b4e4c999ed2 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 11:42:05 -0500 Subject: [PATCH 143/172] Add refactoring kind to status outut. Shorten method name. --- ...EvaluateHybridizeFunctionRefactoringHandler.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index a8fa6d925..b8670cb9f 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -115,9 +115,9 @@ public Object execute(ExecutionEvent event) throws ExecutionException { CSVPrinter optimizableFunctionPrinter = createCSVPrinter(OPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter nonOptimizableFunctionPrinter = createCSVPrinter(NONOPTMIZABLE_CSV_FILENAME, buildAttributeColumnNames()); CSVPrinter errorPrinter = createCSVPrinter(FAILED_PRECONDITIONS_CSV_FILENAME, - buildAttributeColumnNames("severity", "code", "message")); + buildAttributeColumnNames("refactoring", "severity", "code", "message")); CSVPrinter statusPrinter = createCSVPrinter(STATUS_CSV_FILENAME, - buildAttributeColumnNames("severity", "code", "message"));) { + buildAttributeColumnNames("refactoring", "severity", "code", "message"));) { IProject[] pythonProjectsFromEvent = getSelectedPythonProjectsFromEvent(event); monitor.beginTask("Analyzing projects...", pythonProjectsFromEvent.length); @@ -183,11 +183,11 @@ public Object execute(ExecutionEvent event) throws ExecutionException { resultsPrinter.print(errorEntries.size()); // number. - printStatusEntries(errorPrinter, errorEntries); + printStatuses(errorPrinter, errorEntries); // general refactoring statuses. Set generalEntries = getRefactoringStatusEntries(functions, x -> true); - printStatusEntries(statusPrinter, generalEntries); + printStatuses(statusPrinter, generalEntries); // refactoring type counts. for (Refactoring refactoringKind : Refactoring.values()) @@ -237,7 +237,7 @@ private static Set getRefactoringStatusEntries(Set entries) throws IOException { + private static void printStatuses(CSVPrinter printer, Collection entries) throws IOException { for (RefactoringStatusEntry entry : entries) { if (!entry.isFatalError()) { Object correspondingElement = entry.getData(); @@ -248,7 +248,8 @@ private static void printStatusEntries(CSVPrinter printer, Collection Date: Mon, 6 Nov 2023 11:47:10 -0500 Subject: [PATCH 144/172] Shorten warning message. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 2ba3f5068..a172529f9 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -874,7 +874,7 @@ public void check() { this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); - this.addWarning("This function is hybrid but potentially contains Python side-effects."); + this.addWarning("This hybrid function potentially contains Python side-effects."); } } } From 7a4ad0e3ea4d4de4c2f448d13c5de7567d140e97 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 12:24:41 -0500 Subject: [PATCH 145/172] Use function. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index a172529f9..9c014a7de 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -870,7 +870,7 @@ public void check() { this.addFailure(PreconditionFailure.HAS_TENSOR_PARAMETERS, "Functions with tensor parameters may benefit from hybreidization."); - if (this.hasPythonSideEffects) { + if (this.getHasPythonSideEffects()) { this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); From cf217806256ece4fb6b713292e717abb54b56b35 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 12:51:35 -0500 Subject: [PATCH 146/172] Change else block to statement. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 9c014a7de..dbd7f618f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -862,10 +862,9 @@ public void check() { if (!this.getHasPythonSideEffects()) { this.addTransformation(CONVERT_TO_EAGER); this.setPassingPrecondition(P2); - } else { // it has side-effects. + } else // it has side-effects. this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); - } } else { // it has a tensor parameter. this.addFailure(PreconditionFailure.HAS_TENSOR_PARAMETERS, "Functions with tensor parameters may benefit from hybreidization."); From b3df810a765099abe5e9274bb2c27807fcc96eff Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 13:01:26 -0500 Subject: [PATCH 147/172] Fix candidate detection. Can't use a precondition failure because we may have not assigned them. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index b8670cb9f..40ac820ef 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -151,8 +151,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // optimization available functions. These are the "filtered" functions. We consider functions to be candidates iff they // have a tensor-like parameter or are currently hybrid. Set candidates = functions.stream().filter(Function::isHybridizationAvailable) - .filter(f -> f.isHybrid() || f.getEntryMatchingFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS) == null) - .collect(Collectors.toSet()); + .filter(f -> f.isHybrid() || f.getLikelyHasTensorParameter()).collect(Collectors.toSet()); resultsPrinter.print(candidates.size()); // number. // candidate functions. From 97e6a2fc5fed38e4054e40e3d23b8b6a3250a55c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 13:49:35 -0500 Subject: [PATCH 148/172] Fix warning about hybrid functions with side-effects. --- .../hunter/hybridize/core/analysis/Function.java | 10 ++++++++-- .../tests/HybridizeFunctionRefactoringTest.java | 14 +++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index dbd7f618f..b7b5bc5eb 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -872,13 +872,19 @@ public void check() { if (this.getHasPythonSideEffects()) { this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); - - this.addWarning("This hybrid function potentially contains Python side-effects."); } } + + // if we are not de-hybridizing, issue a warning. + if (!this.willDehybridize()) + this.addWarning("This hybrid function potentially contains Python side-effects."); } } + public boolean willDehybridize() { + return this.getTransformations().contains(CONVERT_TO_EAGER); + } + public IDocument getContainingDocument() { return this.getFunctionDefinition().containingDocument; } diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index eee3590cd..f772fd998 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5251,8 +5251,8 @@ public void testPythonSideEffects47() throws Exception { RefactoringStatusEntry error = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.ERROR); assertEquals(PreconditionFailure.HAS_TENSOR_PARAMETERS.getCode(), error.getCode()); - // NOTE: Change to assertEquals(..., 1, ...) once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertEquals("We should warn that the hybrid function is capturing leaked tensors.", 0, + // NOTE: Change to assertEquals(..., 2, ...) once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertEquals("We should warn that the hybrid function is capturing leaked tensors.", 1, Arrays.stream(capturesLeakedTensor.getStatus().getEntries()).map(RefactoringStatusEntry::getSeverity) .filter(s -> s == RefactoringStatus.WARNING).count()); @@ -5261,13 +5261,13 @@ public void testPythonSideEffects47() throws Exception { assertNull(capturesLeakedTensor.getPassingPrecondition()); assertTrue(capturesLeakedTensor.getTransformations().isEmpty()); - Optional warning = Arrays.stream(capturesLeakedTensor.getStatus().getEntries()) - .filter(e -> e.getSeverity() == RefactoringStatus.WARNING).findFirst(); + long warningCount = Arrays.stream(capturesLeakedTensor.getStatus().getEntries()) + .filter(e -> e.getSeverity() == RefactoringStatus.WARNING).count(); - // NOTE: Change to assertFalse when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + // NOTE: Change to assertEquals(..., 2, ...) when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. // NOTE: Add assertEquals(RefactoringStatus.WARNING, entry.getSeverity()) when // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertTrue("Warn about a hybrid function that leaks as a potential tensor.", warning.isEmpty()); + assertEquals("Warn about a hybrid function that leaks as a potential tensor.", 1, warningCount); } @Test @@ -5401,7 +5401,7 @@ public void testPythonSideEffects54() throws Exception { RefactoringStatus status = function.getStatus(); assertTrue("We can't convert something to eager if it has side-effects because that will alter semantics.", status.hasError()); - assertEquals(1, status.getEntries().length); + assertEquals(2, status.getEntries().length); assertEquals(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS.getCode(), status.getEntryWithHighestSeverity().getCode()); assertEquals(Refactoring.OPTIMIZE_HYBRID_FUNCTION, function.getRefactoring()); From ccc8c037ecad2069156f5531071ff8c6fbc0611b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 15:56:10 -0500 Subject: [PATCH 149/172] Fix candidate bug. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 40ac820ef..22d57afb5 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -151,7 +151,9 @@ public Object execute(ExecutionEvent event) throws ExecutionException { // optimization available functions. These are the "filtered" functions. We consider functions to be candidates iff they // have a tensor-like parameter or are currently hybrid. Set candidates = functions.stream().filter(Function::isHybridizationAvailable) - .filter(f -> f.isHybrid() || f.getLikelyHasTensorParameter()).collect(Collectors.toSet()); + .filter(f -> f.isHybrid() != null && f.isHybrid() + || f.getLikelyHasTensorParameter() != null && f.getLikelyHasTensorParameter()) + .collect(Collectors.toSet()); resultsPrinter.print(candidates.size()); // number. // candidate functions. From 9737e688697827b3ceaf674f249443d74a39ae7e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 16:06:15 -0500 Subject: [PATCH 150/172] Organize imports. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index 22d57afb5..d13e179f1 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -45,7 +45,6 @@ import edu.cuny.citytech.refactoring.common.eval.handlers.EvaluateRefactoringHandler; import edu.cuny.hunter.hybridize.core.analysis.Function; import edu.cuny.hunter.hybridize.core.analysis.Function.HybridizationParameters; -import edu.cuny.hunter.hybridize.core.analysis.PreconditionFailure; import edu.cuny.hunter.hybridize.core.analysis.PreconditionSuccess; import edu.cuny.hunter.hybridize.core.analysis.Refactoring; import edu.cuny.hunter.hybridize.core.analysis.Transformation; From f711d94ed254b1d65ff0a3d40b54e5303acdd489 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 16:07:23 -0500 Subject: [PATCH 151/172] Only output nodes in verbose mode. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index b7b5bc5eb..d8700cd33 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -538,7 +538,9 @@ private static Set getNodes(MethodReference methodReference, CallGraph c } LOG.info("Found " + nodes.size() + " node(s) corresponding to: " + methodReference + "."); - LOG.info("Nodes:\n" + nodes.stream().map(Objects::toString).collect(Collectors.joining("\n"))); + + if (VERBOSE) + LOG.info("Nodes:\n" + nodes.stream().map(Objects::toString).collect(Collectors.joining("\n"))); return nodes; } From df04cadc45f756f0ab68fca94cba5edb5c38949f Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 16:10:46 -0500 Subject: [PATCH 152/172] Update comment. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index d8700cd33..0f2db4ed2 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -373,7 +373,8 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis> mod = modRef.computeMod(callGraph, pointerAnalysis); // Get the nodes corresponding to this function's declaration. NOTE: There can be multiple nodes for a function declaration under - // the current representation. It seems that there is a declaration node for each call to the function. + // the current representation. It seems that there is a declaration node for each call to the function. Each node has a different + // calling context. Set nodes = this.getNodes(callGraph); if (nodes.isEmpty()) From 8306682b90de60b2105416dc3ad77a46e1b1afae Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 16:15:52 -0500 Subject: [PATCH 153/172] Organize imports. --- .../hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index f772fd998..aedb1a02d 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.stream.Collectors; From 36f73156451d0518a04b6d58a4dd1917e48d3f64 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 16:30:28 -0500 Subject: [PATCH 154/172] Use only one call graph node per function declaration. --- .../hybridize/core/analysis/Function.java | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 0f2db4ed2..54c7b2868 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -380,28 +380,28 @@ public void inferPythonSideEffects(CallGraph callGraph, PointerAnalysis modSet = mod.get(cgNode); - LOG.info("Found " + modSet.size() + " original modified location(s)."); - modSet.forEach(pk -> LOG.info("Original modified location: " + pk + ".")); - - // Filter out the modified locations. - Set filteredModSet = this.filterSideEffects(modSet, callGraph, pointerAnalysis); - LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); - filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); - - // Log the locations we are removing. - SetView removed = Sets.difference(Sets.newHashSet(modSet), filteredModSet); - LOG.info("Removed " + removed.size() + " locations."); - removed.forEach(pk -> LOG.info("Removed modified location: " + pk + ".")); - - if (!filteredModSet.isEmpty()) { - this.setHasPythonSideEffects(TRUE); - LOG.info(this + " has side-effects."); - return; - } + // Only consider the first node. The calling context shouldn't matter for us right now. + CGNode cgNode = nodes.iterator().next(); + + // Get the locations (pointers) modified by this function. + OrdinalSet modSet = mod.get(cgNode); + LOG.info("Found " + modSet.size() + " original modified location(s)."); + modSet.forEach(pk -> LOG.info("Original modified location: " + pk + ".")); + + // Filter out the modified locations. + Set filteredModSet = this.filterSideEffects(modSet, callGraph, pointerAnalysis); + LOG.info("Found " + filteredModSet.size() + " filtered modified location(s)."); + filteredModSet.forEach(pk -> LOG.info("Filtered modified location: " + pk + ".")); + + // Log the locations we are removing. + SetView removed = Sets.difference(Sets.newHashSet(modSet), filteredModSet); + LOG.info("Removed " + removed.size() + " locations."); + removed.forEach(pk -> LOG.info("Removed modified location: " + pk + ".")); + + if (!filteredModSet.isEmpty()) { + this.setHasPythonSideEffects(TRUE); + LOG.info(this + " has side-effects."); + return; } this.setHasPythonSideEffects(FALSE); @@ -466,19 +466,21 @@ private static boolean allCreationsWithinClosureInteral(MethodReference methodRe if (cgNodes.isEmpty()) throw new IllegalArgumentException("Can't find call graph nodes corresponding to: " + methodReference + "."); - for (CGNode node : cgNodes) // FIXME: I don't think we need multiple nodes here. Also, do we need to consider synthetic nodes? - // check the called functions. - for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { - CGNode next = succNodes.next(); - MethodReference reference = next.getMethod().getReference(); + // Only consider the first node. The only difference should be the calling context, which shouldn't make a difference for us. + CGNode node = cgNodes.iterator().next(); - if (!seen.contains(reference)) { - seen.add(reference); + // check the called functions. + for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { + CGNode next = succNodes.next(); + MethodReference reference = next.getMethod().getReference(); - if (allCreationsWithinClosureInteral(reference, instanceKey, callGraph, seen)) - return true; - } + if (!seen.contains(reference)) { + seen.add(reference); + + if (allCreationsWithinClosureInteral(reference, instanceKey, callGraph, seen)) + return true; } + } return false; } From efd7ce4c2954a6fc4724225dd5b195a332980c3c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 17:45:34 -0500 Subject: [PATCH 155/172] Fix warning. We should warn when it's hybrid and it has Python side-effects regardless if we suggest a refactoring or not. They may not take the refactoring suggestion. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 54c7b2868..b85b97b43 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -880,8 +880,8 @@ public void check() { } } - // if we are not de-hybridizing, issue a warning. - if (!this.willDehybridize()) + // Warn if the function has side-effects. + if (this.getHasPythonSideEffects() != null && this.getHasPythonSideEffects()) this.addWarning("This hybrid function potentially contains Python side-effects."); } } diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index aedb1a02d..4112d105b 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5250,8 +5250,8 @@ public void testPythonSideEffects47() throws Exception { RefactoringStatusEntry error = capturesLeakedTensor.getStatus().getEntryMatchingSeverity(RefactoringStatus.ERROR); assertEquals(PreconditionFailure.HAS_TENSOR_PARAMETERS.getCode(), error.getCode()); - // NOTE: Change to assertEquals(..., 2, ...) once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertEquals("We should warn that the hybrid function is capturing leaked tensors.", 1, + // NOTE: Change to assertEquals(..., 1, ...) once https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + assertEquals("We should warn that the hybrid function is capturing leaked tensors.", 0, Arrays.stream(capturesLeakedTensor.getStatus().getEntries()).map(RefactoringStatusEntry::getSeverity) .filter(s -> s == RefactoringStatus.WARNING).count()); From dd7a6d7969aff035cb4893a8e8d8cd878592ee2c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 17:47:31 -0500 Subject: [PATCH 156/172] Fix test. --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 4112d105b..91e78e4f7 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5263,10 +5263,10 @@ public void testPythonSideEffects47() throws Exception { long warningCount = Arrays.stream(capturesLeakedTensor.getStatus().getEntries()) .filter(e -> e.getSeverity() == RefactoringStatus.WARNING).count(); - // NOTE: Change to assertEquals(..., 2, ...) when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. + // NOTE: Change to assertEquals(..., 1, ...) when https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. // NOTE: Add assertEquals(RefactoringStatus.WARNING, entry.getSeverity()) when // https://github.com/ponder-lab/Hybridize-Functions-Refactoring/issues/281 is fixed. - assertEquals("Warn about a hybrid function that leaks as a potential tensor.", 1, warningCount); + assertEquals("Warn about a hybrid function that leaks as a potential tensor.", 0, warningCount); } @Test From 482329ad06f6103d8b60122dcc3fa66757f9ad18 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Mon, 6 Nov 2023 17:48:00 -0500 Subject: [PATCH 157/172] Use accessor. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index b85b97b43..b8a5467a2 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -857,7 +857,7 @@ public void check() { this.addFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS, "This function has no tensor parameters and may not benefit from hybridization."); - if (this.hasPythonSideEffects) + if (this.getHasPythonSideEffects()) this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "Can't hybridize a function with Python side-effects."); } } else { // Hybrid. Use table 2. From e890d114215390f3021669edec5885fa8a83674e Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 10:17:44 -0500 Subject: [PATCH 158/172] Add caching to Python side-effects. Shaved off ~4 minutes on a large project. --- .../hybridize/core/analysis/Function.java | 44 +++++++++++++++++++ ...HybridizeFunctionRefactoringProcessor.java | 1 + 2 files changed, 45 insertions(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index b8a5467a2..ac2b8c952 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -42,6 +42,7 @@ import org.python.pydev.parser.visitors.NodeUtils; import org.python.pydev.parser.visitors.TypeInfo; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; import com.ibm.wala.cast.ipa.callgraph.AstGlobalPointerKey; @@ -355,6 +356,8 @@ public boolean getReduceRetracingParamExists() { private RefactoringStatus status = new RefactoringStatus(); + private static Map>> creationsCache = Maps.newHashMap(); + public Function(FunctionDefinition fd) { this.functionDefinition = fd; } @@ -450,10 +453,46 @@ private Set filterSideEffects(Iterable modSet, CallGraph private static boolean allCreationsWithinClosure(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph) { Set seen = Sets.newHashSet(); return allCreationsWithinClosureInteral(methodReference, instanceKey, callGraph, seen); + } private static boolean allCreationsWithinClosureInteral(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph, Set seen) { + Map> cache2 = creationsCache.get(methodReference); + + if (cache2 != null) { + Map cache3 = cache2.get(instanceKey); + + if (cache3 != null) { + Boolean result = cache3.get(callGraph); + + if (result != null) + return result; + } + } + + boolean result = allCreationsWithinClosureInteral2(methodReference, instanceKey, callGraph, seen); + + if (cache2 == null) { + cache2 = Maps.newHashMap(); + creationsCache.put(methodReference, cache2); + } + + Map cache3 = cache2.get(instanceKey); + + if (cache3 == null) { + cache3 = Maps.newHashMap(); + cache2.put(instanceKey, cache3); + } + + Boolean previous = cache3.put(callGraph, result); + assert previous == null : "Should be a new key."; + + return result; + } + + private static boolean allCreationsWithinClosureInteral2(MethodReference methodReference, InstanceKey instanceKey, CallGraph callGraph, + Set seen) { seen.add(methodReference); // check this function. @@ -1083,4 +1122,9 @@ public boolean hasOnlyOneFailurePerKind() { public RefactoringStatusEntry getEntryMatchingFailure(PreconditionFailure failure) { return this.getStatus().getEntryMatchingCode(Function.PLUGIN_ID, failure.getCode()); } + + public static void clearCaches() { + creationsCache.clear(); + } + } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index bc53534ec..ba337c4d2 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -294,6 +294,7 @@ protected void clearCaches() { this.getProjectToCallGraphBuilder().clear(); this.getProjectToCallGraph().clear(); this.getProjectToTensorTypeAnalysis().clear(); + Function.clearCaches(); } @Override From a54f674d89b229a5ab2061577becd938ff0260d7 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 10:32:55 -0500 Subject: [PATCH 159/172] Analyze functions in parallel. --- .../HybridizeFunctionRefactoringProcessor.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index ba337c4d2..d15e7ad48 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -194,22 +194,20 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat LOG.info("Checking " + projectFunctions.size() + " function" + (allFunctions.size() > 1 ? "s" : "") + "."); subMonitor.beginTask(Messages.CheckingFunctions, allFunctions.size()); - for (Function func : projectFunctions) { + projectFunctions.parallelStream().forEach(func -> { LOG.info("Checking function: " + func + "."); // Find out if it's hybrid via the tf.function decorator. try { func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); } catch (BadLocationException e) { - throw new CoreException(Status.error("Could not compute hybridization for: : " + func, e)); + throw new RuntimeException("Could not compute hybridization for: " + func + ".", e); } - // TODO: Whether a function has a tensor argument should probably be an initial - // condition: functions w/o such arguments should not be candidates. try { func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); } catch (BadLocationException e) { - throw new CoreException(Status.error("Could not infer tensor parameters for: : " + func, e)); + throw new RuntimeException("Could not infer tensor parameters for: " + func + ".", e); } // Check Python side-effects. @@ -222,7 +220,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // next function. status.merge(func.getStatus()); subMonitor.worked(1); - continue; + return; } // check the function preconditions. @@ -238,7 +236,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat subMonitor.worked(1); assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; - } + }); } return status; From 5bcc9631944acaf21beda78b5cc819852fb534ac Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 10:33:46 -0500 Subject: [PATCH 160/172] Update comment. --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index ac2b8c952..800c7fb4c 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -508,7 +508,7 @@ private static boolean allCreationsWithinClosureInteral2(MethodReference methodR // Only consider the first node. The only difference should be the calling context, which shouldn't make a difference for us. CGNode node = cgNodes.iterator().next(); - // check the called functions. + // check the callees. for (Iterator succNodes = callGraph.getSuccNodes(node); succNodes.hasNext();) { CGNode next = succNodes.next(); MethodReference reference = next.getMethod().getReference(); From c54341ac8a92e455cc20ca62cb6052190ac96ecd Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 12:28:08 -0500 Subject: [PATCH 161/172] Conditionally check Python side-effects. Only check them if the function is a refactoring canddiate. This saves a lot of time. --- .../hybridize/core/analysis/Function.java | 12 +++++------ ...HybridizeFunctionRefactoringProcessor.java | 20 ++++++++++++++++++- .../HybridizeFunctionRefactoringTest.java | 8 +++++++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 800c7fb4c..3994ff5ab 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -887,33 +887,33 @@ public void check() { this.setRefactoring(CONVERT_EAGER_FUNCTION_TO_HYBRID); if (this.getLikelyHasTensorParameter()) { - if (!this.getHasPythonSideEffects()) { + if (this.getHasPythonSideEffects() != null && !this.getHasPythonSideEffects()) { this.addTransformation(Transformation.CONVERT_TO_HYBRID); this.setPassingPrecondition(P1); - } else + } else if (this.getHasPythonSideEffects() != null) // it has side-effects. this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "Can't hybridize a function with Python side-effects."); } else { // no tensor parameters. this.addFailure(PreconditionFailure.HAS_NO_TENSOR_PARAMETERS, "This function has no tensor parameters and may not benefit from hybridization."); - if (this.getHasPythonSideEffects()) + if (this.getHasPythonSideEffects() != null && this.getHasPythonSideEffects()) this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "Can't hybridize a function with Python side-effects."); } } else { // Hybrid. Use table 2. this.setRefactoring(OPTIMIZE_HYBRID_FUNCTION); if (!this.getLikelyHasTensorParameter()) { - if (!this.getHasPythonSideEffects()) { + if (this.getHasPythonSideEffects() != null && !this.getHasPythonSideEffects()) { this.addTransformation(CONVERT_TO_EAGER); this.setPassingPrecondition(P2); - } else // it has side-effects. + } else if (this.getHasPythonSideEffects() != null) // it has side-effects. this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); } else { // it has a tensor parameter. this.addFailure(PreconditionFailure.HAS_TENSOR_PARAMETERS, "Functions with tensor parameters may benefit from hybreidization."); - if (this.getHasPythonSideEffects()) { + if (this.getHasPythonSideEffects() != null && this.getHasPythonSideEffects()) { this.addFailure(PreconditionFailure.HAS_PYTHON_SIDE_EFFECTS, "De-hybridizing a function with Python side-effects may alter semantics."); } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index d15e7ad48..b22a96957 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -97,6 +97,8 @@ private static RefactoringStatus checkParameters(Function func) { */ private boolean dumpCallGraph = Boolean.getBoolean(DUMP_CALL_GRAPH_PROPERTY_KEY); + private boolean alwaysCheckPythonSideEffects; + public HybridizeFunctionRefactoringProcessor() { // Force the use of typeshed. It's an experimental feature of PyDev. InterpreterGeneralPreferences.FORCE_USE_TYPESHED = Boolean.TRUE; @@ -105,6 +107,11 @@ public HybridizeFunctionRefactoringProcessor() { CAstCallGraphUtil.AVOID_DUMP = !this.dumpCallGraph; } + public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffects) { + this(); + this.alwaysCheckPythonSideEffects = alwaysCheckPythonSideEffects; + } + public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet) throws TooManyMatchesException { this(); @@ -121,6 +128,12 @@ public HybridizeFunctionRefactoringProcessor(Set functionDef } } + public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet, boolean alwaysCheckPythonSideEffects) + throws TooManyMatchesException { + this(functionDefinitionSet); + this.alwaysCheckPythonSideEffects = alwaysCheckPythonSideEffects; + } + @Override public RefactoringStatus checkFinalConditions(IProgressMonitor pm, CheckConditionsContext context) throws CoreException, OperationCanceledException { @@ -212,7 +225,8 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat // Check Python side-effects. try { - func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); + if (this.getAlwaysCheckPythonSideEffects() || func.isHybrid() || func.getLikelyHasTensorParameter()) + func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); } catch (UndeterminablePythonSideEffectsException e) { LOG.warn("Unable to infer side-effects of: " + func + ".", e); func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, @@ -325,6 +339,10 @@ public String getProcessorName() { return Messages.Name; } + private boolean getAlwaysCheckPythonSideEffects() { + return this.alwaysCheckPythonSideEffects; + } + @Override public boolean isApplicable() throws CoreException { // TODO Auto-generated method stub diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 91e78e4f7..ae9772425 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -143,6 +143,11 @@ public int getInterpreterType() throws CoreException { private static final String TF_FUNCTION_FQN = "tensorflow.python.eager.def_function.function"; + /** + * Check Python side-effects regardless if it's a candidate. + */ + private static final boolean ALWAYS_CHECK_PYTHON_SIDE_EFFECTS = true; + /** * Add a module to the given {@link IPythonNature}. * @@ -550,7 +555,8 @@ private Set getFunctions(String fileNameWithoutExtension) throws Excep Set inputFunctionDefinitions = availableFunctionDefs.stream() .map(f -> new FunctionDefinition(f, fileNameWithoutExtension, inputTestFile, document, nature)).collect(Collectors.toSet()); - HybridizeFunctionRefactoringProcessor processor = new HybridizeFunctionRefactoringProcessor(inputFunctionDefinitions); + HybridizeFunctionRefactoringProcessor processor = new HybridizeFunctionRefactoringProcessor(inputFunctionDefinitions, + ALWAYS_CHECK_PYTHON_SIDE_EFFECTS); ProcessorBasedRefactoring refactoring = new ProcessorBasedRefactoring(processor); From c7033b1925abfb543b25bc2013120295e7a0cfca Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 12:43:32 -0500 Subject: [PATCH 162/172] Use static import. --- .../refactorings/HybridizeFunctionRefactoringProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index b22a96957..cbcf8506b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -1,5 +1,6 @@ package edu.cuny.hunter.hybridize.core.refactorings; +import static java.lang.Boolean.TRUE; import static org.eclipse.core.runtime.Platform.getLog; import java.io.IOException; @@ -101,7 +102,7 @@ private static RefactoringStatus checkParameters(Function func) { public HybridizeFunctionRefactoringProcessor() { // Force the use of typeshed. It's an experimental feature of PyDev. - InterpreterGeneralPreferences.FORCE_USE_TYPESHED = Boolean.TRUE; + InterpreterGeneralPreferences.FORCE_USE_TYPESHED = TRUE; // Have WALA dump the call graph if appropriate. CAstCallGraphUtil.AVOID_DUMP = !this.dumpCallGraph; From dd65812d4303546e6bc7882a609ec6811782356c Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 14:28:33 -0500 Subject: [PATCH 163/172] Add side-effect test that uses keyword arguments. --- .../HybridizeFunction/testPythonSideEffects56/in/A.py | 9 +++++++++ .../testPythonSideEffects56/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 6 ++++++ 3 files changed, 15 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/A.py new file mode 100644 index 000000000..9da8e1ec1 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/A.py @@ -0,0 +1,9 @@ +def g(p1, p2): + assert p1 == 5 and p2 == 2 + + +def f(): + g(5, p2=2) + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects56/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index ae9772425..fc0f6dfc4 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5434,5 +5434,11 @@ public void testPythonSideEffects55() throws Exception { assertEquals(Collections.singleton(Transformation.CONVERT_TO_EAGER), function.getTransformations()); } + @Test + public void testPythonSideEffects56() throws Exception { + Function f = getFunction("f"); + assertFalse("Keyword argument assignments shouldn't be considered as heap writes.", f.getHasPythonSideEffects()); + } + // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported } From 8b9b3461b9255cbea97d844f1014df27e20e6cf1 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Tue, 7 Nov 2023 15:24:33 -0500 Subject: [PATCH 164/172] Get the fix for https://github.com/wala/ML/issues/103. --- edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF | 2 +- hybridize.target | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF index d1f03f776..3cec2a06c 100644 --- a/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF +++ b/edu.cuny.hunter.hybridize.core/META-INF/MANIFEST.MF @@ -28,7 +28,7 @@ Import-Package: com.google.common.collect, com.ibm.wala.cast.python.loader;version="0.1.0", com.ibm.wala.cast.python.ml.analysis;version="0.1.0", com.ibm.wala.cast.python.ml.client;version="0.1.0", - com.ibm.wala.cast.python.modref;version="0.10.0", + com.ibm.wala.cast.python.modref;version="0.11.0", com.ibm.wala.cast.python.ssa, com.ibm.wala.cast.python.types, com.ibm.wala.cast.python.util, diff --git a/hybridize.target b/hybridize.target index 1c412cc0a..dd279f896 100644 --- a/hybridize.target +++ b/hybridize.target @@ -29,7 +29,7 @@ com.ibm.wala com.ibm.wala.cast.python.ml - 0.11.0-SNAPSHOT + 0.12.0-SNAPSHOT jar From 438007ef7c8a2fb38d88e6345e9155621439aabc Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 09:17:37 -0500 Subject: [PATCH 165/172] Add embedded function test. --- .../HybridizeFunction/testPythonSideEffects57/in/A.py | 11 +++++++++++ .../testPythonSideEffects57/in/requirements.txt | 0 .../tests/HybridizeFunctionRefactoringTest.java | 6 ++++++ 3 files changed, 17 insertions(+) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py new file mode 100644 index 000000000..a91d34221 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py @@ -0,0 +1,11 @@ +def f(): + + def g(): + return 5 + + a = g() + assert a == 5 + + +if __name__ == "__main__": + f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index fc0f6dfc4..e30c23a83 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5440,5 +5440,11 @@ public void testPythonSideEffects56() throws Exception { assertFalse("Keyword argument assignments shouldn't be considered as heap writes.", f.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects57() throws Exception { + Function f = getFunction("f"); + assertFalse("Embedded functions aren't side-effects.", f.getHasPythonSideEffects()); + } + // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported } From fa50a56a665ea6c96a33163a5b7b8929a464d04b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 14:08:18 -0500 Subject: [PATCH 166/172] Optimizable functions should be taken from candidates. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index d13e179f1..f3ab74d2b 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -164,8 +164,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { transformationsPrinter.printRecord(buildAttributeColumnValues(function, transformation)); } - // optimizable functions. - Set optimizableFunctions = processor.getOptimizableFunctions(); + // optimizable candidate functions. + Set optimizableFunctions = Sets.intersection(candidates, processor.getOptimizableFunctions()); resultsPrinter.print(optimizableFunctions.size()); // number. for (Function function : optimizableFunctions) From 936995868458efb28e12e4530cc6dea62fc52a02 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 14:08:18 -0500 Subject: [PATCH 167/172] Optimizable functions should be taken from candidates. --- .../handlers/EvaluateHybridizeFunctionRefactoringHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index d13e179f1..f3ab74d2b 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -164,8 +164,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { transformationsPrinter.printRecord(buildAttributeColumnValues(function, transformation)); } - // optimizable functions. - Set optimizableFunctions = processor.getOptimizableFunctions(); + // optimizable candidate functions. + Set optimizableFunctions = Sets.intersection(candidates, processor.getOptimizableFunctions()); resultsPrinter.print(optimizableFunctions.size()); // number. for (Function function : optimizableFunctions) From 48d75811e718c52cb243f62668bd5aa2606ac691 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 16:35:14 -0500 Subject: [PATCH 168/172] Add decorated test. --- .../testPythonSideEffects57/in/A.py | 3 +-- .../testPythonSideEffects58/in/A.py | 14 ++++++++++++++ .../testPythonSideEffects58/in/requirements.txt | 1 + .../tests/HybridizeFunctionRefactoringTest.java | 6 ++++++ 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py index a91d34221..9daede908 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects57/in/A.py @@ -7,5 +7,4 @@ def g(): assert a == 5 -if __name__ == "__main__": - f() +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/A.py new file mode 100644 index 000000000..a2e17dc04 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/A.py @@ -0,0 +1,14 @@ +import tensorflow as tf + + +def f(): + + @tf.function + def g(): + return 5 + + a = g() + assert a == 5 + + +f() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/requirements.txt new file mode 100644 index 000000000..b154f958f --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPythonSideEffects58/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index e30c23a83..5a67c2500 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -5446,5 +5446,11 @@ public void testPythonSideEffects57() throws Exception { assertFalse("Embedded functions aren't side-effects.", f.getHasPythonSideEffects()); } + @Test + public void testPythonSideEffects58() throws Exception { + Function f = getFunction("f"); + assertFalse("Decorated embedded functions aren't side-effects.", f.getHasPythonSideEffects()); + } + // TODO: Left off at: https://www.tensorflow.org/guide/function#recursive_tffunctions_are_not_supported } From c4ab93b254e6e426faa94d586175b365299dc00b Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 17:14:32 -0500 Subject: [PATCH 169/172] Add option to always check Python side-effects for the evaluator. --- .../edu/cuny/hunter/hybridize/core/utils/Util.java | 4 ++-- .../EvaluateHybridizeFunctionRefactoringHandler.java | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java index 1e333ea5d..39231bbe6 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/utils/Util.java @@ -48,10 +48,10 @@ public static Refactoring createRefactoring(Set functionDefi return new ProcessorBasedRefactoring(new HybridizeFunctionRefactoringProcessor(functionDefinitions)); } - public static HybridizeFunctionRefactoringProcessor createHybridizeFunctionRefactoring(IProject[] projects) + public static HybridizeFunctionRefactoringProcessor createHybridizeFunctionRefactoring(IProject[] projects, boolean alwaysCheckPythonSideEffects) throws ExecutionException, CoreException, IOException { Set functionDefinitions = getFunctionDefinitions(Arrays.asList(projects)); - return new HybridizeFunctionRefactoringProcessor(functionDefinitions); + return new HybridizeFunctionRefactoringProcessor(functionDefinitions, alwaysCheckPythonSideEffects); } public static Set getFunctionDefinitions(Iterable iterable) diff --git a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java index f3ab74d2b..c02e08016 100644 --- a/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java +++ b/edu.cuny.hunter.hybridize.eval/src/edu/cuny/hunter/hybridize/eval/handlers/EvaluateHybridizeFunctionRefactoringHandler.java @@ -72,6 +72,8 @@ public class EvaluateHybridizeFunctionRefactoringHandler extends EvaluateRefacto private static final String PERFORM_CHANGE_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.performChange"; + private static final String ALWAYS_CHECK_PYTHON_SIDE_EFFECTS_PROPERTY_KEY = "edu.cuny.hunter.hybridize.eval.alwaysCheckPythonSideEffects"; + private static String[] buildAttributeColumnNames(String... additionalColumnNames) { String[] primaryColumns = new String[] { "subject", "function", "module", "relative path" }; List ret = new ArrayList<>(Arrays.asList(primaryColumns)); @@ -89,6 +91,8 @@ private static Object[] buildAttributeColumnValues(Function function, Object... return ret.toArray(Object[]::new); } + private boolean alwaysCheckPythonSideEffects = Boolean.getBoolean(ALWAYS_CHECK_PYTHON_SIDE_EFFECTS_PROPERTY_KEY); + @Override public Object execute(ExecutionEvent event) throws ExecutionException { Job.create("Evaluating Hybridize Functions refactoring...", monitor -> { @@ -129,7 +133,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException { TimeCollector resultsTimeCollector = new TimeCollector(); resultsTimeCollector.start(); - HybridizeFunctionRefactoringProcessor processor = createHybridizeFunctionRefactoring(new IProject[] { project }); + HybridizeFunctionRefactoringProcessor processor = createHybridizeFunctionRefactoring(new IProject[] { project }, + this.getAlwaysCheckPythonSideEffects()); resultsTimeCollector.stop(); // run the precondition checking. @@ -339,4 +344,8 @@ private static IProject getProject(Object obj) { return null; } + + public boolean getAlwaysCheckPythonSideEffects() { + return alwaysCheckPythonSideEffects; + } } From 68b2225dcb23af4c565f79bd48f377cc960c5753 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 17:33:22 -0500 Subject: [PATCH 170/172] Look for a specific function. --- ...HybridizeFunctionRefactoringProcessor.java | 90 ++++++++++--------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index cbcf8506b..70af9baa9 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -208,50 +208,52 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat LOG.info("Checking " + projectFunctions.size() + " function" + (allFunctions.size() > 1 ? "s" : "") + "."); subMonitor.beginTask(Messages.CheckingFunctions, allFunctions.size()); - projectFunctions.parallelStream().forEach(func -> { - LOG.info("Checking function: " + func + "."); - - // Find out if it's hybrid via the tf.function decorator. - try { - func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); - } catch (BadLocationException e) { - throw new RuntimeException("Could not compute hybridization for: " + func + ".", e); - } - - try { - func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); - } catch (BadLocationException e) { - throw new RuntimeException("Could not infer tensor parameters for: " + func + ".", e); - } - - // Check Python side-effects. - try { - if (this.getAlwaysCheckPythonSideEffects() || func.isHybrid() || func.getLikelyHasTensorParameter()) - func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); - } catch (UndeterminablePythonSideEffectsException e) { - LOG.warn("Unable to infer side-effects of: " + func + ".", e); - func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, - "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); - // next function. - status.merge(func.getStatus()); - subMonitor.worked(1); - return; - } - - // check the function preconditions. - func.check(); - - status.merge(checkParameters(func)); - subMonitor.checkCanceled(); - - status.merge(checkDecorators(func)); - subMonitor.checkCanceled(); - - status.merge(func.getStatus()); - subMonitor.worked(1); - - assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; - }); + projectFunctions.stream() + .filter(f -> f.getIdentifier().equals("main") && f.getContainingModuleName().equals("pretrain_paired_tf")) + .forEach(func -> { + LOG.info("Checking function: " + func + "."); + + // Find out if it's hybrid via the tf.function decorator. + try { + func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); + } catch (BadLocationException e) { + throw new RuntimeException("Could not compute hybridization for: " + func + ".", e); + } + + try { + func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); + } catch (BadLocationException e) { + throw new RuntimeException("Could not infer tensor parameters for: " + func + ".", e); + } + + // Check Python side-effects. + try { + if (this.getAlwaysCheckPythonSideEffects() || func.isHybrid() || func.getLikelyHasTensorParameter()) + func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); + } catch (UndeterminablePythonSideEffectsException e) { + LOG.warn("Unable to infer side-effects of: " + func + ".", e); + func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, + "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); + // next function. + status.merge(func.getStatus()); + subMonitor.worked(1); + return; + } + + // check the function preconditions. + func.check(); + + status.merge(checkParameters(func)); + subMonitor.checkCanceled(); + + status.merge(checkDecorators(func)); + subMonitor.checkCanceled(); + + status.merge(func.getStatus()); + subMonitor.worked(1); + + assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; + }); } return status; From 9075f4e13c349daf05e34859908035cefefe4799 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Wed, 8 Nov 2023 22:58:17 -0500 Subject: [PATCH 171/172] Revert "Look for a specific function." This reverts commit 68b2225dcb23af4c565f79bd48f377cc960c5753. --- ...HybridizeFunctionRefactoringProcessor.java | 90 +++++++++---------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index 70af9baa9..cbcf8506b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -208,52 +208,50 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat LOG.info("Checking " + projectFunctions.size() + " function" + (allFunctions.size() > 1 ? "s" : "") + "."); subMonitor.beginTask(Messages.CheckingFunctions, allFunctions.size()); - projectFunctions.stream() - .filter(f -> f.getIdentifier().equals("main") && f.getContainingModuleName().equals("pretrain_paired_tf")) - .forEach(func -> { - LOG.info("Checking function: " + func + "."); - - // Find out if it's hybrid via the tf.function decorator. - try { - func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); - } catch (BadLocationException e) { - throw new RuntimeException("Could not compute hybridization for: " + func + ".", e); - } - - try { - func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); - } catch (BadLocationException e) { - throw new RuntimeException("Could not infer tensor parameters for: " + func + ".", e); - } - - // Check Python side-effects. - try { - if (this.getAlwaysCheckPythonSideEffects() || func.isHybrid() || func.getLikelyHasTensorParameter()) - func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); - } catch (UndeterminablePythonSideEffectsException e) { - LOG.warn("Unable to infer side-effects of: " + func + ".", e); - func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, - "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); - // next function. - status.merge(func.getStatus()); - subMonitor.worked(1); - return; - } - - // check the function preconditions. - func.check(); - - status.merge(checkParameters(func)); - subMonitor.checkCanceled(); - - status.merge(checkDecorators(func)); - subMonitor.checkCanceled(); - - status.merge(func.getStatus()); - subMonitor.worked(1); - - assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; - }); + projectFunctions.parallelStream().forEach(func -> { + LOG.info("Checking function: " + func + "."); + + // Find out if it's hybrid via the tf.function decorator. + try { + func.computeHybridization(subMonitor.split(IProgressMonitor.UNKNOWN)); + } catch (BadLocationException e) { + throw new RuntimeException("Could not compute hybridization for: " + func + ".", e); + } + + try { + func.inferTensorTensorParameters(analysis, subMonitor.split(IProgressMonitor.UNKNOWN)); + } catch (BadLocationException e) { + throw new RuntimeException("Could not infer tensor parameters for: " + func + ".", e); + } + + // Check Python side-effects. + try { + if (this.getAlwaysCheckPythonSideEffects() || func.isHybrid() || func.getLikelyHasTensorParameter()) + func.inferPythonSideEffects(callGraph, builder.getPointerAnalysis()); + } catch (UndeterminablePythonSideEffectsException e) { + LOG.warn("Unable to infer side-effects of: " + func + ".", e); + func.addFailure(PreconditionFailure.UNDETERMINABLE_SIDE_EFFECTS, + "Can't infer side-effects, most likely due to a call graph issue caused by a decorator or a missing function call."); + // next function. + status.merge(func.getStatus()); + subMonitor.worked(1); + return; + } + + // check the function preconditions. + func.check(); + + status.merge(checkParameters(func)); + subMonitor.checkCanceled(); + + status.merge(checkDecorators(func)); + subMonitor.checkCanceled(); + + status.merge(func.getStatus()); + subMonitor.worked(1); + + assert func.hasOnlyOneFailurePerKind() : "Count failures only once."; + }); } return status; From 12a721e40950a9c9afd700f9472695cfa38008fd Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 9 Nov 2023 10:51:04 -0500 Subject: [PATCH 172/172] Run tests sequentially. Add comment. --- ...HybridizeFunctionRefactoringProcessor.java | 44 +++++++++++++++++-- .../HybridizeFunctionRefactoringTest.java | 8 +++- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java index cbcf8506b..5d1cbb6be 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/refactorings/HybridizeFunctionRefactoringProcessor.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.core.resources.IProject; import org.eclipse.core.runtime.CoreException; @@ -25,7 +26,7 @@ import org.eclipse.ltk.core.refactoring.participants.CheckConditionsContext; import org.eclipse.ltk.core.refactoring.participants.RefactoringParticipant; import org.eclipse.ltk.core.refactoring.participants.SharableParticipants; -import org.python.pydev.ast.refactoring.TooManyMatchesException; +import org.python.pydev.ast.refactoring.TooManyMatchesException; /* FIXME: This exception sounds too low-level. */ import org.python.pydev.core.preferences.InterpreterGeneralPreferences; import com.ibm.wala.cast.ipa.callgraph.CAstCallGraphUtil; @@ -100,6 +101,8 @@ private static RefactoringStatus checkParameters(Function func) { private boolean alwaysCheckPythonSideEffects; + private boolean processFunctionsInParallel = true; + public HybridizeFunctionRefactoringProcessor() { // Force the use of typeshed. It's an experimental feature of PyDev. InterpreterGeneralPreferences.FORCE_USE_TYPESHED = TRUE; @@ -113,7 +116,13 @@ public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffect this.alwaysCheckPythonSideEffects = alwaysCheckPythonSideEffects; } - public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet) throws TooManyMatchesException { + public HybridizeFunctionRefactoringProcessor(boolean alwaysCheckPythonSideEffects, boolean processFunctionsInParallel) { + this(alwaysCheckPythonSideEffects); + this.processFunctionsInParallel = processFunctionsInParallel; + } + + public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet) + throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ { this(); // Convert the FunctionDefs to Functions. @@ -130,11 +139,17 @@ public HybridizeFunctionRefactoringProcessor(Set functionDef } public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet, boolean alwaysCheckPythonSideEffects) - throws TooManyMatchesException { + throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ { this(functionDefinitionSet); this.alwaysCheckPythonSideEffects = alwaysCheckPythonSideEffects; } + public HybridizeFunctionRefactoringProcessor(Set functionDefinitionSet, boolean alwaysCheckPythonSideEffects, + boolean processFunctionsInParallel) throws TooManyMatchesException /* FIXME: This exception sounds too low-level. */ { + this(functionDefinitionSet, alwaysCheckPythonSideEffects); + this.processFunctionsInParallel = processFunctionsInParallel; + } + @Override public RefactoringStatus checkFinalConditions(IProgressMonitor pm, CheckConditionsContext context) throws CoreException, OperationCanceledException { @@ -208,7 +223,7 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat LOG.info("Checking " + projectFunctions.size() + " function" + (allFunctions.size() > 1 ? "s" : "") + "."); subMonitor.beginTask(Messages.CheckingFunctions, allFunctions.size()); - projectFunctions.parallelStream().forEach(func -> { + this.getStream(projectFunctions).forEach(func -> { LOG.info("Checking function: " + func + "."); // Find out if it's hybrid via the tf.function decorator. @@ -257,6 +272,18 @@ private RefactoringStatus checkFunctions(IProgressMonitor monitor) throws Operat return status; } + /** + * Returns a {@link Stream} of {@link Function}s. Properties of the stream are dependent on the state of this + * {@link HybridizeFunctionRefactoringProcessor}. + * + * @param functions The {@link Set} of {@link Function}s from which to derive a {@link Stream}. + * @return A potentially parallel {@link Stream} of {@link Function}s. + */ + private Stream getStream(Set functions) { + Stream stream = functions.stream(); + return this.getProcessFunctionsInParallel() ? stream.parallel() : stream; + } + private TensorTypeAnalysis computeTensorTypeAnalysis(EclipsePythonProjectTensorAnalysisEngine engine, PythonSSAPropagationCallGraphBuilder builder) throws CancelException { Map projectToTensorTypeAnalysis = this.getProjectToTensorTypeAnalysis(); @@ -372,4 +399,13 @@ protected boolean getDumpCallGraph() { public Map getProjectToTensorTypeAnalysis() { return projectToTensorTypeAnalysis; } + + /** + * True iff project functions should be processed in parallel. Otherwise, they are processed sequentially. + * + * @return True iff project functions should be processed in parallel. + */ + private boolean getProcessFunctionsInParallel() { + return this.processFunctionsInParallel; + } } diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 5a67c2500..523ba0308 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -148,6 +148,12 @@ public int getInterpreterType() throws CoreException { */ private static final boolean ALWAYS_CHECK_PYTHON_SIDE_EFFECTS = true; + /** + * Whether we should run the function processing in parallel. Running in parallel makes the logs difficult to read and doesn't offer + * much in way of speedup since each test has only a few {@link Function}s. + */ + private static final boolean PROCESS_FUNCTIONS_IN_PARALLEL = false; + /** * Add a module to the given {@link IPythonNature}. * @@ -556,7 +562,7 @@ private Set getFunctions(String fileNameWithoutExtension) throws Excep .map(f -> new FunctionDefinition(f, fileNameWithoutExtension, inputTestFile, document, nature)).collect(Collectors.toSet()); HybridizeFunctionRefactoringProcessor processor = new HybridizeFunctionRefactoringProcessor(inputFunctionDefinitions, - ALWAYS_CHECK_PYTHON_SIDE_EFFECTS); + ALWAYS_CHECK_PYTHON_SIDE_EFFECTS, PROCESS_FUNCTIONS_IN_PARALLEL); ProcessorBasedRefactoring refactoring = new ProcessorBasedRefactoring(processor);