From df69fd5df0ae45d90704d1dee6f14b13bee0d971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sun, 11 Dec 2022 22:58:32 -0500 Subject: [PATCH 01/43] parsing positional arguments for tf.function decorator --- .../hybridize/core/analysis/Function.java | 112 ++++++++++++------ 1 file changed, 75 insertions(+), 37 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 2ac357d6..6a16b3f8 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -121,44 +121,82 @@ public HybridizationParameters(IProgressMonitor monitor) throws TooManyMatchesEx // tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters. if (tfFunctionDecorator.func instanceof Call) { Call callFunction = (Call) tfFunctionDecorator.func; - // We only care about the actual keywords for now. - // TODO: Parse positional arguments (#108). - keywordType[] keywords = callFunction.keywords; - for (keywordType keyword : keywords) { - if (keyword.arg instanceof NameTok) { - NameTok name = (NameTok) keyword.arg; - if (name.id.equals(FUNC)) - // Found parameter func - this.funcParamExists = true; - else if (name.id.equals(INPUT_SIGNATURE)) - // Found parameter input_signature - this.inputSignatureParamExists = true; - else if (name.id.equals(AUTOGRAPH)) - // Found parameter autograph - this.autoGraphParamExists = true; - // The version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - // The version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) - // Found parameter reduce_retracing - // or experimental_relax_shapes - this.reduceRetracingParamExists = true; - else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS)) - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; + + // Using keywords instead of positional arguments + if (callFunction.args.length == 0) { + keywordType[] keywords = callFunction.keywords; + for (keywordType keyword : keywords) { + if (keyword.arg instanceof NameTok) { + NameTok name = (NameTok) keyword.arg; + if (name.id.equals(FUNC)) + // Found parameter func + this.funcParamExists = true; + else if (name.id.equals(INPUT_SIGNATURE)) + // Found parameter input_signature + this.inputSignatureParamExists = true; + else if (name.id.equals(AUTOGRAPH)) + // Found parameter autograph + this.autoGraphParamExists = true; + // The version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + // The version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) + // Found parameter reduce_retracing + // or experimental_relax_shapes + this.reduceRetracingParamExists = true; + else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS)) + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + } } + } else { + /** + * Positional arguments for tf.function as per the documentation of TF 2.9: tf.function( func=None, + * input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, + * experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, + * experimental_follow_type_hints=None + */ + exprType[] arguments = callFunction.args; + if (arguments.length >= 1) + // Found parameter func + this.funcParamExists = true; + if (arguments.length >= 2) + // Found parameter input_signature + this.inputSignatureParamExists = true; + if (arguments.length >= 3) + // Found parameter autograph + this.autoGraphParamExists = true; + if (arguments.length >= 4) + // Found parameter jit_compile + this.jitCompileParamExists = true; + if (arguments.length >= 5) + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + if (arguments.length >= 6) + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + if (arguments.length >= 7) + // Found parameter experimental_relax_shapes (deprecated) + this.reduceRetracingParamExists = true; + if (arguments.length >= 8) + // Found parameter experimental_compile (deprecated) + this.jitCompileParamExists = true; + if (arguments.length >= 9) + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + } } // else, tf.function is used without parameters. } From 3b881a6543482ce809e9bb9bd71775315a7244be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sun, 11 Dec 2022 23:14:25 -0500 Subject: [PATCH 02/43] fixing comments --- .../cuny/hunter/hybridize/core/analysis/Function.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 6a16b3f8..c59a62bb 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -162,12 +162,11 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) } } } else { - /** - * Positional arguments for tf.function as per the documentation of TF 2.9: tf.function( func=None, - * input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, - * experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, - * experimental_follow_type_hints=None - */ + // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function( func=None, + // input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, + // experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, + // experimental_follow_type_hints=None + exprType[] arguments = callFunction.args; if (arguments.length >= 1) // Found parameter func From 2899de38355b6491f6626ed0af24cab4bb00dca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 12 Dec 2022 01:32:30 -0500 Subject: [PATCH 03/43] adding tests --- .../hybridize/core/analysis/Function.java | 4 +- .../testPositionalParameters/in/A.py | 8 + .../in/requirements.txt | 1 + .../testPositionalParameters10/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters2/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters3/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters4/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters5/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters6/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters7/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters8/in/A.py | 9 + .../in/requirements.txt | 1 + .../testPositionalParameters9/in/A.py | 12 ++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 190 ++++++++++++++++++ 22 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c59a62bb..da4f258d 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -170,7 +170,9 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) exprType[] arguments = callFunction.args; if (arguments.length >= 1) // Found parameter func - this.funcParamExists = true; + // Since we are parsing decorators, the first positional argument is `None` + // which is the default value of func, so we should not put as true + this.funcParamExists = false; if (arguments.length >= 2) // Found parameter input_signature this.inputSignatureParamExists = true; diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py new file mode 100644 index 00000000..383ea9e2 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py @@ -0,0 +1,8 @@ +import tensorflow as tf + +@tf.function(None) +def test(): + pass + +if __name__ == '__main__': + test() \ No newline at end of file diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py new file mode 100644 index 00000000..397b8256 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None, False) +def test(x: tf.Tensor): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py new file mode 100644 index 00000000..36b622e2 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),)) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py new file mode 100644 index 00000000..36eb5f1e --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py new file mode 100644 index 00000000..fd684f96 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py new file mode 100644 index 00000000..14af1511 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py new file mode 100644 index 00000000..7215dfb4 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix") +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py new file mode 100644 index 00000000..0a67bc50 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py new file mode 100644 index 00000000..0a67bc50 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py new file mode 100644 index 00000000..eddff3a2 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py @@ -0,0 +1,12 @@ +import tensorflow as tf + +# experimental_compile cannot be True or False because you get the following error ValueError: Cannot specify both 'experimental_compile' and 'jit_compile'. +# That is why I put the value as None. + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 8d824243..cdf7b6ea 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -787,6 +787,196 @@ public void testComputeParameters9() throws Exception { && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } + /** + * Test for #106. This tests whether we can parse one tf.function positional argument. + */ + @Test + public void testPositionalParameters() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse two tf.function positional arguments. + */ + @Test + public void testPositionalParameters2() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse three tf.function positional arguments. + */ + @Test + public void testPositionalParameters3() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse four tf.function positional arguments. + */ + @Test + public void testPositionalParameters4() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() + && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse five tf.function positional arguments. + */ + @Test + public void testPositionalParameters5() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse six tf.function positional arguments. + */ + @Test + public void testPositionalParameters6() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse seven tf.function positional arguments. + */ + @Test + public void testPositionalParameters7() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse eight tf.function positional arguments. + */ + @Test + public void testPositionalParameters8() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse nine tf.function positional arguments. + */ + @Test + public void testPositionalParameters9() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse ten tf.function positional arguments. + */ + @Test + public void testPositionalParameters10() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + /** * This simply tests whether we have the correct qualified name. */ From 731b66434d3da5094b5dc7fd3bc0bc60356408e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 12 Dec 2022 01:43:58 -0500 Subject: [PATCH 04/43] fix trailing whitespace --- .../HybridizeFunction/testPositionalParameters/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters10/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters2/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters3/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters4/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters5/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters6/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters7/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters8/in/A.py | 2 +- .../HybridizeFunction/testPositionalParameters9/in/A.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py index 383ea9e2..468be2fe 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None) -def test(): +def test(): pass if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py index 397b8256..bb4f86a7 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None, False) -def test(x: tf.Tensor): +def test(x: tf.Tensor): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py index 36b622e2..654eab9f 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),)) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py index 36eb5f1e..84399043 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py index fd684f96..faaecdc5 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py index 14af1511..9b9f77a7 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py index 7215dfb4..09dfb19a 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix") -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py index 0a67bc50..a7df0c25 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py index 0a67bc50..a7df0c25 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py @@ -1,7 +1,7 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) -def test(x): +def test(x): return x if __name__ == '__main__': diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py index eddff3a2..08980378 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py @@ -4,7 +4,7 @@ # That is why I put the value as None. @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None) -def test(x): +def test(x): return x if __name__ == '__main__': From a4cf2cd364a8a90664a669642cf2a3296a4ef27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 12 Dec 2022 02:08:54 -0500 Subject: [PATCH 05/43] fix --- .../hunter/hybridize/core/analysis/Function.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index da4f258d..7727f70f 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -168,6 +168,7 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // experimental_follow_type_hints=None exprType[] arguments = callFunction.args; + if (arguments.length >= 1) // Found parameter func // Since we are parsing decorators, the first positional argument is `None` @@ -183,18 +184,21 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter jit_compile this.jitCompileParamExists = true; if (arguments.length >= 5) + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + if (arguments.length >= 6) // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - if (arguments.length >= 6) + if (arguments.length >= 7) // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - if (arguments.length >= 7) + if (arguments.length >= 8) // Found parameter experimental_relax_shapes (deprecated) this.reduceRetracingParamExists = true; - if (arguments.length >= 8) + if (arguments.length >= 9) // Found parameter experimental_compile (deprecated) this.jitCompileParamExists = true; - if (arguments.length >= 9) + if (arguments.length >= 10) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; From d6b16edae678d3938ca5d3496b4efcabd47f7589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 13 Dec 2022 00:01:38 -0500 Subject: [PATCH 06/43] not counting the default values as existing --- .../hybridize/core/analysis/Function.java | 139 +++++++++++++----- .../testPositionalParameters11/in/A.py | 8 + .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 19 +++ 4 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 7727f70f..aed2fb65 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -15,6 +15,7 @@ import org.python.pydev.parser.jython.ast.Attribute; import org.python.pydev.parser.jython.ast.Call; import org.python.pydev.parser.jython.ast.FunctionDef; +import org.python.pydev.parser.jython.ast.Name; import org.python.pydev.parser.jython.ast.NameTok; import org.python.pydev.parser.jython.ast.argumentsType; import org.python.pydev.parser.jython.ast.decoratorsType; @@ -168,39 +169,111 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // experimental_follow_type_hints=None exprType[] arguments = callFunction.args; - - if (arguments.length >= 1) - // Found parameter func - // Since we are parsing decorators, the first positional argument is `None` - // which is the default value of func, so we should not put as true - this.funcParamExists = false; - if (arguments.length >= 2) - // Found parameter input_signature - this.inputSignatureParamExists = true; - if (arguments.length >= 3) - // Found parameter autograph - this.autoGraphParamExists = true; - if (arguments.length >= 4) - // Found parameter jit_compile - this.jitCompileParamExists = true; - if (arguments.length >= 5) - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - if (arguments.length >= 6) - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - if (arguments.length >= 7) - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - if (arguments.length >= 8) - // Found parameter experimental_relax_shapes (deprecated) - this.reduceRetracingParamExists = true; - if (arguments.length >= 9) - // Found parameter experimental_compile (deprecated) - this.jitCompileParamExists = true; - if (arguments.length >= 10) - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; + int position = 1; + + for (exprType argument : arguments) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (position == 1) { + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter func + this.funcParamExists = true; + } + if (position == 2) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter input_signature + this.inputSignatureParamExists = true; + } + if (position == 3) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "True") + break; + } + // Found parameter autograph + this.autoGraphParamExists = true; + } + if (position == 4) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter jit_compile + this.jitCompileParamExists = true; + } + if (position == 5) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "False") + break; + } + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + } + if (position == 6) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + } + if (position == 7) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + } + if (position == 8) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter experimental_relax_shapes (deprecated) + this.reduceRetracingParamExists = true; + } + if (position == 9) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter experimental_compile (deprecated) + this.jitCompileParamExists = true; + } + if (position == 10) { + // Default value of tf.function, we don't want to classify the parameter as existing + if (argument instanceof Name) { + Name nameArgument = (Name) argument; + if (nameArgument.id == "None") + break; + } + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + } + position++; + } } } // else, tf.function is used without parameters. diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py new file mode 100644 index 00000000..5e373fb8 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py @@ -0,0 +1,8 @@ +import tensorflow as tf + +@tf.function(None,None,True,None,False,None,None,None, None,None) +def test(): + pass + +if __name__ == '__main__': + test() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/requirements.txt new file mode 100644 index 00000000..b154f958 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index cdf7b6ea..638dbe1b 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -977,6 +977,25 @@ public void testPositionalParameters10() throws Exception { && args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } + /** + * Test for #106. This tests that if tf.function has positional arguments set as default, we do not count them as existing. + */ + @Test + public void testPositionalParameters11() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + /** * This simply tests whether we have the correct qualified name. */ From aeca4898695d3bc2b34b6be230e05d19f90de697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 13 Dec 2022 00:42:02 -0500 Subject: [PATCH 07/43] fix, might need to tidy up --- .../hybridize/core/analysis/Function.java | 111 +++++++++++------- 1 file changed, 70 insertions(+), 41 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index aed2fb65..eb89f0b3 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -163,7 +163,7 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) } } } else { - // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function( func=None, + // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function(func=None, // input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, // experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, // experimental_follow_type_hints=None @@ -176,101 +176,130 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) if (position == 1) { if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") + // Found parameter func + this.funcParamExists = true; + } else { + // Found parameter func + this.funcParamExists = true; } - // Found parameter func - this.funcParamExists = true; } if (position == 2) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") { + // Found parameter input_signature + this.inputSignatureParamExists = true; + } + } else { + // Found parameter input_signature + this.inputSignatureParamExists = true; } - // Found parameter input_signature - this.inputSignatureParamExists = true; } if (position == 3) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "True") - break; + if (nameArgument.id != "True") { + // Found parameter autograph + this.autoGraphParamExists = true; + } + } else { + // Found parameter autograph + this.autoGraphParamExists = true; } - // Found parameter autograph - this.autoGraphParamExists = true; } if (position == 4) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") { + // Found parameter jit_compile + this.jitCompileParamExists = true; + } + } else { + // Found parameter jit_compile + this.jitCompileParamExists = true; } - // Found parameter jit_compile - this.jitCompileParamExists = true; } if (position == 5) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "False") - break; + if (nameArgument.id != "False") { + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + } + } else { + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; } - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; } if (position == 6) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") { + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + } + } else { + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; } - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; } if (position == 7) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id == "None") { + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + } + } else { + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; } - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; } if (position == 8) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id == "None") { + // Found parameter experimental_relax_shapes (deprecated) + this.reduceRetracingParamExists = true; + } + } else { + // Found parameter experimental_relax_shapes (deprecated) + this.reduceRetracingParamExists = true; } - // Found parameter experimental_relax_shapes (deprecated) - this.reduceRetracingParamExists = true; } if (position == 9) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") { + // Found parameter experimental_compile (deprecated) + this.jitCompileParamExists = true; + } + } else { + // Found parameter experimental_compile (deprecated) + this.jitCompileParamExists = true; } - // Found parameter experimental_compile (deprecated) - this.jitCompileParamExists = true; } if (position == 10) { // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") - break; + if (nameArgument.id != "None") { + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + } + } else { + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; } - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; } position++; } From 61f1b1bf2717b8b9f5ec09935f5ad41bc40fd19a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 13 Dec 2022 00:51:57 -0500 Subject: [PATCH 08/43] fix --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index eb89f0b3..56f20888 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -253,7 +253,7 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") { + if (nameArgument.id != "None") { // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; } @@ -266,7 +266,7 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Default value of tf.function, we don't want to classify the parameter as existing if (argument instanceof Name) { Name nameArgument = (Name) argument; - if (nameArgument.id == "None") { + if (nameArgument.id != "None") { // Found parameter experimental_relax_shapes (deprecated) this.reduceRetracingParamExists = true; } From 109e0e466539e296259d856752a93c79463bfe3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Wed, 14 Dec 2022 18:08:01 -0500 Subject: [PATCH 09/43] Progress --- .../hybridize/core/analysis/Function.java | 241 ++++++++++-------- .../hunter/hybridize/core/analysis/Util.java | 53 +++- 2 files changed, 171 insertions(+), 123 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index af2457ca..68b48ebd 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -4,11 +4,13 @@ import java.io.File; import java.util.Objects; +import java.util.Set; import org.eclipse.core.runtime.ILog; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.IDocument; +import org.python.pydev.ast.codecompletion.revisited.visitors.Definition; import org.python.pydev.core.IPythonNature; import org.python.pydev.core.docutils.PySelection; import org.python.pydev.parser.jython.ast.Attribute; @@ -106,6 +108,15 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Will contain the last tf.function decorator decoratorsType tfFunctionDecorator = null; + // Declaring definitions of the decorator + Set declaringDefinitions = null; + + // Python source arguments from the declaring function + exprType[] declaringArguments = null; + + // Declaring definition of the decorator + Definition declaringDefinition = null; + // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { IDocument document = Function.this.getContainingDocument(); @@ -114,13 +125,30 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Save the hybrid decorator try { if (Function.isHybrid(decorator, Function.this.containingModuleName, Function.this.containingFile, selection, - Function.this.nature, monitor)) // TODO: Cache this from a previous call (#118). + Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118). tfFunctionDecorator = decorator; + declaringDefinitions = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, + Function.this.containingFile, Function.this.nature, monitor); + } } catch (AmbiguousDeclaringModuleException e) { throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e); } } // We expect to have the last tf.function decorator in tfFunctionDecorator + // Getting the definition, there should only be one in the set. + if (declaringDefinitions != null) { + declaringDefinition = declaringDefinitions.iterator().next(); + } + + // Getting the arguments from TensorFlow source + if (declaringDefinition != null) { + if (declaringDefinition.ast instanceof FunctionDef) { + FunctionDef declaringFunctionDefinition = (FunctionDef) declaringDefinition.ast; + argumentsType declaringArgumentTypes = declaringFunctionDefinition.args; + declaringArguments = declaringArgumentTypes.args; + } + } + if (tfFunctionDecorator != null) // tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters. if (tfFunctionDecorator.func instanceof Call) { @@ -141,13 +169,13 @@ else if (name.id.equals(INPUT_SIGNATURE)) else if (name.id.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; - // The version of the API we are using allows + // The latest version of the API we are using allows // parameter names jit_compile and // deprecated name experimental_compile else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - // The version of the API we are using allows + // The latest version of the API we are using allows // parameter names reduce_retracing // and deprecated name experimental_relax_shapes else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) @@ -172,139 +200,128 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // experimental_follow_type_hints=None exprType[] arguments = callFunction.args; - int position = 1; - - for (exprType argument : arguments) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (position == 1) { - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") - // Found parameter func - this.funcParamExists = true; - } else { - // Found parameter func - this.funcParamExists = true; + Name argumentName = null; + String argumentId = null; + + for (int i = 0; i < arguments.length; i++) { + + // Getting the arguments from the definition + if (declaringArguments != null) { + if (declaringArguments[i] instanceof Name) { + argumentName = (Name) declaringArguments[i]; + argumentId = argumentName.id; } } - if (position == 2) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { + + if (argumentId != null) { + // Matching the arguments from the definition and the arguments from the code being analyzed. + if (argumentId.equals(FUNC)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter func + this.funcParamExists = true; + } else { + // Found parameter func + this.funcParamExists = true; + } + } else if (argumentId.equals(INPUT_SIGNATURE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter input_signature + this.inputSignatureParamExists = true; + } else { // Found parameter input_signature this.inputSignatureParamExists = true; } - } else { - // Found parameter input_signature - this.inputSignatureParamExists = true; - } - } - if (position == 3) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "True") { + } else if (argumentId.equals(AUTOGRAPH)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "True") + // Found parameter autograph + this.autoGraphParamExists = true; + } else { // Found parameter autograph this.autoGraphParamExists = true; } - } else { - // Found parameter autograph - this.autoGraphParamExists = true; - } - } - if (position == 4) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { - // Found parameter jit_compile + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + } else if (argumentId.equals(JIT_COMPILE) || argumentId.equals(EXPERIMENTAL_COMPILE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + } else { + // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; } - } else { - // Found parameter jit_compile - this.jitCompileParamExists = true; - } - } - if (position == 5) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "False") { + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + } else if (argumentId.equals(REDUCE_RETRACING)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "False") + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + } else { // Found parameter reduce_retracing this.reduceRetracingParamExists = true; } - } else { - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - } - } - if (position == 6) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { + } else if (argumentId.equals(EXPERIMENTAL_RELAX_SHAPES)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter experimental_relax_shapes + this.reduceRetracingParamExists = true; + } else { + // Found parameter experimental_relax_shapes + this.reduceRetracingParamExists = true; + } + } else if (argumentId.equals(EXPERIMENTAL_IMPLEMENTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + } else { // Found parameter experimental_implements this.experimentalImplementsParamExists = true; } - } else { - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - } - } - if (position == 7) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { + } else if (argumentId.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + } else { // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; } - } else { - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - } - } - if (position == 8) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { - // Found parameter experimental_relax_shapes (deprecated) - this.reduceRetracingParamExists = true; - } - } else { - // Found parameter experimental_relax_shapes (deprecated) - this.reduceRetracingParamExists = true; - } - } - if (position == 9) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { - // Found parameter experimental_compile (deprecated) - this.jitCompileParamExists = true; - } - } else { - // Found parameter experimental_compile (deprecated) - this.jitCompileParamExists = true; - } - } - if (position == 10) { - // Default value of tf.function, we don't want to classify the parameter as existing - if (argument instanceof Name) { - Name nameArgument = (Name) argument; - if (nameArgument.id != "None") { + } else if (argumentId.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + } else { // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; } - } else { - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; } } - position++; } } diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java index bf7093ab..26b00a0c 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java @@ -33,26 +33,21 @@ public class Util { private static final ILog LOG = getLog(Util.class); - + /** - * Get the name of the module defining the entity described in the given {@link PySelection}. + * Get the name of the definition of the entity described in the given {@link PySelection}. * * @param selection The {@link PySelection} in question. * @param containingModName The name of the module containing the {@link PySelection}. * @param containingFile The {@link File} containing the module. * @param nature The {@link IPythonNature} to use. * @param monitor The IProgressMonitor to use. - * @return The name of the module defining the given {@link PySelection}. + * @return The definition of {@link PySelection}. * @throws AmbiguousDeclaringModuleException On ambiguous definitions found. * @throws BadLocationException On a parsing error. */ - public static String getDeclaringModuleName(PySelection selection, String containingModName, File containingFile, IPythonNature nature, - IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException { - monitor.beginTask("Getting declaring module name.", 1); - - LOG.info(String.format("Getting declaring module name for selection: %s in line: %s, module: %s, file: %s, and project: %s.", - selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName, containingFile, - nature.getProject())); + public static Set getDeclaringDefinition(PySelection selection, String containingModName, File containingFile, + IPythonNature nature, IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException { RefactoringRequest request = new RefactoringRequest(containingFile, selection, nature); @@ -78,13 +73,49 @@ public static String getDeclaringModuleName(PySelection selection, String contai containingFile.getName(), nature.getProject())); // Collect the potential declaring module names. - Set potentialDeclaringModuleNames = new HashSet<>(); + Set potentialDeclaringDefinitions = new HashSet<>(); // for each match. for (ItemPointer itemPointer : pointers) { Definition definition = itemPointer.definition; LOG.info("Found definition: " + definition + "."); + // add it to the set of found module names. + potentialDeclaringDefinitions.add(definition); + } + + return potentialDeclaringDefinitions; + + } + + /** + * Get the name of the module defining the entity described in the given {@link PySelection}. + * + * @param selection The {@link PySelection} in question. + * @param containingModName The name of the module containing the {@link PySelection}. + * @param containingFile The {@link File} containing the module. + * @param nature The {@link IPythonNature} to use. + * @param monitor The IProgressMonitor to use. + * @return The name of the module defining the given {@link PySelection}. + * @throws AmbiguousDeclaringModuleException On ambiguous definitions found. + * @throws BadLocationException On a parsing error. + */ + public static String getDeclaringModuleName(PySelection selection, String containingModName, File containingFile, IPythonNature nature, + IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException { + monitor.beginTask("Getting declaring module name.", 1); + + LOG.info(String.format("Getting declaring module name for selection: %s in line: %s, module: %s, file: %s, and project: %s.", + selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName, containingFile, + nature.getProject())); + + Set definitions = getDeclaringDefinition(selection, containingModName, containingFile, nature, monitor); + + // Collect the potential declaring module names. + Set potentialDeclaringModuleNames = new HashSet<>(); + + // for each definition. + for (Definition definition : definitions) { + IModule module = definition.module; LOG.info(String.format("Found module: %s.", module)); From b7d9d5cf5cecd85e4d085e3999c746db69d5e064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Wed, 14 Dec 2022 18:15:46 -0500 Subject: [PATCH 10/43] fix trailing whitespace --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 68b48ebd..c89be121 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -34,7 +34,6 @@ * @author Tatiana Castro Vélez */ public class Function extends RefactorableProgramEntity { - /** * Parameters that may be passed to a tf.fuction decorator. Parameter descriptions found at: * https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/function @@ -113,10 +112,10 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Python source arguments from the declaring function exprType[] declaringArguments = null; - + // Declaring definition of the decorator Definition declaringDefinition = null; - + // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { IDocument document = Function.this.getContainingDocument(); From 6dbbadd6890a5d2b4a40296449c51fe100c8b0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 15 Dec 2022 06:37:25 -0500 Subject: [PATCH 11/43] Throwing exceptions --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index c89be121..525b3689 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -190,6 +190,10 @@ else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; + else { + throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) + + " is not supported in this tool. This tool supports up to v2.9"); + } } } } else { @@ -319,6 +323,9 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; } + } else { + throw new IllegalArgumentException(String.format("The tf.function argument in position " + i + + " is not supported. This tool supports up to v2.9")); } } } From e5c5b80017b4fb8ce703890f3680451c046f5598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 16 Dec 2022 01:11:46 -0400 Subject: [PATCH 12/43] adding tests for another tf version --- .../testPositionalParameters12/in/A.py | 8 +++++++ .../in/requirements.txt | 1 + .../HybridizeFunctionRefactoringTest.java | 21 ++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py new file mode 100644 index 00000000..d605b2fe --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py @@ -0,0 +1,8 @@ +import tensorflow as tf + +@tf.function(None,(tf.TensorSpec(shape=[None], dtype=tf.float32),),True, True, "google.matmul_low_rank_matrix") +def test(): + pass + +if __name__ == '__main__': + test() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt new file mode 100644 index 00000000..14e2508b --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.6.0 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 8a2ac477..5a1e8f9b 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -803,7 +803,7 @@ public void testComputeParameters12() throws Exception { && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } - + /** * Test for #106. This tests whether we can parse one tf.function positional argument. */ @@ -1013,6 +1013,25 @@ public void testPositionalParameters11() throws Exception { && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } + /** + * Test for #106. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.6). + */ + @Test + public void testPositionalParameters12() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + /** * This simply tests whether we have the correct qualified name. */ From 1046cff21915c9a8ac9bd1c1124e4d069b189341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 16 Dec 2022 21:56:13 -0400 Subject: [PATCH 13/43] changing test's tf.function version and some formatting --- .../HybridizeFunction/testPositionalParameters12/in/A.py | 2 +- .../testPositionalParameters12/in/requirements.txt | 2 +- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py index d605b2fe..b0a470ee 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py @@ -1,6 +1,6 @@ import tensorflow as tf -@tf.function(None,(tf.TensorSpec(shape=[None], dtype=tf.float32),),True, True, "google.matmul_low_rank_matrix") +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), True, True, "google.matmul_low_rank_matrix") def test(): pass diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt index 14e2508b..75f6aa97 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt @@ -1 +1 @@ -tensorflow==2.6.0 +tensorflow==2.8.0 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 5a1e8f9b..726b1356 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -1014,7 +1014,7 @@ public void testPositionalParameters11() throws Exception { } /** - * Test for #106. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.6). + * Test for #106. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.8). */ @Test public void testPositionalParameters12() throws Exception { From 8c489aa5db61fbb103983d1bc4c8cfe5835ee3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 16 Dec 2022 22:09:26 -0400 Subject: [PATCH 14/43] fixing python code --- .../HybridizeFunction/testPositionalParameters12/in/A.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py index b0a470ee..1d06ea38 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py @@ -1,8 +1,9 @@ import tensorflow as tf @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), True, True, "google.matmul_low_rank_matrix") -def test(): - pass +def test(x): + return x if __name__ == '__main__': - test() + number = tf.constant([1.0, 1.0]) + test(number) From 130f807afc61a6abb0fb974604022d678fc8d39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 16 Dec 2022 23:20:22 -0400 Subject: [PATCH 15/43] fixing to a non-default value --- .../HybridizeFunction/testPositionalParameters12/in/A.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py index 1d06ea38..38a64eab 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py @@ -1,6 +1,6 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), True, True, "google.matmul_low_rank_matrix") +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, "google.matmul_low_rank_matrix") def test(x): return x From 55cadea3d8e2cc1e7009db74b47079c01a9f41b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sat, 17 Dec 2022 00:27:47 -0400 Subject: [PATCH 16/43] update --- .../hybridize/core/analysis/Function.java | 237 +++++++++--------- .../testComputeParameters13/in/A.py | 9 + .../in/requirements.txt | 0 .../HybridizeFunctionRefactoringTest.java | 16 ++ 4 files changed, 147 insertions(+), 115 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 525b3689..2c46e317 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -3,6 +3,7 @@ import static org.eclipse.core.runtime.Platform.getLog; import java.io.File; +import java.util.ArrayList; import java.util.Objects; import java.util.Set; @@ -34,6 +35,7 @@ * @author Tatiana Castro Vélez */ public class Function extends RefactorableProgramEntity { + /** * Parameters that may be passed to a tf.fuction decorator. Parameter descriptions found at: * https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/function @@ -110,12 +112,6 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Declaring definitions of the decorator Set declaringDefinitions = null; - // Python source arguments from the declaring function - exprType[] declaringArguments = null; - - // Declaring definition of the decorator - Definition declaringDefinition = null; - // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { IDocument document = Function.this.getContainingDocument(); @@ -134,11 +130,17 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep } } // We expect to have the last tf.function decorator in tfFunctionDecorator + // Declaring definition of the decorator + Definition declaringDefinition = null; + // Getting the definition, there should only be one in the set. if (declaringDefinitions != null) { declaringDefinition = declaringDefinitions.iterator().next(); } + // Python source arguments from the declaring definition + exprType[] declaringArguments = null; + // Getting the arguments from TensorFlow source if (declaringDefinition != null) { if (declaringDefinition.ast instanceof FunctionDef) { @@ -148,6 +150,19 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep } } + // Python source arguments from the declaring definition + ArrayList argumentIdDeclaringDefintion = new ArrayList<>(); + + // Getting the arguments from the definition + if (declaringArguments != null) { + for (exprType declaredArgument : declaringArguments) { + if (declaredArgument instanceof Name) { + Name argumentName = (Name) declaredArgument; + argumentIdDeclaringDefintion.add(argumentName.id); + } + } + } + if (tfFunctionDecorator != null) // tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters. if (tfFunctionDecorator.func instanceof Call) { @@ -159,35 +174,37 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep for (keywordType keyword : keywords) { if (keyword.arg instanceof NameTok) { NameTok name = (NameTok) keyword.arg; - if (name.id.equals(FUNC)) + if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter func this.funcParamExists = true; - else if (name.id.equals(INPUT_SIGNATURE)) + else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter input_signature this.inputSignatureParamExists = true; - else if (name.id.equals(AUTOGRAPH)) + else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter autograph this.autoGraphParamExists = true; // The latest version of the API we are using allows // parameter names jit_compile and // deprecated name experimental_compile - else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) + else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) + && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; // The latest version of the API we are using allows // parameter names reduce_retracing // and deprecated name experimental_relax_shapes - else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) + else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) + && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter reduce_retracing // or experimental_relax_shapes this.reduceRetracingParamExists = true; - else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS)) + else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) + else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) + else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; else { @@ -203,133 +220,123 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // experimental_follow_type_hints=None exprType[] arguments = callFunction.args; - Name argumentName = null; - String argumentId = null; for (int i = 0; i < arguments.length; i++) { - // Getting the arguments from the definition - if (declaringArguments != null) { - if (declaringArguments[i] instanceof Name) { - argumentName = (Name) declaringArguments[i]; - argumentId = argumentName.id; - } - } + String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); - if (argumentId != null) { - // Matching the arguments from the definition and the arguments from the code being analyzed. - if (argumentId.equals(FUNC)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter func - this.funcParamExists = true; - } else { + // Matching the arguments from the definition and the arguments from the code being analyzed. + if (argumentDeclaringDefinition.equals(FUNC)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter func this.funcParamExists = true; - } - } else if (argumentId.equals(INPUT_SIGNATURE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter input_signature - this.inputSignatureParamExists = true; - } else { + } else { + // Found parameter func + this.funcParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter input_signature this.inputSignatureParamExists = true; - } - } else if (argumentId.equals(AUTOGRAPH)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "True") - // Found parameter autograph - this.autoGraphParamExists = true; - } else { + } else { + // Found parameter input_signature + this.inputSignatureParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "True") // Found parameter autograph this.autoGraphParamExists = true; - } - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - } else if (argumentId.equals(JIT_COMPILE) || argumentId.equals(EXPERIMENTAL_COMPILE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - } else { + } else { + // Found parameter autograph + this.autoGraphParamExists = true; + } + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + } else if (argumentDeclaringDefinition.equals(JIT_COMPILE) + || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - } - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - } else if (argumentId.equals(REDUCE_RETRACING)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "False") - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - } else { + } else { + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + } + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + } else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "False") // Found parameter reduce_retracing this.reduceRetracingParamExists = true; - } - } else if (argumentId.equals(EXPERIMENTAL_RELAX_SHAPES)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_relax_shapes - this.reduceRetracingParamExists = true; - } else { + } else { + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_relax_shapes this.reduceRetracingParamExists = true; - } - } else if (argumentId.equals(EXPERIMENTAL_IMPLEMENTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - } else { + } else { + // Found parameter experimental_relax_shapes + this.reduceRetracingParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - } - } else if (argumentId.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - } else { + } else { + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - } - } else if (argumentId.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - } else { + } else { + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; - } } else { - throw new IllegalArgumentException(String.format("The tf.function argument in position " + i - + " is not supported. This tool supports up to v2.9")); + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; } + } else { + throw new IllegalArgumentException(String.format( + "The tf.function argument in position " + i + " is not supported. This tool supports up to v2.9")); } } - } } // else, tf.function is used without parameters. } diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py new file mode 100644 index 00000000..a5d7ce38 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py @@ -0,0 +1,9 @@ +import tensorflow as tf + +@tf.function(reduce_retracing=True) +def test(x): + return x + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + test(number) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 726b1356..c535dbe5 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -804,6 +804,22 @@ public void testComputeParameters12() throws Exception { } + /** + * Test for #30. Contains an incorrect/invalid parameter name. + */ + @Test(expected = IllegalArgumentException.class) + public void testComputeParameters13() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + } + /** * Test for #106. This tests whether we can parse one tf.function positional argument. */ From c83eccfd288d3f8e4d423723fb971a3a27f21395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sat, 17 Dec 2022 00:29:55 -0400 Subject: [PATCH 17/43] adding requirement --- .../testComputeParameters13/in/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt index e69de29b..75f6aa97 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.8.0 From 4361e2e7c3821fe2124891136f267e50b318bc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sat, 17 Dec 2022 01:15:22 -0400 Subject: [PATCH 18/43] update --- .../hybridize/core/analysis/Function.java | 289 +++++++++--------- .../HybridizeFunctionRefactoringTest.java | 30 +- 2 files changed, 159 insertions(+), 160 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 2c46e317..f358acef 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -169,173 +169,172 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep Call callFunction = (Call) tfFunctionDecorator.func; // Using keywords instead of positional arguments - if (callFunction.args.length == 0) { - keywordType[] keywords = callFunction.keywords; - for (keywordType keyword : keywords) { - if (keyword.arg instanceof NameTok) { - NameTok name = (NameTok) keyword.arg; - if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter func - this.funcParamExists = true; - else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter input_signature - this.inputSignatureParamExists = true; - else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter autograph - this.autoGraphParamExists = true; - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) - && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) - && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter reduce_retracing - // or experimental_relax_shapes - this.reduceRetracingParamExists = true; - else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - else { - throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) - + " is not supported in this tool. This tool supports up to v2.9"); - } + keywordType[] keywords = callFunction.keywords; + for (keywordType keyword : keywords) { + if (keyword.arg instanceof NameTok) { + NameTok name = (NameTok) keyword.arg; + if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter func + this.funcParamExists = true; + else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter input_signature + this.inputSignatureParamExists = true; + else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter autograph + this.autoGraphParamExists = true; + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) + && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) + && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter reduce_retracing + // or experimental_relax_shapes + this.reduceRetracingParamExists = true; + else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + else { + throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) + + " is not supported in this tool. This tool supports up to v2.9"); } } - } else { - // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function(func=None, - // input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, - // experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, - // experimental_follow_type_hints=None - - exprType[] arguments = callFunction.args; - - for (int i = 0; i < arguments.length; i++) { - - String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); - - // Matching the arguments from the definition and the arguments from the code being analyzed. - if (argumentDeclaringDefinition.equals(FUNC)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter func - this.funcParamExists = true; - } else { + } + + // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function(func=None, + // input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, + // experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, + // experimental_follow_type_hints=None + + exprType[] arguments = callFunction.args; + + for (int i = 0; i < arguments.length; i++) { + + String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); + + // Matching the arguments from the definition and the arguments from the code being analyzed. + if (argumentDeclaringDefinition.equals(FUNC)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter func this.funcParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter input_signature - this.inputSignatureParamExists = true; - } else { + } else { + // Found parameter func + this.funcParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter input_signature this.inputSignatureParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "True") - // Found parameter autograph - this.autoGraphParamExists = true; - } else { + } else { + // Found parameter input_signature + this.inputSignatureParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "True") // Found parameter autograph this.autoGraphParamExists = true; - } - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - } else if (argumentDeclaringDefinition.equals(JIT_COMPILE) - || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - } else { + } else { + // Found parameter autograph + this.autoGraphParamExists = true; + } + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + } else if (argumentDeclaringDefinition.equals(JIT_COMPILE) + || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - } - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - } else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "False") - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - } else { + } else { + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + } + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + } else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "False") // Found parameter reduce_retracing this.reduceRetracingParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_relax_shapes - this.reduceRetracingParamExists = true; - } else { + } else { + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_relax_shapes this.reduceRetracingParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - } else { + } else { + // Found parameter experimental_relax_shapes + this.reduceRetracingParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - } else { + } else { + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - } else { + } else { + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + } + } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { + // Not considering the default values + if (arguments[i] instanceof Name) { + Name nameArgument = (Name) arguments[i]; + if (nameArgument.id != "None") // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; - } } else { - throw new IllegalArgumentException(String.format( - "The tf.function argument in position " + i + " is not supported. This tool supports up to v2.9")); + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; } + } else { + throw new IllegalArgumentException(String.format( + "The tf.function argument in position " + i + " is not supported. This tool supports up to v2.9")); + } } } // else, tf.function is used without parameters. diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index c535dbe5..cb26946a 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -804,21 +804,21 @@ public void testComputeParameters12() throws Exception { } - /** - * Test for #30. Contains an incorrect/invalid parameter name. - */ - @Test(expected = IllegalArgumentException.class) - public void testComputeParameters13() throws Exception { - Set functions = this.getFunctions(); - assertNotNull(functions); - assertEquals(1, functions.size()); - Function function = functions.iterator().next(); - assertNotNull(function); - - Function.HybridizationParameters args = function.getHybridizationParameters(); - assertNotNull(args); - - } +// /** +// * Test for #30. Contains an incorrect/invalid parameter name. +// */ +// @Test(expected = IllegalArgumentException.class) +// public void testComputeParameters13() throws Exception { +// Set functions = this.getFunctions(); +// assertNotNull(functions); +// assertEquals(1, functions.size()); +// Function function = functions.iterator().next(); +// assertNotNull(function); +// +// Function.HybridizationParameters args = function.getHybridizationParameters(); +// assertNotNull(args); +// +// } /** * Test for #106. This tests whether we can parse one tf.function positional argument. From 53c44f1f5f3f72c04a9d97bbc83cae8986ada5ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Sat, 17 Dec 2022 01:22:28 -0400 Subject: [PATCH 19/43] fix checkstyle --- .../tests/HybridizeFunctionRefactoringTest.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index cb26946a..726b1356 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -804,22 +804,6 @@ public void testComputeParameters12() throws Exception { } -// /** -// * Test for #30. Contains an incorrect/invalid parameter name. -// */ -// @Test(expected = IllegalArgumentException.class) -// public void testComputeParameters13() throws Exception { -// Set functions = this.getFunctions(); -// assertNotNull(functions); -// assertEquals(1, functions.size()); -// Function function = functions.iterator().next(); -// assertNotNull(function); -// -// Function.HybridizationParameters args = function.getHybridizationParameters(); -// assertNotNull(args); -// -// } - /** * Test for #106. This tests whether we can parse one tf.function positional argument. */ From 23ae949c8bb3616a801a69528bbb3b76677d4448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 3 Jan 2023 17:11:06 -0400 Subject: [PATCH 20/43] update --- .../hybridize/core/analysis/Function.java | 100 +++++++++--------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index f358acef..8ef64995 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -168,60 +168,9 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep if (tfFunctionDecorator.func instanceof Call) { Call callFunction = (Call) tfFunctionDecorator.func; - // Using keywords instead of positional arguments - keywordType[] keywords = callFunction.keywords; - for (keywordType keyword : keywords) { - if (keyword.arg instanceof NameTok) { - NameTok name = (NameTok) keyword.arg; - if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter func - this.funcParamExists = true; - else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter input_signature - this.inputSignatureParamExists = true; - else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter autograph - this.autoGraphParamExists = true; - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) - && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) - && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter reduce_retracing - // or experimental_relax_shapes - this.reduceRetracingParamExists = true; - else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - else { - throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) - + " is not supported in this tool. This tool supports up to v2.9"); - } - } - } - - // Positional arguments for tf.function as per the documentation of TF 2.9: tf.function(func=None, - // input_signature=None, autograph=True, jit_compile=None, reduce_retracing=False, experimental_implements=None, - // experimental_autograph_options=None, experimental_relax_shapes=None, experimental_compile=None, - // experimental_follow_type_hints=None - + // Processing positional arguments for tf.function a exprType[] arguments = callFunction.args; - for (int i = 0; i < arguments.length; i++) { - String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); // Matching the arguments from the definition and the arguments from the code being analyzed. @@ -337,6 +286,53 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDe } } + + // Processing keywords arguments + // If we have keyword parameter, afterwards, we cannot have positional parameters because it would result in invalid + // Python code. + keywordType[] keywords = callFunction.keywords; + for (keywordType keyword : keywords) { + if (keyword.arg instanceof NameTok) { + NameTok name = (NameTok) keyword.arg; + if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter func + this.funcParamExists = true; + else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter input_signature + this.inputSignatureParamExists = true; + else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter autograph + this.autoGraphParamExists = true; + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) + && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) + && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter reduce_retracing + // or experimental_relax_shapes + this.reduceRetracingParamExists = true; + else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; + else { + throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) + + " is not supported in this tool. This tool supports up to v2.9"); + } + } + } } // else, tf.function is used without parameters. } From 94a005f7914a9364a5a701d373ae74ee81174e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 9 Jan 2023 10:24:16 -0400 Subject: [PATCH 21/43] fixing comments --- .../hybridize/core/analysis/Function.java | 4 ---- .../HybridizeFunctionRefactoringTest.java | 24 +++++++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 8ef64995..5e178644 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -280,10 +280,6 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; } - } else { - throw new IllegalArgumentException(String.format( - "The tf.function argument in position " + i + " is not supported. This tool supports up to v2.9")); - } } diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 726b1356..5fb355fe 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -569,7 +569,7 @@ public void testAmbiguousDefinition() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument input_signature. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument input_signature. */ @Test public void testComputeParameters() throws Exception { @@ -588,7 +588,7 @@ public void testComputeParameters() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument experimental_autograph_options + * Test for #30. This simply tests whether we can parse the tf.function keyword argument experimental_autograph_options */ @Test public void testComputeParameters2() throws Exception { @@ -608,7 +608,7 @@ public void testComputeParameters2() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument experimental_follow_type_hints. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument experimental_follow_type_hints. */ @Test public void testComputeParameters3() throws Exception { @@ -627,7 +627,7 @@ public void testComputeParameters3() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument experimental_implements. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument experimental_implements. */ @Test public void testComputeParameters4() throws Exception { @@ -646,7 +646,7 @@ public void testComputeParameters4() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument jit_compile. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument jit_compile. */ @Test public void testComputeParameters5() throws Exception { @@ -665,7 +665,7 @@ public void testComputeParameters5() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument reduce_retracing. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument reduce_retracing. */ @Test public void testComputeParameters6() throws Exception { @@ -684,7 +684,7 @@ public void testComputeParameters6() throws Exception { } /** - * Test for #30. This simply tests whether we can parse the tf.function argument autograph. + * Test for #30. This simply tests whether we can parse the tf.function keyword argument autograph. */ @Test public void testComputeParameters7() throws Exception { @@ -722,7 +722,7 @@ public void testComputeParameters8() throws Exception { } /** - * Test for #30. This simply tests whether we can parse tf.function arguments when we have multiple. + * Test for #30. This simply tests whether we can parse tf.function keyword arguments when we have multiple. */ @Test public void testComputeParameters9() throws Exception { @@ -741,7 +741,7 @@ public void testComputeParameters9() throws Exception { } /** - * Test for #30. Test custom decorator with the same parameter names as tf.function. + * Test for #30. Test custom decorator with the same keyword parameter names as tf.function. */ @Test public void testComputeParameters10() throws Exception { @@ -762,8 +762,8 @@ public void testComputeParameters10() throws Exception { } /** - * Test for #30. Test custom decorator with the same parameter names as tf.function and a tf.function (total of two decorators) and only - * count the parameters from the tf.function decorator. + * Test for #30. Test custom decorator with the same keyword parameter names as tf.function and a tf.function (total: two decorators) + * and only count the keyword parameters from the tf.function decorator. */ @Test public void testComputeParameters11() throws Exception { @@ -785,7 +785,7 @@ public void testComputeParameters11() throws Exception { } /** - * Test for #30. Tests two different tf.functions. Should only count the parameters of the last one. + * Test for #30. Tests two different tf.functions. Should only count the keyword parameters of the last one. */ @Test public void testComputeParameters12() throws Exception { From a3cfae637e8814f110015b71ae43dea72c08dca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 9 Jan 2023 17:13:10 -0400 Subject: [PATCH 22/43] progress on comments --- .../testPositionalParameters/in/A.py | 6 +- .../testPositionalParameters10/in/A.py | 13 ++-- .../testPositionalParameters11/in/A.py | 9 +-- .../testPositionalParameters12/in/A.py | 11 +-- .../in/requirements.txt | 2 +- .../testPositionalParameters13/in/A.py | 11 +++ .../in/requirements.txt | 1 + .../testPositionalParameters14/in/A.py | 11 +++ .../in/requirements.txt | 1 + .../testPositionalParameters2/in/A.py | 11 +-- .../testPositionalParameters3/in/A.py | 4 +- .../testPositionalParameters4/in/A.py | 4 +- .../testPositionalParameters5/in/A.py | 4 +- .../testPositionalParameters6/in/A.py | 4 +- .../testPositionalParameters7/in/A.py | 6 +- .../testPositionalParameters8/in/A.py | 2 + .../testPositionalParameters9/in/A.py | 5 +- .../HybridizeFunctionRefactoringTest.java | 68 +++++++++++++++---- 18 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/requirements.txt create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/A.py create mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py index 468be2fe..a9556c00 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py @@ -1,8 +1,10 @@ import tensorflow as tf -@tf.function(None) + +@tf.function def test(): pass + if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py index bb4f86a7..d59e32ec 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py @@ -1,9 +1,14 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None, False) -def test(x: tf.Tensor): +# experimental_compile cannot be True or False because you get the following error ValueError: Cannot specify both 'experimental_compile' and 'jit_compile'. +# That is why I put the value as None. + + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None) +def test(x): return x + if __name__ == '__main__': - number = tf.constant([1.0, 1.0]) - print(test(number)) + number = tf.constant([1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py index 5e373fb8..bb4f86a7 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py @@ -1,8 +1,9 @@ import tensorflow as tf -@tf.function(None,None,True,None,False,None,None,None, None,None) -def test(): - pass +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None, False) +def test(x: tf.Tensor): + return x if __name__ == '__main__': - test() + number = tf.constant([1.0, 1.0]) + print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py index 38a64eab..f27c0fdf 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py @@ -1,9 +1,10 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, "google.matmul_low_rank_matrix") -def test(x): - return x + +@tf.function(None, None, True, None, False, None, None, None, None, None) +def test(): + pass + if __name__ == '__main__': - number = tf.constant([1.0, 1.0]) - test(number) + test() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt index 75f6aa97..b154f958 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt @@ -1 +1 @@ -tensorflow==2.8.0 +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/A.py new file mode 100644 index 00000000..1db4200b --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/A.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, "google.matmul_low_rank_matrix") +def test(x): + return x + + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + test(number) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/requirements.txt new file mode 100644 index 00000000..75f6aa97 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.8.0 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/A.py new file mode 100644 index 00000000..0148e095 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/A.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), autograph=True) +def test(x): + return x + + +if __name__ == '__main__': + number = tf.constant([1.0, 1.0]) + test(number) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt new file mode 100644 index 00000000..75f6aa97 --- /dev/null +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt @@ -0,0 +1 @@ +tensorflow==2.8.0 diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py index 654eab9f..9ec0253d 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters2/in/A.py @@ -1,9 +1,10 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),)) -def test(x): - return x + +@tf.function(None) +def test(): + pass + if __name__ == '__main__': - number = tf.constant([1.0, 1.0]) - print(test(number)) + test() diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py index 84399043..9954b0f0 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters3/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False) + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),)) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0, 1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py index faaecdc5..3f5f5735 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters4/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True) + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0, 1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py index 9b9f77a7..f99166ec 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters5/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True) + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0, 1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py index 09dfb19a..23b75626 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters6/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix") + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0, 1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py index a7df0c25..1b0f5e25 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters7/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) + +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix") def test(x): return x + if __name__ == '__main__': - number = tf.constant([1.0]) + number = tf.constant([1.0, 1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py index a7df0c25..d5417224 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters8/in/A.py @@ -1,9 +1,11 @@ import tensorflow as tf + @tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py index 08980378..56491ef5 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters9/in/A.py @@ -1,12 +1,11 @@ import tensorflow as tf -# experimental_compile cannot be True or False because you get the following error ValueError: Cannot specify both 'experimental_compile' and 'jit_compile'. -# That is why I put the value as None. -@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None) +@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True) def test(x): return x + if __name__ == '__main__': number = tf.constant([1.0]) print(test(number)) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 5fb355fe..a39ae021 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -805,7 +805,7 @@ public void testComputeParameters12() throws Exception { } /** - * Test for #106. This tests whether we can parse one tf.function positional argument. + * Test for #106. This tests a tf.function with no arguments. */ @Test public void testPositionalParameters() throws Exception { @@ -824,7 +824,7 @@ public void testPositionalParameters() throws Exception { } /** - * Test for #106. This tests whether we can parse two tf.function positional arguments. + * Test for #106. This tests whether we can parse one tf.function positional argument. */ @Test public void testPositionalParameters2() throws Exception { @@ -837,6 +837,29 @@ public void testPositionalParameters2() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); + // In this test, we have only the first positional argument `func`. If we want to use tf.function as a decorator, that first + // parameter + // should be None. Since we are only capturing the parameters that have non-default values, this test asserts that we don't have any + // of the parameters present. + assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } + + /** + * Test for #106. This tests whether we can parse two tf.function positional arguments. + */ + @Test + public void testPositionalParameters3() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + assertTrue(args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); @@ -846,7 +869,7 @@ public void testPositionalParameters2() throws Exception { * Test for #106. This tests whether we can parse three tf.function positional arguments. */ @Test - public void testPositionalParameters3() throws Exception { + public void testPositionalParameters4() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -865,7 +888,7 @@ public void testPositionalParameters3() throws Exception { * Test for #106. This tests whether we can parse four tf.function positional arguments. */ @Test - public void testPositionalParameters4() throws Exception { + public void testPositionalParameters5() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -884,7 +907,7 @@ public void testPositionalParameters4() throws Exception { * Test for #106. This tests whether we can parse five tf.function positional arguments. */ @Test - public void testPositionalParameters5() throws Exception { + public void testPositionalParameters6() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -903,7 +926,7 @@ public void testPositionalParameters5() throws Exception { * Test for #106. This tests whether we can parse six tf.function positional arguments. */ @Test - public void testPositionalParameters6() throws Exception { + public void testPositionalParameters7() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -922,7 +945,7 @@ public void testPositionalParameters6() throws Exception { * Test for #106. This tests whether we can parse seven tf.function positional arguments. */ @Test - public void testPositionalParameters7() throws Exception { + public void testPositionalParameters8() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -933,7 +956,7 @@ public void testPositionalParameters7() throws Exception { assertNotNull(args); assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } @@ -941,7 +964,7 @@ public void testPositionalParameters7() throws Exception { * Test for #106. This tests whether we can parse eight tf.function positional arguments. */ @Test - public void testPositionalParameters8() throws Exception { + public void testPositionalParameters9() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -960,7 +983,7 @@ public void testPositionalParameters8() throws Exception { * Test for #106. This tests whether we can parse nine tf.function positional arguments. */ @Test - public void testPositionalParameters9() throws Exception { + public void testPositionalParameters10() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -979,7 +1002,7 @@ public void testPositionalParameters9() throws Exception { * Test for #106. This tests whether we can parse ten tf.function positional arguments. */ @Test - public void testPositionalParameters10() throws Exception { + public void testPositionalParameters11() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -998,7 +1021,7 @@ public void testPositionalParameters10() throws Exception { * Test for #106. This tests that if tf.function has positional arguments set as default, we do not count them as existing. */ @Test - public void testPositionalParameters11() throws Exception { + public void testPositionalParameters12() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -1017,7 +1040,7 @@ public void testPositionalParameters11() throws Exception { * Test for #106. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.8). */ @Test - public void testPositionalParameters12() throws Exception { + public void testPositionalParameters13() throws Exception { Set functions = this.getFunctions(); assertNotNull(functions); assertEquals(1, functions.size()); @@ -1031,6 +1054,25 @@ public void testPositionalParameters12() throws Exception { && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } + + /** + * Test for #125. This test mixing positional and keyword arguments. + */ + @Test + public void testPositionalParameters14() throws Exception { + Set functions = this.getFunctions(); + assertNotNull(functions); + assertEquals(1, functions.size()); + Function function = functions.iterator().next(); + assertNotNull(function); + + Function.HybridizationParameters args = function.getHybridizationParameters(); + assertNotNull(args); + + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() + && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + } /** * This simply tests whether we have the correct qualified name. From b145a53d61386bb75d89fc114ad7451c81565e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 9 Jan 2023 18:27:42 -0400 Subject: [PATCH 23/43] progress --- .../testPositionalParameters14/in/requirements.txt | 2 +- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt index 75f6aa97..b154f958 100644 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt +++ b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt @@ -1 +1 @@ -tensorflow==2.8.0 +tensorflow==2.9.3 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index a39ae021..da4ba097 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -956,7 +956,7 @@ public void testPositionalParameters8() throws Exception { assertNotNull(args); assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); } From 72d53c65712b068da2c41389628e247e4ac978a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 10 Jan 2023 10:45:09 -0500 Subject: [PATCH 24/43] removing code that has to do with keywords, not positionals --- .../hybridize/core/analysis/Function.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 5e178644..3f82c92b 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -290,43 +290,37 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep for (keywordType keyword : keywords) { if (keyword.arg instanceof NameTok) { NameTok name = (NameTok) keyword.arg; - if (name.id.equals(FUNC) && argumentIdDeclaringDefintion.contains(name.id)) + if (name.id.equals(FUNC)) // Found parameter func this.funcParamExists = true; - else if (name.id.equals(INPUT_SIGNATURE) && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(INPUT_SIGNATURE)) // Found parameter input_signature this.inputSignatureParamExists = true; - else if (name.id.equals(AUTOGRAPH) && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; // The latest version of the API we are using allows // parameter names jit_compile and // deprecated name experimental_compile - else if ((name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) - && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; // The latest version of the API we are using allows // parameter names reduce_retracing // and deprecated name experimental_relax_shapes - else if ((name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) - && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) // Found parameter reduce_retracing // or experimental_relax_shapes this.reduceRetracingParamExists = true; - else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS) && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(EXPERIMENTAL_IMPLEMENTS)) // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS) && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS) && argumentIdDeclaringDefintion.contains(name.id)) + else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; - else { - throw new IllegalArgumentException(String.format("The tf.function argument " + name.id) - + " is not supported in this tool. This tool supports up to v2.9"); - } } } } // else, tf.function is used without parameters. From d05963b7f691fd8c3c7379b7afc9fc5f08fed8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 10 Jan 2023 10:51:56 -0500 Subject: [PATCH 25/43] Adding newline --- .../hybridize/tests/HybridizeFunctionRefactoringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index da4ba097..bb310920 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -1842,4 +1842,4 @@ public void testHasLikelyTensorParameter10() throws Exception { // TODO: Test arbitrary expression. // TODO: Test cast/assert statements? // TODO: Test tf.Tensor-like things? -} \ No newline at end of file +} From fffd0d0199cfe73e49e9cc59b4f84da7f220ecc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 13 Jan 2023 12:43:50 -0500 Subject: [PATCH 26/43] removing code that accounts for default values, adding documentation, removing a test case that doesn't have to do with this PR --- .../hybridize/core/analysis/Function.java | 162 +++++------------- .../testComputeParameters13/in/A.py | 9 - .../in/requirements.txt | 1 - .../HybridizeFunctionRefactoringTest.java | 42 +++-- 4 files changed, 67 insertions(+), 147 deletions(-) delete mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py delete mode 100644 edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 3f82c92b..3567d5ca 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -63,42 +63,47 @@ public class HybridizationParameters { private static final String FUNC = "func"; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph. + * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph. Available in TF version [2.0,2.11]. */ private boolean autoGraphParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_follow_type_hints. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_follow_type_hints. Available in TF version + * [2.4,2.11]. */ private boolean experimentaFollowTypeHintsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_autograph_options. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_autograph_options. Available in TF version + * [2.0,2.11]. */ private boolean experimentalAutographOptionsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_implements. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_implements. Available in TF version + * [2.1,2.11]. */ private boolean experimentalImplementsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter func. + * True iff this {@link Function}'s {@link decoratorsType} has parameter func. Available in TF version [2.0,2.11]. */ private boolean funcParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter input_signature. + * True iff this {@link Function}'s {@link decoratorsType} has parameter input_signature. Available in TF version [2.0,2.11]. */ private boolean inputSignatureParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter jit_compile. + * True iff this {@link Function}'s {@link decoratorsType} has parameter jit_compile. Available in TF version [2.5,2.11]. + * Previously, experimental_compile which is available TF version [2.1,2.11]. */ private boolean jitCompileParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter reduce_retracing. + * True iff this {@link Function}'s {@link decoratorsType} has parameter reduce_retracing. Available in TF version [2.9,2.11]. + * Previously, experimental_relax_shapes which is available TF version [2.0,2.11]. */ private boolean reduceRetracingParamExists; @@ -174,113 +179,40 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); // Matching the arguments from the definition and the arguments from the code being analyzed. - if (argumentDeclaringDefinition.equals(FUNC)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter func - this.funcParamExists = true; - } else { - // Found parameter func - this.funcParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter input_signature - this.inputSignatureParamExists = true; - } else { - // Found parameter input_signature - this.inputSignatureParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "True") - // Found parameter autograph - this.autoGraphParamExists = true; - } else { - // Found parameter autograph - this.autoGraphParamExists = true; - } - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile - } else if (argumentDeclaringDefinition.equals(JIT_COMPILE) - || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - } else { - // Found parameter jit_compile/experimental_compile - this.jitCompileParamExists = true; - } - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes - } else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "False") - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - } else { - // Found parameter reduce_retracing - this.reduceRetracingParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_relax_shapes - this.reduceRetracingParamExists = true; - } else { - // Found parameter experimental_relax_shapes - this.reduceRetracingParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - } else { - // Found parameter experimental_implements - this.experimentalImplementsParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - } else { - // Found parameter experimental_autograph_options - this.experimentalAutographOptionsParamExists = true; - } - } else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) { - // Not considering the default values - if (arguments[i] instanceof Name) { - Name nameArgument = (Name) arguments[i]; - if (nameArgument.id != "None") - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - } else { - // Found parameter experimental_follow_type_hints - this.experimentaFollowTypeHintsParamExists = true; - } - } + if (argumentDeclaringDefinition.equals(FUNC)) + // Found parameter func + this.funcParamExists = true; + else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) + // Found parameter input_signature + this.inputSignatureParamExists = true; + else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) + // Found parameter autograph + this.autoGraphParamExists = true; + // The latest version of the API we are using allows + // parameter names jit_compile and + // deprecated name experimental_compile + else if (argumentDeclaringDefinition.equals(JIT_COMPILE) + || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) + // Found parameter jit_compile/experimental_compile + this.jitCompileParamExists = true; + // The latest version of the API we are using allows + // parameter names reduce_retracing + // and deprecated name experimental_relax_shapes + else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) + // Found parameter reduce_retracing + this.reduceRetracingParamExists = true; + else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) + // Found parameter experimental_relax_shapes + this.reduceRetracingParamExists = true; + else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) + // Found parameter experimental_implements + this.experimentalImplementsParamExists = true; + else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) + // Found parameter experimental_autograph_options + this.experimentalAutographOptionsParamExists = true; + else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) + // Found parameter experimental_follow_type_hints + this.experimentaFollowTypeHintsParamExists = true; } // Processing keywords arguments diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py deleted file mode 100644 index a5d7ce38..00000000 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/A.py +++ /dev/null @@ -1,9 +0,0 @@ -import tensorflow as tf - -@tf.function(reduce_retracing=True) -def test(x): - return x - -if __name__ == '__main__': - number = tf.constant([1.0, 1.0]) - test(number) diff --git a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt b/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt deleted file mode 100644 index 75f6aa97..00000000 --- a/edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testComputeParameters13/in/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tensorflow==2.8.0 diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index bb310920..8a119cf7 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -838,12 +838,10 @@ public void testPositionalParameters2() throws Exception { assertNotNull(args); // In this test, we have only the first positional argument `func`. If we want to use tf.function as a decorator, that first - // parameter - // should be None. Since we are only capturing the parameters that have non-default values, this test asserts that we don't have any - // of the parameters present. + // parameter should be None. assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -862,7 +860,7 @@ public void testPositionalParameters3() throws Exception { assertTrue(args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -881,7 +879,7 @@ public void testPositionalParameters4() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -900,7 +898,7 @@ public void testPositionalParameters5() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -919,7 +917,7 @@ public void testPositionalParameters6() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -938,7 +936,7 @@ public void testPositionalParameters7() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -957,7 +955,7 @@ public void testPositionalParameters8() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -976,7 +974,7 @@ public void testPositionalParameters9() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -995,7 +993,7 @@ public void testPositionalParameters10() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -1014,11 +1012,11 @@ public void testPositionalParameters11() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** - * Test for #106. This tests that if tf.function has positional arguments set as default, we do not count them as existing. + * Test for #106. This tests that if tf.function has positional arguments set as default, we do count them as existing. */ @Test public void testPositionalParameters12() throws Exception { @@ -1031,9 +1029,9 @@ public void testPositionalParameters12() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() - && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() + && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** @@ -1052,9 +1050,9 @@ public void testPositionalParameters13() throws Exception { assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } - + /** * Test for #125. This test mixing positional and keyword arguments. */ @@ -1069,9 +1067,9 @@ public void testPositionalParameters14() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() - && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); } /** From 49065c9eee38e001fdbf667062061d493586f7ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 13 Jan 2023 14:02:36 -0500 Subject: [PATCH 27/43] removing a character from a comment --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 3567d5ca..441db494 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -173,7 +173,7 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep if (tfFunctionDecorator.func instanceof Call) { Call callFunction = (Call) tfFunctionDecorator.func; - // Processing positional arguments for tf.function a + // Processing positional arguments for tf.function exprType[] arguments = callFunction.args; for (int i = 0; i < arguments.length; i++) { String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); From e52d371e365aa84a53a26a032d8bd6b5b8340b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 19 Jan 2023 00:37:16 -0500 Subject: [PATCH 28/43] changing the versions to constants --- .../hybridize/core/analysis/Function.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 441db494..402e6599 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -42,68 +42,73 @@ public class Function extends RefactorableProgramEntity { */ public class HybridizationParameters { + // Available in TF version [2.4,2.11]. private static final String EXPERIMENTAL_FOLLOW_TYPE_HINTS = "experimental_follow_type_hints"; + // Available in TF version [2.0,2.11]. private static final String EXPERIMENTAL_AUTOGRAPH_OPTIONS = "experimental_autograph_options"; + // Available in TF version [2.1,2.11]. private static final String EXPERIMENTAL_IMPLEMENTS = "experimental_implements"; + // Available in TF version [2.9,2.11]. Previously, experimental_relax_shapes which is available TF version [2.0,2.11]. private static final String REDUCE_RETRACING = "reduce_retracing"; + // Available in TF version [2.0,2.11]. private static final String EXPERIMENTAL_RELAX_SHAPES = "experimental_relax_shapes"; + // Available in TF version [2.1,2.11]. private static final String EXPERIMENTAL_COMPILE = "experimental_compile"; + // Available in TF version [2.5,2.11]. Previously, experimental_compile which is available TF version [2.1,2.11]. private static final String JIT_COMPILE = "jit_compile"; + // Available in TF version [2.0,2.11]. private static final String AUTOGRAPH = "autograph"; + // Available in TF version [2.0,2.11]. private static final String INPUT_SIGNATURE = "input_signature"; + // Available in TF version [2.0,2.11]. private static final String FUNC = "func"; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph. Available in TF version [2.0,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph. */ private boolean autoGraphParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_follow_type_hints. Available in TF version - * [2.4,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_follow_type_hints. */ private boolean experimentaFollowTypeHintsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_autograph_options. Available in TF version - * [2.0,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_autograph_options. */ private boolean experimentalAutographOptionsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_implements. Available in TF version - * [2.1,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter experimental_implements. */ private boolean experimentalImplementsParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter func. Available in TF version [2.0,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter func. */ private boolean funcParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter input_signature. Available in TF version [2.0,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter input_signature. */ private boolean inputSignatureParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter jit_compile. Available in TF version [2.5,2.11]. - * Previously, experimental_compile which is available TF version [2.1,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter jit_compile. */ private boolean jitCompileParamExists; /** - * True iff this {@link Function}'s {@link decoratorsType} has parameter reduce_retracing. Available in TF version [2.9,2.11]. - * Previously, experimental_relax_shapes which is available TF version [2.0,2.11]. + * True iff this {@link Function}'s {@link decoratorsType} has parameter reduce_retracing. */ private boolean reduceRetracingParamExists; From ca0c4d24b823eff91aad28aa53b1261bb81f647a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 19 Jan 2023 00:44:11 -0500 Subject: [PATCH 29/43] add comments regarding the declaring defintions --- .../hunter/hybridize/core/analysis/Function.java | 14 ++++++++------ .../cuny/hunter/hybridize/core/analysis/Util.java | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 402e6599..58f12711 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -119,8 +119,9 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Will contain the last tf.function decorator decoratorsType tfFunctionDecorator = null; - // Declaring definitions of the decorator - Set declaringDefinitions = null; + // Declaring definitions of the decorator, if it contains multiple definitions there might be more than one in this set. Since + // we are dealing with tf.function, we expect only one. + Set declaringDefinitionSet = null; // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { @@ -132,7 +133,8 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep if (Function.isHybrid(decorator, Function.this.containingModuleName, Function.this.containingFile, selection, Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118). tfFunctionDecorator = decorator; - declaringDefinitions = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, + // Returns the set of potential declaring definitions of the selection. + declaringDefinitionSet = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor); } } catch (AmbiguousDeclaringModuleException e) { @@ -140,12 +142,12 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep } } // We expect to have the last tf.function decorator in tfFunctionDecorator - // Declaring definition of the decorator + // Declaring definition of the decorator. We get the single definition of tf.function. Definition declaringDefinition = null; // Getting the definition, there should only be one in the set. - if (declaringDefinitions != null) { - declaringDefinition = declaringDefinitions.iterator().next(); + if (declaringDefinitionSet != null) { + declaringDefinition = declaringDefinitionSet.iterator().next(); } // Python source arguments from the declaring definition diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java index 26b00a0c..7e761292 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java @@ -35,7 +35,7 @@ public class Util { private static final ILog LOG = getLog(Util.class); /** - * Get the name of the definition of the entity described in the given {@link PySelection}. + * Get the set of potential declaring definitions of the entity described in the given {@link PySelection}. * * @param selection The {@link PySelection} in question. * @param containingModName The name of the module containing the {@link PySelection}. From ebd388085b1f4d2f808e95b771d088fbbec36c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 19 Jan 2023 00:53:34 -0500 Subject: [PATCH 30/43] changing variable name --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 58f12711..9a852ac3 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -121,7 +121,7 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Declaring definitions of the decorator, if it contains multiple definitions there might be more than one in this set. Since // we are dealing with tf.function, we expect only one. - Set declaringDefinitionSet = null; + Set potentialDeclaringDefinitionSet = null; // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { @@ -134,7 +134,7 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118). tfFunctionDecorator = decorator; // Returns the set of potential declaring definitions of the selection. - declaringDefinitionSet = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, + potentialDeclaringDefinitionSet = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor); } } catch (AmbiguousDeclaringModuleException e) { @@ -146,8 +146,8 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep Definition declaringDefinition = null; // Getting the definition, there should only be one in the set. - if (declaringDefinitionSet != null) { - declaringDefinition = declaringDefinitionSet.iterator().next(); + if (potentialDeclaringDefinitionSet != null) { + declaringDefinition = potentialDeclaringDefinitionSet.iterator().next(); } // Python source arguments from the declaring definition From 19a8a7221b474629902b26efe7fbdb33385bb36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 23 Jan 2023 14:52:17 -0500 Subject: [PATCH 31/43] fixing issue #, changing order of assert clause --- .../HybridizeFunctionRefactoringTest.java | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java index 0357be24..c96e75d3 100644 --- a/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java +++ b/edu.cuny.hunter.hybridize.tests/test cases/edu/cuny/hunter/hybridize/tests/HybridizeFunctionRefactoringTest.java @@ -827,7 +827,7 @@ public void testComputeParameters12() throws Exception { } /** - * Test for #106. This tests a tf.function with no arguments. + * Test for #108. This tests a tf.function with no arguments. */ @Test public void testPositionalParameters() throws Exception { @@ -840,13 +840,13 @@ public void testPositionalParameters() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + assertTrue(!args.hasFuncParam() && !args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && !args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse one tf.function positional argument. + * Test for #108. This tests whether we can parse one tf.function positional argument. */ @Test public void testPositionalParameters2() throws Exception { @@ -861,13 +861,13 @@ public void testPositionalParameters2() throws Exception { // In this test, we have only the first positional argument `func`. If we want to use tf.function as a decorator, that first // parameter should be None. - assertTrue(!args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + assertTrue(args.hasFuncParam() && !args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse two tf.function positional arguments. + * Test for #108. This tests whether we can parse two tf.function positional arguments. */ @Test public void testPositionalParameters3() throws Exception { @@ -880,13 +880,13 @@ public void testPositionalParameters3() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && !args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse three tf.function positional arguments. + * Test for #108. This tests whether we can parse three tf.function positional arguments. */ @Test public void testPositionalParameters4() throws Exception { @@ -899,13 +899,13 @@ public void testPositionalParameters4() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse four tf.function positional arguments. + * Test for #108. This tests whether we can parse four tf.function positional arguments. */ @Test public void testPositionalParameters5() throws Exception { @@ -918,13 +918,13 @@ public void testPositionalParameters5() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() - && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse five tf.function positional arguments. + * Test for #108. This tests whether we can parse five tf.function positional arguments. */ @Test public void testPositionalParameters6() throws Exception { @@ -937,13 +937,13 @@ public void testPositionalParameters6() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse six tf.function positional arguments. + * Test for #108. This tests whether we can parse six tf.function positional arguments. */ @Test public void testPositionalParameters7() throws Exception { @@ -956,13 +956,13 @@ public void testPositionalParameters7() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse seven tf.function positional arguments. + * Test for #108. This tests whether we can parse seven tf.function positional arguments. */ @Test public void testPositionalParameters8() throws Exception { @@ -975,13 +975,13 @@ public void testPositionalParameters8() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse eight tf.function positional arguments. + * Test for #108. This tests whether we can parse eight tf.function positional arguments. */ @Test public void testPositionalParameters9() throws Exception { @@ -994,13 +994,13 @@ public void testPositionalParameters9() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse nine tf.function positional arguments. + * Test for #108. This tests whether we can parse nine tf.function positional arguments. */ @Test public void testPositionalParameters10() throws Exception { @@ -1013,13 +1013,13 @@ public void testPositionalParameters10() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests whether we can parse ten tf.function positional arguments. + * Test for #108. This tests whether we can parse ten tf.function positional arguments. */ @Test public void testPositionalParameters11() throws Exception { @@ -1032,13 +1032,13 @@ public void testPositionalParameters11() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests that if tf.function has positional arguments set as default, we do count them as existing. + * Test for #108. This tests that if tf.function has positional arguments set as default, we do count them as existing. */ @Test public void testPositionalParameters12() throws Exception { @@ -1051,13 +1051,13 @@ public void testPositionalParameters12() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() - && args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && args.hasExperimentalAutographOptParam() + && args.hasExperimentalFollowTypeHintsParam()); } /** - * Test for #106. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.8). + * Test for #108. This tests an earlier tf.function version that has different positional arguments as v2.9 (using v2.8). */ @Test public void testPositionalParameters13() throws Exception { @@ -1070,9 +1070,9 @@ public void testPositionalParameters13() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() && !args.hasReduceRetracingParam() - && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && args.hasJitCompileParam() + && !args.hasReduceRetracingParam() && args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() + && !args.hasExperimentalFollowTypeHintsParam()); } /** @@ -1089,9 +1089,9 @@ public void testPositionalParameters14() throws Exception { Function.HybridizationParameters args = function.getHybridizationParameters(); assertNotNull(args); - assertTrue(args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() + assertTrue(args.hasFuncParam() && args.hasInputSignatureParam() && args.hasAutoGraphParam() && !args.hasJitCompileParam() && !args.hasReduceRetracingParam() && !args.hasExperimentalImplementsParam() && !args.hasExperimentalAutographOptParam() - && !args.hasExperimentalFollowTypeHintsParam() && args.hasFuncParam()); + && !args.hasExperimentalFollowTypeHintsParam()); } /** From f7cd804b0d91e4f6465219838465e02e67c9e337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 26 Jan 2023 16:15:19 -0500 Subject: [PATCH 32/43] Update --- .../hunter/hybridize/core/analysis/Function.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 9a852ac3..e541e629 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -5,7 +5,6 @@ import java.io.File; import java.util.ArrayList; import java.util.Objects; -import java.util.Set; import org.eclipse.core.runtime.ILog; import org.eclipse.core.runtime.IProgressMonitor; @@ -121,7 +120,7 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Declaring definitions of the decorator, if it contains multiple definitions there might be more than one in this set. Since // we are dealing with tf.function, we expect only one. - Set potentialDeclaringDefinitionSet = null; + Definition declaringDefinition = null; // Iterate through the decorators of the function for (decoratorsType decorator : decoratorArray) { @@ -134,22 +133,14 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118). tfFunctionDecorator = decorator; // Returns the set of potential declaring definitions of the selection. - potentialDeclaringDefinitionSet = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, - Function.this.containingFile, Function.this.nature, monitor); + declaringDefinition = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, + Function.this.containingFile, Function.this.nature, monitor).iterator().next(); } } catch (AmbiguousDeclaringModuleException e) { throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e); } } // We expect to have the last tf.function decorator in tfFunctionDecorator - // Declaring definition of the decorator. We get the single definition of tf.function. - Definition declaringDefinition = null; - - // Getting the definition, there should only be one in the set. - if (potentialDeclaringDefinitionSet != null) { - declaringDefinition = potentialDeclaringDefinitionSet.iterator().next(); - } - // Python source arguments from the declaring definition exprType[] declaringArguments = null; From a835e8bd4bd1a19d923640eee2a48838eb9d8532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 24 Feb 2023 12:04:39 -0500 Subject: [PATCH 33/43] Pass build --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 939927e8..7d7dbbd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,9 @@ before_install: - sudo update-alternatives --auto python3 - sudo apt install -y python3.10-distutils - curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + - export PATH=`echo $PATH | tr ":" "\n" | grep -v "/opt/pyenv/shims" | tr "\n" ":"` + - export PATH=`echo $PATH | tr ":" "\n" | grep -v "/home/travis/.phpenv/shims" | tr "\n" ":"` + - PATH="$PATH:/usr/bin/python3" services: - xvfb install: From 594ea9d2a85b200e449224f262a65bdfe11aeba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 24 Feb 2023 12:34:55 -0500 Subject: [PATCH 34/43] Adding more comments --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 +++- .../src/edu/cuny/hunter/hybridize/core/analysis/Util.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 39b3bd1a..96ab99a4 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -177,6 +177,8 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Processing positional arguments for tf.function exprType[] arguments = callFunction.args; for (int i = 0; i < arguments.length; i++) { + // We iterate over the tf.function's parameters positions. From the position we are currently evaluating (i), we use + // the tf.function's definition (argumentIdDeclaringDefintion) to verify which parameter we are analyzing. String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); // Matching the arguments from the definition and the arguments from the code being analyzed. @@ -218,7 +220,7 @@ else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Processing keywords arguments // If we have keyword parameter, afterwards, we cannot have positional parameters because it would result in invalid - // Python code. + // Python code. This is why we check the keywords last. keywordType[] keywords = callFunction.keywords; for (keywordType keyword : keywords) { if (keyword.arg instanceof NameTok) { diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java index 7e761292..677b6566 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java @@ -72,7 +72,7 @@ public static Set getDeclaringDefinition(PySelection selection, Stri selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName, containingFile.getName(), nature.getProject())); - // Collect the potential declaring module names. + // Collect the potential declaring definition names. Set potentialDeclaringDefinitions = new HashSet<>(); // for each match. @@ -80,7 +80,7 @@ public static Set getDeclaringDefinition(PySelection selection, Stri Definition definition = itemPointer.definition; LOG.info("Found definition: " + definition + "."); - // add it to the set of found module names. + // add it to the set of found definition names. potentialDeclaringDefinitions.add(definition); } From c182a370e52f48a8614be691b17af7b7d126435a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Mon, 6 Mar 2023 10:27:53 -0500 Subject: [PATCH 35/43] Updates on requested changes --- .../hybridize/core/analysis/Function.java | 132 +++++++++++------- .../hunter/hybridize/core/analysis/Util.java | 3 +- 2 files changed, 83 insertions(+), 52 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 96ab99a4..640fbb19 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -5,6 +5,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Objects; +import java.util.Set; import org.eclipse.core.runtime.ILog; import org.eclipse.core.runtime.IProgressMonitor; @@ -44,34 +45,54 @@ public class Function extends RefactorableProgramEntity { */ public class HybridizationParameters { - // Available in TF version [2.4,2.11]. + /** + * Available in TF version [2.4,2.11]. + */ private static final String EXPERIMENTAL_FOLLOW_TYPE_HINTS = "experimental_follow_type_hints"; - // Available in TF version [2.0,2.11]. + /** + * Available in TF version [2.0,2.11]. + */ private static final String EXPERIMENTAL_AUTOGRAPH_OPTIONS = "experimental_autograph_options"; - // Available in TF version [2.1,2.11]. + /** + * Available in TF version [2.1,2.11]. + */ private static final String EXPERIMENTAL_IMPLEMENTS = "experimental_implements"; - // Available in TF version [2.9,2.11]. Previously, experimental_relax_shapes which is available TF version [2.0,2.11]. + /** + * Available in TF version [2.9,2.11]. Previously, experimental_relax_shapes which is available TF version [2.0,2.11]. + */ private static final String REDUCE_RETRACING = "reduce_retracing"; - // Available in TF version [2.0,2.11]. + /** + * Available in TF version [2.0,2.11]. + */ private static final String EXPERIMENTAL_RELAX_SHAPES = "experimental_relax_shapes"; - // Available in TF version [2.1,2.11]. + /** + * Available in TF version [2.1,2.11]. + */ private static final String EXPERIMENTAL_COMPILE = "experimental_compile"; - // Available in TF version [2.5,2.11]. Previously, experimental_compile which is available TF version [2.1,2.11]. + /** + * Available in TF version [2.5,2.11]. Previously, experimental_compile which is available TF version [2.1,2.11]. + */ private static final String JIT_COMPILE = "jit_compile"; - // Available in TF version [2.0,2.11]. + /** + * Available in TF version [2.0,2.11]. + */ private static final String AUTOGRAPH = "autograph"; - // Available in TF version [2.0,2.11]. + /** + * Available in TF version [2.0,2.11]. + */ private static final String INPUT_SIGNATURE = "input_signature"; - // Available in TF version [2.0,2.11]. + /** + * Available in TF version [2.0,2.11]. + */ private static final String FUNC = "func"; /** @@ -136,38 +157,18 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118). tfFunctionDecorator = decorator; // Returns the set of potential declaring definitions of the selection. - declaringDefinition = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, - Function.this.containingFile, Function.this.nature, monitor).iterator().next(); + Set potentialDeclaringDefitinion = Util.getDeclaringDefinition(selection, + Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor); + if (potentialDeclaringDefitinion.size() == 1) + declaringDefinition = potentialDeclaringDefitinion.iterator().next(); } } catch (AmbiguousDeclaringModuleException e) { throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e); } } // We expect to have the last tf.function decorator in tfFunctionDecorator - // Python source arguments from the declaring definition - exprType[] declaringArguments = null; - - // Getting the arguments from TensorFlow source - if (declaringDefinition != null) { - if (declaringDefinition.ast instanceof FunctionDef) { - FunctionDef declaringFunctionDefinition = (FunctionDef) declaringDefinition.ast; - argumentsType declaringArgumentTypes = declaringFunctionDefinition.args; - declaringArguments = declaringArgumentTypes.args; - } - } - - // Python source arguments from the declaring definition - ArrayList argumentIdDeclaringDefintion = new ArrayList<>(); - - // Getting the arguments from the definition - if (declaringArguments != null) { - for (exprType declaredArgument : declaringArguments) { - if (declaredArgument instanceof Name) { - Name argumentName = (Name) declaredArgument; - argumentIdDeclaringDefintion.add(argumentName.id); - } - } - } + // Getting tf.functions Python definition arguments. + ArrayList argumentIdDeclaringDefintion = getTfFunctionPythonDefinitionArguments(declaringDefinition); if (tfFunctionDecorator != null) // tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters. @@ -176,9 +177,9 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Processing positional arguments for tf.function exprType[] arguments = callFunction.args; + // We iterate over the tf.function's parameters positions. for (int i = 0; i < arguments.length; i++) { - // We iterate over the tf.function's parameters positions. From the position we are currently evaluating (i), we use - // the tf.function's definition (argumentIdDeclaringDefintion) to verify which parameter we are analyzing. + // From the position i, we use the tf.function's definition to verify which parameter we are analyzing. String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); // Matching the arguments from the definition and the arguments from the code being analyzed. @@ -191,16 +192,14 @@ else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile + // In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and + // deprecated name experimental_compile. else if (argumentDeclaringDefinition.equals(JIT_COMPILE) || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes + // In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing + // and deprecated name experimental_relax_shapes. else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) // Found parameter reduce_retracing this.reduceRetracingParamExists = true; @@ -234,15 +233,13 @@ else if (name.id.equals(INPUT_SIGNATURE)) else if (name.id.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; - // The latest version of the API we are using allows - // parameter names jit_compile and - // deprecated name experimental_compile + // In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and + // deprecated name experimental_compile. else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - // The latest version of the API we are using allows - // parameter names reduce_retracing - // and deprecated name experimental_relax_shapes + // In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing + // and deprecated name experimental_relax_shapes. else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) // Found parameter reduce_retracing // or experimental_relax_shapes @@ -261,6 +258,41 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) } // else, tf.function is used without parameters. } + /** + * Get the tf.function parameter names from the {@link Definition}. + * + * @param declaringDefinition The Definition to use. + * @return An array with the names of the arguments given by {@link Definition}. + */ + ArrayList getTfFunctionPythonDefinitionArguments(Definition declaringDefinition) { + // Python source arguments from the declaring definition + exprType[] declaringArguments = null; + + // Getting the arguments from TensorFlow source + if (declaringDefinition != null) { + if (declaringDefinition.ast instanceof FunctionDef) { + FunctionDef declaringFunctionDefinition = (FunctionDef) declaringDefinition.ast; + argumentsType declaringArgumentTypes = declaringFunctionDefinition.args; + declaringArguments = declaringArgumentTypes.args; + } + } + + // Python source arguments from the declaring definition + ArrayList argumentIdDeclaringDefintion = new ArrayList<>(); + + // Getting the arguments from the definition + if (declaringArguments != null) { + for (exprType declaredArgument : declaringArguments) { + if (declaredArgument instanceof Name) { + Name argumentName = (Name) declaredArgument; + argumentIdDeclaringDefintion.add(argumentName.id); + } + } + } + + return argumentIdDeclaringDefintion; + } + /** * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph. * diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java index 677b6566..b85c0b12 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java @@ -33,7 +33,7 @@ public class Util { private static final ILog LOG = getLog(Util.class); - + /** * Get the set of potential declaring definitions of the entity described in the given {@link PySelection}. * @@ -85,7 +85,6 @@ public static Set getDeclaringDefinition(PySelection selection, Stri } return potentialDeclaringDefinitions; - } /** From d3239b44bed182979b47d84451228b4454486ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 10 Mar 2023 15:10:32 -0500 Subject: [PATCH 36/43] update --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 640fbb19..4ad629a8 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -159,7 +159,9 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Returns the set of potential declaring definitions of the selection. Set potentialDeclaringDefitinion = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor); - if (potentialDeclaringDefitinion.size() == 1) + // If it has an element we store it in a variable. If it doesn't, the variable would be null and the code below + // handles this if it is the case. + if (potentialDeclaringDefitinion.iterator().hasNext()) declaringDefinition = potentialDeclaringDefitinion.iterator().next(); } } catch (AmbiguousDeclaringModuleException e) { @@ -215,6 +217,8 @@ else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; + else + throw new IllegalArgumentException("Unable to process tf.function argument."); } // Processing keywords arguments @@ -253,6 +257,8 @@ else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; + else + throw new IllegalArgumentException("Unable to process tf.function argument."); } } } // else, tf.function is used without parameters. From 9a60cbfb8656a02d5336562af80b76f8c348db64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 10 Mar 2023 15:14:56 -0500 Subject: [PATCH 37/43] update --- .../src/edu/cuny/hunter/hybridize/core/analysis/Function.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 4ad629a8..d9d0fa44 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -194,13 +194,13 @@ else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; - // In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and // deprecated name experimental_compile. else if (argumentDeclaringDefinition.equals(JIT_COMPILE) || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - // In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing // and deprecated name experimental_relax_shapes. else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) // Found parameter reduce_retracing From 1cb750256d2429a05e03cf4b64026a81dc16bea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 10 Mar 2023 16:06:21 -0500 Subject: [PATCH 38/43] Renaming for clarity --- .../hybridize/core/analysis/Function.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index d9d0fa44..f7656a64 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -177,44 +177,42 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep if (tfFunctionDecorator.func instanceof Call) { Call callFunction = (Call) tfFunctionDecorator.func; - // Processing positional arguments for tf.function - exprType[] arguments = callFunction.args; // We iterate over the tf.function's parameters positions. - for (int i = 0; i < arguments.length; i++) { + for (int i = 0; i < callFunction.args.length; i++) { // From the position i, we use the tf.function's definition to verify which parameter we are analyzing. - String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i); + String evaluatedArgument = argumentIdDeclaringDefintion.get(i); // Matching the arguments from the definition and the arguments from the code being analyzed. - if (argumentDeclaringDefinition.equals(FUNC)) + if (evaluatedArgument.equals(FUNC)) // Found parameter func this.funcParamExists = true; - else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE)) + else if (evaluatedArgument.equals(INPUT_SIGNATURE)) // Found parameter input_signature this.inputSignatureParamExists = true; - else if (argumentDeclaringDefinition.equals(AUTOGRAPH)) + else if (evaluatedArgument.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and // deprecated name experimental_compile. - else if (argumentDeclaringDefinition.equals(JIT_COMPILE) - || argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE)) + else if (evaluatedArgument.equals(JIT_COMPILE) + || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing // and deprecated name experimental_relax_shapes. - else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING)) + else if (evaluatedArgument.equals(REDUCE_RETRACING)) // Found parameter reduce_retracing this.reduceRetracingParamExists = true; - else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES)) + else if (evaluatedArgument.equals(EXPERIMENTAL_RELAX_SHAPES)) // Found parameter experimental_relax_shapes this.reduceRetracingParamExists = true; - else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS)) + else if (evaluatedArgument.equals(EXPERIMENTAL_IMPLEMENTS)) // Found parameter experimental_implements this.experimentalImplementsParamExists = true; - else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) + else if (evaluatedArgument.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; - else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) + else if (evaluatedArgument.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; else From f7106114f7330f4838e9eb09da20e78ea0303d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 10 Mar 2023 17:08:05 -0500 Subject: [PATCH 39/43] update --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index f7656a64..e34b58ce 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -194,8 +194,7 @@ else if (evaluatedArgument.equals(AUTOGRAPH)) this.autoGraphParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and // deprecated name experimental_compile. - else if (evaluatedArgument.equals(JIT_COMPILE) - || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) + else if (evaluatedArgument.equals(JIT_COMPILE) || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing @@ -235,12 +234,12 @@ else if (name.id.equals(INPUT_SIGNATURE)) else if (name.id.equals(AUTOGRAPH)) // Found parameter autograph this.autoGraphParamExists = true; - // In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and // deprecated name experimental_compile. else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE)) // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; - // In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing // and deprecated name experimental_relax_shapes. else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES)) // Found parameter reduce_retracing From 567369029473901d27f16794745786fe2abc32b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Fri, 17 Mar 2023 13:00:45 -0400 Subject: [PATCH 40/43] update --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index e34b58ce..27dce719 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -159,10 +159,12 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Returns the set of potential declaring definitions of the selection. Set potentialDeclaringDefitinion = Util.getDeclaringDefinition(selection, Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor); - // If it has an element we store it in a variable. If it doesn't, the variable would be null and the code below - // handles this if it is the case. + + // If it has an element, we store the definition in a variable. if (potentialDeclaringDefitinion.iterator().hasNext()) declaringDefinition = potentialDeclaringDefitinion.iterator().next(); + else + throw new IllegalStateException("Can't determine tf.function decorator defintion."); } } catch (AmbiguousDeclaringModuleException e) { throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e); @@ -267,7 +269,7 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) * @param declaringDefinition The Definition to use. * @return An array with the names of the arguments given by {@link Definition}. */ - ArrayList getTfFunctionPythonDefinitionArguments(Definition declaringDefinition) { + private ArrayList getTfFunctionPythonDefinitionArguments(Definition declaringDefinition) { // Python source arguments from the declaring definition exprType[] declaringArguments = null; From 5d72476f76610c5fe4eea0d3aeebefd01a472f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 21 Mar 2023 22:20:57 -0400 Subject: [PATCH 41/43] Changing comments, removing keyword args else --- .../hunter/hybridize/core/analysis/Function.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 27dce719..164dae1a 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -143,7 +143,7 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep decoratorsType tfFunctionDecorator = null; // Declaring definitions of the decorator, if it contains multiple definitions there might be more than one in this set. Since - // we are dealing with tf.function, we expect only one. + // we are dealing with tf.function, we check we it has one definition. Definition declaringDefinition = null; // Iterate through the decorators of the function @@ -186,35 +186,26 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep // Matching the arguments from the definition and the arguments from the code being analyzed. if (evaluatedArgument.equals(FUNC)) - // Found parameter func this.funcParamExists = true; else if (evaluatedArgument.equals(INPUT_SIGNATURE)) - // Found parameter input_signature this.inputSignatureParamExists = true; else if (evaluatedArgument.equals(AUTOGRAPH)) - // Found parameter autograph this.autoGraphParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and // deprecated name experimental_compile. else if (evaluatedArgument.equals(JIT_COMPILE) || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) - // Found parameter jit_compile/experimental_compile this.jitCompileParamExists = true; // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing // and deprecated name experimental_relax_shapes. else if (evaluatedArgument.equals(REDUCE_RETRACING)) - // Found parameter reduce_retracing this.reduceRetracingParamExists = true; else if (evaluatedArgument.equals(EXPERIMENTAL_RELAX_SHAPES)) - // Found parameter experimental_relax_shapes this.reduceRetracingParamExists = true; else if (evaluatedArgument.equals(EXPERIMENTAL_IMPLEMENTS)) - // Found parameter experimental_implements this.experimentalImplementsParamExists = true; else if (evaluatedArgument.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) - // Found parameter experimental_autograph_options this.experimentalAutographOptionsParamExists = true; else if (evaluatedArgument.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) - // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; else throw new IllegalArgumentException("Unable to process tf.function argument."); @@ -256,8 +247,6 @@ else if (name.id.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) // Found parameter experimental_follow_type_hints this.experimentaFollowTypeHintsParamExists = true; - else - throw new IllegalArgumentException("Unable to process tf.function argument."); } } } // else, tf.function is used without parameters. From 54ba275aef1fdcc868f793d8dc2ba3a098ff4f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Tue, 21 Mar 2023 22:37:21 -0400 Subject: [PATCH 42/43] Checking that the size of the parse tffunction args is less than definition, more general method name, more information on exception messages --- .../hybridize/core/analysis/Function.java | 73 ++++++++++--------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 35aa227f..66e77c0e 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -172,44 +172,51 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep } // We expect to have the last tf.function decorator in tfFunctionDecorator // Getting tf.functions Python definition arguments. - ArrayList argumentIdDeclaringDefintion = getTfFunctionPythonDefinitionArguments(declaringDefinition); + ArrayList argumentIdDeclaringDefintion = getPythonDefinitionArguments(declaringDefinition); if (tfFunctionDecorator != null) // tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters. if (tfFunctionDecorator.func instanceof Call) { Call callFunction = (Call) tfFunctionDecorator.func; + exprType[] tfFunctionPositionalArgs = callFunction.args; + // We iterate over the tf.function's parameters positions. - for (int i = 0; i < callFunction.args.length; i++) { - // From the position i, we use the tf.function's definition to verify which parameter we are analyzing. - String evaluatedArgument = argumentIdDeclaringDefintion.get(i); - - // Matching the arguments from the definition and the arguments from the code being analyzed. - if (evaluatedArgument.equals(FUNC)) - this.funcParamExists = true; - else if (evaluatedArgument.equals(INPUT_SIGNATURE)) - this.inputSignatureParamExists = true; - else if (evaluatedArgument.equals(AUTOGRAPH)) - this.autoGraphParamExists = true; - // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and - // deprecated name experimental_compile. - else if (evaluatedArgument.equals(JIT_COMPILE) || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) - this.jitCompileParamExists = true; - // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing - // and deprecated name experimental_relax_shapes. - else if (evaluatedArgument.equals(REDUCE_RETRACING)) - this.reduceRetracingParamExists = true; - else if (evaluatedArgument.equals(EXPERIMENTAL_RELAX_SHAPES)) - this.reduceRetracingParamExists = true; - else if (evaluatedArgument.equals(EXPERIMENTAL_IMPLEMENTS)) - this.experimentalImplementsParamExists = true; - else if (evaluatedArgument.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) - this.experimentalAutographOptionsParamExists = true; - else if (evaluatedArgument.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) - this.experimentaFollowTypeHintsParamExists = true; - else - throw new IllegalArgumentException("Unable to process tf.function argument."); - } + if (tfFunctionPositionalArgs.length <= argumentIdDeclaringDefintion.size()) { + for (int i = 0; i < tfFunctionPositionalArgs.length; i++) { + // From the position i, we use the tf.function's definition to verify which parameter we are analyzing. + String evaluatedArgument = argumentIdDeclaringDefintion.get(i); + + // Matching the arguments from the definition and the arguments from the code being analyzed. + if (evaluatedArgument.equals(FUNC)) + this.funcParamExists = true; + else if (evaluatedArgument.equals(INPUT_SIGNATURE)) + this.inputSignatureParamExists = true; + else if (evaluatedArgument.equals(AUTOGRAPH)) + this.autoGraphParamExists = true; + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names jit_compile and + // deprecated name experimental_compile. + else if (evaluatedArgument.equals(JIT_COMPILE) || evaluatedArgument.equals(EXPERIMENTAL_COMPILE)) + this.jitCompileParamExists = true; + // In our accepted interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing + // and deprecated name experimental_relax_shapes. + else if (evaluatedArgument.equals(REDUCE_RETRACING)) + this.reduceRetracingParamExists = true; + else if (evaluatedArgument.equals(EXPERIMENTAL_RELAX_SHAPES)) + this.reduceRetracingParamExists = true; + else if (evaluatedArgument.equals(EXPERIMENTAL_IMPLEMENTS)) + this.experimentalImplementsParamExists = true; + else if (evaluatedArgument.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS)) + this.experimentalAutographOptionsParamExists = true; + else if (evaluatedArgument.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) + this.experimentaFollowTypeHintsParamExists = true; + else + throw new IllegalArgumentException("Unable to process tf.function argument at position " + i); + } + } else + throw new IllegalArgumentException( + "Unable to process tf.function argument. The number of arguments " + tfFunctionPositionalArgs.length + + " exceeds the accepted number of argumets " + argumentIdDeclaringDefintion.size()); // Processing keywords arguments // If we have keyword parameter, afterwards, we cannot have positional parameters because it would result in invalid @@ -253,12 +260,12 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS)) } /** - * Get the tf.function parameter names from the {@link Definition}. + * Get the parameter names from the {@link Definition}. * * @param declaringDefinition The Definition to use. * @return An array with the names of the arguments given by {@link Definition}. */ - private ArrayList getTfFunctionPythonDefinitionArguments(Definition declaringDefinition) { + private ArrayList getPythonDefinitionArguments(Definition declaringDefinition) { // Python source arguments from the declaring definition exprType[] declaringArguments = null; From afd01ffc02e29590b803b67768befa81ee7665c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tatiana=20Castro-V=C3=A9lez?= Date: Thu, 23 Mar 2023 09:45:22 -0400 Subject: [PATCH 43/43] Adding more info --- .../edu/cuny/hunter/hybridize/core/analysis/Function.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java index 66e77c0e..4d6564f8 100644 --- a/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java +++ b/edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java @@ -164,7 +164,10 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep if (potentialDeclaringDefitinion.iterator().hasNext()) declaringDefinition = potentialDeclaringDefitinion.iterator().next(); else - throw new IllegalStateException("Can't determine tf.function decorator defintion."); + throw new IllegalStateException(String.format( + "Can't find declaring definition for selection: %s in line: %s, file: %s, and project: %s.", + selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), + Function.this.containingFile.getName(), Function.this.nature.getProject())); } } catch (AmbiguousDeclaringModuleException e) { throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e);