Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update R8 and enable Dalvik tests on JDK 21 #1353

Merged
merged 7 commits into from
Dec 29, 2023
Merged

Update R8 and enable Dalvik tests on JDK 21 #1353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
66c79bd
WIP
msridhar Dec 8, 2023
78e8c74
WIP
msridhar Dec 21, 2023
09a28ea
Merge branch 'master' into update-r8
msridhar Dec 29, 2023
4b9d854
WIP
msridhar Dec 29, 2023
48920ae
more
msridhar Dec 29, 2023
ef6c025
cleanup
msridhar Dec 29, 2023
f7a167c
add docs
msridhar Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions core/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,20 +312,19 @@ artifacts.add(collectTestDataJar.name, collectTestData.map { it.destinationDirec

////////////////////////////////////////////////////////////////////////
//
// collect "com.ibm.wala.core.testdata_1.0.0a.jar"
// collect "com.ibm.wala.core.testdata_1.0.0a.jar" for Dalvik tests
//

val collectTestDataA by
val collectTestDataAForDalvik by
tasks.registering(Jar::class) {
archiveFileName = "com.ibm.wala.core.testdata_1.0.0a.jar"
from(compileTestSubjectsJava)
from("classes")
includeEmptyDirs = false
destinationDirectory = layout.buildDirectory.dir(name)
exclude(
// This is an invalid class so don't include it; it causes D8 to crash
"**/CodeDeleted.class",
"**/SortingExample.class",
"**/A.class",
)
}

Expand Down Expand Up @@ -386,7 +385,7 @@ val dalvikTestResources: Configuration by configurations.creating { isCanBeResol

listOf(
collectJLex,
collectTestDataA,
collectTestDataAForDalvik,
downloadJavaCup,
extractBcel,
)
Expand Down
6 changes: 0 additions & 6 deletions dalvik/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,6 @@ if (isWindows) tasks.named<Test>("test") { exclude("**/droidbench/**") }
else sourceSets.test.configure { resources.srcDir(unpackDroidBench) }

tasks.named<Test>("test") {
if (JavaVersion.current() == JavaVersion.VERSION_21) {
// Disable the task for JDK 21 for now. We have test failures due to a required r8 upgrade
// that introduces some new behaviors we don't expect
// See https://github.com/wala/WALA/issues/1349
enabled = false
}
maxHeapSize = "800M"

outputs.files(
Expand Down
87 changes: 46 additions & 41 deletions dalvik/src/test/java/com/ibm/wala/dalvik/test/callGraph/JVMLDalvikComparisonTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,8 @@
import com.ibm.wala.ipa.callgraph.Entrypoint;
import com.ibm.wala.ipa.callgraph.impl.Util;
import com.ibm.wala.ipa.callgraph.propagation.InstanceKey;
import com.ibm.wala.ipa.callgraph.propagation.LocalPointerKey;
import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis;
import com.ibm.wala.ipa.callgraph.propagation.ReturnValueKey;
import com.ibm.wala.ipa.callgraph.propagation.SSAPropagationCallGraphBuilder;
import com.ibm.wala.ipa.callgraph.propagation.cfa.ExceptionReturnValueKey;
import com.ibm.wala.ipa.cha.ClassHierarchy;
import com.ibm.wala.ipa.cha.ClassHierarchyException;
import com.ibm.wala.ipa.cha.ClassHierarchyFactory;
Expand All @@ -40,15 +37,12 @@
import com.ibm.wala.types.MethodReference;
import com.ibm.wala.util.CancelException;
import com.ibm.wala.util.collections.HashSetFactory;
import com.ibm.wala.util.collections.Iterator2Collection;
import com.ibm.wala.util.collections.Pair;
import com.ibm.wala.util.intset.IntSet;
import com.ibm.wala.util.intset.IntSetUtil;
import com.ibm.wala.util.intset.MutableIntSet;
import com.ibm.wala.util.intset.OrdinalSet;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Set;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -103,6 +97,22 @@ private static Set<Pair<CGNode, CGNode>> edgeDiff(

private static void test(String mainClass, String javaScopeFile)
throws IllegalArgumentException, IOException, CancelException, ClassHierarchyException {
test(mainClass, javaScopeFile, false);
}

/**
* Run tests to compare the call graphs computing with the JVM bytecode frontend vs the Dalvik
* frontend
*
* @param mainClass main class for the test
* @param javaScopeFile scope file for the test
* @param allowExtraJavaCGEdges if true, allow extra edges in the JVM bytecode frontend call
* graph. This flag is temporarily required due to issues with JLex caused by
* https://issuetracker.google.com/issues/316744331. TODO: remove this flag once the D8 issue
* is fixed.
*/
private static void test(String mainClass, String javaScopeFile, boolean allowExtraJavaCGEdges)
throws IllegalArgumentException, IOException, CancelException, ClassHierarchyException {
Pair<CallGraph, PointerAnalysis<InstanceKey>> java = makeJavaBuilder(javaScopeFile, mainClass);

AnalysisScope javaScope = java.fst.getClassHierarchy().getScope();
Expand All @@ -114,13 +124,15 @@ private static void test(String mainClass, String javaScopeFile)
Set<MethodReference> androidMethods = applicationMethods(android.fst);
Set<MethodReference> javaMethods = applicationMethods(java.fst);

Iterator<Pair<CGNode, CGNode>> javaExtraEdges =
edgeDiff(java.fst, android.fst, false).iterator();
assert !checkEdgeDiff(android, androidMethods, javaMethods, javaExtraEdges);
if (!allowExtraJavaCGEdges) {
Set<Pair<CGNode, CGNode>> javaExtraEdges = edgeDiff(java.fst, android.fst, false);
assert !checkEdgeDiff(android, androidMethods, javaMethods, javaExtraEdges)
: "found extra edges in Java call graph";
}

Iterator<Pair<CGNode, CGNode>> androidExtraEdges =
edgeDiff(android.fst, java.fst, true).iterator();
assert !checkEdgeDiff(java, javaMethods, androidMethods, androidExtraEdges);
Set<Pair<CGNode, CGNode>> androidExtraEdges = edgeDiff(android.fst, java.fst, true);
assert !checkEdgeDiff(java, javaMethods, androidMethods, androidExtraEdges)
: "found extra edges in Android call graph";

checkSourceLines(java.fst, android.fst);
}
Expand Down Expand Up @@ -177,36 +189,29 @@ private static void checkSourceLines(CallGraph java, CallGraph android) {
}

private static boolean checkEdgeDiff(
Pair<CallGraph, PointerAnalysis<InstanceKey>> android,
Set<MethodReference> androidMethods,
Set<MethodReference> javaMethods,
Iterator<Pair<CGNode, CGNode>> javaExtraEdges) {
Pair<CallGraph, PointerAnalysis<InstanceKey>> firstResult,
Set<MethodReference> methodsInFirst,
Set<MethodReference> methodsInSecond,
Set<Pair<CGNode, CGNode>> extraEdgesInSecond) {
boolean fail = false;
if (javaExtraEdges.hasNext()) {
if (!extraEdgesInSecond.isEmpty()) {
fail = true;
Set<MethodReference> javaExtraNodes = HashSetFactory.make(javaMethods);
javaExtraNodes.removeAll(androidMethods);

System.err.println(Iterator2Collection.toSet(javaExtraEdges));
System.err.println(javaExtraNodes);

System.err.println(android.fst);

for (CGNode n : android.fst) {
System.err.println("### " + n);
if (n.getIR() != null) {
System.err.println(n.getIR());

System.err.println("return: " + android.snd.getPointsToSet(new ReturnValueKey(n)));
System.err.println(
"exceptions: " + android.snd.getPointsToSet(new ExceptionReturnValueKey(n)));
for (int i = 1; i < n.getIR().getSymbolTable().getMaxValueNumber(); i++) {
LocalPointerKey x = new LocalPointerKey(n, i);
OrdinalSet<InstanceKey> s = android.snd.getPointsToSet(x);
if (s != null && !s.isEmpty()) {
System.err.println(i + ": " + s);
}
}
Set<MethodReference> extraMethodsInSecond = HashSetFactory.make(methodsInSecond);
extraMethodsInSecond.removeAll(methodsInFirst);

System.err.println(extraEdgesInSecond);
System.err.println(extraMethodsInSecond);

CallGraph firstCG = firstResult.fst;
for (Pair<CGNode, CGNode> p : extraEdgesInSecond) {
System.err.println("### " + p.fst);
System.err.println("### " + p.snd);
System.err.println("### " + p.fst.getIR());
System.err.println("====");
Set<CGNode> nodes = firstCG.getNodes(p.fst.getMethod().getReference());
for (CGNode n : nodes) {
System.err.println("### " + n);
System.err.println("### " + n.getIR());
}
}
}
Expand All @@ -216,7 +221,7 @@ private static boolean checkEdgeDiff(
@Test
public void testJLex()
throws ClassHierarchyException, IllegalArgumentException, IOException, CancelException {
test(TestConstants.JLEX_MAIN, TestConstants.JLEX);
test(TestConstants.JLEX_MAIN, TestConstants.JLEX, true);
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ ktfmt = "0.44"
spotless = "6.23.3"

[libraries]
android-tools = "com.android.tools:r8:2.2.42"
android-tools = "com.android.tools:r8:8.2.39"
ant = "org.apache.ant:ant:1.10.14"
assertj-core = "org.assertj:assertj-core:3.24.2"
commons-cli = "commons-cli:commons-cli:1.5.0"
Expand Down
Loading