ponder-lab · bgprudhomme · Oct 18, 2024 · Oct 18, 2024 · Oct 18, 2024 · Oct 19, 2024
diff --git a/hazelcast/pom.xml b/hazelcast/pom.xml
@@ -40,6 +40,8 @@
         <pax.exam.version>2.6.0</pax.exam.version>
         <pax.runner.version>1.8.6</pax.runner.version>
         <javax.inject.version>1</javax.inject.version>
+        <jmh.version>1.37</jmh.version>
+        <uberjar.name>benchmarks</uberjar.name>
     </properties>
 
     <build>
@@ -203,6 +205,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <executions>
                     <execution>
+                        <id>default-shade</id>
                         <phase>package</phase>
                         <goals>
                             <goal>shade</goal>
@@ -298,6 +301,68 @@
                             </filters>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>jmh-shade</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>${uberjar.name}</finalName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <!--
+                                        Shading signed JARs will fail without this.
+                                        http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
+                                    -->
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>run-benchmarks</id>
+                        <phase>integration-test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <classpathScope>test</classpathScope>
+                            <executable>java</executable>
+                            <arguments>
+                                <argument>-classpath</argument>
+                                <classpath />
+                                <argument>org.openjdk.jmh.Main</argument>
+                                <argument>.*</argument>
+                                <argument>-bm</argument>
+                                <argument>avgt</argument>
+                                <argument>-f</argument>
+                                <argument>1</argument>
+                                <argument>-tu</argument>
+                                <argument>ms</argument>
+                                <argument>-rff</argument>
+                                <argument>jmh-results_legacy.csv</argument>
+                                <argument>-o</argument>
+                                <argument>jmh-results_legacy.txt</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>
@@ -570,6 +635,18 @@
     </build>
 
     <dependencies>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <!-- Needed for TPC -->
         <dependency>
             <groupId>com.hazelcast</groupId>

diff --git a/hazelcast/src/test/java/com/hazelcast/client/ClientRegressionWithMockNetworkTest.java b/hazelcast/src/test/java/com/hazelcast/client/ClientRegressionWithMockNetworkTest.java
@@ -85,13 +85,36 @@
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
 @RunWith(HazelcastParallelClassRunner.class)
 @Category({QuickTest.class, ParallelJVMTest.class})
+@State(Scope.Benchmark)
+@Warmup(iterations = 5)
+@BenchmarkMode({ Mode.AverageTime })
+@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:+UseG1GC" })
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class ClientRegressionWithMockNetworkTest extends HazelcastTestSupport {
 
+    @Param({"10", "50", "100", "500", "1000", "5000"})
+    int threadCount;
+
     private final TestHazelcastFactory hazelcastFactory = new TestHazelcastFactory();
 
     @After
+    @TearDown(Level.Invocation)
     public void cleanup() {
         hazelcastFactory.terminateAll();
     }
@@ -731,6 +754,7 @@ public void testClientReconnect_thenCheckRequestsAreRetriedWithoutException() {
     }
 
     @Test
+    @Benchmark
     public void testClusterShutdown_thenCheckOperationsNotHanging() throws Exception {
         HazelcastInstance hazelcastInstance = hazelcastFactory.newHazelcastInstance();
 
@@ -746,7 +770,6 @@ public void testClusterShutdown_thenCheckOperationsNotHanging() throws Exception
         int mapSize = 1000;
 
         CountDownLatch clientStartedDoingRequests = new CountDownLatch(1);
-        int threadCount = 100;
 
         CountDownLatch testFinishedLatch = new CountDownLatch(threadCount);
 

diff --git a/hazelcast/src/test/java/com/hazelcast/client/cluster/ClientClusterStateTest.java b/hazelcast/src/test/java/com/hazelcast/client/cluster/ClientClusterStateTest.java
@@ -48,17 +48,41 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
 @RunWith(HazelcastSerialClassRunner.class)
 @Category({QuickTest.class, ParallelJVMTest.class})
+@State(Scope.Benchmark)
+@Warmup(iterations = 5)
+@BenchmarkMode({ Mode.AverageTime })
+@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:+UseG1GC" })
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class ClientClusterStateTest {
 
+    @Param({"10", "50", "100", "500", "1000", "5000"})
+    int numThreads;
+
     private TestHazelcastFactory factory;
 
     private HazelcastInstance[] instances;
 
     private HazelcastInstance instance;
 
     @Before
+    @Setup(Level.Invocation)
     public void before() {
         factory = new TestHazelcastFactory();
         instances = factory.newInstances(new Config(), 4);
@@ -69,6 +93,7 @@ public void before() {
     }
 
     @After
+    @TearDown(Level.Invocation)
     public void after() {
         factory.terminateAll();
     }
@@ -144,6 +169,7 @@ public void testClient_canExecuteOperations_whenClusterState_goesBackToActive_fr
     }
 
     @Test
+    @Benchmark
     public void testClusterShutdownDuringMapPutAll() {
         warmUpPartitions(instances);
         waitAllForSafeState(instances);
@@ -160,7 +186,6 @@ public void testClusterShutdownDuringMapPutAll() {
             values.put(value, value);
         }
 
-        final int numThreads = 10;
         final CountDownLatch threadsFinished = new CountDownLatch(numThreads);
         final CountDownLatch threadsStarted = new CountDownLatch(numThreads);
 

diff --git a/hazelcast/src/test/java/com/hazelcast/client/util/ClientStateListenerTest.java b/hazelcast/src/test/java/com/hazelcast/client/util/ClientStateListenerTest.java
@@ -34,6 +34,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 import static com.hazelcast.client.config.ClientConnectionStrategyConfig.ReconnectMode.ASYNC;
 import static com.hazelcast.client.config.ClientConnectionStrategyConfig.ReconnectMode.OFF;
@@ -48,12 +49,36 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
 @RunWith(HazelcastParallelClassRunner.class)
 @Category({QuickTest.class, ParallelJVMTest.class})
+@State(Scope.Benchmark)
+@Warmup(iterations = 5)
+@BenchmarkMode({ Mode.AverageTime })
+@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:+UseG1GC" })
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class ClientStateListenerTest extends ClientTestSupport {
+
+    @Param({"10", "50", "100", "500", "1000", "5000"})
+    int numThreads;
+
     private final TestHazelcastFactory hazelcastFactory = new TestHazelcastFactory();
 
     @After
+    @TearDown(Level.Invocation)
     public void cleanup() {
         hazelcastFactory.terminateAll();
     }
@@ -137,8 +162,8 @@ public void testClientReconnectModeAsyncConnected() throws InterruptedException
     }
 
     @Test(timeout = MINUTE * 10)
+    @Benchmark
     public void testClientReconnectModeAsyncConnectedMultipleThreads() {
-        int numThreads = 10;
         ExecutorService executor = Executors.newFixedThreadPool(numThreads);
 
         ClientConfig clientConfig = new ClientConfig();

diff --git a/hazelcast/src/test/java/com/hazelcast/cluster/JoinStressTest.java b/hazelcast/src/test/java/com/hazelcast/cluster/JoinStressTest.java
@@ -64,15 +64,40 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
 @RunWith(HazelcastSerialClassRunner.class)
 @Category(NightlyTest.class)
+@State(Scope.Benchmark)
+@Warmup(iterations = 5)
+@BenchmarkMode({ Mode.AverageTime })
+@Fork(value = 1, jvmArgs = { "-Xms1G", "-Xmx1G", "-XX:+UseG1GC" })
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class JoinStressTest extends HazelcastTestSupport {
 
+    @Param({"10", "50", "100", "500", "1000", "5000"})
+    int count;
+
     private static final long TEN_MINUTES_IN_MILLIS = 10 * 60 * 1000L;
     private ILogger logger = Logger.getLogger(JoinStressTest.class);
 
     @Before
     @After
+    @Setup(Level.Invocation)
+    @TearDown(Level.Invocation)
     public void tearDown() throws Exception {
         HazelcastInstanceFactory.terminateAll();
     }
@@ -89,7 +114,6 @@ public void testMulticastJoinWithManyNodes() throws InterruptedException {
 
     @Test(timeout = TEN_MINUTES_IN_MILLIS)
     public void testJoinCompletesCorrectlyWhenMultipleNodesStartedParallel() {
-        int count = 10;
         final TestHazelcastInstanceFactory factory = new TestHazelcastInstanceFactory(count);
         final HazelcastInstance[] instances = new HazelcastInstance[count];
         final CountDownLatch latch = new CountDownLatch(count);
@@ -244,6 +268,7 @@ private void initNetworkConfig(NetworkConfig networkConfig, int basePort, int po
     }
 
     @Test(timeout = 300000)
+    @Benchmark
     public void testJoinWhenMemberClosedInBetween() throws InterruptedException {
         //Test is expecting to all can join safely.
         // On the failed case the last opened instance throws java.lang.IllegalStateException: Node failed to start!
@@ -254,11 +279,10 @@ public void testJoinWhenMemberClosedInBetween() throws InterruptedException {
         HazelcastInstance i4 = Hazelcast.newHazelcastInstance(config);
 
         final IMap<Integer, Integer> map = i4.getMap("a");
-        int numThreads = 40;
         final int loop = 5000;
 
-        Thread[] threads = new Thread[numThreads];
-        for (int i = 0; i < numThreads; i++) {
+        Thread[] threads = new Thread[count];
+        for (int i = 0; i < count; i++) {
             threads[i] = new Thread(() -> {
                 Random random = new Random();
                 for (int j = 0; j < loop; j++) {
@@ -285,7 +309,7 @@ public void testJoinWhenMemberClosedInBetween() throws InterruptedException {
         //Should not throw java.lang.IllegalStateException: Node failed to start!
         Hazelcast.newHazelcastInstance(config);
 
-        for (int i = 0; i < numThreads; i++) {
+        for (int i = 0; i < count; i++) {
             threads[i].join();
         }
     }