POC for integration tests

adot-testbed/README.md

@@ -0,0 +1,12 @@
+# Introduction
+
+The tests require that AWS credentias are sent to the container under test. Please follow this procedure.
+
+
+```
+export AWS_REGION=us-west-2
+export AWS_ACCESS_KEY_ID=<key>
+export AWS_SECRET_ACCESS_KEY=<secret>
+export AWS_SESSION_TOKEN=<session>
+./gradlew test --rerun-tasks --info
+```

adot-testbed/app/build.gradle.kts

@@ -0,0 +1,47 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a sample Java application project to get you started.
+ * For more details on building Java & JVM projects, please refer to https://docs.gradle.org/8.2/userguide/building_java_projects.html in the Gradle documentation.
+ */
+
+plugins {
+    // Apply the application plugin to add support for building a CLI application in Java.
+    application
+}
+
+repositories {
+    // Use Maven Central for resolving dependencies.
+    mavenCentral()
+}
+
+dependencies {
+    // Use JUnit Jupiter for testing.
+    testImplementation("org.junit.jupiter:junit-jupiter:5.9.2")
+    testImplementation("org.testcontainers:testcontainers:1.19.0")
+    testImplementation("org.testcontainers:junit-jupiter:1.19.0")
+    testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+    testImplementation("org.slf4j:slf4j-simple:1.7.36")
+    testImplementation(platform("software.amazon.awssdk:bom:2.20.156"))
+    testImplementation("software.amazon.awssdk:cloudwatchlogs")
+    testImplementation("com.github.rholder:guava-retrying:2.0.0")
+    testImplementation("org.assertj:assertj-core:3.24.2")
+}
+
+// Apply a specific Java toolchain to ease working on different environments.
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+application {
+    // Define the main class for the application.
+    mainClass.set("adot.testbed.App")
+}
+
+tasks.named<Test>("test") {
+    // Use JUnit Platform for unit tests.
+    useJUnitPlatform()
+    systemProperty("adot.testbed.localcreds", System.getProperty("adot.testbed.localcreds"))
+}

adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java

@@ -0,0 +1,123 @@
+package software.amazon.adot.testbed;
+import java.io.FileWriter;
+/*
+ * This Java source file was generated by the Gradle 'init' task.
+ */
+
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
+import net.bytebuddy.asm.Advice;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
+import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@Testcontainers(disabledWithoutDocker = true)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class LogsTests {
+    private static final String LOCAL_CREDENTIALS = System.getProperty("adot.testbed.localcreds");
+    private final String TEST_IMAGE = "public.ecr.aws/aws-otel-test/adot-collector-integration-test:v0.33.1-3fe9d45";
+    private final Logger collectorLogger = LoggerFactory.getLogger("collector");
+    private final File tempFile = File.createTempFile("testlog", ".log");
+
+
+    protected final GenericContainer<?> collector = new GenericContainer<>(TEST_IMAGE)
+            .withExposedPorts(4317)
+            .withCopyFileToContainer(MountableFile.forClasspathResource("/config.yaml"), "/etc/collector/config.yaml")
+            .withFileSystemBind(tempFile.getAbsolutePath(), "/logs/logs.log")
+            .withLogConsumer(new Slf4jLogConsumer(collectorLogger))
+            .waitingFor(Wait.forLogMessage(".*Serving Prometheus metrics.*", 1))
+            .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.filelog.receiver,+adot.awscloudwatchlogs.exporter,+adot.file_storage.extension");
+
+
+    @BeforeAll
+    void setup() throws Exception {
+        if (LOCAL_CREDENTIALS != null && !LOCAL_CREDENTIALS.isEmpty()) {
+            System.out.println(LOCAL_CREDENTIALS);
+            collector.withCopyFileToContainer(MountableFile.forHostPath(LOCAL_CREDENTIALS), "/home/aoc/.aws/");
+        } else {
+            collector.withEnv(System.getenv());
+        }
+        collector.start();
+    }
+
+    @AfterAll
+            void tearDown() throws Exception {
+        collector.stop();
+    }
+
+    LogsTests() throws Exception {
+    }
+
+    @Test void testSendLogs() throws Exception {
+       var fileWriter = new FileWriter(tempFile);
+       var lines = new HashSet<String>();
+
+        IntStream.range(0, 10).forEach( x -> {
+            try {
+                String line = UUID.randomUUID().toString();
+                lines.add(line);
+                fileWriter.write(line + "\n");
+
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        fileWriter.flush();
+
+
+        var cwClient = CloudWatchLogsClient.builder()
+                .build();
+
+        RetryerBuilder.<Void>newBuilder()
+                .retryIfException()
+                .retryIfRuntimeException()
+                .retryIfExceptionOfType(org.opentest4j.AssertionFailedError.class)
+                .withWaitStrategy(WaitStrategies.fixedWait(10, TimeUnit.SECONDS))
+                .withStopStrategy(StopStrategies.stopAfterAttempt(5))
+                .build()
+                .call(() -> {
+                    var now = Instant.now();
+                    var start = now.minus(Duration.ofMinutes(2));
+                    var end = now.plus(Duration.ofMinutes(2));
+                    var response = cwClient.getLogEvents(GetLogEventsRequest.builder().logGroupName("/test/logs")
+                            .logStreamName("logstream-tests")
+                            .startTime(start.toEpochMilli())
+                            .endTime(end.toEpochMilli())
+                            .build());
+
+                    var events = response.events();
+                    var receivedMessages = events.stream().map( x -> x.message() ).collect(Collectors.toSet());
+                    assertThat(receivedMessages.containsAll(lines)).isTrue();
+                    return null;
+                });
+
+    }
+}

adot-testbed/app/src/test/resources/config.yaml

@@ -0,0 +1,55 @@
+extensions:
+  pprof:
+    endpoint: 0.0.0.0:1777
+
+receivers:
+  filelog:
+    include: [/logs/*.log]
+    encoding: "ascii"
+    start_at: beginning
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'collector-monitoring'
+          scrape_interval: 10s
+          static_configs:
+            - targets: ['localhost:8888']
+processors:
+  batch:
+exporters:
+  logging:
+    verbosity: detailed
+  awscloudwatchlogs:
+    log_group_name: "/test/logs"
+    log_stream_name: "logstream-tests"
+    raw_log: true
+
+
+  awsemf:
+    log_group_name: "emf-metrics-log-group"
+    log_stream_name: "emf-metrics-log-stream"
+    metric_declarations:
+      - dimensions: [[],[]]
+        metric_name_selectors:
+          - "^otelcol_process.*"
+      - dimensions: [[receiver],[]]
+        metric_name_selectors:
+          - "^otelcol_receiver.*"
+      - dimensions: [[exporter],[]]
+        metric_name_selectors:
+          - "^otelcol_exporter.*"
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+#      processors: [batch]
+      exporters: [logging,awscloudwatchlogs]
+    metrics:
+      receivers: [prometheus]
+      exporters: [awsemf]
+  extensions: [pprof]
+  telemetry:
+    logs:
+      level: info
+    metrics:
+      level: detailed

adot-testbed/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists