Add Integration Tests for the Log components (#1426)

* POC for integration tests

* Initial commit

* Modify to add sample tests for log4j and syslog

* Add Json Test case and config

* Change the aws credentials mount

* Change retention and log group names

* Use Env variable for Unique log stream

* Change Variable name to appropriate

* Adjsut imports

* Use Env for Creds

* Add PR build check

* exclude tests

* Changes to default Test image, Session_token usage, exposed port removal

* Change plugin type to library

* Remove Unused/duplicate

* Use modified Feature gates in collector

* Extract body field for validation, remove raw_log

* Rename to logStreamName, Remove logging Exporter in pipeline

* Remove unused constructor

* Use the feature gate names from collector

* Add Assertion that helps to check the exact elements

---------

Co-authored-by: Raphael Silva <[email protected]>

.github/workflows/pr-build.yml

@@ -68,8 +68,23 @@ jobs:
         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
         restore-keys: |
           ${{ runner.os }}-node-
-    - name: test
+    - name: Test CDK
       working-directory: cdk_infra/
       run: |
         npm install
         npm run test
+  adot-testbed:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+      - name: adot-testbed build
+        working-directory: adot-testbed/
+        run: gradle build -x test

adot-testbed/README.md

@@ -0,0 +1,12 @@
+# Introduction
+
+The tests require that AWS credentials are sent to the container under test. Please follow this procedure.
+
+
+```
+export AWS_REGION=us-west-2
+export AWS_ACCESS_KEY_ID=<key>
+export AWS_SECRET_ACCESS_KEY=<secret>
+export AWS_SESSION_TOKEN=<session>
+./gradlew test --rerun-tasks --info
+```

adot-testbed/app/build.gradle.kts

@@ -0,0 +1,42 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a sample Java application project to get you started.
+ * For more details on building Java & JVM projects, please refer to https://docs.gradle.org/8.2/userguide/building_java_projects.html in the Gradle documentation.
+ */
+
+plugins {
+    // Apply the application plugin to add support for building a CLI application in Java.
+    id ("java-library")
+}
+
+repositories {
+    // Use Maven Central for resolving dependencies.
+    mavenCentral()
+}
+
+dependencies {
+    // Use JUnit Jupiter for testing.
+    testImplementation("org.junit.jupiter:junit-jupiter:5.9.2")
+    testImplementation("org.testcontainers:testcontainers:1.19.0")
+    testImplementation("org.testcontainers:junit-jupiter:1.19.0")
+    testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+    testImplementation("org.slf4j:slf4j-simple:1.7.36")
+    testImplementation(platform("software.amazon.awssdk:bom:2.20.156"))
+    testImplementation("software.amazon.awssdk:cloudwatchlogs")
+    testImplementation("com.github.rholder:guava-retrying:2.0.0")
+    testImplementation("org.assertj:assertj-core:3.24.2")
+    testImplementation("com.fasterxml.jackson.core:jackson-databind:2.13.0")
+}
+
+// Apply a specific Java toolchain to ease working on different environments.
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+tasks.named<Test>("test") {
+    // Use JUnit Platform for unit tests.
+    useJUnitPlatform()
+}

adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java

@@ -0,0 +1,170 @@
+package software.amazon.adot.testbed;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
+import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashSet;
+import java.util.UUID;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@Testcontainers(disabledWithoutDocker = true)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class LogsTests {
+    private static final String TEST_IMAGE = System.getenv("TEST_IMAGE") != null && !System.getenv("TEST_IMAGE").isEmpty()
+        ? System.getenv("TEST_IMAGE")
+        : "public.ecr.aws/aws-observability/aws-otel-collector:latest";
+    private final Logger collectorLogger = LoggerFactory.getLogger("collector");
+    private static final String uniqueID = UUID.randomUUID().toString();
+
+    private GenericContainer<?> collector;
+
+    private GenericContainer<?> createAndStartCollector(String configFilePath, String logFilePath, String logStreamName) throws IOException {
+
+        // Create an environment variable map
+        Map<String, String> envVariables = new HashMap<>();
+        envVariables.put("LOG_STREAM_NAME", logStreamName);
+        //Set credentials
+        envVariables.put("AWS_REGION", System.getenv("AWS_REGION"));
+        envVariables.put("AWS_ACCESS_KEY_ID", System.getenv("AWS_ACCESS_KEY_ID"));
+        envVariables.put("AWS_SECRET_ACCESS_KEY", System.getenv("AWS_SECRET_ACCESS_KEY"));
+        // Check if AWS_SESSION_TOKEN is not null before adding it
+        if (System.getenv("AWS_SESSION_TOKEN") != null) {
+            envVariables.put("AWS_SESSION_TOKEN", System.getenv("AWS_SESSION_TOKEN"));
+        }
+
+        var collector = new GenericContainer<>(TEST_IMAGE)
+            .withCopyFileToContainer(MountableFile.forClasspathResource(configFilePath), "/etc/collector/config.yaml")
+            .withLogConsumer(new Slf4jLogConsumer(collectorLogger))
+            .waitingFor(Wait.forLogMessage(".*Everything is ready. Begin running and processing data.*", 1))
+            .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.receiver.filelog,+adot.exporter.awscloudwatchlogs,+adot.extension.file_storage")
+            .withEnv(envVariables);
+
+        //Mount the log file for the file log receiver to parse
+        collector.withCopyFileToContainer(MountableFile.forClasspathResource(logFilePath), logFilePath );
+
+        collector.start();
+        collector.waitingFor(Wait.forHealthcheck());
+        return collector;
+    }
+
+    @Test
+    void testSyslog() throws Exception {
+        String logStreamName = "rfcsyslog-logstream-" + uniqueID;
+        collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", "/logs/RFC5424.log", logStreamName);
+
+        validateLogs(logStreamName , "/logs/RFC5424.log");
+        collector.stop();
+    }
+
+    @Test
+    void testLog4j() throws Exception {
+        String logStreamName = "log4j-logstream-" + uniqueID;
+        collector = createAndStartCollector("/configurations/config-log4j.yaml", "/logs/log4j.log", logStreamName);
+
+        validateLogs(logStreamName , "/logs/log4j.log");
+        collector.stop();
+    }
+
+    @Test
+    void testJson() throws Exception {
+        String logStreamName = "json-logstream-" + uniqueID;
+        collector = createAndStartCollector("/configurations/config-json.yaml", "/logs/testingJSON.log", logStreamName);
+
+        validateLogs(logStreamName , "/logs/testingJSON.log");
+        collector.stop();
+    }
+
+    void validateLogs(String testLogStreamName, String logFilePath) throws Exception {
+        var file = new File(logFilePath);
+        var lines = new HashSet<String>();
+
+        try (InputStream inputStream = getClass().getResourceAsStream(logFilePath);
+             BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                lines.add(line);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Error reading from the file: " + logFilePath, e);
+        }
+
+        var cwClient = CloudWatchLogsClient.builder()
+            .build();
+
+        var objectMapper = new ObjectMapper();
+
+        RetryerBuilder.<Void>newBuilder()
+            .retryIfException()
+            .retryIfRuntimeException()
+            .retryIfExceptionOfType(org.opentest4j.AssertionFailedError.class)
+            .withWaitStrategy(WaitStrategies.fixedWait(10, TimeUnit.SECONDS))
+            .withStopStrategy(StopStrategies.stopAfterAttempt(5))
+            .build()
+            .call(() -> {
+                var now = Instant.now();
+                var start = now.minus(Duration.ofMinutes(2));
+                var end = now.plus(Duration.ofMinutes(2));
+                var response = cwClient.getLogEvents(GetLogEventsRequest.builder().logGroupName("adot-testbed/logs-component-testing/logs")
+                    .logStreamName(testLogStreamName)
+                    .startTime(start.toEpochMilli())
+                    .endTime(end.toEpochMilli())
+                    .build());
+
+                var events = response.events();
+                var receivedMessages = events.stream().map(x -> x.message()).collect(Collectors.toSet());
+
+                // Extract the "body" field from each received message that is received from cloudwatch in JSON Format
+                var messageToValidate = receivedMessages.stream()
+                    .map(message -> {
+                        try {
+                            JsonNode jsonNode = objectMapper.readTree(message);
+                            return jsonNode.get("body").asText();
+                        } catch (Exception e) {
+                            return null;
+                        }
+                    })
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toSet());
+
+                //Validate body field in JSON-messageToValidate with actual log line from the log file.
+                assertThat(messageToValidate.containsAll(lines)).isTrue();
+                assertThat(messageToValidate).containsExactlyInAnyOrderElementsOf(lines);
+                return null;
+            });
+    }
+
+}

adot-testbed/app/src/test/resources/configurations/config-json.yaml

@@ -0,0 +1,19 @@
+receivers:
+  filelog:
+    include: [/logs/testingJSON.log]
+    encoding: utf-8
+    start_at: beginning
+    operators:
+      - type: json_parser
+
+exporters:
+  awscloudwatchlogs:
+    log_group_name: "adot-testbed/logs-component-testing/logs"
+    log_stream_name: ${LOG_STREAM_NAME}
+    log_retention: 7
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [awscloudwatchlogs]

adot-testbed/app/src/test/resources/configurations/config-log4j.yaml

@@ -0,0 +1,16 @@
+receivers:
+  filelog:
+    include: [/logs/log4j.log]
+    start_at: beginning
+
+exporters:
+  awscloudwatchlogs:
+    log_group_name: "adot-testbed/logs-component-testing/logs"
+    log_stream_name: ${LOG_STREAM_NAME}
+    log_retention: 7
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [awscloudwatchlogs]

adot-testbed/app/src/test/resources/configurations/config-rfcsyslog.yaml

@@ -0,0 +1,19 @@
+receivers:
+  filelog:
+    include: [/logs/RFC5424.log]
+    start_at: beginning
+    operators:
+      - type: syslog_parser
+        protocol: rfc5424
+
+exporters:
+  awscloudwatchlogs:
+    log_group_name: "adot-testbed/logs-component-testing/logs"
+    log_stream_name: ${LOG_STREAM_NAME}
+    log_retention: 7
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [awscloudwatchlogs]

adot-testbed/app/src/test/resources/logs/RFC5424.log

@@ -0,0 +1,5 @@
+<165>1 2023-19-22T18:09:11Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] First entry...
+<166>1 2023-19-22T18:09:11Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] Second Entry...
+<167>1 2023-19-22T18:09:11Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] Third Entry...
+<168>1 2023-19-22T18:09:11Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] Fourth Entry...
+<169>1 2023-19-22T18:09:11Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] Fourth Entry...

adot-testbed/app/src/test/resources/logs/log4j.log

@@ -0,0 +1,5 @@
+[otel.javaagent 2023-09-25 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- First Entry
+[otel.javaagent 2023-09-26 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Second Entry
+[otel.javaagent 2023-09-27 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Third Entry
+[otel.javaagent 2023-09-28 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fourth Entry
+[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fifth Entry

adot-testbed/app/src/test/resources/logs/testingJSON.log

@@ -0,0 +1 @@
+{"Records":[{"eventVersion":"1.08","userIdentity":{"type":"AssumedRole","principalId":"AROAZFA5SFCJPQHLSBPVK:i-069fd4e7e8a3255b8","arn":"arn:aws:sts::777777777777:assumed-role/terraform-22222222222222222222222221/i-069fd4e7e8a3255b8","accountId":"777777777777","accessKeyId":"AAAAAAAAAAAAAAAAAAA","sessionContext":{"sessionIssuer":{"type":"Role","principalId":"AROAZFA5SFCJPQHLSBPVK","arn":"arn:aws:iam::777777777777:role/terraform-22222222222222222222222221","accountId":"777777777777","userName":"terraform-22222222222222222222222221"},"webIdFederationData":{},"attributes":{"creationDate":"2023-09-26T20:43:59Z","mfaAuthenticated":"false"},"ec2RoleDelivery":"2.0"}},"eventTime":"2023-09-26T23:59:00Z","eventSource":"ssm.amazonaws.com","eventName":"ListInstanceAssociations","awsRegion":"us-west-2","sourceIPAddress":"34.212.88.55","userAgent":"aws-sdk-go/1.44.260 (go1.18.3; linux; amd64) amazon-ssm-agent/","requestParameters":{"instanceId":"i-069fd4e7e8a3255b8","maxResults":20},"responseElements":null,"requestID":"a23f9ca8-a29a-44bc-8731-50f6444b47eb","eventID":"58e2ac71-365e-4264-ad10-5e050a8b0bfa","readOnly":true,"resources":[{"accountId":"777777777777","ARN":"arn:aws:ec2:us-west-2:777777777777:instance/i-069fd4e7e8a3255b8"}],"eventType":"AwsApiCall","managementEvent":true,"recipientAccountId":"777777777777","eventCategory":"Management","tlsDetails":{"tlsVersion":"TLSv1.2","cipherSuite":"ECDHE-RSA-AES128-GCM-SHA256","clientProvidedHostHeader":"ssm.us-west-2.amazonaws.com"}}]}

adot-testbed/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists