Clean up.

lib/s3lib/dynamodb_export_test.go

@@ -1,6 +1,7 @@
 package s3lib
 
 import (
+	"github.com/aws/aws-sdk-go-v2/service/dynamodbstreams/types"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
@@ -29,3 +30,72 @@ func TestAssertType(t *testing.T) {
 		assert.ErrorContains(t, err, "expected type bool, got string")
 	}
 }
+
+const validDynamoDBJSON = `{
+    "Item": {
+        "account_id": {
+            "S": "account-9825"
+        },
+        "user_id": {
+            "S": "user_id_1"
+        },
+        "is_null": {
+            "NULL": true
+        },
+        "sample_list": {
+            "L": [
+                {
+                    "S": "item1"
+                },
+                {
+                    "N": "2"
+                }
+            ]
+        },
+        "flag": {
+            "BOOL": false
+        },
+        "string_set": {
+            "SS": [
+                "value2",
+                "value44",
+                "value55",
+                "value66",
+                "value1"
+            ]
+        },
+        "random_number": {
+            "N": "4851973137566368817"
+        },
+        "number_set": {
+            "NS": [
+                "3",
+                "2",
+                "1"
+            ]
+        },
+        "sample_map": {
+            "M": {
+                "key1": {
+                    "S": "value1"
+                },
+                "key2": {
+                    "N": "2"
+                }
+            }
+        }
+    }
+}`
+
+func TestParseDynamoDBJSON(t *testing.T) {
+	{
+		// Valid JSON
+		value, err := parseDynamoDBJSON([]byte(validDynamoDBJSON))
+		assert.NoError(t, err)
+		assert.Equal(t, 9, len(value))
+
+		// Check keys
+		value, isOk := value["account_id"].(*types.AttributeValueMemberS{})
+		assert.True(t, isOk)
+	}
+}

lib/s3lib/s3lib.go

@@ -99,8 +99,7 @@ func (s *S3Client) StreamJsonGzipFile(ctx context.Context, file S3File, ch chan<
 	scanner.Buffer(buf, maxBufferSize)
 
 	for scanner.Scan() {
-		line := scanner.Bytes()
-		output, err := parseDynamoDBJSON(line)
+		output, err := parseDynamoDBJSON(scanner.Bytes())
 		if err != nil {
 			return fmt.Errorf("failed to parse dynamodb json: %w", err)
 		}