From 17e5dfb2d1026e86edba18a39e69ad69045b5157 Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Tue, 6 Feb 2024 16:56:27 -0300 Subject: [PATCH] Add s3 handling reference links --- integrations/stdin_to_securitylake.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrations/stdin_to_securitylake.py b/integrations/stdin_to_securitylake.py index 1604bc2ed9ebc..d176bb38b004e 100755 --- a/integrations/stdin_to_securitylake.py +++ b/integrations/stdin_to_securitylake.py @@ -7,16 +7,19 @@ import time import json from datetime import datetime -from pyarrow import parquet, Table +from pyarrow import parquet, Table, fs block_ending = { "block_ending": True } +s3 = fs.S3FileSystem(region='eu-west-3') + def map_to_ocsf(): ## Code that translates fields to OCSF def encode_parquet(list): ### We can write directly to S3 from pyarrow: ### https://arrow.apache.org/docs/python/filesystems.html#s3 + ### https://arrow.apache.org/docs/python/generated/pyarrow.fs.S3FileSystem.html#pyarrow.fs.S3FileSystem.open_output_stream ### ### Credentials can be stored in /root/.aws/credentials ### https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html