Merge branch 'master' into cert

NEONScience · Dec 14, 2024 · 5f5a96e · 5f5a96e
2 parents 0dc3a32 + 56e902f
commit 5f5a96e
Showing 1 changed file with 34 additions and 36 deletions.
diff --git a/pipe/aepg600m/aepg600m_data_source_trino.yaml b/pipe/aepg600m/aepg600m_data_source_trino.yaml
@@ -97,42 +97,40 @@ transform:
     fi
     set +x
 
-    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-    # Export L0 data to bucket
-    # if [[ -d "$OUT_PATH/$SOURCE_TYPE" ]]; then
-    #   linkdir=$(mktemp -d)
-    #   shopt -s globstar
-    #   out_parquet_glob="${OUT_PATH}/**/*.parquet"
-    #   # Example: /pfs/out/li191r/2023/01/01/12345/data/file.parquet
-    #   echo "Linking output files to ${linkdir}"
-    #   #set -x
-    #   for f in $out_parquet_glob; do
-    #     # Parse the path
-    #     [[ "$f" =~ ^$OUT_PATH/(.*)/([0-9]+)/([0-9]+)/([0-9]+)/(.*)/data/(.*)$ ]]
-    #     fsourcetype="${BASH_REMATCH[1]}"
-    #     fyear="${BASH_REMATCH[2]}"
-    #     fmonth="${BASH_REMATCH[3]}"
-    #     fday="${BASH_REMATCH[4]}"
-    #     fsourceid="${BASH_REMATCH[5]}"
-    #     fname="${BASH_REMATCH[6]}"
-    #     outdir="${linkdir}/v1/${fsourcetype}/ms=${fyear}-${fmonth}/source_id=${fsourceid}"
-    #     mkdir -p "${outdir}"
-    #     ln -s "${f}" "${outdir}/${fname}"
-    #   done
-    #   #set +x
-    #   echo "Syncing files to bucket"
-    #   rclone \
-    #     --no-check-dest \
-    #     --copy-links \
-    #     --gcs-bucket-policy-only \
-    #     --gcs-no-check-bucket \
-    #     copy \
-    #     "${linkdir}" \
-    #     ":gcs://${BUCKET_NAME}"
-    #   echo "Removing temporary files"
-    #   rm -rf $linkdir
-    # fi
+    Export L0 data to bucket
+    if [[ -d "$OUT_PATH/$SOURCE_TYPE" ]]; then
+      linkdir=$(mktemp -d)
+      shopt -s globstar
+      out_parquet_glob="${OUT_PATH}/**/*.parquet"
+      # Example: /pfs/out/li191r/2023/01/01/12345/data/file.parquet
+      echo "Linking output files to ${linkdir}"
+      #set -x
+      for f in $out_parquet_glob; do
+        # Parse the path
+        [[ "$f" =~ ^$OUT_PATH/(.*)/([0-9]+)/([0-9]+)/([0-9]+)/(.*)/data/(.*)$ ]]
+        fsourcetype="${BASH_REMATCH[1]}"
+        fyear="${BASH_REMATCH[2]}"
+        fmonth="${BASH_REMATCH[3]}"
+        fday="${BASH_REMATCH[4]}"
+        fsourceid="${BASH_REMATCH[5]}"
+        fname="${BASH_REMATCH[6]}"
+        outdir="${linkdir}/v1/${fsourcetype}/ms=${fyear}-${fmonth}/source_id=${fsourceid}"
+        mkdir -p "${outdir}"
+        ln -s "${f}" "${outdir}/${fname}"
+      done
+      #set +x
+      echo "Syncing files to bucket"
+      rclone \
+        --no-check-dest \
+        --copy-links \
+        --gcs-bucket-policy-only \
+        --gcs-no-check-bucket \
+        copy \
+        "${linkdir}" \
+        ":gcs://${BUCKET_NAME}"
+      echo "Removing temporary files"
+      rm -rf $linkdir
+    fi
     EOF
   env:
     # Static environment variables for data conversion step