Merge branch 'master' into cert

NEONScience · Dec 14, 2024 · 4c06cd9 · 4c06cd9
2 parents d42a306 + 96fcdb6
commit 4c06cd9
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 104 deletions.
diff --git a/.github/workflows/CERT_aepg600m_site_list.yml b/.github/workflows/CERT_aepg600m_site_list.yml
@@ -4,7 +4,7 @@ on:
     branches:
       - 'cert'
     paths:
-      - 'pipe/aepg600m/site-list.json'
+      - 'pipe/aepg600m/site-list*.json'
   workflow_dispatch: {} # Allows trigger of workflow from web interface
 jobs:
   put-file:
@@ -15,8 +15,8 @@ jobs:
       PACH_TOKEN: ${{ secrets.RepoOwnerPachydermCert }}
       REPO: aepg600m_site_list # Pachyderm repo
       BRANCH: master 
-      IN_PATHS: 'pipe/aepg600m/site-list.json' # Comma-separated list (no spaces) to one or more paths or directories. Length must match OUT_PATHS. If directory, all files in directory will be placed in pachyderm at corresponding entry of OUT_PATHS. 
-      OUT_PATHS: 'site-list.json' # Comma-separated list (no spaces) of corresponding path(s) to place the files(s) in Pachyderm. Must be same length as IN_PATHS. If corresponding entry in IN_PATHS is a file, specify to the file. If corresponding entry in IN_PATHS is a directory, specify to the directory. 
+      IN_PATHS: 'pipe/aepg600m/site-list-heated-kafka.json,pipe/aepg600m/site-list-nonheated-kafka.json,pipe/aepg600m/site-list-heated-trino.json,pipe/aepg600m/site-list-nonheated-trino.json' # Comma-separated list (no spaces) to one or more paths or directories. Length must match OUT_PATHS. If directory, all files in directory will be placed in pachyderm at corresponding entry of OUT_PATHS. 
+      OUT_PATHS: 'site-list-heated-kafka.json,site-list-nonheated-kafka.json,site-list-heated-trino.json,site-list-nonheated-trino.json' # Comma-separated list (no spaces) of corresponding path(s) to place the files(s) in Pachyderm. Must be same length as IN_PATHS. If corresponding entry in IN_PATHS is a file, specify to the file. If corresponding entry in IN_PATHS is a directory, specify to the directory. 
     steps:
       - uses: actions/checkout@v4
       - run: ls -la

diff --git a/.github/workflows/DEV_aepg600m_site_list.yml b/.github/workflows/DEV_aepg600m_site_list.yml
@@ -4,7 +4,7 @@ on:
     branches:
       - 'master'
     paths:
-      - 'pipe/aepg600m/site-list.json'
+      - 'pipe/aepg600m/site-list*.json'
   workflow_dispatch: {} # Allows trigger of workflow from web interface
 jobs:
   put-file:
@@ -15,8 +15,8 @@ jobs:
       PACH_TOKEN: ${{ secrets.RepoOwnerPachydermDev }}
       REPO: aepg600m_site_list # Pachyderm repo
       BRANCH: master 
-      IN_PATHS: 'pipe/aepg600m/site-list.json' # Comma-separated list (no spaces) to one or more paths or directories. Length must match OUT_PATHS. If directory, all files in directory will be placed in pachyderm at corresponding entry of OUT_PATHS. 
-      OUT_PATHS: 'site-list.json' # Comma-separated list (no spaces) of corresponding path(s) to place the files(s) in Pachyderm. Must be same length as IN_PATHS. If corresponding entry in IN_PATHS is a file, specify to the file. If corresponding entry in IN_PATHS is a directory, specify to the directory. 
+      IN_PATHS: 'pipe/aepg600m/site-list-heated-kafka.json,pipe/aepg600m/site-list-nonheated-kafka.json,pipe/aepg600m/site-list-heated-trino.json,pipe/aepg600m/site-list-nonheated-trino.json' # Comma-separated list (no spaces) to one or more paths or directories. Length must match OUT_PATHS. If directory, all files in directory will be placed in pachyderm at corresponding entry of OUT_PATHS. 
+      OUT_PATHS: 'site-list-heated-kafka.json,site-list-nonheated-kafka.json,site-list-heated-trino.json,site-list-nonheated-trino.json' # Comma-separated list (no spaces) of corresponding path(s) to place the files(s) in Pachyderm. Must be same length as IN_PATHS. If corresponding entry in IN_PATHS is a file, specify to the file. If corresponding entry in IN_PATHS is a directory, specify to the directory. 
     steps:
       - uses: actions/checkout@v4
       - run: ls -la

diff --git a/pipe/aepg600m/aepg600m_location_loader.yaml b/pipe/aepg600m/aepg600m_location_loader.yaml
@@ -19,6 +19,7 @@ transform:
     SOURCE_TYPE_2: aepg600m_heated
     OUT_PATH: /pfs/out
     LOG_LEVEL: INFO
+    ERR_PATH: /pfs/out/errored_datums
   secrets:
   - name: pdr-secret
     mount_path: /var/db_secret

diff --git a/pipe/aepg600m/site-list.json b/pipe/aepg600m/site-list.json