Merge pull request #2254 from AllenInstitute/bugfix/2254-nwb-export

Fix nwb_version so that dandi can read it

tools/documentation/Dockerfile

@@ -1,5 +1,5 @@
 FROM debian:bookworm
-MAINTAINER Thomas Braun [email protected]
+LABEL org.opencontainers.image.authors="Thomas Braun [email protected]"
 
 ENV DEBIAN_FRONTEND=noninteractive
 

tools/ftp-upload/Dockerfile

@@ -1,20 +1,21 @@
-FROM debian:bullseye
+FROM debian:bookworm
 LABEL org.opencontainers.image.authors="Thomas Braun [email protected]"
 
-RUN DEBIAN_FRONTEND=noninteractive \
-    apt-get update && \
-    apt-get install -y \
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update &&   \
+    apt-get install -y  \
         ca-certificates \
-        lftp \
-        openssl && \
+        lftp            \
+        openssl &&      \
     apt-get clean
 
 ARG UNAME=ci
-ARG UID=1000
-ARG GID=1000
+ARG USERID=1000
+ARG GROUPID=1000
 
-RUN groupadd -g ${GID} -o ${UNAME} && \
-    useradd -m -u ${UID} -g ${GID} -o -s /bin/bash ${UNAME}
+RUN groupadd -g ${GROUPID} -o ${UNAME} && \
+    useradd -m -u ${USERID} -g ${GROUPID} -o -s /bin/bash ${UNAME}
 
 USER ${UNAME}
 

tools/ftp-upload/download-files.sh

@@ -59,8 +59,8 @@ top_level=$(git rev-parse --show-toplevel)
 # build containter
 echo "##[group]Build Docker container 'ftp-upload'"
 docker build \
-    --build-arg UID=$(id -u) \
-    --build-arg GID=$(id -g) \
+    --build-arg USERID=$(id -u) \
+    --build-arg GROUPID=$(id -g) \
     -t ftp-upload \
     $top_level/tools/ftp-upload
 echo "##[endgroup]"

tools/ftp-upload/upload-files.sh

@@ -49,8 +49,8 @@ top_level=$(git rev-parse --show-toplevel)
 # build containter
 echo "##[group]Build Docker container 'ftp-upload'"
 docker build \
-    --build-arg UID=$(id -u) \
-    --build-arg GID=$(id -g) \
+    --build-arg USERID=$(id -u) \
+    --build-arg GROUPID=$(id -g) \
     -t ftp-upload \
     $top_level/tools/ftp-upload
 echo "##[endgroup]"

tools/nwb-read-tests/Dockerfile

@@ -1,5 +1,5 @@
-FROM debian:bullseye
-MAINTAINER Thomas Braun [email protected]
+FROM debian:bookworm
+LABEL org.opencontainers.image.authors="Thomas Braun [email protected]"
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -10,15 +10,10 @@ RUN apt-get update &&                          \
       python3-minimal                          \
       python3-pip                              \
       python3-setuptools                       \
-      python3-wheel &&                         \
+      python3-venv                             \
+      &&                                       \
     rm -rf /var/lib/apt/lists/*
 
-ARG PACKAGE_WITH_VERSION
-
-RUN pip3 install $PACKAGE_WITH_VERSION
-
-RUN pip3 check
-
 ARG USERID
 ARG GROUPID
 
@@ -29,3 +24,13 @@ RUN useradd -u $USERID -g $GROUPID -ms /bin/bash ci
 USER ci
 
 WORKDIR /home/ci
+
+COPY requirements.txt .
+
+# https://stackoverflow.com/a/75696359
+RUN python -m venv --system-site-packages /home/ci/.venv && \
+    . /home/ci/.venv/bin/activate                        && \
+    pip3 install -r requirements.txt
+
+# https://stackoverflow.com/a/56286534
+ENV PATH=/home/ci/.venv/bin:${PATH}

tools/nwb-read-tests/nwbv2-read-test.py

@@ -24,17 +24,27 @@ def checkFile(path):
         print(f"The file {path} does not exist.", file=sys.stderr)
         return 1
 
-    # 1.) Validation
+    # 1.) pynwb Validation
     comp = run(["python", "-m", "pynwb.validate", path],
                stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=120)
 
     if comp.returncode != 0:
-        print(f"Validation output: {comp.stdout}", file=sys.stderr)
+        print(f"pynwb validation output: {comp.stdout}", file=sys.stderr)
         return 1
 
-    print(f"Validation output: {comp.stdout}", file=sys.stdout)
+    print(f"pynwb validation output: {comp.stdout}", file=sys.stdout)
 
-    # 2.) Read test
+    # 2.) dandi Validation
+    comp = run(["dandi", "validate", "--ignore", "(NWBI|DANDI)", path],
+               stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=120)
+
+    if comp.returncode != 0:
+        print(f"dandi validation output: {comp.stdout}", file=sys.stderr)
+        return 1
+
+    print(f"dandi validation output: {comp.stdout}", file=sys.stdout)
+
+    # 3.) Read test
     with NWBHDF5IO(path, mode='r', load_namespaces=True) as io:
         nwbfile = io.read()
 
@@ -56,7 +66,7 @@ def checkFile(path):
             print(f"epochs.treelevel: {nwbfile.epochs[:, 'treelevel']}")
             print(f"epochs.timeseries: {nwbfile.epochs[:, 'timeseries']}")
 
-    # check that pynwb/hdmf can read our object IDs
+    # 4. Check that pynwb/hdmf can read our object IDs
     with h5py.File(path, 'r') as f:
         root_object_id_hdf5 = to_str(f["/"].attrs["object_id"])
 

tools/nwb-read-tests/requirements.in

@@ -0,0 +1,2 @@
+pynwb==2.8.1
+dandi==0.63.0