Initial commit with partial progress

.github/workflows/test.yaml

@@ -0,0 +1,187 @@
+
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches: [main, develop]
+    tags: ['*']
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  BASE_IMG_REPO: ubuntu
+  BASE_IMG_TAG: 22.04
+  NARG: 2
+
+jobs:
+
+  test_docker:
+
+    runs-on: ubuntu-latest
+
+    env:
+      TAG: gha-test
+      INT_TAG: gha-int-test
+
+    # This job runs for all events that trigger this workflow
+
+    steps:
+
+      - name: Concatenate env variables
+        run: |
+          echo "PSYLLID_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }}" >> $GITHUB_ENV
+#          echo "PSYLLID_INT_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.INT_TAG }}" >> $GITHUB_ENV
+
+      - name: Checkout the repo 
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: setup_buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Build
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          load: true
+          build-args: |
+            base_image=${{ env.BASE_IMG_REPO }}
+            base_tag=${{ env.BASE_IMG_TAG }}
+            psyllid_tag=${{ env.TAG }}
+            build_type=Debug
+            enable_testing=TRUE
+            narg=${{ env.NARG }}
+          platforms: linux/amd64
+          tags: ${{ env.PSYLLID_TAG }}
+
+#      - name: Build Integration Tests
+#        id: build_int_tests
+#        uses: docker/build-push-action@v5
+#        with:
+#          context: ./testing/integration
+#          push: false
+#          load: true
+#          build-args: |
+#            img_user=${{ env.REGISTRY }}/driplineorg
+#            img_repo=dripline-cpp
+#            img_tag=${{ env.TAG }}
+#          platforms: linux/amd64
+#          tags: ${{ env.DL_CPP_INT_TAG }}
+
+#      - name: Unit Tests
+#        run: |
+#          docker run --rm ${{ env.PSYLLID_INT_TAG }} bash -c "
+#            cd /usr/local/build/testing
+#            ./run_tests"
+
+#      - name: Integration Tests
+#        run: |
+#          cd testing/integration
+#          ./do-testing.sh ${{ env.INT_TAG }}
+
+# For debugging
+#      - name: Setup tmate session
+#        if: ${{ ! success() }}
+#        uses: mxschmitt/action-tmate@v3
+
+
+
+
+
+
+
+
+
+jobs:
+
+  docker-build-and-publish:
+
+    name: Build and push the docker images
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [Dev, Prod]
+        include:
+          - build: Dev
+            tag-suffix: '-dev'
+            build-type: Debug
+          - build: Prod
+            tag-suffix: ''
+            build-type: Release
+
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          # Need to avoid a shallow clone (fetch-depth=1) so that the lookup of the tag works
+          fetch-depth: 0
+
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          flavor: |
+            latest=auto
+            suffix=${{ matrix.tag-suffix }},onlatest=true
+          tags: |
+            type=semver,pattern={{raw}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      - name: Get previous tag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        id: tag_name
+        with:
+          fallback: beta
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: setup_buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          buildkitd-flags: --debug
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build
+        id: build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          push: ${{ github.event_name == 'push' && ( contains(github.ref, 'refs/tags/') || github.ref == 'refs/heads/develop' ) }}
+          build-args: |
+            base_image=${{ env.BASE_IMG_REPO }}
+            base_tag=${{ env.BASE_IMG_TAG }}
+            psyllid_tag=${{ steps.tag_name.outputs.tag }}
+            build_type=${{ matrix.build-type }}
+            narg=${{ env.NARG }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          platforms: linux/amd64
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }}

Dockerfile.withfpa

@@ -0,0 +1,83 @@
+ARG base_image=ubuntu
+ARG base_tag=22.04
+
+# Base image with environment variables set
+FROM ${base_image}:${base_tag} AS base
+
+# Set bash as the default shell
+SHELL ["/bin/bash", "-c"]
+
+ARG psyllid_tag=beta
+ARG psyllid_subdir=psyllid
+ARG build_type=Release
+
+ENV P8_ROOT=/usr/local/p8
+ENV PSYLLID_TAG=${psyllid_tag}
+ENV PSYLLID_INSTALL_PREFIX=${P8_ROOT}/${psyllid_subdir}/${PSYLLID_TAG}
+
+ENV PATH="${PATH}:${PSYLLID_INSTALL_PREFIX}"
+
+# Build image with dev dependencies
+FROM base AS build
+
+RUN apt-get update &&\
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential \
+        cmake \
+        git \
+        openssl \
+        libfftw3-dev \
+        libboost-chrono-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
+        libhdf5-dev \
+        librabbitmq-dev \
+        libyaml-cpp-dev \
+        rapidjson-dev \
+        &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/* &&\
+    /bin/true
+
+COPY . /tmp_source
+
+## store cmake args because we'll need to run twice (known package_builder issue)
+## use `extra_cmake_args` to add or replace options at build time; CMAKE_CONFIG_ARGS_LIST are defaults
+ARG extra_cmake_args=""
+ENV CMAKE_CONFIG_ARGS_LIST="\
+      -D CMAKE_BUILD_TYPE=$build_type \
+      -D CMAKE_INSTALL_PREFIX:PATH=$PSYLLID_BUILD_PREFIX \
+      -D Psyllid_ENABLE_FPA=TRUE \
+      ${extra_cmake_args} \
+      ${OS_CMAKE_ARGS} \
+      "
+
+RUN mkdir -p /tmp_source/build &&\
+    cd /tmp_source/build &&\
+    cmake ${CMAKE_CONFIG_ARGS_LIST} .. &&\
+    cmake ${CMAKE_CONFIG_ARGS_LIST} .. &&\
+    make install &&\
+    /bin/true
+
+# Final production image
+FROM base
+
+RUN apt-get update &&\
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential \
+        libssl3 \
+        libfftw3-double3 \
+        libboost-chrono1.74.0 \
+        libboost-filesystem1.74.0 \
+        libboost-system1.74.0 \
+        libhdf5-cpp-103 \
+        librabbitmq4 \
+        libyaml-cpp0.7 \
+        rapidjson-dev \
+        &&\
+    apt-get clean &&\
+    rm -rf /var/lib/apt/lists/* &&\
+    /bin/true
+
+# for now we must grab the extra dependency content as well as psyllid itself
+COPY --from=build $PSYLLID_BUILD_PREFIX $PSYLLID_BUILD_PREFIX

testing/docker-compose.yaml

@@ -0,0 +1,38 @@
+# This compose file lauches a RabbitMQ broker with both the HTTP port (15672) and the broker communication port (5672) mapped to the host.
+# Use this file if you want to run Psyllid on the host with a broker running in a container.
+
+services:
+  # a rabbit broker
+  rabbit-broker:
+    image: rabbitmq:3-management
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u guest:guest http://rabbit-broker:15672/api/overview &> /dev/null || exit 1"]
+
+  psyllid:
+    image: ghcr.io/project8/psyllid:${IMG_TAG:-latest}
+    depends_on:
+      rabbit-broker:
+        condition: service_healthy
+    volumes:
+      - ./str_1ch_dataprod.yaml:/configs/str_1ch_dataprod.yaml
+    command: >
+      bash -c "psyllid -vv --auth-file /auths.json -c /configs/str_1ch_dataprod.yaml"
+#      bash -c "psyllid -vv --auth-file /auths.json -c /configs/str_1ch_dataprod.yaml dripline.broker=rabbit-broker"
+    configs:
+      - auths.json
+
+#  tester:
+#    image: ghcr.io/driplineorg/dripline:latest
+#    depends_on:
+#      - psyllid
+#    command: >
+#      bash -c "dl-agent --auth-file /auths.json "
+
+configs:
+  auths.json:
+    file: ../docker/authentications.json
+
+

testing/str_1ch_dataprod.yaml

@@ -0,0 +1,53 @@
+dripline:
+#    broker: rabbit-broker
+    queue: psyllid
+
+post-to-slack: false
+
+#on-startup:
+#    - type: set
+#      key: filename
+#      payload:
+#          values: ["test_output.egg"]
+#    - type: cmd
+#      key: start-run
+#      payload:
+#            blah:
+
+daq:
+    activate-at-startup: true
+    n-files: 1
+    max-file-size-mb: 500
+
+streams:
+    ch0:
+        preset:
+            type: str-1ch-dataprod
+            nodes:
+              - { type: data-producer,          name: dprd }
+              - { type: tf-roach-receiver,      name: tfrr }
+              - { type: streaming-writer,       name: strw }
+              - { type: term-freq-data,         name: termf }
+#              - { type: term-time-data,         name: termt }
+            connections:
+              - "dprd.out_0:tfrr.in_0"
+#              - "tfrr.out_0:termt.in_0"
+              - "tfrr.out_0:strw.in_0"
+              - "tfrr.out_1:termf.in_0"
+
+        device:
+            n-channels: 1
+            bit-depth: 8
+            data-type-size: 1
+            sample-size: 2
+            record-size: 4096
+            acq-rate: 100 # MHz
+            v-offset: 0.0
+            v-range: 0.5
+
+        dprd:
+            length: 10
+
+        strw:
+            file-num: 0
+