diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..6f37c76 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,187 @@ + +name: Test + +on: + pull_request: + push: + branches: [main, develop] + tags: ['*'] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + BASE_IMG_REPO: ubuntu + BASE_IMG_TAG: 22.04 + NARG: 2 + +jobs: + + test_docker: + + runs-on: ubuntu-latest + + env: + TAG: gha-test + INT_TAG: gha-int-test + + # This job runs for all events that trigger this workflow + + steps: + + - name: Concatenate env variables + run: | + echo "PSYLLID_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }}" >> $GITHUB_ENV +# echo "PSYLLID_INT_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.INT_TAG }}" >> $GITHUB_ENV + + - name: Checkout the repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: setup_buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Build + id: build + uses: docker/build-push-action@v5 + with: + context: . + push: false + load: true + build-args: | + base_image=${{ env.BASE_IMG_REPO }} + base_tag=${{ env.BASE_IMG_TAG }} + psyllid_tag=${{ env.TAG }} + build_type=Debug + enable_testing=TRUE + narg=${{ env.NARG }} + platforms: linux/amd64 + tags: ${{ env.PSYLLID_TAG }} + +# - name: Build Integration Tests +# id: build_int_tests +# uses: docker/build-push-action@v5 +# with: +# context: ./testing/integration +# push: false +# load: true +# build-args: | +# img_user=${{ env.REGISTRY }}/driplineorg +# img_repo=dripline-cpp +# img_tag=${{ env.TAG }} +# platforms: linux/amd64 +# tags: ${{ env.DL_CPP_INT_TAG }} + +# - name: Unit Tests +# run: | +# docker run --rm ${{ env.PSYLLID_INT_TAG }} bash -c " +# cd /usr/local/build/testing +# ./run_tests" + +# - name: Integration Tests +# run: | +# cd testing/integration +# ./do-testing.sh ${{ env.INT_TAG }} + +# For debugging +# - name: Setup tmate session +# if: ${{ ! success() }} +# uses: mxschmitt/action-tmate@v3 + + + + + + + + + +jobs: + + docker-build-and-publish: + + name: Build and push the docker images + + strategy: + fail-fast: false + matrix: + build: [Dev, Prod] + include: + - build: Dev + tag-suffix: '-dev' + build-type: Debug + - build: Prod + tag-suffix: '' + build-type: Release + + runs-on: ubuntu-latest + steps: + + - name: Check out the repo + uses: actions/checkout@v3 + with: + submodules: recursive + # Need to avoid a shallow clone (fetch-depth=1) so that the lookup of the tag works + fetch-depth: 0 + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }} + flavor: | + latest=auto + suffix=${{ matrix.tag-suffix }},onlatest=true + tags: | + type=semver,pattern={{raw}} + type=ref,event=branch + type=ref,event=pr + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + + - name: Get previous tag + uses: "WyriHaximus/github-action-get-previous-tag@v1" + id: tag_name + with: + fallback: beta + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + id: setup_buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: --debug + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build + id: build + uses: docker/build-push-action@v3 + with: + context: . + file: Dockerfile + push: ${{ github.event_name == 'push' && ( contains(github.ref, 'refs/tags/') || github.ref == 'refs/heads/develop' ) }} + build-args: | + base_image=${{ env.BASE_IMG_REPO }} + base_tag=${{ env.BASE_IMG_TAG }} + psyllid_tag=${{ steps.tag_name.outputs.tag }} + build_type=${{ matrix.build-type }} + narg=${{ env.NARG }} + tags: ${{ steps.docker_meta.outputs.tags }} + platforms: linux/amd64 + + - name: Release + uses: softprops/action-gh-release@v1 + if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} diff --git a/Dockerfile.withfpa b/Dockerfile.withfpa new file mode 100644 index 0000000..b55fc7e --- /dev/null +++ b/Dockerfile.withfpa @@ -0,0 +1,83 @@ +ARG base_image=ubuntu +ARG base_tag=22.04 + +# Base image with environment variables set +FROM ${base_image}:${base_tag} AS base + +# Set bash as the default shell +SHELL ["/bin/bash", "-c"] + +ARG psyllid_tag=beta +ARG psyllid_subdir=psyllid +ARG build_type=Release + +ENV P8_ROOT=/usr/local/p8 +ENV PSYLLID_TAG=${psyllid_tag} +ENV PSYLLID_INSTALL_PREFIX=${P8_ROOT}/${psyllid_subdir}/${PSYLLID_TAG} + +ENV PATH="${PATH}:${PSYLLID_INSTALL_PREFIX}" + +# Build image with dev dependencies +FROM base AS build + +RUN apt-get update &&\ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + cmake \ + git \ + openssl \ + libfftw3-dev \ + libboost-chrono-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libhdf5-dev \ + librabbitmq-dev \ + libyaml-cpp-dev \ + rapidjson-dev \ + &&\ + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* &&\ + /bin/true + +COPY . /tmp_source + +## store cmake args because we'll need to run twice (known package_builder issue) +## use `extra_cmake_args` to add or replace options at build time; CMAKE_CONFIG_ARGS_LIST are defaults +ARG extra_cmake_args="" +ENV CMAKE_CONFIG_ARGS_LIST="\ + -D CMAKE_BUILD_TYPE=$build_type \ + -D CMAKE_INSTALL_PREFIX:PATH=$PSYLLID_BUILD_PREFIX \ + -D Psyllid_ENABLE_FPA=TRUE \ + ${extra_cmake_args} \ + ${OS_CMAKE_ARGS} \ + " + +RUN mkdir -p /tmp_source/build &&\ + cd /tmp_source/build &&\ + cmake ${CMAKE_CONFIG_ARGS_LIST} .. &&\ + cmake ${CMAKE_CONFIG_ARGS_LIST} .. &&\ + make install &&\ + /bin/true + +# Final production image +FROM base + +RUN apt-get update &&\ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libssl3 \ + libfftw3-double3 \ + libboost-chrono1.74.0 \ + libboost-filesystem1.74.0 \ + libboost-system1.74.0 \ + libhdf5-cpp-103 \ + librabbitmq4 \ + libyaml-cpp0.7 \ + rapidjson-dev \ + &&\ + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* &&\ + /bin/true + +# for now we must grab the extra dependency content as well as psyllid itself +COPY --from=build $PSYLLID_BUILD_PREFIX $PSYLLID_BUILD_PREFIX diff --git a/testing/docker-compose.yaml b/testing/docker-compose.yaml new file mode 100644 index 0000000..d51bec4 --- /dev/null +++ b/testing/docker-compose.yaml @@ -0,0 +1,38 @@ +# This compose file lauches a RabbitMQ broker with both the HTTP port (15672) and the broker communication port (5672) mapped to the host. +# Use this file if you want to run Psyllid on the host with a broker running in a container. + +services: + # a rabbit broker + rabbit-broker: + image: rabbitmq:3-management + ports: + - "5672:5672" + - "15672:15672" + healthcheck: + test: ["CMD-SHELL", "curl -u guest:guest http://rabbit-broker:15672/api/overview &> /dev/null || exit 1"] + + psyllid: + image: ghcr.io/project8/psyllid:${IMG_TAG:-latest} + depends_on: + rabbit-broker: + condition: service_healthy + volumes: + - ./str_1ch_dataprod.yaml:/configs/str_1ch_dataprod.yaml + command: > + bash -c "psyllid -vv --auth-file /auths.json -c /configs/str_1ch_dataprod.yaml" +# bash -c "psyllid -vv --auth-file /auths.json -c /configs/str_1ch_dataprod.yaml dripline.broker=rabbit-broker" + configs: + - auths.json + +# tester: +# image: ghcr.io/driplineorg/dripline:latest +# depends_on: +# - psyllid +# command: > +# bash -c "dl-agent --auth-file /auths.json " + +configs: + auths.json: + file: ../docker/authentications.json + + diff --git a/testing/str_1ch_dataprod.yaml b/testing/str_1ch_dataprod.yaml new file mode 100644 index 0000000..6c801a1 --- /dev/null +++ b/testing/str_1ch_dataprod.yaml @@ -0,0 +1,53 @@ +dripline: +# broker: rabbit-broker + queue: psyllid + +post-to-slack: false + +#on-startup: +# - type: set +# key: filename +# payload: +# values: ["test_output.egg"] +# - type: cmd +# key: start-run +# payload: +# blah: + +daq: + activate-at-startup: true + n-files: 1 + max-file-size-mb: 500 + +streams: + ch0: + preset: + type: str-1ch-dataprod + nodes: + - { type: data-producer, name: dprd } + - { type: tf-roach-receiver, name: tfrr } + - { type: streaming-writer, name: strw } + - { type: term-freq-data, name: termf } +# - { type: term-time-data, name: termt } + connections: + - "dprd.out_0:tfrr.in_0" +# - "tfrr.out_0:termt.in_0" + - "tfrr.out_0:strw.in_0" + - "tfrr.out_1:termf.in_0" + + device: + n-channels: 1 + bit-depth: 8 + data-type-size: 1 + sample-size: 2 + record-size: 4096 + acq-rate: 100 # MHz + v-offset: 0.0 + v-range: 0.5 + + dprd: + length: 10 + + strw: + file-num: 0 + \ No newline at end of file