Merge pull request #12 from perfsonar/prometheus-integration

Prometheus integration

.github/workflows/publish_docker.yml

@@ -0,0 +1,74 @@
+name: perfSONAR Logstash Image
+
+# This action builds Docker images of logstash then 
+# pushes result to GitHub Container Registry.
+
+on:
+  push:
+    branches: [ '*' ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ 'main' ]
+
+env:
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_BASE: ${{ github.repository }}
+  BUILD_DIR: perfsonar-logstash/perfsonar-logstash
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker images
+        id: build-and-push
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ env.BUILD_DIR }}
+          file: ${{ env.BUILD_DIR }}/docker/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/publish_docker_prometheus.yml

@@ -0,0 +1,75 @@
+name: Prometheus Logstash Image
+
+# This action builds Docker images of logstash then 
+# pushes result to GitHub Container Registry.
+
+on:
+  push:
+    branches: [ '*' ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ 'main' ]
+
+env:
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_BASE: ${{ github.repository }}
+  BUILD_DIR: perfsonar-logstash/perfsonar-logstash
+  IMAGE_SUBNAME: prometheus
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE }}-${{ env.IMAGE_SUBNAME }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker images
+        id: build-and-push
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ env.BUILD_DIR }}
+          file: ${{ env.BUILD_DIR }}/docker/Dockerfile-${{ env.IMAGE_SUBNAME }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -5,9 +5,9 @@
 .vscode/
 .env
 artifacts/
-docker-compose.yml
 ansible/vars/env.yml
 unibuild-log
 unibuild-work/
 unibuild-repo/
 multiarch_build/
+DOSB/

README.md

@@ -1,45 +1,34 @@
 # perfSONAR Logstash Pipeline
 
+This contains a set of Logstash pipelines used by the perfSONAR project to process data and store in OpenSearch. It contains the following pipelines:
 
-**UNDER CONSTRUCTION: NOT FOR PRODUCTION USE**
-
-## Build and use Docker image for development
-
-```
-make local # only need to do this first time to copy config in place
-make #run this everytime you make change to pipeline
-docker-compose up -d
-```
-
-## RPMS for centos7
-```
-#Build rpm and test install in a container
-make centos7
-
-##verify RPM install
-docker-compose -f docker-compose.qa.yml up -d centos7
-docker-compose -f docker-compose.qa.yml exec centos7 bash
-tail -f /var/log/logstash/logstash-plain.log 
-```
-
-## Debian package
-```
-#Build logstash .deb
-make release
-make build
-git clone https://github.com/perfsonar/debian-docker-buildmachines.git ../debian-docker-buildmachines
-pwd=$(pwd)
-cd ../debian-docker-buildmachines
-./build-in-docker logstash
-cd $pwd
-mkdir -p artifacts/debian/
-cp ../build_results/*deb artifacts/debian/
-
-
-#Test it with perfsonar-testpoint
-docker-compose -f docker-compose.debian.yml build
-docker-compose -f docker-compose.debian.yml up -d
-docker-compose -f docker-compose.debian.yml exec debian bash
-apt install perfsonar-testpoint
-pscheduler task --archive '{"archiver":"http","data":{"schema":2,"_url":"http://localhost:11283","op":"put","_headers":{"content-type":"application/json"}}}' rtt --dest localhost
-```
+- **perfSONAR Pipeline**: This pipeline runs an HTTP listener that accepts measurement results being archived by pscheduler.
+
+- **Prometheus Pipeline**: This is a pipeline for reading generic Prometheus Metrics and storing them in OpenSearch. 
+
+# Docker Images
+
+Each pipeline is built as a separate docker image. Both having a number of customization options:
+
+## Docker Environment Variables
+
+- **opensearch_output_host** - The hostname of the OpenSearch instance
+- **opensearch_output_user** - The username used to authenticate to OpenSearch
+- **opensearch_output_password** - The password used to authenticate to OpenSearch
+
+
+## Docker Volumes
+
+### perfSONAR 
+- **/usr/lib/perfsonar/logstash/pipeline/99-outputs.conf** - Override this to define a custom output filter if you do not want to use the default output.
+
+### Prometheus
+- **/usr/lib/perfsonar/logstash/prometheus_pipeline/*.conf** - Override any file in the .conf directory. You will want to add at least one input filter likely an HTTP poller to grab the data. You can override *99-outputs.conf* if you want a custom output filter. 
+
+# OS Packages
+
+A full compliment of OS packages can be built using [unibuild](https://github.com/perfsonar/unibuild). They consist of the following:
+
+- **perfsonar-logstash** - This package contains both pipelines but only the perfSONAR pipeline is enabled by default. The Prometheus pipeline needs inputs defined which is generally handled by other perfSONAR packages (like perfsonar-archive) or auto-generated. You can enable manually with the script `/usr/lib/perfsonar/logstash/scripts/enable_prometheus_pipeline.py`
+
+- **perfsonar-logstash-output-plugin** - Install the OpenSearch output plugin since not included by default with logstash. 

perfsonar-logstash/perfsonar-logstash/Makefile

@@ -18,11 +18,16 @@ ifndef SYSTEMDPATH
 	@false
 endif
 	mkdir -p ${ROOTPATH}/pipeline
+	mkdir -p ${ROOTPATH}/prometheus_pipeline
 	mkdir -p ${ROOTPATH}/ruby
+	mkdir -p ${ROOTPATH}/java/maven
 	mkdir -p ${ROOTPATH}/scripts
 	mkdir -p ${CONFIGPATH}
 	mkdir -p ${SYSTEMDPATH}/logstash.service.d
 	cp -r pipeline/* ${ROOTPATH}/pipeline
+	cp -r prometheus_pipeline/* ${ROOTPATH}/prometheus_pipeline
 	cp -r ruby/* ${ROOTPATH}/ruby
+	cp -r java/* ${ROOTPATH}/java
 	cp -r scripts/* ${ROOTPATH}/scripts
 	cp -r systemd/* ${SYSTEMDPATH}/logstash.service.d/
+	mvn -Dmaven.repo.local=${ROOTPATH}/java/maven -f ${ROOTPATH}/java/pom.xml dependency:resolve