From aa211fe7a50fe261aedbb5f9ada25303e1c9808f Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 19:19:02 -0400 Subject: [PATCH 01/13] Removing old dockerfiles --- Debian_dockerfile | 31 ----------------------------- Dockerfile | 5 ----- docker-compose.debian.yml | 24 ----------------------- docker-compose.qa.yml | 13 ------------- docker-compose.unibuild.yml | 16 --------------- docker/centos7/Dockerfile | 39 ------------------------------------- 6 files changed, 128 deletions(-) delete mode 100644 Debian_dockerfile delete mode 100644 Dockerfile delete mode 100644 docker-compose.debian.yml delete mode 100644 docker-compose.qa.yml delete mode 100644 docker-compose.unibuild.yml delete mode 100644 docker/centos7/Dockerfile diff --git a/Debian_dockerfile b/Debian_dockerfile deleted file mode 100644 index 6f4e6de..0000000 --- a/Debian_dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM jrei/systemd-debian:10 - -ENV container docker - -# Install necessary packages for testing -RUN apt-get update -y && \ - apt-get install -y wget apt-transport-https curl gnupg unzip - -COPY . /app/logstash/ - -# Logstash package pre configurations -RUN cd /app/logstash/ && \ - wget https://artifacts.elastic.co/downloads/logstash/logstash-oss-7.16.1-amd64.deb && \ - apt-get install -y ./logstash-oss*.deb - -# Install perfsonar logstash and output plugin -RUN cd /app/logstash/artifacts/debian/ && \ - apt-get install -y ./perfsonar-logstash-output-plugin_*.deb && \ - apt-get install -y ./perfsonar-logstash_*.deb - -VOLUME ["/var/lib/pgsql", "/var/log", "/etc/rsyslog.d" ] - -EXPOSE 123/udp 443 861 862 5000 5001 5101 5201 5890-5900 8760-9960/tcp 8760-9960/udp 18760-19960/tcp 18760-19960/udp - -# Install perfsonar testpoint -RUN cd /etc/apt/sources.list.d/ && \ - curl -o perfsonar-release.list http://downloads.perfsonar.net/debian/perfsonar-release.list && \ - curl http://downloads.perfsonar.net/debian/perfsonar-official.gpg.key | apt-key add - && \ - apt-get update -y - #apt-get update -y && \ - #apt-get install -y perfsonar-testpoint diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e3feeb4..0000000 --- a/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM logstash:7.11.2 - -COPY ./pipeline/* /usr/share/logstash/pipeline/ -COPY ./pipeline_etc/ /etc/perfsonar/logstash -COPY ./ruby/ /usr/lib/perfsonar/logstash/ruby \ No newline at end of file diff --git a/docker-compose.debian.yml b/docker-compose.debian.yml deleted file mode 100644 index 7733443..0000000 --- a/docker-compose.debian.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '3.7' -services: - debian: - build: - context: . - dockerfile: Debian_dockerfile - environment: - - container=docker - - TZ=UTC - #network_mode: "host" - restart: on-failure - tmpfs: - - /tmp:exec - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - working_dir: /app - privileged: true - tty: true - ports: - - "5601:5601" - - "9200:9200" - - "9300:9300" diff --git a/docker-compose.qa.yml b/docker-compose.qa.yml deleted file mode 100644 index 670af56..0000000 --- a/docker-compose.qa.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '3.7' -services: - centos7: - image: 'perfsonar/logstash/qa:centos7' - build: - context: . - dockerfile: docker/centos7/Dockerfile - tmpfs: - - /run - - /tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - working_dir: /app \ No newline at end of file diff --git a/docker-compose.unibuild.yml b/docker-compose.unibuild.yml deleted file mode 100644 index a48f851..0000000 --- a/docker-compose.unibuild.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.8' -services: - alma8: - image: perfsonar/unibuild-alma8:latest - build: - context: ../ - dockerfile: ./docker-envs/Dockerfile-alma8 - volumes: - - .:/app - debian10: - image: perfsonar/unibuild-debian10:latest - build: - context: ../ - dockerfile: ./docker-envs/Dockerfile-debian10 - volumes: - - .:/app diff --git a/docker/centos7/Dockerfile b/docker/centos7/Dockerfile deleted file mode 100644 index 738b830..0000000 --- a/docker/centos7/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM centos:7 -ENV container docker - -#cleanup to enable systemd -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ - systemd-tmpfiles-setup.service ] || rm -f $i; done); \ - rm -f /lib/systemd/system/multi-user.target.wants/*;\ - rm -f /etc/systemd/system/*.wants/*;\ - rm -f /lib/systemd/system/local-fs.target.wants/*; \ - rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ - rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ - rm -f /lib/systemd/system/basic.target.wants/*;\ - rm -f /lib/systemd/system/anaconda.target.wants/*; - -#Install build environment dependencies -RUN yum update -y && \ - yum install -y epel-release make rpmbuild rpmdevtools && \ - yum clean all && \ - mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} && \ - echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros - -# Copy code to /app -COPY . /app - -#Build RPM -RUN cd /app && \ - make dist && \ - mv perfsonar-logstash-output-plugin-*.tar.gz ~/rpmbuild/SOURCES/ && \ - mv perfsonar-logstash-*.tar.gz ~/rpmbuild/SOURCES/ && \ - rpmbuild -bs perfsonar-logstash-output-plugin.spec && \ - rpmbuild -ba perfsonar-logstash-output-plugin.spec && \ - rpmbuild -bs perfsonar-logstash.spec && \ - rpmbuild -ba perfsonar-logstash.spec - -#shared volumes -VOLUME /sys/fs/cgroup - -#Keep container running -CMD ["/usr/sbin/init"] From 25a93301c211484385bf2356ff6a2cdf4390fe3e Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 19:37:42 -0400 Subject: [PATCH 02/13] Initial commit of prometheus pipeline with docker images --- .../perfsonar-logstash/docker/Dockerfile | 18 +++ .../docker/Dockerfile-prometheus | 24 ++++ .../perfsonar-logstash/java/pom.xml | 17 +++ .../pipeline_etc/logstash_sysconfig | 5 +- .../prometheus_pipeline/02-formatting.conf | 23 ++++ .../prometheus_pipeline/99-outputs.conf | 13 ++ .../perfsonar-logstash/ruby/lib/ratecache.rb | 33 +++++ .../ruby/prometheus_parse.rb | 119 ++++++++++++++++++ .../scripts/enable_prometheus_pipeline.py | 26 ++++ 9 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 perfsonar-logstash/perfsonar-logstash/docker/Dockerfile create mode 100644 perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus create mode 100644 perfsonar-logstash/perfsonar-logstash/java/pom.xml create mode 100644 perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/02-formatting.conf create mode 100644 perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/99-outputs.conf create mode 100644 perfsonar-logstash/perfsonar-logstash/ruby/lib/ratecache.rb create mode 100644 perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb create mode 100644 perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile new file mode 100644 index 0000000..cc5f913 --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.9.0 + +#Copy files +COPY ./pipeline/* /usr/lib/perfsonar/logstash/pipeline/ +COPY ./pipeline_etc/ /etc/perfsonar/logstash +COPY ./ruby/ /usr/lib/perfsonar/logstash/ruby +COPY ./scripts/ /usr/lib/perfsonar/logstash/scripts + +#install dependencies +USER root +RUN apt-get update -y && apt-get install -y python3 python3-yaml +#make config look like non-docker so scripts work +RUN mkdir /etc/logstash &&\ + ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml +USER logstash + +#Generate pipelines.yml +RUN python3 /usr/lib/perfsonar/logstash/scripts/update_logstash_pipeline_yml.py \ No newline at end of file diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus new file mode 100644 index 0000000..4266086 --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus @@ -0,0 +1,24 @@ +FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.9.0 + +#Copy files +COPY ./prometheus_pipeline/* /usr/lib/perfsonar/logstash/prometheus_pipeline/ +COPY ./pipeline_etc/ /etc/perfsonar/logstash +COPY ./ruby/ /usr/lib/perfsonar/logstash/ruby +COPY ./scripts/ /usr/lib/perfsonar/logstash/scripts +COPY ./java/ /usr/lib/perfsonar/logstash/java + +#Install caching library +USER root +RUN apt-get update -y && apt-get install -y maven python3 python3-yaml +RUN cd /usr/lib/perfsonar/logstash/java &&\ + mkdir maven &&\ + mvn -Dmaven.repo.local=/usr/lib/perfsonar/logstash/java/maven dependency:resolve &&\ + chown -R logstash:root /usr/lib/perfsonar/logstash/java/maven + +#make config look like non-docker so scripts work +RUN mkdir /etc/logstash &&\ + ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml +USER logstash + +#Generate pipelines.yml +RUN python3 /usr/lib/perfsonar/logstash/scripts/enable_prometheus_pipeline.py diff --git a/perfsonar-logstash/perfsonar-logstash/java/pom.xml b/perfsonar-logstash/perfsonar-logstash/java/pom.xml new file mode 100644 index 0000000..63f1aa8 --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/java/pom.xml @@ -0,0 +1,17 @@ + + 4.0.0 + net.perfsonar + prometheus-pipeline + 1.0 + jar + prometheus-pipeline + https://perfsonar.net + + + + com.github.ben-manes.caffeine + caffeine + 3.1.1 + + + \ No newline at end of file diff --git a/perfsonar-logstash/perfsonar-logstash/pipeline_etc/logstash_sysconfig b/perfsonar-logstash/perfsonar-logstash/pipeline_etc/logstash_sysconfig index 442f811..15ff0fa 100644 --- a/perfsonar-logstash/perfsonar-logstash/pipeline_etc/logstash_sysconfig +++ b/perfsonar-logstash/perfsonar-logstash/pipeline_etc/logstash_sysconfig @@ -1,6 +1,5 @@ ## Logstash environment variables. log_level=info -opensearch_output_host=https://localhost:9200 -opensearch_output_user=pscheduler_logstash -opensearch_output_password=pscheduler_logstash XPACK_MONITORING_ENABLED=False +RATE_CACHE_MAX_ENTRIES=10000000 +RATE_CACHE_EXPIRES=600 diff --git a/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/02-formatting.conf b/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/02-formatting.conf new file mode 100644 index 0000000..f1e19df --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/02-formatting.conf @@ -0,0 +1,23 @@ +filter { + # Add meta id + mutate { + rename => { + "[http_poller_metadata][name]" => "[meta][id]" + } + } + + # Remove extra http_poller_metadata + mutate { + remove_field => ["http_poller_metadata"] + } + + # Parse and split metrics + ruby { + path => '/usr/lib/perfsonar/logstash/ruby/prometheus_parse.rb' + } + + # Remove leftover message field + mutate { + remove_field => ["message"] + } +} \ No newline at end of file diff --git a/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/99-outputs.conf b/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/99-outputs.conf new file mode 100644 index 0000000..99232f2 --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/prometheus_pipeline/99-outputs.conf @@ -0,0 +1,13 @@ +output { + #If has [type] create index using that in name. + if [type] { + opensearch { + hosts => ["${opensearch_output_host}"] + ssl_certificate_verification => false + user => "${opensearch_output_user}" + password => "${opensearch_output_password}" + action => "create" + index => "prometheus_%{[type]}" + } + } +} diff --git a/perfsonar-logstash/perfsonar-logstash/ruby/lib/ratecache.rb b/perfsonar-logstash/perfsonar-logstash/ruby/lib/ratecache.rb new file mode 100644 index 0000000..f1d5c8d --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/ruby/lib/ratecache.rb @@ -0,0 +1,33 @@ +#Require every jar from the maven directory +folder = '/usr/lib/perfsonar/logstash/java/maven/' +Dir["#{folder}/**/*.jar"].each { |jar| require jar } + +#import classes we want to use +java_import 'com.github.benmanes.caffeine.cache.LoadingCache' +java_import 'java.time.Duration' + +class RateCache + #Instantiate cache + @@cache = Java::ComGithubBenmanesCaffeineCache::Caffeine.newBuilder().maximumSize(ENV['RATE_CACHE_MAX_ENTRIES'].to_i).expireAfterWrite(Duration.ofSeconds(ENV['RATE_CACHE_EXPIRES'].to_i)).build() + + def self.get(k) + return @@cache.getIfPresent(k) + end + + def self.put(k, v) + @@cache.put(k, v) + end + + def self.delete(k) + @@cache.invalidate(k) + end + + def self.stats() + return @@cache.stats() + end + + def self.size() + return @@cache.estimatedSize() + end + +end \ No newline at end of file diff --git a/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb new file mode 100644 index 0000000..41364a0 --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb @@ -0,0 +1,119 @@ +require '/usr/lib/perfsonar/logstash/ruby/lib/ratecache.rb' + +# def register(params) +# #Require one of these - preference given to values if given +# @values = params["values"] +# end + +def _parse_labels(label_str) + label_obj = {} + if not label_str.nil? then + label_str.split(",").each do |label_pair| + label_pair_arr = label_pair.split("=", 2) + next if label_pair_arr.length() != 2 + label_obj[label_pair_arr[0]] = label_pair_arr[1].delete_prefix('"').delete_suffix('"') + end + end + label_key = label_obj.sort.to_s + return label_key,label_obj +end + +def _filter(event) + prom_data = event.get("[message]") + + #### + # The following will need to be configured externally + meta_metrics = { + "node_uname_info" => "uname", + "node_dmi_info "=> "dmi", + "node_os_info" => "os" + } + skip_list = [ "^node_scrape_*", "^go_*" ] + #### + + formatted_records = {} + formatted_metadata = {} + data_type_map = {} + meta_id = event.get("[meta][id]") + record_type = event.get("[type]") + rate_key_prefix = "#{meta_id}::#{record_type}" + curr_time = event.get("@timestamp").to_i + prom_data.each_line do |prom_line| + if type_matches = prom_line.match(/^\#\s*TYPE\s+([a-zA-Z_:][a-zA-Z0-9_:]*)\s+(\w+)\s*$/) then + #we don't actually use this, but seems useful when support for histograms gets added + data_type_map[type_matches[1]] = type_matches[2] + elsif type_matches = prom_line.start_with?("#") then + next + elsif metric_matches = prom_line.match(/^([a-zA-Z_:][a-zA-Z0-9_:]*)({(([a-zA-Z_][a-zA-Z0-9_]*=.*),?)*})?\s+(.+)$/) then + metric_name = metric_matches[1] + skip_metric = false + skip_list.each do |sl| + if metric_name.match(Regexp.new(sl)) then + skip_metric = true + break + end + end + next if skip_metric + metric_labels = metric_matches[3] + metric_value = { "val" => metric_matches[5].to_f } + label_key, label_obj = _parse_labels(metric_labels) + + #Calculate the rate - currently bucket size not normalized + rate_key = "#{rate_key_prefix}::#{metric_name}::#{label_key}" + prev_rate = RateCache.get(rate_key) + if prev_rate && prev_rate.key?("ts") && prev_rate.key?("v") then + time_range = curr_time - prev_rate["ts"] + metric_value["delta"] = metric_value["val"] - prev_rate["v"] + metric_value["rate"] = metric_value["delta"]/time_range if time_range > 0 + end + RateCache.put(rate_key, {"ts" => curr_time, "v" => metric_value["val"]}) + #Check if we want to use this as metadata or just normal metrics + if meta_metrics.has_key?(metric_name) then + formatted_metadata[meta_metrics[metric_name]] = label_obj + else + if not formatted_records.has_key?(label_key) then + formatted_records[label_key] = { + "meta" => { "id" => meta_id, "labels" => label_obj }, + "values" => {} + } + end + formatted_records[label_key]["values"][metric_name] = metric_value + end + end + end + + #Add metadata to records and format as logstash events + event.set("[message]", nil) + events = [] + formatted_records.each do |fr_k, fr_v| + e = event.clone() + + formatted_metadata.each do |md_k, md_v| + fr_v["meta"][md_k] = md_v + end + e.set("meta", fr_v["meta"]) + e.set("values", fr_v["values"] ) + events.append(e) + end + + return events +end + +## +# Handles exception in call to main _filter function. +# May want to get rid of this if causes performance issues, +# but provides catch-all so exception does not kill logstash +# and provides more helpful output when exceptions do occur. +def filter(event) + #catch exception + begin + result = _filter(event) + return result + rescue => exception + logger.error("Caught ruby exception in prometheus_parse.rb #{exception.message}") + event.cancel() + end + + return [event] +end + diff --git a/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py b/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py new file mode 100644 index 0000000..de6b8be --- /dev/null +++ b/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import yaml + +#initialize +pipelines_yml = [] + +#open the file +with open('/etc/logstash/pipelines.yml') as file: + pipelines_yml = yaml.safe_load(file) + pipeline_exists = False + +#look for pscheduler pipeline +for pipeline in pipelines_yml: + if pipeline is not None and pipeline.get("pipeline.id", None) == "prometheus": + pipeline_exists = True + break + +#if not exists, then add it and output to file +if not pipeline_exists: + pipelines_yml.append({ + "pipeline.id": "prometheus", + "path.config": "/usr/lib/perfsonar/logstash/prometheus_pipeline/*.conf" + }) + with open('/etc/logstash/pipelines.yml', 'w') as file: + file.write(yaml.dump(pipelines_yml, default_flow_style=False)) From a63ebb148c0eac3e9a3a85c4d9cd5728937e7313 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 21:04:10 -0400 Subject: [PATCH 03/13] Adding docker image build --- .github/workflows/publish_docker.yml | 74 ++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/publish_docker.yml diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml new file mode 100644 index 0000000..bafe501 --- /dev/null +++ b/.github/workflows/publish_docker.yml @@ -0,0 +1,74 @@ +name: Logstash Docker Images + +# This action builds Docker images of logstash then +# pushes result to GitHub Container Registry. + +on: + push: + branches: [ '*' ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ 'main' ] + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_BASE: ${{ github.repository }} + BUILD_DIR: perfsonar-logstash/perfsonar-logstash + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker images + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: ${{ env.BUILD_DIR }} + file: ${{ env.BUILD_DIR }}/docker/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From c96c86034b0faa09373ba4ece4e833b4b3f958af Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 21:09:10 -0400 Subject: [PATCH 04/13] Adding docker image build for prometheus pipeline --- .../workflows/publish_docker_prometheus.yml | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/publish_docker_prometheus.yml diff --git a/.github/workflows/publish_docker_prometheus.yml b/.github/workflows/publish_docker_prometheus.yml new file mode 100644 index 0000000..952ced8 --- /dev/null +++ b/.github/workflows/publish_docker_prometheus.yml @@ -0,0 +1,75 @@ +name: Logstash Docker Images + +# This action builds Docker images of logstash then +# pushes result to GitHub Container Registry. + +on: + push: + branches: [ '*' ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ 'main' ] + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_BASE: ${{ github.repository }} + BUILD_DIR: perfsonar-logstash/perfsonar-logstash + IMAGE_SUBNAME: prometheus + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE }}-${{ env.IMAGE_SUBNAME }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker images + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: ${{ env.BUILD_DIR }} + file: ${{ env.BUILD_DIR }}/docker/Dockerfile-${{ env.IMAGE_SUBNAME }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From fa48f88c7bd3de6ee00bd5e62a0cf8f818892dfc Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 21:12:20 -0400 Subject: [PATCH 05/13] Cleanup for action names --- .github/workflows/publish_docker.yml | 2 +- .github/workflows/publish_docker_prometheus.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index bafe501..adad576 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -1,4 +1,4 @@ -name: Logstash Docker Images +name: perfSONAR Logstash Image # This action builds Docker images of logstash then # pushes result to GitHub Container Registry. diff --git a/.github/workflows/publish_docker_prometheus.yml b/.github/workflows/publish_docker_prometheus.yml index 952ced8..80f750e 100644 --- a/.github/workflows/publish_docker_prometheus.yml +++ b/.github/workflows/publish_docker_prometheus.yml @@ -1,4 +1,4 @@ -name: Logstash Docker Images +name: Prometheus Logstash Image # This action builds Docker images of logstash then # pushes result to GitHub Container Registry. From e45cc4eb5fa997b25ed3ae913ee32ba53f4051c1 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Sun, 27 Aug 2023 21:25:42 -0400 Subject: [PATCH 06/13] Packaging for prometheus pipeline --- perfsonar-logstash/perfsonar-logstash/Makefile | 5 +++++ .../perfsonar-logstash/unibuild-packaging/deb/control | 2 +- .../unibuild-packaging/deb/perfsonar-logstash.install | 2 ++ .../perfsonar-logstash/unibuild-packaging/deb/rules | 5 +++-- .../unibuild-packaging/rpm/perfsonar-logstash.spec | 8 +++++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/perfsonar-logstash/perfsonar-logstash/Makefile b/perfsonar-logstash/perfsonar-logstash/Makefile index 5327a61..b13b25e 100644 --- a/perfsonar-logstash/perfsonar-logstash/Makefile +++ b/perfsonar-logstash/perfsonar-logstash/Makefile @@ -18,12 +18,17 @@ ifndef SYSTEMDPATH @false endif mkdir -p ${ROOTPATH}/pipeline + mkdir -p ${ROOTPATH}/prometheus_pipeline mkdir -p ${ROOTPATH}/ruby + mkdir -p ${ROOTPATH}/java/maven mkdir -p ${ROOTPATH}/scripts mkdir -p ${CONFIGPATH} mkdir -p ${SYSTEMDPATH}/logstash.service.d cp -r pipeline/* ${ROOTPATH}/pipeline + cp -r prometheus_pipeline/* ${ROOTPATH}/prometheus_pipeline cp -r ruby/* ${ROOTPATH}/ruby + cp -r java/* ${ROOTPATH}/java cp -r scripts/* ${ROOTPATH}/scripts cp -r pipeline_etc/* ${CONFIGPATH} cp -r systemd/* ${SYSTEMDPATH}/logstash.service.d/ + mvn -Dmaven.repo.local=${ROOTPATH}/java/maven -f ${ROOTPATH}/java/pom.xml dependency:resolve diff --git a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/control b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/control index 5ea2388..76c5b96 100644 --- a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/control +++ b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/control @@ -3,7 +3,7 @@ Maintainer: perfSONAR developers Section: net Priority: optional Standards-Version: 4.3.0 -Build-Depends: debhelper (>= 10) +Build-Depends: debhelper (>= 10), maven Homepage: http://www.perfsonar.net Package: perfsonar-logstash diff --git a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/perfsonar-logstash.install b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/perfsonar-logstash.install index cf5ea56..ba5edbf 100644 --- a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/perfsonar-logstash.install +++ b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/perfsonar-logstash.install @@ -1,4 +1,6 @@ pipeline/* /usr/lib/perfsonar/logstash/pipeline/ +prometheus_pipeline/* /usr/lib/perfsonar/logstash/prometheus_pipeline/ scripts/* /usr/lib/perfsonar/logstash/scripts/ ruby/* /usr/lib/perfsonar/logstash/ruby/ +java/* /usr/lib/perfsonar/logstash/java/ pipeline_etc/logstash_sysconfig /etc/perfsonar/logstash/ diff --git a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/rules b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/rules index 4bd2a4f..fb8795e 100644 --- a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/rules +++ b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/deb/rules @@ -9,5 +9,6 @@ override_dh_auto_build: override_dh_auto_install: - - + dh_install + mkdir -p $(CURDIR)/debian/perfsonar-logstash/usr/lib/perfsonar/logstash/java/maven + mvn -Dmaven.repo.local=$(CURDIR)/debian/perfsonar-logstash/usr/lib/perfsonar/logstash/java/maven -f $(CURDIR)/debian/perfsonar-logstash/usr/lib/perfsonar/logstash/java/pom.xml dependency:resolve \ No newline at end of file diff --git a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/rpm/perfsonar-logstash.spec b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/rpm/perfsonar-logstash.spec index 4e9ba10..85c934b 100644 --- a/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/rpm/perfsonar-logstash.spec +++ b/perfsonar-logstash/perfsonar-logstash/unibuild-packaging/rpm/perfsonar-logstash.spec @@ -1,7 +1,9 @@ %define install_base /usr/lib/perfsonar %define logstash_base %{install_base}/logstash %define ruby_base %{logstash_base}/ruby +%define java_base %{logstash_base}/java %define pipeline_base %{logstash_base}/pipeline +%define prometheus_base %{logstash_base}/prometheus_pipeline %define scripts_base %{logstash_base}/scripts %define config_base /etc/perfsonar/logstash @@ -23,6 +25,7 @@ Requires: logstash-oss Requires: perfsonar-common Requires: perfsonar-logstash-output-plugin Requires(post): python3 +BuildRequires: maven %if 0%{?el7} Requires(post): python36-PyYAML %else @@ -69,6 +72,7 @@ fi %license LICENSE %config(noreplace) %{pipeline_base}/01-inputs.conf %config(noreplace) %{pipeline_base}/99-outputs.conf +%config(noreplace) %{prometheus_base}/99-outputs.conf %config(noreplace) %{config_base}/logstash_sysconfig %config(noreplace) %{_sysconfdir}/systemd/system/logstash.service.d/* %attr(0755, perfsonar, perfsonar) %{scripts_base}/* @@ -77,7 +81,9 @@ fi #Add below when the day comes we have something that doesn't start with 0 or 9 #%{pipeline_base}/[1-8][0-9]-*.conf #{pipeline_base}/9[0-8]-*.conf -%{ruby_base}/*.rb +%{prometheus_base}/02-formatting.conf +%{ruby_base}/* +%{java_base}/* %changelog * Sun Mar 21 2021 andy@es.net 4.4.0-0.0.a1 From c29ebc9fdfeccd9aeff88036bd19c1ac180c03ce Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Mon, 28 Aug 2023 07:41:34 -0400 Subject: [PATCH 07/13] Updating README --- README.md | 73 +++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 5e8cb59..d8f0ab9 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,34 @@ # perfSONAR Logstash Pipeline +This contains a set of Logstash pipelines used by the perfSONAR project to process data and store in OpenSearch. It contains the following pipelines: -**UNDER CONSTRUCTION: NOT FOR PRODUCTION USE** - -## Build and use Docker image for development - -``` -make local # only need to do this first time to copy config in place -make #run this everytime you make change to pipeline -docker-compose up -d -``` - -## RPMS for centos7 -``` -#Build rpm and test install in a container -make centos7 - -##verify RPM install -docker-compose -f docker-compose.qa.yml up -d centos7 -docker-compose -f docker-compose.qa.yml exec centos7 bash -tail -f /var/log/logstash/logstash-plain.log -``` - -## Debian package -``` -#Build logstash .deb -make release -make build -git clone https://github.com/perfsonar/debian-docker-buildmachines.git ../debian-docker-buildmachines -pwd=$(pwd) -cd ../debian-docker-buildmachines -./build-in-docker logstash -cd $pwd -mkdir -p artifacts/debian/ -cp ../build_results/*deb artifacts/debian/ - - -#Test it with perfsonar-testpoint -docker-compose -f docker-compose.debian.yml build -docker-compose -f docker-compose.debian.yml up -d -docker-compose -f docker-compose.debian.yml exec debian bash -apt install perfsonar-testpoint -pscheduler task --archive '{"archiver":"http","data":{"schema":2,"_url":"http://localhost:11283","op":"put","_headers":{"content-type":"application/json"}}}' rtt --dest localhost -``` +- **perfSONAR Pipeline**: This pipeline runs an HTTP listener that accepts measurement results being archived by pscheduler. + +- **Prometheus Pipeline**: This is a pipeline for reading generic Prometheus Metrics and storing them in OpenSearch. + +# Docker Images + +Each pipeline is built as a separate docker image. Both having a number of customization options: + +## Docker Environment Variables + +- **opensearch_output_host** - The hostname of the OpenSearch instance +- **opensearch_output_user** - The username used to authenticate to OpenSearch +- **opensearch_output_password** - The password used to authenticate to OpenSearch + + +## Docker Volumes + +### perfSONAR +- **/usr/lib/perfsonar/logstash/pipeline/99-outputs.conf** - Override this to define a custom output filter if you do not want to use the default output. + +### Prometheus +- **/usr/lib/perfsonar/logstash/prometheus_pipeline/.conf** - Override any file in the .conf directory. You will want to add at least one input filter likely an HTTP poller to grab the data. You can override *99-outputs.conf* if you want a custom output filter. + +# OS Packages + +A full compliment of OS packages can be built using [unibuild](https://github.com/perfsonar/unibuild). They consist of the following: + +- **perfsonar-logstash** - This package contains both pipelines but only the perfSONAR pipeline is enabled by default. The Prometheus pipeline needs inputs defined which is generally handled by other perfSONAR packages (like perfsonar-archive) or auto-generated. You can enable manually with the script `/usr/lib/perfsonar/logstash/scripts/enable_prometheus_pipeline.py` + +- **perfsonar-logstash-output-plugin** - Install the OpenSearch output plugin since not included by default with logstash. From 47d8202fdfd9efc7db87bb7a97468da960e3f3f6 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Mon, 28 Aug 2023 07:42:58 -0400 Subject: [PATCH 08/13] Updating README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8f0ab9..9a08cd3 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Each pipeline is built as a separate docker image. Both having a number of custo - **/usr/lib/perfsonar/logstash/pipeline/99-outputs.conf** - Override this to define a custom output filter if you do not want to use the default output. ### Prometheus -- **/usr/lib/perfsonar/logstash/prometheus_pipeline/.conf** - Override any file in the .conf directory. You will want to add at least one input filter likely an HTTP poller to grab the data. You can override *99-outputs.conf* if you want a custom output filter. +- **/usr/lib/perfsonar/logstash/prometheus_pipeline/*.conf** - Override any file in the .conf directory. You will want to add at least one input filter likely an HTTP poller to grab the data. You can override *99-outputs.conf* if you want a custom output filter. # OS Packages From f6df5348f41f09aaddd3a34fe3b4d5b9432c3912 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Mon, 28 Aug 2023 16:45:52 -0700 Subject: [PATCH 09/13] Small update to prometheus pipeline config --- .../perfsonar-logstash/scripts/enable_prometheus_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py b/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py index de6b8be..df3ef8d 100644 --- a/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py +++ b/perfsonar-logstash/perfsonar-logstash/scripts/enable_prometheus_pipeline.py @@ -20,7 +20,8 @@ if not pipeline_exists: pipelines_yml.append({ "pipeline.id": "prometheus", - "path.config": "/usr/lib/perfsonar/logstash/prometheus_pipeline/*.conf" + "path.config": "/usr/lib/perfsonar/logstash/prometheus_pipeline/*.conf", + "pipeline.ecs_compatibility": "disabled" }) with open('/etc/logstash/pipelines.yml', 'w') as file: file.write(yaml.dump(pipelines_yml, default_flow_style=False)) From 1d4a990b7c825d79e38ffa0879f1fad35d0c4230 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Mon, 28 Aug 2023 19:50:57 -0700 Subject: [PATCH 10/13] Adding nil check to prometheus parser --- perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb index 41364a0..0478141 100644 --- a/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb +++ b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb @@ -38,6 +38,9 @@ def _filter(event) record_type = event.get("[type]") rate_key_prefix = "#{meta_id}::#{record_type}" curr_time = event.get("@timestamp").to_i + if prom_data.nil? then + return [] + end prom_data.each_line do |prom_line| if type_matches = prom_line.match(/^\#\s*TYPE\s+([a-zA-Z_:][a-zA-Z0-9_:]*)\s+(\w+)\s*$/) then #we don't actually use this, but seems useful when support for histograms gets added From 00c9edcc26e1aa09e066f85357aadd084a8c2dfb Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Mon, 28 Aug 2023 19:53:10 -0700 Subject: [PATCH 11/13] Adding previous check to better location --- .../perfsonar-logstash/ruby/prometheus_parse.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb index 0478141..670e4c4 100644 --- a/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb +++ b/perfsonar-logstash/perfsonar-logstash/ruby/prometheus_parse.rb @@ -20,6 +20,9 @@ def _parse_labels(label_str) def _filter(event) prom_data = event.get("[message]") + if prom_data.nil? then + return [] + end #### # The following will need to be configured externally @@ -38,9 +41,6 @@ def _filter(event) record_type = event.get("[type]") rate_key_prefix = "#{meta_id}::#{record_type}" curr_time = event.get("@timestamp").to_i - if prom_data.nil? then - return [] - end prom_data.each_line do |prom_line| if type_matches = prom_line.match(/^\#\s*TYPE\s+([a-zA-Z_:][a-zA-Z0-9_:]*)\s+(\w+)\s*$/) then #we don't actually use this, but seems useful when support for histograms gets added From 0a2e2e1ae93ad5d2e8c00c56d5340761e4c52052 Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Wed, 30 Aug 2023 16:45:34 -0700 Subject: [PATCH 12/13] Clearing out pipelines.yml in docker image --- perfsonar-logstash/perfsonar-logstash/docker/Dockerfile | 3 ++- .../perfsonar-logstash/docker/Dockerfile-prometheus | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile index cc5f913..91f13a0 100644 --- a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile @@ -11,7 +11,8 @@ USER root RUN apt-get update -y && apt-get install -y python3 python3-yaml #make config look like non-docker so scripts work RUN mkdir /etc/logstash &&\ - ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml + echo "" > /usr/share/logstash/config/pipelines.yml &&\ + ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml USER logstash #Generate pipelines.yml diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus index 4266086..393742c 100644 --- a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus @@ -17,6 +17,7 @@ RUN cd /usr/lib/perfsonar/logstash/java &&\ #make config look like non-docker so scripts work RUN mkdir /etc/logstash &&\ + echo "" > /usr/share/logstash/config/pipelines.yml &&\ ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml USER logstash From 9f0dcdf3947605e7837a4163ee044594a98aad8f Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Wed, 30 Aug 2023 16:50:14 -0700 Subject: [PATCH 13/13] Clearing out pipelines.yml in docker image round 2 --- perfsonar-logstash/perfsonar-logstash/docker/Dockerfile | 2 +- .../perfsonar-logstash/docker/Dockerfile-prometheus | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile index 91f13a0..e1b8521 100644 --- a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile @@ -11,7 +11,7 @@ USER root RUN apt-get update -y && apt-get install -y python3 python3-yaml #make config look like non-docker so scripts work RUN mkdir /etc/logstash &&\ - echo "" > /usr/share/logstash/config/pipelines.yml &&\ + echo "[]" > /usr/share/logstash/config/pipelines.yml &&\ ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml USER logstash diff --git a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus index 393742c..025b998 100644 --- a/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus +++ b/perfsonar-logstash/perfsonar-logstash/docker/Dockerfile-prometheus @@ -17,7 +17,7 @@ RUN cd /usr/lib/perfsonar/logstash/java &&\ #make config look like non-docker so scripts work RUN mkdir /etc/logstash &&\ - echo "" > /usr/share/logstash/config/pipelines.yml &&\ + echo "[]" > /usr/share/logstash/config/pipelines.yml &&\ ln -s /usr/share/logstash/config/pipelines.yml /etc/logstash/pipelines.yml USER logstash