Redo probes base on Jobber (cron replacement)

probes/Dockerfile

@@ -1,20 +1,44 @@
-# Copyright European Organization for Nuclear Research (CERN) 2017
+# Copyright European Organization for Nuclear Research (CERN) 2020
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 #
 # Authors:
-# - Eric Vaandering, <[email protected]>, 2019
+# - Eric Vaandering, <[email protected]>, 2020
 
-FROM rucio/rucio-daemons:latest
+FROM centos:7
 
-RUN yum install -y git \
-    && yum clean all && rm -rf /var/cache/yum
+WORKDIR /tmp
+ADD oic.rpm /tmp
 
-ADD run-probes.sh /
+RUN yum install -y epel-release.noarch && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+RUN yum upgrade -y && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+RUN yum install -y python-pip libaio gcc python-devel.x86_64 openssl-devel.x86_64 MySQL-python git && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+RUN rpm -i /tmp/oic.rpm; \
+    echo "/usr/lib/oracle/12.2/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \
+    ldconfig
+
+RUN rpm -i https://github.com/dshearer/jobber/releases/download/v1.4.0/jobber-1.4.0-1.el7.x86_64.rpm
+
+RUN pip install --upgrade pip setuptools
+RUN rm -rf /usr/lib/python2.7/site-packages/ipaddress*
+RUN pip install --pre rucio[oracle,mysql,postgresql]
+RUN pip install j2cli psycopg2-binary
+RUN pip install cx_oracle==6.3.1 PyMySQL
 
 WORKDIR /
 RUN git clone https://github.com/rucio/probes.git
 
+ADD rucio.cfg.j2 /tmp/
+
+ADD run-probes.sh /
+
 ENTRYPOINT ["/run-probes.sh"]

probes/dot-jobber

@@ -0,0 +1,59 @@
+## This is your jobfile: use it to tell Jobber what jobs you want it to
+## run on your behalf.  For details of what you can specify here,
+## please see https://dshearer.github.io/jobber/doc/.
+##
+## It consists of two sections: "prefs" and "jobs".  In "prefs" you can
+## set various general settings.  In "jobs", you define your jobs.
+
+version: 1.4
+
+prefs:
+  ## You can have the Jobber daemon keep a log of various activities
+  ## with the "logPath" setting; the log will be written to the given
+  ## path (if the path is relative, it will be interpreted relative to
+  ## your home directory).  Your user account must be able to write to
+  ## the given path.  NOTE: This is NOT where logs about job runs
+  ## are stored --- for that, see the "runLog" setting below.  WARNING:
+  ## Jobber will NOT rotate this file.
+  #logPath: jobber-log
+
+  ## You can specify how info about past runs is stored.  For
+  ## "type: memory" (the default), they are stored in memory and
+  ## are lost when the Jobber service stops.
+  #runLog:
+  #    type: memory
+  #    maxLen: 100  # the max number of entries to remember
+
+  ## For "type: file", past run logs are stored on disk.  The log file is
+  ## rotated when it reaches a size of 'maxFileLen' MB.  Up to
+  ## 'maxHistories' historical run logs (that is, not including the
+  ## current one) are kept.
+  #runLog:
+  #    type: file
+  #    path: /tmp/claudius
+  #    maxFileLen: 50m  # in MB
+  #    maxHistories: 5
+
+resultSinks:
+  #- &programSink
+  #  type: program
+  #  path: /home/handleError.sh
+
+  #- &systemEmailSink
+  #  type: system-email
+
+  #- &filesystemSink
+  #  type: filesystem
+  #  path: /path/to/dir
+  #  data: [stdout, stderr]
+  #  maxAgeDays: 10
+
+jobs:
+  ## This section must contain a YAML sequence of maps like the following:
+  #DailyBackup:
+  #    cmd: backup daily  # shell command to execute
+  #    time: '* * * * * *'  # SEC MIN HOUR MONTH_DAY MONTH WEEK_DAY.
+  #    onError: Continue  # what to do when the job has an error: Stop, Backoff, or Continue
+  #    notifyOnError: [*programSink]  # what to do with result when job has an error
+  #    notifyOnFailure: [*systemEmailSink, *programSink]  # what to do with result when the job stops due to errors
+  #    notifyOnSuccess: [*filesystemSink]  # what to do with result when the job succeeds

probes/rucio.cfg.j2

@@ -0,0 +1,121 @@
+[accounts]
+special_accounts = {{ RUCIO_CFG_ACCOUNTS_SPECIAL_ACCOUNTS | default('panda, tier0') }}
+
+[common]
+logdir = {{ RUCIO_CFG_COMMON_LOGDIR | default('/var/log/rucio') }}
+loglevel = {{ RUCIO_CFG_COMMON_LOGLEVEL | default('DEBUG') }}
+mailtemplatedir = {{ RUCIO_CFG_COMMON_MAILTEMPLATEDIR | default('/opt/rucio/etc/mail_templates') }}
+
+[database]
+default = {{ RUCIO_CFG_DATABASE_DEFAULT | default('sqlite:////tmp/rucio.db') }}
+{% if RUCIO_CFG_DATABASE_SCHEMA is defined %}
+schema = {{ RUCIO_CFG_DATABASE_SCHEMA }}
+{%- endif %}
+pool_reset_on_return = {{ RUCIO_CFG_DATABASE_POOL_RESET_ON_RETURN | default('rollback') }}
+echo = {{ RUCIO_CFG_DATABASE_ECHO | default('0') }}
+pool_recycle = {{ RUCIO_CFG_DATABASE_POOL_RECYCLE | default('600') }}
+{% if RUCIO_CFG_DATABASE_POOL_SIZE is defined %}pool_size = {{ RUCIO_CFG_DATABASE_POOL_SIZE }}{% endif %}
+{% if RUCIO_CFG_DATABASE_POOL_TIMEOUT is defined %}pool_timeout = {{ RUCIO_CFG_DATABASE_POOL_TIMEOUT }}{% endif %}
+{% if RUCIO_CFG_DATABASE_MAX_OVERFLOW is defined %}max_overflow = {{ RUCIO_CFG_DATABASE_MAX_OVERFLOW }}{% endif %}
+{% if RUCIO_CFG_DATABASE_POWUSERACCOUNT is defined %}powuseraccount = {{ RUCIO_CFG_DATABASE_POWUSERACCOUNT }}{% endif %}
+{% if RUCIO_CFG_DATABASE_POWUSERPASSWORD is defined %}powuserpassword = {{ RUCIO_CFG_DATABASE_POWUSERPASSWORD }}{% endif %}
+
+[monitor]
+carbon_server = {{ RUCIO_CFG_MONITOR_CARBON_SERVER | default('localhost') }}
+carbon_port = {{ RUCIO_CFG_MONITOR_CARBON_PORT | default('8125') }}
+user_scope = {{ RUCIO_CFG_MONITOR_USER_SCOPE | default('default_docker') }}
+
+[policy]
+permission = {{ RUCIO_CFG_POLICY_PERMISSION | default('generic') }}
+schema = {{ RUCIO_CFG_POLICY_SCHEMA | default('generic') }}
+lfn2pfn_algorithm_default = {{ RUCIO_CFG_POLICY_LFN2PFN_ALGORITHM_DEFAULT | default('hash') }}
+{% if RUCIO_CFG_POLICY_LFN2PFN_MODULE is defined %}lfn2pfn_module = {{ RUCIO_CFG_POLICY_LFN2PFN_MODULE }}{% endif %}
+
+support = {{ RUCIO_CFG_POLICY_SUPPORT | default('https://github.com/rucio/rucio/issues/') }}
+support_rucio = {{ RUCIO_CFG_POLICY_SUPPORT_RUCIO | default('https://github.com/rucio/rucio/issues/') }}
+
+[automatix]
+{% if RUCIO_CFG_AUTOMATIX_SITES is defined %}sites = {{ RUCIO_CFG_AUTOMATIX_SITES }}{% endif %}
+{% if RUCIO_CFG_AUTOMATIX_SLEEP_TIME is defined %}sleep_time = {{ RUCIO_CFG_AUTOMATIX_SLEEP_TIME }}{% endif %}
+{% if RUCIO_CFG_AUTOMATIX_DATASET_LIFETIME is defined %}dataset_lifetime = {{ RUCIO_CFG_AUTOMATIX_DATASET_LIFETIME }}{% endif %}
+{% if RUCIO_CFG_AUTOMATIX_SET_METADATA is defined %}set_metadata = {{ RUCIO_CFG_AUTOMATIX_SET_METADATA }}{% endif %}
+
+[auditor]
+results = {{ RUCIO_CFG_AUDITOR_RESULTS | default('/opt/rucio/auditor/results/') }}
+cache = {{ RUCIO_CFG_AUDITOR_CACHE | default('/opt/rucio/auditor/cache') }}
+
+[conveyor]
+scheme = {{ RUCIO_CFG_CONVEYOR_SCHEME | default('srm,gsiftp,root,http,https') }}
+transfertool = {{ RUCIO_CFG_CONVEYOR_TRANSFERTOOL | default('fts3') }}
+ftshosts = {{ RUCIO_CFG_CONVEYOR_FTSHOSTS | default('https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446') }}
+cacert = {{ RUCIO_CFG_CONVEYOR_CACERT | default('/opt/rucio/etc/web/ca.crt') }}
+usercert = {{ RUCIO_CFG_CONVEYOR_USERCERT | default('/opt/rucio/tools/x509up') }}
+{% if RUCIO_CFG_CONVEYOR_CACHE_TIME is defined %}cache_time = {{ RUCIO_CFG_CONVEYOR_CACHE_TIME }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_USE_DETERMINISTIC_ID is defined %}use_deterministic_id = {{ RUCIO_CFG_CONVEYOR_USE_DETERMINISTIC_ID }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_POLL_TIMEOUT is defined %}poll_timeout = {{ RUCIO_CFG_CONVEYOR_POLL_TIMEOUT }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_SUBMIT_TIMEOUT is defined %}submit_timeout = {{ RUCIO_CFG_CONVEYOR_SUBMIT_TIMEOUT }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_BRING_ONLINE is defined %}bring_online = {{ RUCIO_CFG_CONVEYOR_BRING_ONLINE }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_QUEUE_MODE is defined %}queue_mode = {{ RUCIO_CFG_CONVEYOR_QUEUE_MODE }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_USING_MEMCACHE is defined %}using_memcache = {{ RUCIO_CFG_CONVEYOR_USING_MEMCACHE }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_FTSMONHOSTS is defined %}ftsmonhosts = {{ RUCIO_CFG_CONVEYOR_FTSMONHOSTS }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_USER_ACTIVITIES is defined %}user_activities = {{ RUCIO_CFG_CONVEYOR_USER_ACTIVITIES }}{% endif %}
+{% if RUCIO_CFG_CONVEYOR_USER_TRANSFERS is defined %}user_transfers = {{ RUCIO_CFG_CONVEYOR_USER_TRANSFERS }}{% endif %}
+
+[messaging-fts3]
+port = {{ RUCIO_CFG_MESSAGING_FTS3_PORT | default('61123') }}
+ssl_key_file = {{ RUCIO_CFG_MESSAGING_FTS3_SSL_KEY_FILE | default('/etc/grid-security/hostkey.pem') }}
+ssl_cert_file = {{ RUCIO_CFG_MESSAGING_FTS3_SSL_CERT_FILE | default('/etc/grid-security/hostcert.pem') }}
+destination = {{ RUCIO_CFG_MESSAGING_FTS3_DESTINATION | default('/topic/transfer.fts_monitoring_queue_state') }}
+brokers = {{ RUCIO_CFG_MESSAGING_FTS3_BROKERS | default('dashb-test-mb.cern.ch') }}
+voname = {{ RUCIO_CFG_MESSAGING_FTS3_VONAME | default('atlas') }}
+
+[messaging-hermes]
+{% if RUCIO_CFG_MESSAGING_HERMES_USERNAME is defined %}username = {{ RUCIO_CFG_MESSAGING_HERMES_USERNAME }}{% endif %}
+{% if RUCIO_CFG_MESSAGING_HERMES_PASSWORD is defined %}password = {{ RUCIO_CFG_MESSAGING_HERMES_PASSWORD }}{% endif %}
+port = {{ RUCIO_CFG_MESSAGING_HERMES_PORT | default('61023') }}
+{% if RUCIO_CFG_MESSAGING_HERMES_NONSSL_PORT is defined %}nonssl_port = {{ RUCIO_CFG_MESSAGING_HERMES_NONSSL_PORT }}{% endif %}
+{% if RUCIO_CFG_MESSAGING_HERMES_USE_SSL is defined %}use_ssl = {{ RUCIO_CFG_MESSAGING_HERMES_USE_SSL }}{% endif %}
+ssl_key_file = {{ RUCIO_CFG_MESSAGING_HERMES_SSL_KEY_FILE | default('/etc/grid-security/hostkey.pem') }}
+ssl_cert_file = {{ RUCIO_CFG_MESSAGING_HERMES_SSL_CERT_FILE | default('/etc/grid-security/hostcert.pem') }}
+destination = {{ RUCIO_CFG_MESSAGING_HERMES_DESTINATION | default('/topic/rucio.events') }}
+brokers = {{ RUCIO_CFG_MESSAGING_HERMES_BROKERS | default('atlas-test-mb.cern.ch') }}
+voname = {{ RUCIO_CFG_MESSAGING_HERMES_VONAME | default('atlas') }}
+email_from = {{ RUCIO_CFG_MESSAGING_HERMES_EMAIL_FROM | default('Rucio <[email protected]>') }}
+{% if RUCIO_CFG_MESSAGING_HERMES_EMAIL_TEST is defined %}email_test = {{ RUCIO_CFG_MESSAGING_HERMES_EMAIL_TEST }}{% endif %}
+
+[tracer-kronos]
+brokers= {{ RUCIO_CFG_TRACER_KRONOS_BROKERS | default('atlas-test-mb.cern.ch') }}
+port= {{ RUCIO_CFG_TRACER_KRONOS_PORT | default('61013') }}
+ssl_key_file = {{ RUCIO_CFG_TRACER_SSL_KEY_FILE | default('/etc/grid-security/hostkey.pem') }}
+ssl_cert_file = {{ RUCIO_CFG_TRACER_SSL_CERT_FILE | default('/etc/grid-security/hostcert.pem') }}
+queue = {{ RUCIO_CFG_TRACER_QUEUE | default('/queue/Consumer.kronos.rucio.tracer') }}
+prefetch_size = {{ RUCIO_CFG_TRACER_PREFETCH_SIZE | default('10') }}
+chunksize = {{ RUCIO_CFG_TRACER_CHUNKSIZE | default('10') }}
+subscription_id = {{ RUCIO_CFG_TRACER_SUBSCRIPTION_ID | default('rucio-tracer-listener') }}
+use_ssl = {{ RUCIO_CFG_TRACER_USE_SSL | default('False') }}
+reconnect_attempts = {{ RUCIO_CFG_TRACER_RECONNECT_ATTEMPTS | default('100') }}
+excluded_usrdns = {{ RUCIO_CFG_TRACER_EXCLUDED_USRDNS | default('') }}
+{% if RUCIO_CFG_TRACER_KRONOS_USERNAME is defined %}username = {{ RUCIO_CFG_TRACER_KRONOS_USERNAME }}{% endif %}
+{% if RUCIO_CFG_TRACER_KRONOS_PASSWORD is defined %}password = {{ RUCIO_CFG_TRACER_KRONOS_PASSWORD }}{% endif %}
+dataset_wait = {{ RUCIO_CFG_TRACER_DATASET_WAIT | default('60') }}
+
+[transmogrifier]
+maxdids = {{ RUCIO_CFG_TRANSMOGRIFIER_MAXDIDS | default('100000') }}
+
+[messaging-cache]
+port = {{ RUCIO_CFG_MESSAGING_CACHE_PORT | default('61123') }}
+ssl_key_file = {{ RUCIO_CFG_MESSAGING_CACHE_SSL_KEY_FILE | default('/etc/grid-security/hostkey.pem') }}
+ssl_cert_file = {{ RUCIO_CFG_MESSAGING_CACHE_SSL_CERT_FILE | default('/etc/grid-security/hostcert.pem') }}
+destination = {{ RUCIO_CFG_MESSAGING_CACHE_DESTINATION | default('/topic/rucio.cache') }}
+brokers = {{ RUCIO_CFG_MESSAGING_CACHE_BROKERS | default('dashb-test-mb.cern.ch') }}
+voname = {{ RUCIO_CFG_MESSAGING_CACHE_VONAME | default('atlas') }}
+account = {{ RUCIO_CFG_MESSAGING_CACHE_ACCOUNT | default('ddm') }}
+
+[credentials]
+gcs = {{ RUCIO_CFG_CREDENTIALS_GCS | default('/opt/rucio/etc/google-cloud-storage-test.json') }}
+signature_lifetime = {{ RUCIO_CFG_CREDENTIALS_SIGNATURE_LIFETIME | default('3600') }}
+
+[nagios]
+proxy = {{ RUCIO_CFG_NAGIOS_PROXY | default('/opt/proxy/x509up') }}
+fts_vo = {{ RUCIO_CFG_NAGIOS_FTS_VO | default('atlas') }}
+update_distances = {{ RUCIO_CFG_NAGIOS_UPDATE_DISTANCES | default('True') }}

probes/run-probes.sh

@@ -1,5 +1,7 @@
 #! /bin/bash
 
+mkdir -p /opt/rucio/etc/
+
 if [ -f /opt/rucio/etc/rucio.cfg ]; then
     echo "rucio.cfg already mounted."
 else
@@ -13,19 +15,14 @@ if [ ! -z "$RUCIO_PRINT_CFG" ]; then
     echo ""
 fi
 
-cd /probes/common
+cp /etc/jobber-config/dot-jobber.yaml /root/.jobber
 
-echo "Will run probes:"
-echo $PROBES
+echo "Starting Jobber"
+/usr/local/libexec/jobbermaster &
 
-# Accept probe names space separated and run each in turn
-IFS=' ' read -r -a probes <<< $PROBES
+sleep 5
 
-for probe in "${probes[@]}"
-do
-    echo
-    echo $probe
-    ./${probe}
-done
+echo
+echo "============= Jobber log file ============="
 
-sleep 60
+tail -f /var/log/jobber-runs