Added logserver and fixed Glidein log token and URL propagation

Added php and python example for a server receiving Glidein logs

Fixed JWT generation and GLIDEIN_LOG_RECIPIENTS_FACTORY param for Factory logserver, improved also code and docstrings, and added documentation for logging and log server

Refactored the logging utility to have all public functions starting with 'glog_'
Added an example custom script, logging_test.sh, using Glidein logging

Fixed mod_ssl dependency, fixed ReleaseManagerLib bugs for RPM building, and replaced the deprecated optparse with argparse

Added the glideinwms-logserver RPM and adjusted ReleaseManagerLib for it

Thanks to @namrathaurs for the review and the suggestion to remove all the example secrets

TODO: changes to url_dirs.desc require factory upgrade. Should a reconfig be sufficient?

.reuse/dep5

@@ -13,6 +13,10 @@ Files: .codecov.yml .coveragerc .editorconfig .gitattributes .gitignore .gitmodu
 Copyright: 2009 Fermi Research Alliance, LLC
 License: Apache-2.0
 
+Files: logserver/logging_config.json logserver/composer.json
+Copyright: 2009 Fermi Research Alliance, LLC
+License: Apache-2.0
+
 Files: .github/ISSUE_TEMPLATE/* bigfiles/* etc/* config/* creation/templates/*.service creation/templates/*.cron creation/templates/*.timer doc/tags*txt lib/logging.conf test/bats/fixtures/* unittests/*.fixture unittests/fixtures/*
 Copyright: 2009 Fermi Research Alliance, LLC
 License: Apache-2.0

CHANGELOG.md

@@ -3,14 +3,13 @@ SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC
 SPDX-License-Identifier: Apache-2.0
 -->
 
-## Changes Since Last Release OR vX.Y.Z \[yyyy-mm-dd\]
+## v3.10.9 \[2024-12-23\]
 
-Changes since the last release
+Fixed the Glidein logging and added an sample log server
 
 ### New features / functionalities
 
--   item one of the list
--   item N
+-   Added custom log server example (glideinwms-logging) (Issue #398, PR #467)
 
 ### Changed defaults / behaviours
 
@@ -24,6 +23,7 @@ Changes since the last release
 
 -   Fixed early truncation in log files configuration and inconsistent documentation (Issue #464, PR #462, PR #463)
 -   Removed confusing tac broken pipe messages from the Glidein stderr (PR #465)
+-   Fixed JWT logging credentials not transferred to the Glidein. This includes removal of DictFile.append() and use of add_environment() for JWT tokens (Issue #398, PR #467)
 -   Fixed quotes in glidein command line unpacking and replaced deprecated add_config_line commands (PR #468)
 
 ### Testing / Development

build/ReleaseManager/ReleaseManagerLib.py

@@ -38,11 +38,11 @@ def __init__(self, ver, srcDir, relDir, rc, rpmRel):
             self.rpmVersion = self.versionToRPMVersion(ver)
             self.rpmbuildDir = os.path.join(self.releaseDir, "rpmbuild")
             self.rpmOSVersion = self.getElVersion()
+            # SRPM name has "el" also for AlmaLinux
             self.srpmFile = os.path.join(
                 self.rpmbuildDir,
                 "SRPMS",
-                "glideinwms-%s-%s.%s%s.src.rpm"
-                % (self.rpmVersion, self.rpmRelease, self.rpmOSVersion[0], self.rpmOSVersion[1]),
+                f"glideinwms-{self.rpmVersion}-{self.rpmRelease}.{self.rpmOSVersion[0]}{self.rpmOSVersion[1]}.src.rpm",
             )
             self.buildRPMs = bool(which("rpmbuild"))
             if not self.buildRPMs:
@@ -81,22 +81,34 @@ def getElVersion(self):
             # Deprecated - distname, version, id = distro.linux_distribution()
             distname = distro.name()  # If full_distribution_name is false, the result of distro.id()
             version = distro.version()
-            id = distro.codename()
+            dist_id = distro.codename()
         else:
             # TODO: remove the else branch once Py3.6 is no more supported
-            distname, version, id = platform.linux_distribution()  # pylint: disable=no-member
-        distmap = {"Fedora": "fc", "Scientific Linux": "el", "Red Hat": "el", "CentOS Stream": "el"}
+            distname, version, dist_id = platform.linux_distribution()  # pylint: disable=no-member
+        # Check if mock profiles changed
+        # As of Dec 2024 on AlmaLinux9: alma+epel-..., rhel+epel-..., centos-stream+epel-...
+        # No profile has epel-... (maybe rhel-7 for sl7)
+        distmap = {
+            "Fedora": ("fc", "fedora"),
+            "Scientific Linux": ("el", "epel"),
+            "Red Hat": ("el", "rhel+epel"),
+            "CentOS Stream": ("el", "centos-stream+epel"),
+            "AlmaLinux": ("el", "alma+epel"),
+            "RockyLinux": ("el", "rocky+epel"),
+        }
         dist = None
+        el_profile = None
         for d in distmap:
             if distname.startswith(d):
-                dist = distmap[d]
+                dist = distmap[d][0]
+                el_profile = distmap[d][1]
                 break
         if dist is None:
             raise Exception("Unsupported distribution: %s" % distname)
         else:
             el_string = dist
         major_version = version.split(".")[0]
-        return (el_string, major_version)
+        return el_string, major_version, el_profile
 
     def addTask(self, task):
         self.tasks.append(task)
@@ -277,9 +289,10 @@ def checksumRelease(self, chksumFile, exclude):
 
 
 class TaskRPM(TaskTar):
-    def __init__(self, rel, python_version, use_mock=True):
+    def __init__(self, rel, python_version, use_mock=True, verbose=False):
         TaskTar.__init__(self, rel)
         self.name = "GlideinwmsRPM"
+        self.verbose = verbose
         self.use_mock = use_mock
         self.python_version = python_version
         self.releaseFile = os.path.join(self.release.releaseDir, self.releaseFilename)
@@ -288,6 +301,7 @@ def __init__(self, rel, python_version, use_mock=True):
         self.specFile = os.path.join(self.release.rpmbuildDir, "SPECS", "glideinwms.spec")
         # self.rpmmacrosFile = os.path.join(os.path.expanduser('~'),
         self.rpmmacrosFile = os.path.join(os.path.dirname(self.release.rpmbuildDir), ".rpmmacros")
+        # Files in build/pkg/rpm to copy in the SOURCES directory
         self.sourceFilenames = [
             "chksum.sh",
             "factory_startup",
@@ -300,6 +314,7 @@ def __init__(self, rel, python_version, use_mock=True):
             "gwms-factory.sysconfig",
             "gwms-frontend.conf.httpd",
             "gwms-frontend.sysconfig",
+            "gwms-logserver.conf.httpd",
         ]
         self.rpmMacros = {
             "_topdir": self.release.rpmbuildDir,
@@ -318,6 +333,8 @@ def createRPMBuildDirs(self):
         rpm_dirs = ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]
         for dirname in rpm_dirs:
             create_dir(os.path.join(self.release.rpmbuildDir, dirname))
+        if self.verbose:
+            print(f"RPM build directories created in {self.release.rpmbuildDir}")
 
     def createSpecFile(self):
         # No error checking because we want to fail in case of errors
@@ -351,33 +368,50 @@ def buildSRPM(self):
         cmd = "rpmbuild -bs %s" % self.specFile
         for m in self.rpmMacros:
             cmd = f'{cmd} --define "{m} {self.rpmMacros[m]}"'
+        if self.verbose:
+            print(f"Building source RPM: {cmd}")
         execute_cmd(cmd)
 
     def buildRPM(self):
-        cmd = "mock -r epel-{}-x86_64 --macro-file={} -i {}".format(
+        # Check if mock profiles changed
+        # As of Dec 2024 on AlmaLinux9: alma+epel-..., rhel+epel-..., centos-stream+epel-...
+        # No profile has epel-... (maybe rhel-7 for sl7)
+        cmd = "mock -r {}-{}-x86_64 --macro-file={} -i {}".format(
+            self.release.rpmOSVersion[2],
             self.release.rpmOSVersion[1],
             self.rpmmacrosFile,
             self.python_version,
         )
+        if self.verbose:
+            print(f"Build mock environment: {cmd}")
         execute_cmd(cmd)
-        cmd = "mock --no-clean -r epel-{}-x86_64 --macro-file={} --resultdir={}/RPMS rebuild {}".format(
+        cmd = "mock --no-clean -r {}-{}-x86_64 --macro-file={} --resultdir={}/RPMS rebuild {}".format(
+            self.release.rpmOSVersion[2],
             self.release.rpmOSVersion[1],
             self.rpmmacrosFile,
             self.release.rpmbuildDir,
             self.release.srpmFile,
         )
+        if self.verbose:
+            print(f"Build RPM with mock: {cmd}")
         execute_cmd(cmd)
 
     def buildRPMWithRPMBuild(self):
         cmd = "rpmbuild -bb %s" % self.specFile
         for m in self.rpmMacros:
             cmd = f'{cmd} --define "{m} {self.rpmMacros[m]}"'
+        if self.verbose:
+            print(f"Build RPM without mock: {cmd}")
         execute_cmd(cmd)
 
     def execute(self):
         if not self.release.buildRPMs:
             self.status = "SKIPPED"
         else:
+            if self.verbose:
+                print(
+                    f"Building RPM (version:{self.release.rpmVersion}, release:{self.release.rpmRelease}, use_mock:{self.use_mock})"
+                )
             # First build the source tarball
             # TaskTar.execute(self)
 

build/ReleaseManager/release.py

@@ -6,7 +6,8 @@
 
 import sys
 import os
-import optparse
+import argparse
+from argparse import RawDescriptionHelpFormatter
 
 # Necessary to allow relative import when started as executable
 if __name__ == "__main__" and __package__ is None:
@@ -42,7 +43,7 @@ def manager_version():
 
 
 def usage():
-    help = [
+    help_str = [
         "%s <version> <SourceDir> <ReleaseDir>" % os.path.basename(sys.argv[0]),
         "NOTE that this script works on the files in your current directory tree",
         "- no git operations like clone/checkout are performed",
@@ -64,113 +65,92 @@ def usage():
         "release.py --release-version=3_2_11 --source-dir=/home/parag/glideinwms --release-dir=/var/tmp/release --rpm-release=3",
         "",
     ]
-    return "\n".join(help)
+    return "\n".join(help_str)
 
 
 def parse_opts(argv):
-    parser = optparse.OptionParser(usage=usage(), version=manager_version(), conflict_handler="resolve")
-    parser.add_option(
+    parser = argparse.ArgumentParser(
+        prog=os.path.basename(sys.argv[0]),
+        description=usage(),
+        conflict_handler="resolve",
+        formatter_class=RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
         "--release-version",
-        dest="relVersion",
         action="store",
+        required=True,
         metavar="<release version>",
         help="GlideinWMS version to release (format w/ underscores, for tarball, RPM version derived from it)",
     )
-    parser.add_option(
+    parser.add_argument(
         "--source-dir",
-        dest="srcDir",
         action="store",
+        required=True,
         metavar="<source directory>",
-        help="directory containing the glideinwms source code",
+        help="directory containing the GlideinWMS source code",
     )
-    parser.add_option(
+    parser.add_argument(
         "--release-dir",
-        dest="relDir",
         default="/tmp/release",
         action="store",
         metavar="<release directory>",
         help="directory to store release tarballs and webpages",
     )
-    parser.add_option(
-        "--rc", dest="rc", default=None, action="store", metavar="<Release Candidate Number>", help="Release Candidate"
+    parser.add_argument(
+        "--rc", default=None, action="store", metavar="<Release Candidate Number>", help="Release Candidate"
     )
-    parser.add_option(
+    parser.add_argument(
         "--rpm-release",
-        dest="rpmRel",
         default=None,
         action="store",
         metavar="<RPM Release Number>",
         help="RPM Release Number",
     )
-    parser.add_option(
+    parser.add_argument(
         "--no-mock", dest="use_mock", action="store_false", help="Set to use rpmbuild instead of mock to build the RPM"
     )
-    parser.add_option(
+    parser.add_argument(
         "--python-version",
-        dest="python_version",
         default="python36",
         action="store",
         metavar="<Python version>",
         help="Python version (default: python36)",
     )
+    parser.add_argument("--verbose", action="store_true", help="Set to see more details of the release building")
+    parser.add_argument("-v", "--version", action="version", version=manager_version())
 
-    if len(argv) == 2 and argv[1] in ["-v", "--version"]:
-        parser.print_version()
-        sys.exit()
-    if len(argv) < 4:
-        print("ERROR: Insufficient arguments specified")
-        parser.print_help()
-        sys.exit(1)
-    options, remainder = parser.parse_args(argv)
-    if len(remainder) > 1:
-        parser.print_help()
-    if not required_args_present(options):
-        print("ERROR: Missing required arguments")
-        parser.print_help()
-        sys.exit(1)
+    options = parser.parse_args()
     return options
 
 
-def required_args_present(options):
-    try:
-        if (options.relVersion is None) or (options.srcDir is None) or (options.relDir is None):
-            return False
-    except AttributeError:
-        return False
-    return True
-
-
-#   check_required_args
-
-
-# def main(ver, srcDir, relDir):
 def main(argv):
     options = parse_opts(argv)
     # sys.exit(1)
-    ver = options.relVersion
-    srcDir = options.srcDir
-    relDir = options.relDir
+    ver = options.release_version
+    src_dir = options.source_dir
+    rel_dir = options.release_dir
     rc = options.rc
-    rpmRel = options.rpmRel
+    rpm_rel = options.rpm_release
     python_version = options.python_version
     use_mock = options.use_mock
+    is_verbose = options.verbose
 
     print("___________________________________________________________________")
     print("Creating following GlideinWMS release")
     print(
         "Version=%s\nSourceDir=%s\nReleaseDir=%s\nReleaseCandidate=%s\nRPMRelease=%s\nPython=%s"
-        % (ver, srcDir, relDir, rc, rpmRel, python_version)
+        % (ver, src_dir, rel_dir, rc, rpm_rel, python_version)
     )
     print("___________________________________________________________________")
     print()
-    rel = ReleaseManagerLib.Release(ver, srcDir, relDir, rc, rpmRel)
+    rel = ReleaseManagerLib.Release(ver, src_dir, rel_dir, rc, rpm_rel)
 
     rel.addTask(ReleaseManagerLib.TaskClean(rel))
     rel.addTask(ReleaseManagerLib.TaskSetupReleaseDir(rel))
     rel.addTask(ReleaseManagerLib.TaskVersionFile(rel))
     rel.addTask(ReleaseManagerLib.TaskTar(rel))
     rel.addTask(ReleaseManagerLib.TaskDocumentation(rel))
-    rel.addTask(ReleaseManagerLib.TaskRPM(rel, python_version, use_mock))
+    rel.addTask(ReleaseManagerLib.TaskRPM(rel, python_version, use_mock, is_verbose))
 
     rel.executeTasks()
     rel.printReport()