From 4f65f0ae22dc04a7cafc8e9b5595c46e942f23d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Gilbert?= <gaetan.gilbert@skyskimmer.net>
Date: Fri, 13 Dec 2024 20:23:07 +0100
Subject: [PATCH] CI: add job printing stats about the current pipeline

---
 .gitlab-ci.yml                       | 13 +++++-
 dev/ci/docker/edge_ubuntu/Dockerfile |  2 +
 dev/tools/pipeline-stats.py          | 62 ++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100755 dev/tools/pipeline-stats.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ae2b9e9ce722..880e602b2988 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -27,6 +27,7 @@ stages:
   - build-2
   - build-3+
   - deploy
+  - stats
 
 # We set "needs" to contain all transitive dependencies. We include the
 # transitive dependencies as otherwise we don't get their artifacts
@@ -40,7 +41,7 @@ variables:
   # echo $(md5sum dev/ci/docker/old_ubuntu_lts/Dockerfile | head -c 10)
   # echo $(md5sum dev/ci/docker/edge_ubuntu/Dockerfile | head -c 10)
   BASE_CACHEKEY: "old_ubuntu_lts-V2024-10-11-f72b1aa7c6"
-  EDGE_CACHEKEY: "edge_ubuntu-V2024-12-02-8a4b2ae476"
+  EDGE_CACHEKEY: "edge_ubuntu-V2024-12-13-5c140221f4"
   BASE_IMAGE: "$CI_REGISTRY_IMAGE:$BASE_CACHEKEY"
   EDGE_IMAGE: "$CI_REGISTRY_IMAGE:$EDGE_CACHEKEY"
 
@@ -1271,3 +1272,13 @@ doc:ci-refman:
       - _build/log
       - _build/default/doc/refman-html
       - _build/default/doc/refman-pdf
+
+pipeline-stats:
+  image: $EDGE_IMAGE
+  extends: .auto-use-tags
+  stage: stats
+  dependencies: []
+  before_script: []
+  script:
+  - dev/tools/pipeline-stats.py
+  when: always
diff --git a/dev/ci/docker/edge_ubuntu/Dockerfile b/dev/ci/docker/edge_ubuntu/Dockerfile
index 8a9ee075dec5..3ae83a174a34 100644
--- a/dev/ci/docker/edge_ubuntu/Dockerfile
+++ b/dev/ci/docker/edge_ubuntu/Dockerfile
@@ -18,6 +18,8 @@ RUN apt-get update -qq && apt-get install --no-install-recommends -y -qq \
         # Dependencies of stdlib and sphinx doc
         texlive-latex-extra texlive-fonts-recommended texlive-xetex latexmk \
         python3-pip python3-setuptools python3-pexpect python3-bs4 fonts-freefont-otf \
+        # pipeline-stats
+        python3-gitlab python3-tabulate \
         # Dependencies of source-doc and coq-makefile
         texlive-latex-extra texlive-science tipa \
         # Dependencies of HB (test suite)
diff --git a/dev/tools/pipeline-stats.py b/dev/tools/pipeline-stats.py
new file mode 100755
index 000000000000..cfcbf91df678
--- /dev/null
+++ b/dev/tools/pipeline-stats.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import os
+import gitlab
+from tabulate import tabulate
+
+gt = gitlab.Gitlab(url="https://gitlab.inria.fr", private_token=os.getenv("PIPELINE_STATS_TOKEN"), api_version="4")
+
+prj = gt.projects.get("coq/coq")
+
+pipeline_id=os.getenv("CI_PIPELINE_ID")
+pipeline = prj.pipelines.get(pipeline_id)
+
+def pptime(seconds):
+    if seconds >= 60 * 60:
+        hours = seconds / (60 * 60)
+        minutes = (seconds % (60 * 60)) / 60
+        return f"{hours:.0f}h {minutes:.0f}min"
+    elif seconds >= 60:
+        minutes = seconds / 60
+        rest = seconds % 60
+        return f"{minutes:.0f}min {rest:.0f}s"
+    else:
+        return f"{seconds:.0f}s"
+
+def ppsize(size, decimal_places=2):
+    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']:
+        if size < 1024.0 or unit == 'PiB':
+            break
+        size /= 1024.0
+    return f"{size:.{decimal_places}f} {unit}"
+
+res=[]
+total_time=0.0
+total_size=0
+total_log_size=0
+
+for j in pipeline.jobs.list(iterator=True):
+    if j.duration is None:
+        continue # non-finished job, eg bench or pipeline stats
+    size=0
+    log_size=0
+    if 'artifacts' in j.attributes:
+        for art in j.attributes['artifacts']:
+            if art['file_type'] == 'trace':
+                log_size += art['size']
+            else:
+                size += art['size']
+    res += [[j.name, j.duration, size, log_size, j.id]]
+    total_time += j.duration
+    total_size += size
+    total_log_size += log_size
+
+res += [['total', total_time, total_size, total_log_size, pipeline_id]]
+
+def sortkey(v):
+    return v[1]
+
+ppres = [ [v[0], pptime(v[1]), ppsize(v[2]), ppsize(v[3]), v[4]]
+          for v in sorted(res, key=sortkey) ]
+
+print(tabulate(ppres, headers=['name', 'duration', 'artifacts size', 'log size', 'id'], tablefmt='orgtbl'))