squad-stats-report

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: set ts=4
#
# Copyright 2022-present Linaro Limited
#
# SPDX-License-Identifier: MIT


import argparse
import json
import logging
import os
import sys
from collections import defaultdict
from datetime import date, timedelta
from pathlib import Path

from squad_client.core.api import SquadApi
from squad_client.core.models import ALL, Squad
from squad_client.utils import getid

squad_host_url = "https://qa-reports.linaro.org/"
SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url))

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

KNOWN_ARCHITECTURES = [
    "arc",
    "arm",
    "arm64",
    "i386",
    "mips",
    "parisc",
    "powerpc",
    "riscv",
    "s390",
    "sh",
    "sparc",
    "x86_64",
]

KNOWN_DEVICES = [
    "bcm2711-rpi-4-b",
    "dragonboard-410c",
    "dragonboard-820c",
    "dragonboard-845c",
    "fvp-aemva",
    "hi6220-hikey",
    "i386",
    "juno-r2",
    "nxp-ls2088",
    "qemu_arm",
    "qemu_arm64",
    "qemu_i386",
    "qemu_x86_64",
    "qemu-arm64",
    "qemu-arm64be",
    "qemu-armv5",
    "qemu-armv7",
    "qemu-armv7be",
    "qemu-i386",
    "qemu-mips32",
    "qemu-mips32el",
    "qemu-mips64",
    "qemu-mips64el",
    "qemu-ppc32",
    "qemu-ppc64",
    "qemu-ppc64le",
    "qemu-riscv32",
    "qemu-riscv64",
    "qemu-s390",
    "qemu-sh4",
    "qemu-sparc64",
    "qemu-x86_64",
    "x15",
    "x86",
]

KNOWN_SUITES = [
    "kselftest-android",
    "kselftest-arm64",
    "kselftest-breakpoints",
    "kselftest-capabilities",
    "kselftest-cgroup",
    "kselftest-clone3",
    "kselftest-core",
    "kselftest-cpu-hotplug",
    "kselftest-cpufreq",
    "kselftest-drivers-dma-buf",
    "kselftest-efivarfs",
    "kselftest-filesystems-binderfs",
    "kselftest-filesystems",
    "kselftest-firmware",
    "kselftest-fpu",
    "kselftest-futex",
    "kselftest-gpio",
    "kselftest-ipc",
    "kselftest-ir",
    "kselftest-kcmp",
    "kselftest-lib",
    "kselftest-membarrier",
    "kselftest-memfd",
    "kselftest-memory-hotplug",
    "kselftest-mincore",
    "kselftest-mount",
    "kselftest-mqueue",
    "kselftest-net-forwarding",
    "kselftest-netfilter",
    "kselftest-nsfs",
    "kselftest-openat2",
    "kselftest-pid_namespace",
    "kselftest-pidfd",
    "kselftest-proc",
    "kselftest-pstore",
    "kselftest-rseq",
    "kselftest-rtc",
    "kselftest-seccomp",
    "kselftest-sigaltstack",
    "kselftest-size",
    "kselftest-splice",
    "kselftest-static_keys",
    "kselftest-sync",
    "kselftest-sysctl",
    "kselftest-tc-testing",
    "kselftest-timens",
    "kselftest-timers",
    "kselftest-tmpfs",
    "kselftest-tpm2",
    "kselftest-user",
    "kselftest-vm",
    "kselftest-zram",
    "kunit",
    "kvm-unit-tests",
    "libhugetlbfs",
    "log-parser-boot",
    "log-parser-test",
    "ltp-cap_bounds",
    "ltp-commands",
    "ltp-containers",
    "ltp-cpuhotplug",
    "ltp-crypto",
    "ltp-cve",
    "ltp-dio",
    "ltp-fcntl-locktests",
    "ltp-filecaps",
    "ltp-fs",
    "ltp-fs_bind",
    "ltp-fs_perms_simple",
    "ltp-fsx",
    "ltp-hugetlb",
    "ltp-io",
    "ltp-ipc",
    "ltp-math",
    "ltp-nptl",
    "ltp-open-posix-tests",
    "ltp-pty",
    "ltp-sched",
    "ltp-securebits",
    "ltp-smoke",
    "ltp-syscalls",
    "ltp-tracing",
    "network-basic-tests",
    "perf",
    "v4l2-compliance",
    "vdso",
]


def parse_args():
    parser = argparse.ArgumentParser(description="Compare builds within SQUAD")

    parser.add_argument(
        "--group",
        required=True,
        help="squad group",
    )

    parser.add_argument(
        "--project",
        required=True,
        help="squad project",
    )

    parser.add_argument(
        "--from-datetime",
        required=True,
        help="Starting date time. Example: 2022-01-01 or 2022-01-01T00:00:00",
    )

    parser.add_argument(
        "--to-datetime",
        required=True,
        help="Ending date time. Example: 2022-12-31 or 2022-12-31T00:00:00",
    )

    parser.add_argument(
        "--filename", help="Name of the output file where results will be written"
    )

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="Display debug messages",
    )

    return parser.parse_args()


def get_number_of_kernel_builts(project, builds):
    suite = project.suite("build")
    envs = project.environments(count=ALL)
    archs = defaultdict(int)
    total = 0
    for build in builds:
        logger.debug(f"Fetching build test from {build.version}")
        tests = build.tests(suite=suite.id, fields="id,environment").values()
        total += len(tests)

        for test in tests:
            for known_arches in KNOWN_ARCHITECTURES:
                env = envs[getid(test.environment)].slug
                if known_arches in env:
                    archs[env] += 1
                    break

    return total, sorted_dict(archs)


def get_devices(environments, all_suites, builds):
    actual_devices = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))

    all_testruns = {e.id: [] for e in environments}
    for build in builds:
        for testrun in build.testruns().values():
            all_testruns[getid(testrun.environment)].append(testrun)

    for env in environments:
        for known_device in KNOWN_DEVICES:
            if known_device in env.slug:
                for testrun in all_testruns[env.id]:
                    for s in testrun.statuses(suite__isnull=False).values():
                        suite = all_suites[s.suite]
                        if suite.slug in KNOWN_SUITES:
                            tests_total = (
                                s.tests_pass
                                + s.tests_skip
                                + s.tests_fail
                                + s.tests_xfail
                            )
                            actual_devices[known_device][suite.slug][
                                "total"
                            ] += tests_total
                            actual_devices[known_device][suite.slug][
                                "pass"
                            ] += s.tests_pass
                            actual_devices[known_device][suite.slug][
                                "skip"
                            ] += s.tests_skip
                            actual_devices[known_device][suite.slug][
                                "fail"
                            ] += s.tests_fail
                            actual_devices[known_device][suite.slug][
                                "xfail"
                            ] += s.tests_xfail

    return actual_devices


def get_total_number_of_tests(builds):
    total = 0
    for build in builds:
        total += build.status.tests_total

    return total


def sorted_dict(d):
    return dict(sorted(d.items(), key=lambda k: k[0]))


def run():
    args = parse_args()
    if args.debug:
        logger.setLevel(level=logging.DEBUG)

    from_datetime = args.from_datetime
    if "T" not in from_datetime:
        from_datetime = f"{from_datetime}T00:00:00"

    to_datetime = args.to_datetime
    if "T" not in to_datetime:
        to_datetime = f"{to_datetime}T23:59:59"

    group = Squad().group(args.group)
    project = group.project(args.project)
    environments = project.environments(count=ALL).values()

    json_dir = "stored_jsons"
    if not os.path.exists(json_dir):
        os.makedirs(json_dir)
        print(f"Created dir: {json_dir}")

    from_date = from_datetime.split("T")[0]
    to_date = to_datetime.split("T")[0]

    from_year = int(from_date.split("-")[0])
    from_month = int(from_date.split("-")[1])
    from_day = int(from_date.split("-")[2])
    to_year = int(to_date.split("-")[0])
    to_month = int(to_date.split("-")[1])
    to_day = int(to_date.split("-")[2])

    first_from_day = True
    a = []
    kernel_pushes = []
    kernel_builts = []
    num_tests = []
    architectures = defaultdict(int)
    devices = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    filename = args.filename or f"{json_dir}/stats-{args.group}-{args.project}.json"
    if os.path.isfile(filename):
        a = json.load(Path(filename).open(encoding="utf-8"))

    tmp_from_date = date(from_year, from_month, from_day)
    end_date = date(to_year, to_month, to_day)
    delta = timedelta(days=1)
    tmp_from_date -= delta
    while tmp_from_date < end_date:
        tmp_to_date = tmp_from_date + delta
        tmp_from_date += delta
        if first_from_day:
            first_from_day = False
            from_time = f"T{from_datetime.split('T')[1]}"
        else:
            from_time = "T00:00:00"
        if tmp_to_date == end_date:
            to_time = f"T{to_datetime.split('T')[1]}"
        else:
            to_time = "T23:59:59"

        # if data already exists
        ask_squad = True
        for entry in a:
            if (
                entry["from_datetime"] == f"{tmp_from_date}{from_time}"
                and entry["to_datetime"] == f"{tmp_to_date}{to_time}"
            ):
                kernel_pushes.append(entry["kernel pushes"])
                kernel_builts.append(entry["kernel builts"])
                num_tests.append(entry["tests"])
                for arch in entry["architectures"]:
                    architectures[arch] += entry["architectures"][arch]

                for dev, suites in entry["devices"].items():
                    for suite in suites:
                        devices[dev][suite]["total"] += entry["devices"][dev][suite][
                            "total"
                        ]
                        devices[dev][suite]["pass"] += entry["devices"][dev][suite][
                            "pass"
                        ]
                        devices[dev][suite]["skip"] += entry["devices"][dev][suite][
                            "skip"
                        ]
                        devices[dev][suite]["fail"] += entry["devices"][dev][suite][
                            "fail"
                        ]
                        devices[dev][suite]["xfail"] += entry["devices"][dev][suite][
                            "xfail"
                        ]
                ask_squad = False
                print(
                    f"Found dates in JSON file {filename}, from_datetime: {tmp_from_date}{from_time}, to_datetime: {tmp_to_date}{to_time}"
                )
                break

        if ask_squad:
            print(
                f"Fetching builds from SQUAD, from_datetime: {tmp_from_date}{from_time}, to_datetime: {tmp_to_date}{to_time}"
            )
            builds = project.builds(
                created_at__lt=f"{tmp_to_date}{to_time}",
                created_at__gt=f"{tmp_from_date}{from_time}",
                count=ALL,
            ).values()
            number_of_kernel_builts, archs = get_number_of_kernel_builts(
                project, builds
            )
            devs = get_devices(environments, project.suites(count=ALL), builds)
            total_tests = get_total_number_of_tests(builds)

            d = {}
            d["from_datetime"] = f"{tmp_from_date}{from_time}"
            d["to_datetime"] = f"{tmp_to_date}{to_time}"
            d["kernel pushes"] = len(builds)
            d["kernel builts"] = number_of_kernel_builts
            d["tests"] = total_tests
            d["architectures"] = archs
            d["devices"] = devs
            a.append(d)

            print(
                f"Write builds to JSON file {filename}, from_datetime: {tmp_from_date}{from_time}, to_datetime: {tmp_to_date}{to_time}"
            )
            Path(filename).write_text(json.dumps(a, indent=4), encoding="utf-8")

            kernel_pushes.append(len(builds))
            kernel_builts.append(number_of_kernel_builts)
            num_tests.append(total_tests)
            for arch in archs:
                architectures[arch] += archs[arch]
            for dev, suites in devs.items():
                for suite in suites:
                    devices[dev][suite]["total"] += devs[dev][suite]["total"]
                    devices[dev][suite]["pass"] += devs[dev][suite]["pass"]
                    devices[dev][suite]["skip"] += devs[dev][suite]["skip"]
                    devices[dev][suite]["fail"] += devs[dev][suite]["fail"]
                    devices[dev][suite]["xfail"] += devs[dev][suite]["xfail"]

    total_kernel_pushes = 0
    for build in kernel_pushes:
        total_kernel_pushes += build
    total_kernel_builts = 0
    for kernel in kernel_builts:
        total_kernel_builts += kernel
    total_tests = 0
    for tests in num_tests:
        total_tests += tests

    archs_str = ""
    arch_lines = list()
    for arch in sorted_dict(architectures):
        arch_lines.append(f"{arch:<10} ({architectures[arch]} builds)")
    archs_str = "\n           ".join(arch_lines)

    devices_str = ""
    device_lines = list()
    for dev in sorted(devices):
        device_lines.append(f"{dev:<20} ({len(devices[dev])} suites)")
    devices_str += "\n           ".join(device_lines)

    report = f"""
        project:       {project.slug}
        from:          {from_datetime}
        to:            {to_datetime}
        kernel pushes: {total_kernel_pushes}
        kernel builts: {total_kernel_builts}
        total tests:   {total_tests}
        architectures:
           {archs_str}
        devices:
           {devices_str}"""
    print(report)


if __name__ == "__main__":
    sys.exit(run())