diff --git a/SPECS/libxml2/CVE-2019-20388.patch b/SPECS/libxml2/CVE-2019-20388.patch deleted file mode 100644 index d62751f6bda..00000000000 --- a/SPECS/libxml2/CVE-2019-20388.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 7ffcd44d7e6c46704f8af0321d9314cd26e0e18a Mon Sep 17 00:00:00 2001 -From: Zhipeng Xie -Date: Tue, 20 Aug 2019 16:33:06 +0800 -Subject: [PATCH] Fix memory leak in xmlSchemaValidateStream - -When ctxt->schema is NULL, xmlSchemaSAXPlug->xmlSchemaPreRun -alloc a new schema for ctxt->schema and set vctxt->xsiAssemble -to 1. Then xmlSchemaVStart->xmlSchemaPreRun initialize -vctxt->xsiAssemble to 0 again which cause the alloced schema -can not be freed anymore. - -Found with libFuzzer. - -Signed-off-by: Zhipeng Xie ---- - xmlschemas.c | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/xmlschemas.c b/xmlschemas.c -index 301c84499..39d92182f 100644 ---- a/xmlschemas.c -+++ b/xmlschemas.c -@@ -28090,7 +28090,6 @@ xmlSchemaPreRun(xmlSchemaValidCtxtPtr vctxt) { - vctxt->nberrors = 0; - vctxt->depth = -1; - vctxt->skipDepth = -1; -- vctxt->xsiAssemble = 0; - vctxt->hasKeyrefs = 0; - #ifdef ENABLE_IDC_NODE_TABLES_TEST - vctxt->createIDCNodeTables = 1; --- -GitLab diff --git a/SPECS/libxml2/CVE-2020-24977.patch b/SPECS/libxml2/CVE-2020-24977.patch deleted file mode 100644 index f8ad63065c5..00000000000 --- a/SPECS/libxml2/CVE-2020-24977.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 50f06b3efb638efb0abd95dc62dca05ae67882c2 Mon Sep 17 00:00:00 2001 -From: Nick Wellnhofer -Date: Fri, 7 Aug 2020 21:54:27 +0200 -Subject: [PATCH] Fix out-of-bounds read with 'xmllint --htmlout' - -Make sure that truncated UTF-8 sequences don't cause an out-of-bounds -array access. - -Thanks to @SuhwanSong and the Agency for Defense Development (ADD) for -the report. - -Fixes #178. ---- - xmllint.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/xmllint.c b/xmllint.c -index f6a8e4636..c647486f3 100644 ---- a/xmllint.c -+++ b/xmllint.c -@@ -528,6 +528,12 @@ static void - xmlHTMLEncodeSend(void) { - char *result; - -+ /* -+ * xmlEncodeEntitiesReentrant assumes valid UTF-8, but the buffer might -+ * end with a truncated UTF-8 sequence. This is a hack to at least avoid -+ * an out-of-bounds read. -+ */ -+ memset(&buffer[sizeof(buffer)-4], 0, 4); - result = (char *) xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer); - if (result) { - xmlGenericError(xmlGenericErrorContext, "%s", result); --- -GitLab diff --git a/SPECS/libxml2/CVE-2020-7595.patch b/SPECS/libxml2/CVE-2020-7595.patch deleted file mode 100644 index b639483f355..00000000000 --- a/SPECS/libxml2/CVE-2020-7595.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 0e1a49c8907645d2e155f0d89d4d9895ac5112b5 Mon Sep 17 00:00:00 2001 -From: Zhipeng Xie -Date: Thu, 12 Dec 2019 17:30:55 +0800 -Subject: [PATCH] Fix infinite loop in xmlStringLenDecodeEntities - -When ctxt->instate == XML_PARSER_EOF,xmlParseStringEntityRef -return NULL which cause a infinite loop in xmlStringLenDecodeEntities - -Found with libFuzzer. - -Signed-off-by: Zhipeng Xie ---- - parser.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/parser.c b/parser.c -index d1c319631..a34bb6cdd 100644 ---- a/parser.c -+++ b/parser.c -@@ -2646,7 +2646,8 @@ xmlStringLenDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, - else - c = 0; - while ((c != 0) && (c != end) && /* non input consuming loop */ -- (c != end2) && (c != end3)) { -+ (c != end2) && (c != end3) && -+ (ctxt->instate != XML_PARSER_EOF)) { - - if (c == 0) break; - if ((c == '&') && (str[1] == '#')) { --- -GitLab diff --git a/SPECS/libxml2/libxml2.signatures.json b/SPECS/libxml2/libxml2.signatures.json index 6b4f7588d14..042e089dbc7 100644 --- a/SPECS/libxml2/libxml2.signatures.json +++ b/SPECS/libxml2/libxml2.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "libxml2-2.9.10.tar.gz": "aafee193ffb8fe0c82d4afef6ef91972cbaf5feea100edc2f262750611b4be1f" + "libxml2-2.9.12.tar.gz": "c8d6681e38c56f172892c85ddc0852e1fd4b53b4209e7f4ebf17f7e2eae71d92" } -} \ No newline at end of file +} diff --git a/SPECS/libxml2/libxml2.spec b/SPECS/libxml2/libxml2.spec index 5e0a590ebee..119cd06f55a 100644 --- a/SPECS/libxml2/libxml2.spec +++ b/SPECS/libxml2/libxml2.spec @@ -2,17 +2,14 @@ %{!?python3_sitelib: %define python3_sitelib %(python3 -c "from distutils.sysconfig import get_python_lib;print(get_python_lib())")} Summary: Libxml2 Name: libxml2 -Version: 2.9.10 -Release: 4%{?dist} +Version: 2.9.12 +Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Mariner Group: System Environment/General Libraries URL: http://www.xmlsoft.org/ Source0: ftp://xmlsoft.org/libxml2/%{name}-%{version}.tar.gz -Patch0: CVE-2019-20388.patch -Patch1: CVE-2020-7595.patch -Patch2: CVE-2020-24977.patch BuildRequires: python2-devel BuildRequires: python2-libs BuildRequires: python3-devel @@ -105,6 +102,9 @@ rm -rf %{buildroot}/* %{_libdir}/cmake/libxml2/libxml2-config.cmake %changelog +* Thu May 27 2021 Mateusz Malisz - 2.9.12-1 +- Update to version 2.9.12 to fix CVE-2021-3517, CVE-2021-3518 and CVE-2021-3537 + * Wed Mar 03 2021 Andrew Phelps - 2.9.10-4 - Skip python tests which are known to be broken. diff --git a/cgmanifest.json b/cgmanifest.json index 7c79dc6c21d..1c2fa07dffb 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -3195,8 +3195,8 @@ "type": "other", "other": { "name": "libxml2", - "version": "2.9.10", - "downloadUrl": "ftp://xmlsoft.org/libxml2/libxml2-2.9.10.tar.gz" + "version": "2.9.12", + "downloadUrl": "ftp://xmlsoft.org/libxml2/libxml2-2.9.12.tar.gz" } } }, diff --git a/toolkit/Makefile b/toolkit/Makefile index 250e46bf86d..804d62ab533 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -23,6 +23,7 @@ INCREMENTAL_TOOLCHAIN ?= n UPDATE_TOOLCHAIN_LIST ?= n REBUILD_PACKAGES ?= y DOWNLOAD_SRPMS ?= n +ALLOW_SRPM_DOWNLOAD_FAIL ?= n REBUILD_TOOLS ?= n RUN_CHECK ?= n USE_UPDATE_REPO ?= y diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index 88a8ac019c2..40a8346bd85 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -137,8 +137,8 @@ tdnf-cli-libs-2.1.0-6.cm1.aarch64.rpm tdnf-devel-2.1.0-6.cm1.aarch64.rpm tdnf-plugin-repogpgcheck-2.1.0-6.cm1.aarch64.rpm createrepo_c-0.11.1-6.cm1.aarch64.rpm -libxml2-2.9.10-4.cm1.aarch64.rpm -libxml2-devel-2.9.10-4.cm1.aarch64.rpm +libxml2-2.9.12-1.cm1.aarch64.rpm +libxml2-devel-2.9.12-1.cm1.aarch64.rpm glib-2.58.0-9.cm1.aarch64.rpm libltdl-2.4.6-5.cm1.aarch64.rpm libltdl-devel-2.4.6-5.cm1.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index fc3312750f7..5d50c211835 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -137,8 +137,8 @@ tdnf-cli-libs-2.1.0-6.cm1.x86_64.rpm tdnf-devel-2.1.0-6.cm1.x86_64.rpm tdnf-plugin-repogpgcheck-2.1.0-6.cm1.x86_64.rpm createrepo_c-0.11.1-6.cm1.x86_64.rpm -libxml2-2.9.10-4.cm1.x86_64.rpm -libxml2-devel-2.9.10-4.cm1.x86_64.rpm +libxml2-2.9.12-1.cm1.x86_64.rpm +libxml2-devel-2.9.12-1.cm1.x86_64.rpm glib-2.58.0-9.cm1.x86_64.rpm libltdl-2.4.6-5.cm1.x86_64.rpm libltdl-devel-2.4.6-5.cm1.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 6d76730f2b7..7904ff02c42 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -218,10 +218,10 @@ libtasn1-debuginfo-4.14-2.cm1.aarch64.rpm libtasn1-devel-4.14-2.cm1.aarch64.rpm libtool-2.4.6-5.cm1.aarch64.rpm libtool-debuginfo-2.4.6-5.cm1.aarch64.rpm -libxml2-2.9.10-4.cm1.aarch64.rpm -libxml2-debuginfo-2.9.10-4.cm1.aarch64.rpm -libxml2-devel-2.9.10-4.cm1.aarch64.rpm -libxml2-python-2.9.10-4.cm1.aarch64.rpm +libxml2-2.9.12-1.cm1.aarch64.rpm +libxml2-debuginfo-2.9.12-1.cm1.aarch64.rpm +libxml2-devel-2.9.12-1.cm1.aarch64.rpm +libxml2-python-2.9.12-1.cm1.aarch64.rpm libxslt-1.1.34-2.cm1.aarch64.rpm libxslt-debuginfo-1.1.34-2.cm1.aarch64.rpm libxslt-devel-1.1.34-2.cm1.aarch64.rpm @@ -331,7 +331,7 @@ python2-test-2.7.18-6.cm1.aarch64.rpm python2-tools-2.7.18-6.cm1.aarch64.rpm python3-cracklib-2.9.7-2.cm1.aarch64.rpm python3-gpg-1.13.1-6.cm1.aarch64.rpm -python3-libxml2-2.9.10-4.cm1.aarch64.rpm +python3-libxml2-2.9.12-1.cm1.aarch64.rpm python3-pwquality-1.4.2-6.cm1.aarch64.rpm python3-rpm-4.14.2-11.cm1.aarch64.rpm python-curses-2.7.18-6.cm1.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 6f67f81a5bb..46c2c1d3fb5 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -218,10 +218,10 @@ libtasn1-debuginfo-4.14-2.cm1.x86_64.rpm libtasn1-devel-4.14-2.cm1.x86_64.rpm libtool-2.4.6-5.cm1.x86_64.rpm libtool-debuginfo-2.4.6-5.cm1.x86_64.rpm -libxml2-2.9.10-4.cm1.x86_64.rpm -libxml2-debuginfo-2.9.10-4.cm1.x86_64.rpm -libxml2-devel-2.9.10-4.cm1.x86_64.rpm -libxml2-python-2.9.10-4.cm1.x86_64.rpm +libxml2-2.9.12-1.cm1.x86_64.rpm +libxml2-debuginfo-2.9.12-1.cm1.x86_64.rpm +libxml2-devel-2.9.12-1.cm1.x86_64.rpm +libxml2-python-2.9.12-1.cm1.x86_64.rpm libxslt-1.1.34-2.cm1.x86_64.rpm libxslt-debuginfo-1.1.34-2.cm1.x86_64.rpm libxslt-devel-1.1.34-2.cm1.x86_64.rpm @@ -331,7 +331,7 @@ python2-test-2.7.18-6.cm1.x86_64.rpm python2-tools-2.7.18-6.cm1.x86_64.rpm python3-cracklib-2.9.7-2.cm1.x86_64.rpm python3-gpg-1.13.1-6.cm1.x86_64.rpm -python3-libxml2-2.9.10-4.cm1.x86_64.rpm +python3-libxml2-2.9.12-1.cm1.x86_64.rpm python3-pwquality-1.4.2-6.cm1.x86_64.rpm python3-rpm-4.14.2-11.cm1.x86_64.rpm python-curses-2.7.18-6.cm1.x86_64.rpm diff --git a/toolkit/scripts/download-packages.sh b/toolkit/scripts/download-packages.sh new file mode 100755 index 00000000000..a8d50190f77 --- /dev/null +++ b/toolkit/scripts/download-packages.sh @@ -0,0 +1,99 @@ +#!/bin/bash -e + +function get_packages { + # First argument is an URL to a directory containing packages + download_url="$1" + + # wget -nv -O - "$download_url" -- Download HTML of package directory and send to stdout (-nv reduces verbosity) + # | grep 'a href' -- Filter out lines that are not links (we're looking for links to rpms) + # | sed -E -e 's: space-separated list of directories in the RPM repository (passed with the -u) (e.g. "base update")' + echo '[OPTIONAL] -h -> print this help dialogue and exit' + echo '[MANDATORY] -t TYPE -> select which type of packages to download. Can provide more than one type, separated by space. The valid types are: x86_64 aarch64 srpms' + echo '[MANDATORY] -u URL -> URL to a root directory of a repository (e.g. https://packages.microsoft.com/cbl-mariner/1.0/prod/)' + echo '[OPTIONAL] -z -> create a tarball for each downloaded package type and clean up' +} + +repository_url= +packages_types= +directories= +tar_packages=0 + +while getopts "d:ht:u:z" OPTIONS; do + case ${OPTIONS} in + d ) directories="$OPTARG" ;; + h ) help; exit 0 ;; + t ) packages_types="$OPTARG" ;; + u ) repository_url=$OPTARG ;; + z ) tar_packages=1 ;; + ? ) echo -e "ERROR: INVALID OPTION.\n\n"; help; exit 1 ;; + esac +done + +if [[ -z "$directories" ]] || [[ -z "$packages_types" ]] || [[ -z "$repository_url" ]]; then + echo -e "ERROR: Arguments '-d', '-t' and '-u' are mandatory!\n\n" + help + exit 2 +fi + +# Remove trailing directory separator, if any +if [[ $repository_url =~ ^.+/$ ]]; then + echo "-- Removing trailing directory separator from $repository_url" + repository_url=`echo $repository_url | head -c -2` +fi + +# For benchmark purposes +before_run=$(date +%s) + +# Iterate over directories and types, downloading the files +for directory in $directories; do + echo "-- Downloading directory $directory..." + for package_type in $packages_types; do + echo "-- Downloading type $package_type for directory $directory..." + + # If these are not srpms, there is additional directory to skip + appendix= + if [[ ! "$package_type" == "srpms" ]]; then + echo "-- Downloading RPMS - adding additional directory." + appendix="/rpms" + fi + + # Appendix contains the slash, if needed. + get_packages "$repository_url"/"$directory"/"$package_type""$appendix" + done +done + +if [[ 1 == $tar_packages ]]; then + make_tarball +fi + +echo "Total execution time:" +after_run=$(date +%s) +date -d@$((before_run - now)) -u +%H:%M:%S diff --git a/toolkit/scripts/filter-packages.sh b/toolkit/scripts/filter-packages.sh new file mode 100755 index 00000000000..ddbab175058 --- /dev/null +++ b/toolkit/scripts/filter-packages.sh @@ -0,0 +1,76 @@ +#!/bin/bash -e + +function filter_packages { + # First argument is a path to the packages directory + rpms_directory="$1" + + # Used later + fields_cutoff=2 + if [[ $ignore_versions -eq 1 ]]; then + fields_cutoff=3 + fi + + duplicates_list=$(find "$rpms_directory" -name '*.rpm' -exec basename {} \; | # Find all the rpms leaving only file names + sort -V | # Sort to stack duplicates together. Sort like version numbers (-V) + rev | # Read backwards to inverse the order of the fields + cut -d'-' -f ${fields_cutoff}- | # Separate into fields by '-' skipping the first two (in case of ignore_versions - 3) + # (actually last - we're inversed) fields (filters out release and extension) + # in case of -i being passed, ignore versions as well. + rev | # Return to normal order + uniq -d | # Print only duplicates + awk '{print}' ORS=' ') # Join lines with space instead of newline + + remove_function="rm -v" + if [[ $pretend -eq 1 ]]; then + remove_function="echo Would remove: " + fi + + for duplicate in $duplicates_list; do + echo "Processing package '$duplicate'." + name_search="${duplicate}-[0-9]*" + + # and remove them (explicitly list removed items) + packages_to_remove=$(find "$rpms_directory" -name "$name_search" |# Find all the RPM files with the same name (different versions treated as different names without -i) + sort -V -r | # Read backwards and sort a version numbers (-V) and print in reverse(-r) to have the latest version at the top. + tail -n +2) # After the sort, the first path is of the latest version. Get a list of all the other RPMs (duplicates), starting line 2. + + for package in $packages_to_remove; do + $remove_function "$package" + done + + # Run a more advanced query again to work correctly when doing a -p run + echo "Left:" $(find "$rpms_directory" -name "$name_search" | sort -V -r | awk '{print}' ORS=' ' | cut -d' ' -f1 ) + echo #newline + done +} + +function help { + echo "Package filter. Filter out (remove) duplicate packages." + echo "Usage:" + echo '[MANDATORY] -d DIR -> path to the directory which contains RPMS (this will) affect all subdirectories' + echo '[OPTIONAL] -h -> print this help dialogue and exit' + echo '[OPTIONAL] -i -> ignore version numbers when looking for the duplicates.' + echo '[OPTIONAL] -p -> pretend mode - just print the packages without removing them' +} + +directory= +pretend=0 +ignore_versions=0 + +while getopts "d:hpi" OPTIONS; do + case ${OPTIONS} in + d ) directory="$OPTARG" ;; + h ) help; exit 0 ;; + i ) ignore_versions=1 ;; + p ) pretend=1 ;; + ? ) echo -e "ERROR: INVALID OPTION.\n\n"; help; exit 1 ;; + esac +done + +if [[ -z "$directory" ]]; then + echo -e "ERROR: Argument '-d' is mandatory!\n\n" + help + exit 2 +fi + +filter_packages "$directory" diff --git a/toolkit/scripts/pkggen.mk b/toolkit/scripts/pkggen.mk index 5a63a941c69..8d4f87ad399 100644 --- a/toolkit/scripts/pkggen.mk +++ b/toolkit/scripts/pkggen.mk @@ -182,13 +182,13 @@ endif $(call print_error,Failed to build: $$(cat $(LOGS_DIR)/pkggen/failures.txt)); } && \ touch $@ -# use temp tarball to avoid tar warning "file changed as we read it" +# use temp tarball to avoid tar warning "file changed as we read it" # that can sporadically occur when tarball is the dir that is compressed compress-rpms: tar -I $(ARCHIVE_TOOL) -cvp -f $(BUILD_DIR)/temp_rpms_tarball.tar.gz -C $(RPMS_DIR)/.. $(notdir $(RPMS_DIR)) mv $(BUILD_DIR)/temp_rpms_tarball.tar.gz $(pkggen_archive) -# use temp tarball to avoid tar warning "file changed as we read it" +# use temp tarball to avoid tar warning "file changed as we read it" # that can sporadically occur when tarball is the dir that is compressed compress-srpms: tar -I $(ARCHIVE_TOOL) -cvp -f $(BUILD_DIR)/temp_srpms_tarball.tar.gz -C $(SRPMS_DIR)/.. $(notdir $(SRPMS_DIR)) diff --git a/toolkit/scripts/srpm_pack.mk b/toolkit/scripts/srpm_pack.mk index 29d5966bcbd..80fc42ccc35 100644 --- a/toolkit/scripts/srpm_pack.mk +++ b/toolkit/scripts/srpm_pack.mk @@ -38,6 +38,33 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag @echo Finished updating $@ ifeq ($(DOWNLOAD_SRPMS),y) + +.SILENT: $(STATUS_FLAGS_DIR)/build_srpms.flag + +ifeq ($(ALLOW_SRPM_DOWNLOAD_FAIL),y) +$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(LOGS_DIR)/pkggen + for spec in $(local_specs); do \ + spec_file=$${spec} && \ + spec_name=$$(basename "$${spec_file}") && \ + srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm 2>"$(LOGS_DIR)/pkggen/$${spec_name}") && \ + log_file="$(LOGS_DIR)/pkggen/$$srpm_file.log" && \ + mkdir -p $(BUILD_SRPMS_DIR) && \ + cd $(BUILD_SRPMS_DIR) && \ + touch $(BUILD_SRPMS_DIR)/$${srpm_file} && \ + for url in $(SRPM_URL_LIST); do \ + wget $${url}/$${srpm_file} \ + $(if $(TLS_CERT),--certificate=$(TLS_CERT)) \ + $(if $(TLS_KEY),--private-key=$(TLS_KEY)) \ + -a $$log_file && \ + break; \ + done && echo "Downloaded $${url}/$${srpm_file}"; \ + done ; \ + echo "Removing empty (failed) SRPMS: " + find $(BUILD_SRPMS_DIR) -type f -empty -delete -print | tee $(LOGS_DIR)/pkggen/deleted-srpms.log + echo "Removed all empty SRPMS. Finished packing." + touch $@ + +else $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_DIR) for spec in $(local_specs); do \ spec_file=$${spec} && \ @@ -56,6 +83,8 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_ $(call print_error,Failed to download $${srpm_file}); } \ done || $(call print_error,Loop in $@ failed) ; \ touch $@ +endif + else $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_sources) $(SPECS_DIR) $(go-srpmpacker) $(go-srpmpacker) \ diff --git a/toolkit/scripts/toolchain.mk b/toolkit/scripts/toolchain.mk index 7e47e3dd40c..610be490e05 100644 --- a/toolkit/scripts/toolchain.mk +++ b/toolkit/scripts/toolchain.mk @@ -25,15 +25,15 @@ toolchain_files = \ $(shell find $(SCRIPTS_DIR)/toolchain/SCRIPTS ) \ $(SCRIPTS_DIR)/toolchain/container/Dockerfile -toolchain_manifest = $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt +TOOLCHAIN_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt # Find the *.rpm corresponding to each of the entries in the manifest # regex operation: (.*\.([^\.]+)\.rpm) extracts *.().rpm" to determine # the exact path of the required rpm # Outputs: $(toolchain_rpms_dir)//..rpm sed_regex_full_path = 's`(.*\.([^\.]+)\.rpm)`$(toolchain_rpms_dir)/\2/\1`p' -toolchain_rpms := $(shell sed -nr $(sed_regex_full_path) < $(toolchain_manifest)) -toolchain_rpms_buildarch := $(shell grep $(build_arch) $(toolchain_manifest)) -toolchain_rpms_noarch := $(shell grep noarch $(toolchain_manifest)) +toolchain_rpms := $(shell sed -nr $(sed_regex_full_path) < $(TOOLCHAIN_MANIFEST)) +toolchain_rpms_buildarch := $(shell grep $(build_arch) $(TOOLCHAIN_MANIFEST)) +toolchain_rpms_noarch := $(shell grep noarch $(TOOLCHAIN_MANIFEST)) $(call create_folder,$(toolchain_build_dir)) $(call create_folder,$(toolchain_downloads_logs_dir)) @@ -165,7 +165,7 @@ ifeq ($(REBUILD_TOOLCHAIN),y) # The basic set of RPMs can always be produced by bootstrapping the toolchain. # Try to skip extracting individual RPMS if the toolchain step has already placed # them into the RPM folder. -$(toolchain_rpms): $(toolchain_manifest) | $(final_toolchain) +$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) | $(final_toolchain) @echo Extracting RPM $@ from toolchain && \ if [ ! -f $@ -o $(final_toolchain) -nt $@ ]; then \ mkdir -p $(dir $@) && \ @@ -185,7 +185,7 @@ $(STATUS_FLAGS_DIR)/toolchain_local_temp.flag: $(TOOLCHAIN_ARCHIVE) $(shell find touch $(BUILD_DIR)/toolchain_temp/* && \ touch $@ -$(toolchain_rpms): $(toolchain_manifest) $(toolchain_local_temp) +$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(toolchain_local_temp) tempFile=$(toolchain_local_temp)/$(notdir $@) && \ if [ ! -f $@ -o $(TOOLCHAIN_ARCHIVE) -nt $@ ]; then \ echo Extracting RPM $@ from toolchain && \ @@ -197,7 +197,7 @@ $(toolchain_rpms): $(toolchain_manifest) $(toolchain_local_temp) else # Download from online package server $(toolchain_rpms): - @rpm_filename="$(notdir $@)" && \ + rpm_filename="$(notdir $@)" && \ rpm_dir="$(dir $@)" && \ log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \ echo "Downloading toolchain RPM: $$rpm_filename" | tee "$$log_file" && \ @@ -207,15 +207,14 @@ $(toolchain_rpms): wget $$url/$$rpm_filename \ $(if $(TLS_CERT),--certificate=$(TLS_CERT)) \ $(if $(TLS_KEY),--private-key=$(TLS_KEY)) \ - -a $$log_file \ - && \ + -a $$log_file && \ echo "Downloaded toolchain RPM: $$rpm_filename" >> $$log_file && \ break; \ done || { \ - echo "\nERROR: Failed to download toolchain package: $$rpm_filename." && \ - echo "ERROR: Last $(toolchain_log_tail_length) lines from log '$$log_file':\n" && \ - tail -n$(toolchain_log_tail_length) $$log_file | sed 's/^/\t/' && \ - $(call print_error,\nToolchain download failed. See above errors for more details.) \ - } + echo "\nERROR: Failed to download toolchain package: $$rpm_filename." && \ + echo "ERROR: Last $(toolchain_log_tail_length) lines from log '$$log_file':\n" && \ + tail -n$(toolchain_log_tail_length) $$log_file | sed 's/^/\t/' && \ + $(call print_error,\nToolchain download failed. See above errors for more details.) \ + } endif endif diff --git a/toolkit/scripts/toolkit.mk b/toolkit/scripts/toolkit.mk index 4a8d79634af..926d96e9ffe 100644 --- a/toolkit/scripts/toolkit.mk +++ b/toolkit/scripts/toolkit.mk @@ -42,3 +42,6 @@ package-toolkit: go-tools echo "$(toolkit_version)" > $(toolkit_release_file) && \ rm -rf $(toolkit_build_dir)/out && \ tar -I $(ARCHIVE_TOOL) -cvp -f $(toolkit_archive) -C $(toolkit_build_dir)/.. $(notdir $(toolkit_build_dir)) + +print-build-summary: + sed -E -n 's:^.+level=info msg="Built \(([^\)]+)\) -> \[(.+)\].+$:\1\t\2:gp' $(LOGS_DIR)/pkggen/rpmbuilding/* | tee $(LOGS_DIR)/pkggen/build-summary.csv diff --git a/toolkit/scripts/tools.mk b/toolkit/scripts/tools.mk index 47ed0924fcc..9830b9659a2 100644 --- a/toolkit/scripts/tools.mk +++ b/toolkit/scripts/tools.mk @@ -125,7 +125,11 @@ clean-chroot-tools: rm -rf $(BUILD_DIR)/worker && \ rm -rf $(BUILD_DIR)/validatechroot -worker_chroot_manifest = $(TOOLCHAIN_MANIFESTS_DIR)/pkggen_core_$(build_arch).txt +# Worker chroot manifest is a file corresponding to the TOOLCHAIN_MANIFEST name. +toolchain_config_name=$(notdir $(TOOLCHAIN_MANIFEST)) +worker_manifest_name=$(shell echo "$(toolchain_config_name)" | sed -E 's:^toolchain:pkggen_core:' ) +worker_chroot_manifest = $(TOOLCHAIN_MANIFESTS_DIR)/$(worker_manifest_name) +#$(TOOLCHAIN_MANIFESTS_DIR)/pkggen_core_$(build_arch).txt # Find the *.rpm corresponding to each of the entries in the manifest # regex operation: (.*\.([^\.]+)\.rpm) extracts *.().rpm" to determine # the exact path of the required rpm diff --git a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go index dbd68437008..aa47bfe7dfb 100644 --- a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go +++ b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go @@ -163,6 +163,7 @@ func resolveSingleNode(cloner *rpmrepocloner.RpmRepoCloner, node *pkggraph.PkgNo if err != nil { logger.Log.Errorf("Failed to clone %s from RPM repo. Error: %s", node, err) } else { + logger.Log.Debugf("Successfully cloned %s.", node) node.State = pkggraph.StateCached } return