Merge pull request #260 from truenas/NAS-132425

NAS-132425 / None / sync truenas/zfs-2.3-release with upstream 2.3.0-rc3 tag

.github/workflows/codeql.yml

@@ -11,7 +11,7 @@ concurrency:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read

.github/workflows/scripts/qemu-1-setup.sh

@@ -18,19 +18,21 @@ ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
 
 # we expect RAM shortage
 cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
+# /etc/ksmtuned.conf - Configuration file for ksmtuned
 # https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
 KSM_MONITOR_INTERVAL=60
 
 # Millisecond sleep between ksm scans for 16Gb server.
 # Smaller servers sleep more, bigger sleep less.
-KSM_SLEEP_MSEC=10
-KSM_NPAGES_BOOST=300
-KSM_NPAGES_DECAY=-50
-KSM_NPAGES_MIN=64
-KSM_NPAGES_MAX=2048
-
-KSM_THRES_COEF=25
-KSM_THRES_CONST=2048
+KSM_SLEEP_MSEC=30
+
+KSM_NPAGES_BOOST=0
+KSM_NPAGES_DECAY=0
+KSM_NPAGES_MIN=1000
+KSM_NPAGES_MAX=25000
+
+KSM_THRES_COEF=80
+KSM_THRES_CONST=8192
 
 LOGFILE=/var/log/ksmtuned.log
 DEBUG=1

.github/workflows/scripts/qemu-2-start.sh

@@ -52,16 +52,16 @@ case "$OS" in
     OSNAME="Debian 12"
     URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
     ;;
-  fedora39)
-    OSNAME="Fedora 39"
-    OSv="fedora39"
-    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
-    ;;
   fedora40)
     OSNAME="Fedora 40"
-    OSv="fedora39"
+    OSv="fedora-unknown"
     URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
     ;;
+  fedora41)
+    OSNAME="Fedora 41"
+    OSv="fedora-unknown"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"
+    ;;
   freebsd13-3r)
     OSNAME="FreeBSD 13.3-RELEASE"
     OSv="freebsd13.0"

.github/workflows/scripts/qemu-3-deps.sh

@@ -13,10 +13,10 @@ function archlinux() {
   echo "##[endgroup]"
 
   echo "##[group]Install Development Tools"
-  sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \
-    fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils parted \
-    pax perf python-packaging python-setuptools qemu-guest-agent ksh samba \
-    sysstat rng-tools rsync wget xxhash
+  sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \
+    fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \
+    parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \
+    samba sysstat rng-tools rsync wget xxhash
   echo "##[endgroup]"
 }
 
@@ -30,11 +30,11 @@ function debian() {
 
   echo "##[group]Install Development Tools"
   sudo apt-get install -y \
-    acl alien attr autoconf bc cpio curl dbench dh-python dkms fakeroot \
-    fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev libaio-dev \
-    libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev libelf-dev \
-    libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev libtool \
-    libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
+    acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \
+    fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \
+    libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \
+    libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
+    libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
     lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
     python3-cffi python3-dev python3-distlib python3-packaging \
     python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
@@ -66,16 +66,23 @@ function rhel() {
   echo "##[endgroup]"
 
   echo "##[group]Install Development Tools"
-  sudo dnf group install -y "Development Tools"
+
+  # Alma wants "Development Tools", Fedora 41 wants "development-tools"
+  if ! sudo dnf group install -y "Development Tools" ; then
+    echo "Trying 'development-tools' instead of 'Development Tools'"
+    sudo dnf group install -y development-tools
+  fi
+
   sudo dnf install -y \
-    acl attr bc bzip2 curl dbench dkms elfutils-libelf-devel fio gdb git \
-    jq kernel-rpm-macros ksh libacl-devel libaio-devel libargon2-devel \
-    libattr-devel libblkid-devel libcurl-devel libffi-devel ncompress \
-    libselinux-devel libtirpc-devel libtool libudev-devel libuuid-devel \
-    lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \
-    python3 python3-cffi python3-devel python3-packaging kernel-devel \
-    python3-setuptools qemu-guest-agent rng-tools rpcgen rpm-build rsync \
-    samba sysstat systemd watchdog wget xfsprogs-devel xxhash zlib-devel
+    acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \
+    gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \
+    libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \
+    ncompress libselinux-devel libtirpc-devel libtool libudev-devel \
+    libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \
+    parted perf python3 python3-cffi python3-devel python3-packaging \
+    kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \
+    rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \
+    zlib-devel
   echo "##[endgroup]"
 }
 
@@ -111,6 +118,7 @@ case "$1" in
     archlinux
     ;;
   debian*)
+    echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
     debian
     echo "##[group]Install Debian specific"
     sudo apt-get install -yq linux-perf dh-sequence-dkms

.github/workflows/scripts/qemu-4-build.sh

@@ -83,7 +83,7 @@ function rpm_build_and_install() {
   echo "##[endgroup]"
 
   echo "##[group]Install"
-  run sudo dnf -y --skip-broken localinstall $(ls *.rpm | grep -v src.rpm)
+  run sudo dnf -y --nobest install $(ls *.rpm | grep -v src.rpm)
   echo "##[endgroup]"
 
 }

.github/workflows/scripts/qemu-5-setup.sh

@@ -14,17 +14,21 @@ PID=$(pidof /usr/bin/qemu-system-x86_64)
 tail --pid=$PID -f /dev/null
 sudo virsh undefine openzfs
 
-# definitions of per operating system
+# default values per test vm:
+VMs=2
+CPU=2
+
+# cpu pinning
+CPUSET=("0,1" "2,3")
+
 case "$OS" in
   freebsd*)
-    VMs=2
-    CPU=3
+    # FreeBSD can't be optimized via ksmtuned
     RAM=6
     ;;
   *)
-    VMs=2
-    CPU=3
-    RAM=7
+    # Linux can be optimized via ksmtuned
+    RAM=8
     ;;
 esac
 
@@ -73,6 +77,7 @@ EOF
     --cpu host-passthrough \
     --virt-type=kvm --hvm \
     --vcpus=$CPU,sockets=1 \
+    --cpuset=${CPUSET[$((i-1))]} \
     --memory $((1024*RAM)) \
     --memballoon model=virtio \
     --graphics none \

.github/workflows/zfs-qemu.yml

@@ -22,8 +22,8 @@ jobs:
       - name: Generate OS config and CI type
         id: os
         run: |
-          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13-4r", "freebsd14-0r", "freebsd14-1s", "ubuntu20", "ubuntu22", "ubuntu24"]'
-          QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora40", "freebsd13-3r", "freebsd14-1r", "ubuntu24"]'
+          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora40", "fedora41", "freebsd13-4r", "freebsd14-0r", "freebsd14-1s", "ubuntu20", "ubuntu22", "ubuntu24"]'
+          QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora41", "freebsd13-3r", "freebsd14-1r", "ubuntu24"]'
           # determine CI type when running on PR
           ci_type="full"
           if ${{ github.event_name == 'pull_request' }}; then
@@ -46,7 +46,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # rhl:     almalinux8, almalinux9, centos-stream9, fedora39, fedora40
+        # rhl:     almalinux8, almalinux9, centos-stream9, fedora40, fedora41
         # debian:  debian11, debian12, ubuntu20, ubuntu22, ubuntu24
         # misc:    archlinux, tumbleweed
         # FreeBSD Release: freebsd13-3r, freebsd13-4r, freebsd14-0r, freebsd14-1r

META

@@ -2,7 +2,7 @@ Meta:          1
 Name:          zfs
 Branch:        1.0
 Version:       2.3.0
-Release:       rc2
+Release:       rc3
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS

cmd/zdb/zdb.c

@@ -1131,7 +1131,7 @@ dump_zap(objset_t *os, uint64_t object, void *data, size_t size)
 		    !!(zap_getflags(zc.zc_zap) & ZAP_FLAG_UINT64_KEY);
 
 		if (key64)
-			(void) printf("\t\t0x%010lx = ",
+			(void) printf("\t\t0x%010" PRIu64 "x = ",
 			    *(uint64_t *)attrp->za_name);
 		else
 			(void) printf("\t\t%s = ", attrp->za_name);
@@ -2152,14 +2152,21 @@ dump_brt(spa_t *spa)
 	if (dump_opt['T'] < 3)
 		return;
 
+	/* -TTT shows a per-vdev histograms; -TTTT shows all entries */
+	boolean_t do_histo = dump_opt['T'] == 3;
+
 	char dva[64];
-	printf("\n%-16s %-10s\n", "DVA", "REFCNT");
+
+	if (!do_histo)
+		printf("\n%-16s %-10s\n", "DVA", "REFCNT");
 
 	for (uint64_t vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) {
 		brt_vdev_t *brtvd = &brt->brt_vdevs[vdevid];
 		if (brtvd == NULL || !brtvd->bv_initiated)
 			continue;
 
+		uint64_t counts[64] = {};
+
 		zap_cursor_t zc;
 		zap_attribute_t *za = zap_attribute_alloc();
 		for (zap_cursor_init(&zc, brt->brt_mos, brtvd->bv_mos_entries);
@@ -2172,14 +2179,26 @@ dump_brt(spa_t *spa)
 			    za->za_integer_length, za->za_num_integers,
 			    &refcnt));
 
-			uint64_t offset = *(const uint64_t *)za->za_name;
+			if (do_histo)
+				counts[highbit64(refcnt)]++;
+			else {
+				uint64_t offset =
+				    *(const uint64_t *)za->za_name;
 
-			snprintf(dva, sizeof (dva), "%" PRIu64 ":%llx", vdevid,
-			    (u_longlong_t)offset);
-			printf("%-16s %-10llu\n", dva, (u_longlong_t)refcnt);
+				snprintf(dva, sizeof (dva), "%" PRIu64 ":%llx",
+				    vdevid, (u_longlong_t)offset);
+				printf("%-16s %-10llu\n", dva,
+				    (u_longlong_t)refcnt);
+			}
 		}
 		zap_cursor_fini(&zc);
 		zap_attribute_free(za);
+
+		if (do_histo) {
+			printf("\nBRT: vdev %" PRIu64
+			    ": DVAs with 2^n refcnts:\n", vdevid);
+			dump_histogram(counts, 64, 0);
+		}
 	}
 }
 
@@ -4266,6 +4285,10 @@ dump_uberblock(uberblock_t *ub, const char *header, const char *footer)
 	(void) printf("\ttimestamp = %llu UTC = %s",
 	    (u_longlong_t)ub->ub_timestamp, ctime(&timestamp));
 
+	char blkbuf[BP_SPRINTF_LEN];
+	snprintf_blkptr(blkbuf, sizeof (blkbuf), &ub->ub_rootbp);
+	(void) printf("\tbp = %s\n", blkbuf);
+
 	(void) printf("\tmmp_magic = %016llx\n",
 	    (u_longlong_t)ub->ub_mmp_magic);
 	if (MMP_VALID(ub)) {

cmd/zed/zed_disk_event.c

@@ -139,7 +139,8 @@ dev_event_nvlist(struct udev_device *dev)
 		 * is /dev/sda.
 		 */
 		struct udev_device *parent_dev = udev_device_get_parent(dev);
-		if ((value = udev_device_get_sysattr_value(parent_dev, "size"))
+		if (parent_dev != NULL &&
+		    (value = udev_device_get_sysattr_value(parent_dev, "size"))
 		    != NULL) {
 			uint64_t numval = DEV_BSIZE;
 

cmd/zfs/zfs_main.c

@@ -2162,6 +2162,7 @@ zfs_do_get(int argc, char **argv)
 	cb.cb_type = ZFS_TYPE_DATASET;
 
 	struct option long_options[] = {
+		{"json", no_argument, NULL, 'j'},
 		{"json-int", no_argument, NULL, ZFS_OPTION_JSON_NUMS_AS_INT},
 		{0, 0, 0, 0}
 	};
@@ -3760,8 +3761,13 @@ collect_dataset(zfs_handle_t *zhp, list_cbdata_t *cb)
 		if (cb->cb_json) {
 			if (pl->pl_prop == ZFS_PROP_NAME)
 				continue;
+			const char *prop_name;
+			if (pl->pl_prop != ZPROP_USERPROP)
+				prop_name = zfs_prop_to_name(pl->pl_prop);
+			else
+				prop_name = pl->pl_user_prop;
 			if (zprop_nvlist_one_property(
-			    zfs_prop_to_name(pl->pl_prop), propstr,
+			    prop_name, propstr,
 			    sourcetype, source, NULL, props,
 			    cb->cb_json_as_int) != 0)
 				nomem();
@@ -3852,6 +3858,7 @@ zfs_do_list(int argc, char **argv)
 	nvlist_t *data = NULL;
 
 	struct option long_options[] = {
+		{"json", no_argument, NULL, 'j'},
 		{"json-int", no_argument, NULL, ZFS_OPTION_JSON_NUMS_AS_INT},
 		{0, 0, 0, 0}
 	};
@@ -7436,9 +7443,15 @@ share_mount(int op, int argc, char **argv)
 	uint_t nthr;
 	jsobj = data = item = NULL;
 
+	struct option long_options[] = {
+		{"json", no_argument, NULL, 'j'},
+		{0, 0, 0, 0}
+	};
+
 	/* check options */
-	while ((c = getopt(argc, argv, op == OP_MOUNT ? ":ajRlvo:Of" : "al"))
-	    != -1) {
+	while ((c = getopt_long(argc, argv,
+	    op == OP_MOUNT ? ":ajRlvo:Of" : "al",
+	    op == OP_MOUNT ? long_options : NULL, NULL)) != -1) {
 		switch (c) {
 		case 'a':
 			do_all = 1;
@@ -8374,8 +8387,14 @@ zfs_do_channel_program(int argc, char **argv)
 	boolean_t sync_flag = B_TRUE, json_output = B_FALSE;
 	zpool_handle_t *zhp;
 
+	struct option long_options[] = {
+		{"json", no_argument, NULL, 'j'},
+		{0, 0, 0, 0}
+	};
+
 	/* check options */
-	while ((c = getopt(argc, argv, "nt:m:j")) != -1) {
+	while ((c = getopt_long(argc, argv, "nt:m:j", long_options,
+	    NULL)) != -1) {
 		switch (c) {
 		case 't':
 		case 'm': {
@@ -9083,7 +9102,13 @@ zfs_do_version(int argc, char **argv)
 	int c;
 	nvlist_t *jsobj = NULL, *zfs_ver = NULL;
 	boolean_t json = B_FALSE;
-	while ((c = getopt(argc, argv, "j")) != -1) {
+
+	struct option long_options[] = {
+		{"json", no_argument, NULL, 'j'},
+		{0, 0, 0, 0}
+	};
+
+	while ((c = getopt_long(argc, argv, "j", long_options, NULL)) != -1) {
 		switch (c) {
 		case 'j':
 			json = B_TRUE;