From ef8b872301c5fbeeea3b0410b369b8f36584cd65 Mon Sep 17 00:00:00 2001
From: Warner Losh <imp@FreeBSD.org>
Date: Wed, 17 Aug 2022 11:33:58 -0600
Subject: [PATCH] stand: Flesh out the zfs image creation

Move the mbr non-geli zfs cases to no-priv creation with makefs / mkimg.
Add comments about the weird thing we do for MBR + ZFS + Legacy.  Add
comments about other architectures. Still need to think through how to
leverage a completed universe to do all the architectures...

Sponsored by:		Netflix
---
 tools/boot/rootgen.sh | 150 +++++++++++++++---------------------------
 1 file changed, 54 insertions(+), 96 deletions(-)

diff --git a/tools/boot/rootgen.sh b/tools/boot/rootgen.sh
index f087afa5ad49..06e8a1ec8245 100755
--- a/tools/boot/rootgen.sh
+++ b/tools/boot/rootgen.sh
@@ -214,35 +214,21 @@ mk_nogeli_mbr_zfs_legacy() {
     bios=$7
     pool=nogeli-mbr-zfs-legacy
 
-    dd if=/dev/zero of=${img} count=1 seek=$((200 * 1024 * 1024 / 512))
-    md=$(mdconfig -f ${img})
-    gpart create -s mbr ${md}
-    gpart add -t freebsd ${md}
-    gpart set -a active -i 1 ${md}
-    gpart create -s bsd ${md}s1
-    gpart add -t freebsd-zfs ${md}s1
-    # install-boot will make this bootable
-    zpool create -O mountpoint=none -R ${mntpt} ${pool} ${md}s1a
-    zpool set bootfs=${pool} ${pool}
-    zfs create -po mountpoint=/ ${pool}/ROOT/default
-    # NB: The online guides go nuts customizing /var and other mountpoints here, no need
-    cpsys ${src} ${mntpt}
-    # need to make a couple of tweaks
-    cat >> ${mntpt}/boot/loader.conf <<EOF
-cryptodev_load=YES
-zfs_load=YES
-EOF
-    cp /boot/kernel/acl_nfs4.ko ${mntpt}/boot/kernel/acl_nfs4.ko
-    cp /boot/kernel/cryptodev.ko ${mntpt}/boot/kernel/cryptodev.ko
-    cp /boot/kernel/zfs.ko ${mntpt}/boot/kernel/zfs.ko
-    # end tweaks
-    zfs umount -f ${pool}/ROOT/default
-    zfs set mountpoint=none ${pool}/ROOT/default
-    zpool set bootfs=${pool}/ROOT/default ${pool}
-    zpool set autoexpand=on ${pool}
-    zpool export ${pool}
-    ${SRCTOP}/tools/boot/install-boot.sh -g ${geli} -s ${scheme} -f ${fs} -b ${bios} -d ${src} ${md}
-    mdconfig -d -u ${md}
+    zfs_extra $src $dst
+    makefs -t zfs -s 200m \
+	-o poolname=${pool} -o bootfs=${pool} -o rootpath=/ \
+	${img}.s1a ${src} ${dst}
+    # The old boot1/boot2 boot split is also used by zfs. We need to extract zfsboot1
+    # from this image. Since there's no room in the mbr format for the rest of the loader,
+    # it will load the zfsboot loader from the reserved for bootloader area of the ZFS volume
+    # being booted, hence the need to dd it into the raw img later.
+    # Please note: zfsboot only works with partition 'a' which must be the root
+    # partition / zfs volume
+    dd if=${src}/boot/zfsboot of=${dst}/zfsboot1 count=1
+    mkimg -s bsd -b ${dst}zfsboot1 -p freebsd-zfs:=${img}.s1a -o ${img}.s1
+    dd if=${src}/boot/zfsboot of=${img}.s1a skip=1 seek=1024
+    mkimg -a 1 -s mbr -b ${src}/boot/mbr -p freebsd:=${img}.s1 -o ${img}
+    rm -rf ${dst}
 }
 
 mk_nogeli_mbr_zfs_uefi() {
@@ -255,36 +241,14 @@ mk_nogeli_mbr_zfs_uefi() {
     bios=$7
     pool=nogeli-mbr-zfs-uefi
 
-    dd if=/dev/zero of=${img} count=1 seek=$((200 * 1024 * 1024 / 512))
-    md=$(mdconfig -f ${img})
-    gpart create -s mbr ${md}
-    gpart add -t efi -s ${espsize}k ${md}
-    gpart add -t freebsd ${md}
-    gpart set -a active -i 2 ${md}
-    gpart create -s bsd ${md}s2
-    gpart add -t freebsd-zfs ${md}s2
-    # install-boot will make this bootable
-    zpool create -O mountpoint=none -R ${mntpt} ${pool} ${md}s2a
-    zpool set bootfs=${pool} ${pool}
-    zfs create -po mountpoint=/ ${pool}/ROOT/default
-    # NB: The online guides go nuts customizing /var and other mountpoints here, no need
-    cpsys ${src} ${mntpt}
-    # need to make a couple of tweaks
-    cat >> ${mntpt}/boot/loader.conf <<EOF
-cryptodev_load=YES
-zfs_load=YES
-EOF
-    cp /boot/kernel/acl_nfs4.ko ${mntpt}/boot/kernel/acl_nfs4.ko
-    cp /boot/kernel/cryptodev.ko ${mntpt}/boot/kernel/cryptodev.ko
-    cp /boot/kernel/zfs.ko ${mntpt}/boot/kernel/zfs.ko
-    # end tweaks
-    zfs umount -f ${pool}/ROOT/default
-    zfs set mountpoint=none ${pool}/ROOT/default
-    zpool set bootfs=${pool}/ROOT/default ${pool}
-    zpool set autoexpand=on ${pool}
-    zpool export ${pool}
-    ${SRCTOP}/tools/boot/install-boot.sh -g ${geli} -s ${scheme} -f ${fs} -b ${bios} -d ${src} ${md}
-    mdconfig -d -u ${md}
+    zfs_extra $src $dst
+    make_esp_file ${img}.s1 ${espsize} ${src}/boot/loader.efi
+    makefs -t zfs -s 200m \
+	-o poolname=${pool} -o bootfs=${pool} -o rootpath=/ \
+	${img}.s2a ${src} ${dst}
+    mkimg -s bsd -b ${dst}zfsboot1 -p freebsd-zfs:=${img}.s2a -o ${img}.s2
+    mkimg -a 1 -s mbr -b ${src}/boot/mbr -p efi:=${img}.s1 -p freebsd:=${img}.s2 -o ${img}
+    rm -rf ${dst}
 }
 
 mk_nogeli_mbr_zfs_both() {
@@ -297,36 +261,21 @@ mk_nogeli_mbr_zfs_both() {
     bios=$7
     pool=nogeli-mbr-zfs-both
 
-    dd if=/dev/zero of=${img} count=1 seek=$((200 * 1024 * 1024 / 512))
-    md=$(mdconfig -f ${img})
-    gpart create -s mbr ${md}
-    gpart add -t efi -s  ${espsize}k ${md}
-    gpart add -t freebsd ${md}
-    gpart set -a active -i 2 ${md}
-    gpart create -s bsd ${md}s2
-    gpart add -t freebsd-zfs ${md}s2
-    # install-boot will make this bootable
-    zpool create -O mountpoint=none -R ${mntpt} ${pool} ${md}s2a
-    zpool set bootfs=${pool} ${pool}
-    zfs create -po mountpoint=/ ${pool}/ROOT/default
-    # NB: The online guides go nuts customizing /var and other mountpoints here, no need
-    cpsys ${src} ${mntpt}
-    # need to make a couple of tweaks
-    cat >> ${mntpt}/boot/loader.conf <<EOF
-cryptodev_load=YES
-zfs_load=YES
-EOF
-    cp /boot/kernel/acl_nfs4.ko ${mntpt}/boot/kernel/acl_nfs4.ko
-    cp /boot/kernel/cryptodev.ko ${mntpt}/boot/kernel/cryptodev.ko
-    cp /boot/kernel/zfs.ko ${mntpt}/boot/kernel/zfs.ko
-    # end tweaks
-    zfs umount -f ${pool}/ROOT/default
-    zfs set mountpoint=none ${pool}/ROOT/default
-    zpool set bootfs=${pool}/ROOT/default ${pool}
-    zpool set autoexpand=on ${pool}
-    zpool export ${pool}
-    ${SRCTOP}/tools/boot/install-boot.sh -g ${geli} -s ${scheme} -f ${fs} -b ${bios} -d ${src} ${md}
-    mdconfig -d -u ${md}
+    zfs_extra $src $dst
+    make_esp_file ${img}.s1 ${espsize} ${src}/boot/loader.efi
+    makefs -t zfs -s 200m \
+	-o poolname=${pool} -o bootfs=${pool} -o rootpath=/ \
+	${img}.s2a ${src} ${dst}
+    # The old boot1/boot2 boot split is also used by zfs. We need to extract zfsboot1
+    # from this image. Since there's no room in the mbr format for the rest of the loader,
+    # it will load the zfsboot loader from the reserved for bootloader area of the ZFS volume
+    # being booted, hence the need to dd it into the raw img later.
+    # Please note: zfsboot only works with partition 'a' which must be the root
+    # partition / zfs volume
+    dd if=${src}/boot/zfsboot of=${dst}/zfsboot1 count=1
+    mkimg -s bsd -b ${dst}zfsboot1 -p freebsd-zfs:=${img}.s2a -o ${img}.s2
+    dd if=${src}/boot/zfsboot of=${img}.s1a skip=1 seek=1024
+    mkimg -a 1 -s mbr -b ${src}/boot/mbr -p efi:=${img}.s1 -p freebsd:=${img}.s2 -o ${img}
 }
 
 mk_geli_gpt_ufs_legacy() {
@@ -779,7 +728,7 @@ fi
 for arch in amd64; do
     for geli in nogeli; do # geli
 	for scheme in gpt mbr; do
-	    for fs in ufs ; do # zfs
+	    for fs in ufs zfs; do
 		for bios in legacy uefi both; do
 		    make_one_image ${arch} ${geli} ${scheme} ${fs} ${bios}
 		done
@@ -787,6 +736,7 @@ for arch in amd64; do
 	done
     done
 done
+    # We should also do a cd image for amd64 here
 echo ${IMGDIR}/all.sh
 
 rmdir ${MNTPT}
@@ -800,24 +750,32 @@ for arch in i386; do
 	for scheme in gpt mbr; do
 	    for fs in ufs zfs; do
 		for bios in legacy; do
+		    # The legacy boot is shared with amd64 so those routines could
+		    # likely be used here.
 		    make_one_image ${arch} ${geli} ${scheme} ${fs} ${bios}
 		done
 	    done
 	done
     done
 done
+    # We should also do a cd image for i386 here
 
 for arch in arm aarch64; do
+    geli=nogeli		# I don't think geli boot works / is supported on arm
     for scheme in gpt mbr; do
-	fs=ufs
-	bios=efi
+      for fs in ufs zfs; do
+	bios=efi # Note: arm has some uboot support with ufs, what to do?
 	make_one_image ${arch} ${geli} ${scheme} ${fs} ${bios}
+      done
     done
 done
 
-for arch in powerpc powerpc64; do
-    for scheme in ppc-wtf; do
-	fs=ufs
+# It's not clear that the nested looping paradigm is best for powerpc
+# due to its diversity.
+for arch in powerpc powerpc64 powerpc64le; do
+    geli=nogeli
+    for scheme in apm gpt; do
+	fs=ufs # zfs + gpt might be supported?
 	for bios in ofw uboot chrp; do
 	    make_one_image ${arch} ${geli} ${scheme} ${fs} ${bios}
 	done
@@ -826,7 +784,7 @@ done
 
 for arch in riscv; do
     geli=nogeli
-    fs=ufs
+    fs=ufs		# Generic ZFS booting support with efi?
     scheme=gpt
     bios=efi
     make_one_image ${arch} ${geli} ${scheme} ${fs} ${bios}