#!/bin/bash -xe PS4='+ ($LINENO) ' # Config # ########## WORKDIR=./work # sudo rm -r this folder if you want to re-run everything DLDIR=./downloads OUTDIR=./out #OUTUSB=/dev/sdb1 DEVEL_MODE=y ROOTCMD=sudo WGET="wget" #"wget --no-check-certificate" # You probably need to tweak version numbers if you have an HTTP 404 - Not found error KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.3.tar.xz KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/v7.0.0/kconfiglib.py KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.174.20_all.udeb UFTP_TARBALL_URL=http://downloads.sourceforge.net/project/uftp-multicast/source-tar/uftp-4.9.7.tar.gz BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-x86_64 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids # Utilities # ############# # From https://landley.net/writing/rootfs-programming.html # Its first argument is the new directory, and the rest of its arguments are executables to copy. function mkchroot { [ $# -lt 2 ] && return 0 dest=$1 shift for i in "$@" do # Get an absolute path for the file p=$i [ "${p:0:1}" == "/" ] || p=$(which $i) || true if [ ! -e "$p" ] then echo "mkchoot not found: $i" return 1 fi # Skip files that already exist at target. [ -s "$dest/$p" ] && continue # Create destination path d=$(echo "$p" | grep -o '.*/') && mkdir -p "$dest/$d" && # Copy file echo + cp --dereference --preserve=mode "$p" "$dest/$p" && cp --dereference --preserve=mode "$p" "$dest/$p" && # Recursively copy shared libraries' shared libraries. mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $? done } # Environement and dependencies # ################################# codename=$(lsb_release -sc || true) if [ "x$codename" != "xbuster" ] then cat >&2 <<EOT This script is tested only on GNU/Linux Debian 10 amd64 (aka Buster). The fastest way to have the right environment is : * download Debian 10 amd64 live * here : http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/ * or here : http://cdimage.debian.org/mirror/cdimage/archive/ * burn it or copy it on a USB stick (as raw, with "sudo cp XX.iso /dev/sdX") * alternatively launch a VM with it * run this script from there EOT exit 1 fi mkdir -p "$WORKDIR" "$DLDIR" "$OUTDIR" if [ ! -e "$DLDIR/apt-update-done" ] then $ROOTCMD apt-get update > "$DLDIR/apt-update-done" fi if [ ! -e "$WORKDIR/apt-install-done" ] then # Dependencies of this script (assuming default debian install or live) $ROOTCMD apt-get install wget libncurses5-dev coreutils syslinux # Dependencies for kernel building $ROOTCMD apt-get install build-essential flex bison # Dependancies for kernel tools [ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \ libdw-dev libaudit-dev libssl-dev libslang2-dev libnuma-dev \ systemtap-sdt-dev python-dev binutils-dev libiberty-dev libbabeltrace-ctf-dev # Optionnally qemu to run the result for santity checking [ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86 # Dependencies to put into the initrd $ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace $ROOTCMD apt-get install partclone fdisk udpcast gdisk efibootmgr tcpdump > "$WORKDIR/apt-install-done" fi # Kernel build setup # ###################### kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL) [ -s "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL" if [ ! -s "$WORKDIR/kernel/Makefile" ] then mkdir -p "$WORKDIR/kernel" tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel" fi if [ ! -s "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" ] then [ -s "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL" [ -s "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL" mkdir -p "$WORKDIR/kernel/scripts/Kconfiglib" patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" && \ cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" fi cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT" #!/usr/bin/env python import sys from kconfiglib import Kconfig, standard_config_filename, TRI_TO_STR, TRISTATE def sset(sym, value=None): # Default value if value == None: if sym.assignable: # find highest possible assignable value (last item of modifiable sorted tuple) value = sym.assignable[-1] else: print('%s is not modifiable at all for now'%sym.name) return True # Sanity check if isinstance(value, (int, long)) and value not in sym.assignable: print('%s can\'t be set to %s for now'%(sym.name,TRI_TO_STR[value])) return True # Idempotency check if isinstance(value, (int, long)): old_value = sym.tri_value else: old_value = sym.str_value if old_value == value: # No more_work return False # Set value if isinstance(value, (int, long)): print('%s=%s [was: %s]'%(sym.name,TRI_TO_STR[value],TRI_TO_STR[old_value])) else: print('%s=%s [was: %s]'%(sym.name,value,old_value)) sym.set_value(value) # plausible more_work to do return True kconf = Kconfig(sys.argv[1]) kconf.load_config(standard_config_filename()) debug = '--debug' in sys.argv; passes = 5 support_xz = 'HAVE_KERNEL_XZ' in kconf.syms print('support_xz == %i'%support_xz) i = 0 more_work = True while more_work and i < passes: more_work = False i += 1 print('Kconfiglib/customize.py pass %i'%i) for sym in kconf.defined_syms: # Default hostname is (none) and could make FreeBSD's dhcpd complain because unallowed '()' if sym.name == 'DEFAULT_HOSTNAME': more_work = sset(sym, 'eficast') or more_work # Embed initrd in the EFI bootable kernel if sym.name == 'INITRAMFS_SOURCE': more_work = sset(sym, '../initrd/') or more_work # Make kernel directly loadable by EFI, add USB3, Dell flash if sym.name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']: more_work = sset(sym) or more_work # Support soft RAID (linux) and hard RAID (some cards) if sym.name in ['DM_RAID', 'SCSI_LOWLEVEL', 'MEGARAID_SAS', 'MEGARAID_NEWGEN']: more_work = sset(sym) or more_work # If --debug passed as arg, make kernel aware of virtual drivers (used for testing eficast on qemu/kvm) if debug and sym.name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']: more_work = sset(sym) or more_work # Disable thing that are unneeded or annoying for the purpose of disk cloning if sym.name in [ 'HAMRADIO', 'HIBERNATION', 'KEYS', 'LOGO', 'NETFILTER', 'NETWORK_FILESYSTEMS', 'PCCARD', 'RFKILL', 'SECURITY', 'SOUND', 'SUSPEND', 'VIRTUALIZATION', 'WIRELESS', 'WLAN']: more_work = sset(sym, 0) or more_work # Compress everything with XZ if available (slower, smaller) if support_xz: if sym.name in ['KERNEL_XZ', 'RD_XZ']: # , 'INITRAMFS_COMPRESSION_XZ']: more_work = sset(sym) or more_work if sym.name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']: more_work = sset(sym, 0) or more_work if sym.name == 'INITRAMFS_COMPRESSION': more_work = sset(sym, '.xz') or more_work # Following generic actions should done only on visible TRISTATE symbols if sym.type == TRISTATE and sym.visibility > 0: # Build all available net/ethernet drivers if True in [ ('drivers/net/ethernet' in node.filename) for node in sym.nodes ]: more_work = sset(sym) or more_work # Try to get everything in kernel, not as a module (1=='m') if sym.tri_value == 1 and sym.assignable and 2 in sym.assignable: more_work = sset(sym) or more_work # Write .config even if some symbols are unset res = kconf.write_config(standard_config_filename()) if i == passes: print('ERROR : can\'t set some of kernel config symbols after %i passes'%passes) res = 1 sys.exit(res) EOT chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py" # Kernel prepare + make tools # ############################### ( cd "$WORKDIR/kernel" if [ ! -s .config ] then make defconfig if [ "x$DEVEL_MODE" == "xy" ] then extra="SCRIPT_ARG=--debug" else extra="" fi make scriptconfig SCRIPT=scripts/Kconfiglib/customize.py $extra fi ) if [ "x$DEVEL_MODE" == "xy" -a ! -s "$WORKDIR/kernel/tools/perf/perf" ] then ( cd "$WORKDIR/kernel" # Workaround : linux-3.16.57 (and others?) have make tools/perf broken, ignore it make tools/perf || true ) fi # Build additionnal tools from source # ####################################### uftp_tarball=$DLDIR/$(basename "$UFTP_TARBALL_URL") [ -s "$uftp_tarball" ] || $WGET -O "$uftp_tarball" "$UFTP_TARBALL_URL" if [ ! -s "$WORKDIR/uftp/uftpd.1" ] then mkdir -p "$WORKDIR/uftp" tar xf "$uftp_tarball" --strip-components=1 -C "$WORKDIR/uftp" fi if [ ! -s "$WORKDIR/uftp/uftpd" ] then ( cd "$WORKDIR/uftp/" make NO_ENCRYPTION=1 ) fi # Initial Ram Disk building (embed in kernel) # ############################################### if [ ! -L "$WORKDIR/initrd/var/lock" ] then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt,root,proc,root,sbin,sys,run/lock,run/uftpd,tmp,var/log} $ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/" $ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock" ln -s "/proc/mounts" "$WORKDIR/initrd/etc/mtab" ln -s "../run" "$WORKDIR/initrd/var/run" ln -s "../run/lock" "$WORKDIR/initrd/var/lock" fi if [ ! -s "$WORKDIR/initrd/bin/busybox" ] then [ -s "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL" cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" chmod +x "$WORKDIR/initrd/bin/busybox" fi if [ ! -L "$WORKDIR/initrd/init" ] then ln -s /bin/busybox "$WORKDIR/initrd/init" fi if [ ! -s "$WORKDIR/initrd/etc/keys.bmap" ] then # When using sudo with password auth, ask and cache pass first $ROOTCMD true # The following compound command will suck at asking pass $ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap" fi if [ ! -s "$WORKDIR/initrd/usr/sbin/partclone.restore" ] then ( set +x PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin" # Diagnostic tools mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump # Manpage display mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl # Cloning tools mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr sfdisk gdisk sgdisk udp-receiver # Some dyn-loaded libraries (ldd will not display them) mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0 cp -ar /lib/terminfo "$WORKDIR/initrd/lib/" ) fi # Workaround : kernel 3.2.102 "make tools/perf" do nothing. Other are broken. Prevent failing if perf missing if [ "x$DEVEL_MODE" == "xy" -a -s "$WORKDIR/kernel/tools/perf/perf"] then ( p="$WORKDIR/kernel/tools/perf/perf" cp -a "$p" "$WORKDIR/initrd/sbin/" set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi if [ ! -f "$WORKDIR/initrd/usr/share/groff/current/man.local" ] then mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/current/font "$WORKDIR/initrd/etc/groff/" cp /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/" cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/" cp -r /usr/share/groff/current/font/devascii "$WORKDIR/initrd/usr/share/groff/current/font/" cp -r /usr/share/groff/current/tmac "$WORKDIR/initrd/usr/share/groff/current/" cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/current/" fi p="$WORKDIR/initrd/usr/bin/uftpd" if [ ! -s "$p" ] then ( cd "$WORKDIR/uftp/" make NO_ENCRYPTION=1 DESTDIR="../initrd" install ) ( set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi p="$WORKDIR/kernel/tools/perf/perf" if [ "x$KERNEL_TOOLS" == "xy" -a ! -s "$p" ] then ( cp -a "$p" "$WORKDIR/initrd/sbin/" set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi if [ ! -s "$WORKDIR/initrd/usr/share/misc/pci.ids" ] then [ -s "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL" [ -s "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL" mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc" cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/" cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/" fi if [ ! -d "$WORKDIR/initrd/lib/firmware" ] then [ -s "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL" dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/" find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v fi cat > "$WORKDIR/initrd/etc/init.d/funcs" <<"EOF" rescue_shell() { echo "Something went wrong. Dropping to a shell." setsid cttyhack /bin/busybox sh sync umount /dev /sys /proc #umount /dev/pts /dev /sys /proc poweroff -d1 -f } tty_prog() { tty=/dev/tty$1; shift while true do echo "(re)spawning $* on $tty" >$tty setsid sh -c "exec $* <$tty >$tty 2>&1" sleep 2 done } network_up() { ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface do echo ip link set dev $iface up ip link set dev $iface up done sleep 8 # PHY link det. + IPv6 DAD & Autoconf } machine_info() { for k in system-manufacturer system-product-name \ baseboard-manufacturer baseboard-product-name \ bios-version bios-release-date do echo $k: $(dmidecode -s $k) done grep -F MemTotal: /proc/meminfo lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p' ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p' lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/' } network_show() { ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p' } start_uftpd() { # FIXME : -I is there as a bug workaround "address already in use" uftpd -q -B 2097152 -x2 -F /run/uftpd.csv -L /run/uftpd.log -t -D /run/uftpd -M ff02::42 -I 2/6 [ $? -eq 0 ] && netstat -nlu | grep -q ':1044 ' && echo -e '\033[42m' ----- UFTPD ready ----- '\033[0m' } EOF cat > "$WORKDIR/initrd/etc/inittab" <<"EOF" # Custom init scripts ::sysinit:/etc/init.d/rcS # Executes as root what is received via uftpd (no security at all) ::respawn:/bin/sinkdo /run/uftpd /run/sink # Standard things follow ::ctrlaltdel:/sbin/reboot ::shutdown:/sbin/swapoff -a ::shutdown:/bin/umount -a -r ::restart:/sbin/init tty2::askfirst:/bin/sh tty3::askfirst:/bin/sh tty4::askfirst:/bin/sh EOF cat > "$WORKDIR/initrd/etc/init.d/rcS" <<"EOF" #!/bin/busybox sh echo "rcS script started" # Load helper functions . /etc/init.d/funcs # Trace execution set -v # Setup links for programs supported by busybox /bin/busybox --install -s || rescue_shell # Mount pseudo-filesystems mount -t proc none /proc || rescue_shell mount -t sysfs none /sys || rescue_shell mount -t devtmpfs -o size=1m none /dev || rescue_shell ln -s /proc/self/fd/2 /dev/stderr #mkdir /dev/pts || rescue_shell #mount -t devpts none /dev/pts || rescue_shell # Allow using most of the RAM for rootfs mount -o remount -o size=80% / # Load keyboard map loadkmap < /etc/keys.bmap network_up efibootmgr -v machine_info network_show start_uftpd EOF chmod +x "$WORKDIR/initrd/etc/init.d/rcS" cat > "$WORKDIR/initrd/bin/sinkcat" <<"EOF" #!/bin/busybox sh if [ $# -ne 1 ] then cat <<EOT Usage: $(basename $0) <sink-dir> Concatenate then delete files as soon they appear in sink-dir to stdout. If multiple files are found in sink, the first in alphabetical order is choosen. <sink-dir> must not exists, this program must create it (avoiding mistakes). Dropping an empty file in dir-sink will clean exit this program. EOT exit 1 fi SINKDIR=$1 mkdir "$SINKDIR" && cd "$SINKDIR" if [ $? -ne 0 ] then echo "Cannot mkdir/chdir to '$SINKDIR'" >&2 exit 2 fi while true do f=$(ls | grep -v '~' | head -n1) if [ -n "$f" ] then if [ -f "$f" -a -r "$f" ] then size=$(stat -c'%s' -- "$f") # Do the actual work on the following line cat -- "$f" && rm -- "$f" >&2 # Normal exit condition if [ $size -eq 0 ] then cd / && rmdir -- "$SINKDIR" >&2 exit 0 fi else echo "'$SINKDIR/$f' is not a readable file" >&2 exit 3 fi fi sleep 1 done EOF chmod +x "$WORKDIR/initrd/bin/sinkcat" cat > "$WORKDIR/initrd/bin/sinkdo" <<"EOF" #!/bin/busybox sh if [ $# -ne 2 ] then cat <<EOT Usage: $(basename $0) <land-dir> <sink-dir> EOT exit 1 fi LANDDIR=$1 SINKDIR=$2 cd "$LANDDIR" || exit 2 curtask="(start)" while true do f=$(ls | grep -v '~' | head -n1) if [ -z "$f" ] then sleep 1 continue fi if [ ! -f "$f" -o ! -r "$f" ] then echo "'$LANDDIR/$f' is not a readable file" >&2 exit 2 fi size=$(stat -c'%s' -- "$f") task=${f:0:2} if [ "$curtask" == "$task" ] then # Next file of an already started task mv -- "$f" "$SINKDIR/" else # Switch to the next task if [ -d "$SINKDIR" ] then # Inform sinkcat about end-of-data touch -- "$SINKDIR/zz" wait res=$? # Error code from "./$f" # sinkcat always rmdir "$SINKDIR" on normal exit if [ $res -ne 0 -o -d "$SINKDIR" ] then echo "Task $task has ran into troubles" >&2 exit 3 fi fi echo "Switching from task $curtask to $task" >&2 curtask=$task if [ "$task" == "99" ] then # Normal exit condition rm -- "$f" >&2 echo "All tasks completed sucessfully" >&2 exit 0 else # Start a new task chmod +x -- "$f" # XXX Checks on $f (is a script ?) sinkcat $SINKDIR | "./$f" & while [ ! -d "$SINKDIR" ]; do sleep 1; done rm -- "$f" >&2 fi fi done EOF chmod +x "$WORKDIR/initrd/bin/sinkdo" # Kernel build (with embed initramfs) # ####################################### ( cd "$WORKDIR/kernel" # Workaround : some kernel version forget to update embed initramfs in certain cases [ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz nproc=$(nproc --all) make -j ${nproc:-4} ) # Copy / run result EFI file # ############################## # Workaround : direct kernel boot with libvirt/virt-manager do a chown on BOOTX64.EFI and cp won't overwrite [ -f "$OUTDIR/BOOTX64.EFI" ] && rm -f "$OUTDIR/BOOTX64.EFI" cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI" if [ -n "$OUTUSB" -a -b "$OUTUSB" ] then mkdir -p "$WORKDIR/mountpoint" mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB" $ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint" $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD umount "$WORKDIR/mountpoint" fi [ "x$DEVEL_MODE" == "xy" ] && qemu-system-x86_64 -M q35 -m 256 -kernel "$OUTDIR/BOOTX64.EFI" -enable-kvm -serial stdio -append "console=ttyAMA0 console=ttyS0"