#!/bin/bash -xe PS4='+ ($LINENO) ' # Config # ########## WORKDIR=./work # sudo rm -r this folder if you want to re-run everything DLDIR=./downloads OUTDIR=./out #OUTUSB=/dev/sdb1 DEVEL_MODE=y ROOTCMD=sudo WGET="wget" #"wget --no-check-certificate" # You probably need to tweak version numbers if you have an HTTP 404 - Not found error KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.3.tar.xz KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/v7.0.0/kconfiglib.py KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.174.20_all.udeb UFTP_TARBALL_URL=http://downloads.sourceforge.net/project/uftp-multicast/source-tar/uftp-4.9.7.tar.gz BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-x86_64 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids # Utilities # ############# # From https://landley.net/writing/rootfs-programming.html # Its first argument is the new directory, and the rest of its arguments are executables to copy. function mkchroot { [ $# -lt 2 ] && return 0 dest=$1 shift for i in "$@" do # Get an absolute path for the file p=$i [ "${p:0:1}" == "/" ] || p=$(which $i) || true if [ ! -e "$p" ] then echo "mkchoot not found: $i" return 1 fi # Skip files that already exist at target. [ -s "$dest/$p" ] && continue # Create destination path d=$(echo "$p" | grep -o '.*/') && mkdir -p "$dest/$d" && # Copy file echo + cp --dereference --preserve=mode "$p" "$dest/$p" && cp --dereference --preserve=mode "$p" "$dest/$p" && # Recursively copy shared libraries' shared libraries. mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $? done } # Environement and dependencies # ################################# codename=$(lsb_release -sc || true) if [ "x$codename" != "xbuster" ] then cat >&2 < "$DLDIR/apt-update-done" fi if [ ! -e "$WORKDIR/apt-install-done" ] then # Dependencies of this script (assuming default debian install or live) $ROOTCMD apt-get install wget libncurses5-dev coreutils syslinux # Dependencies for kernel building $ROOTCMD apt-get install build-essential flex bison # Dependancies for kernel tools [ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \ libdw-dev libaudit-dev libssl-dev libslang2-dev libnuma-dev \ systemtap-sdt-dev python-dev binutils-dev libiberty-dev libbabeltrace-ctf-dev # Optionnally qemu to run the result for santity checking [ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86 # Dependencies to put into the initrd $ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace $ROOTCMD apt-get install partclone fdisk udpcast gdisk efibootmgr tcpdump > "$WORKDIR/apt-install-done" fi # Kernel build setup # ###################### kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL) [ -s "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL" if [ ! -s "$WORKDIR/kernel/Makefile" ] then mkdir -p "$WORKDIR/kernel" tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel" fi if [ ! -s "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" ] then [ -s "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL" [ -s "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL" mkdir -p "$WORKDIR/kernel/scripts/Kconfiglib" patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" && \ cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" fi cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT" #!/usr/bin/env python import sys from kconfiglib import Kconfig, standard_config_filename, TRI_TO_STR, TRISTATE def sset(sym, value=None): # Default value if value == None: if sym.assignable: # find highest possible assignable value (last item of modifiable sorted tuple) value = sym.assignable[-1] else: print('%s is not modifiable at all for now'%sym.name) return True # Sanity check if isinstance(value, (int, long)) and value not in sym.assignable: print('%s can\'t be set to %s for now'%(sym.name,TRI_TO_STR[value])) return True # Idempotency check if isinstance(value, (int, long)): old_value = sym.tri_value else: old_value = sym.str_value if old_value == value: # No more_work return False # Set value if isinstance(value, (int, long)): print('%s=%s [was: %s]'%(sym.name,TRI_TO_STR[value],TRI_TO_STR[old_value])) else: print('%s=%s [was: %s]'%(sym.name,value,old_value)) sym.set_value(value) # plausible more_work to do return True kconf = Kconfig(sys.argv[1]) kconf.load_config(standard_config_filename()) debug = '--debug' in sys.argv; passes = 5 support_xz = 'HAVE_KERNEL_XZ' in kconf.syms print('support_xz == %i'%support_xz) i = 0 more_work = True while more_work and i < passes: more_work = False i += 1 print('Kconfiglib/customize.py pass %i'%i) for sym in kconf.defined_syms: # Default hostname is (none) and could make FreeBSD's dhcpd complain because unallowed '()' if sym.name == 'DEFAULT_HOSTNAME': more_work = sset(sym, 'eficast') or more_work # Embed initrd in the EFI bootable kernel if sym.name == 'INITRAMFS_SOURCE': more_work = sset(sym, '../initrd/') or more_work # Make kernel directly loadable by EFI, add USB3, Dell flash if sym.name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']: more_work = sset(sym) or more_work # Support soft RAID (linux) and hard RAID (some cards) if sym.name in ['DM_RAID', 'SCSI_LOWLEVEL', 'MEGARAID_SAS', 'MEGARAID_NEWGEN']: more_work = sset(sym) or more_work # If --debug passed as arg, make kernel aware of virtual drivers (used for testing eficast on qemu/kvm) if debug and sym.name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']: more_work = sset(sym) or more_work # Disable thing that are unneeded or annoying for the purpose of disk cloning if sym.name in [ 'HAMRADIO', 'HIBERNATION', 'KEYS', 'LOGO', 'NETFILTER', 'NETWORK_FILESYSTEMS', 'PCCARD', 'RFKILL', 'SECURITY', 'SOUND', 'SUSPEND', 'VIRTUALIZATION', 'WIRELESS', 'WLAN']: more_work = sset(sym, 0) or more_work # Compress everything with XZ if available (slower, smaller) if support_xz: if sym.name in ['KERNEL_XZ', 'RD_XZ']: # , 'INITRAMFS_COMPRESSION_XZ']: more_work = sset(sym) or more_work if sym.name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']: more_work = sset(sym, 0) or more_work if sym.name == 'INITRAMFS_COMPRESSION': more_work = sset(sym, '.xz') or more_work # Following generic actions should done only on visible TRISTATE symbols if sym.type == TRISTATE and sym.visibility > 0: # Build all available net/ethernet drivers if True in [ ('drivers/net/ethernet' in node.filename) for node in sym.nodes ]: more_work = sset(sym) or more_work # Try to get everything in kernel, not as a module (1=='m') if sym.tri_value == 1 and sym.assignable and 2 in sym.assignable: more_work = sset(sym) or more_work # Write .config even if some symbols are unset res = kconf.write_config(standard_config_filename()) if i == passes: print('ERROR : can\'t set some of kernel config symbols after %i passes'%passes) res = 1 sys.exit(res) EOT chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py" # Kernel prepare + make tools # ############################### ( cd "$WORKDIR/kernel" if [ ! -s .config ] then make defconfig if [ "x$DEVEL_MODE" == "xy" ] then extra="SCRIPT_ARG=--debug" else extra="" fi make scriptconfig SCRIPT=scripts/Kconfiglib/customize.py $extra fi ) if [ "x$DEVEL_MODE" == "xy" -a ! -s "$WORKDIR/kernel/tools/perf/perf" ] then ( cd "$WORKDIR/kernel" # Workaround : linux-3.16.57 (and others?) have make tools/perf broken, ignore it make tools/perf || true ) fi # Build additionnal tools from source # ####################################### uftp_tarball=$DLDIR/$(basename "$UFTP_TARBALL_URL") [ -s "$uftp_tarball" ] || $WGET -O "$uftp_tarball" "$UFTP_TARBALL_URL" if [ ! -s "$WORKDIR/uftp/uftpd.1" ] then mkdir -p "$WORKDIR/uftp" tar xf "$uftp_tarball" --strip-components=1 -C "$WORKDIR/uftp" fi if [ ! -s "$WORKDIR/uftp/uftpd" ] then ( cd "$WORKDIR/uftp/" make NO_ENCRYPTION=1 ) fi # Initial Ram Disk building (embed in kernel) # ############################################### if [ ! -L "$WORKDIR/initrd/var/lock" ] then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt,root,proc,root,sbin,sys,run/lock,run/uftpd,tmp,var/log} $ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/" $ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock" ln -s "/proc/mounts" "$WORKDIR/initrd/etc/mtab" ln -s "../run" "$WORKDIR/initrd/var/run" ln -s "../run/lock" "$WORKDIR/initrd/var/lock" fi if [ ! -s "$WORKDIR/initrd/bin/busybox" ] then [ -s "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL" cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" chmod +x "$WORKDIR/initrd/bin/busybox" fi if [ ! -L "$WORKDIR/initrd/init" ] then ln -s /bin/busybox "$WORKDIR/initrd/init" fi if [ ! -s "$WORKDIR/initrd/etc/keys.bmap" ] then # When using sudo with password auth, ask and cache pass first $ROOTCMD true # The following compound command will suck at asking pass $ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap" fi if [ ! -s "$WORKDIR/initrd/usr/sbin/partclone.restore" ] then ( set +x PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin" # Diagnostic tools mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump # Manpage display mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl # Cloning tools mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr sfdisk gdisk sgdisk udp-receiver # Some dyn-loaded libraries (ldd will not display them) mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0 cp -ar /lib/terminfo "$WORKDIR/initrd/lib/" ) fi # Workaround : kernel 3.2.102 "make tools/perf" do nothing. Other are broken. Prevent failing if perf missing if [ "x$DEVEL_MODE" == "xy" -a -s "$WORKDIR/kernel/tools/perf/perf"] then ( p="$WORKDIR/kernel/tools/perf/perf" cp -a "$p" "$WORKDIR/initrd/sbin/" set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi if [ ! -f "$WORKDIR/initrd/usr/share/groff/current/man.local" ] then mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/current/font "$WORKDIR/initrd/etc/groff/" cp /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/" cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/" cp -r /usr/share/groff/current/font/devascii "$WORKDIR/initrd/usr/share/groff/current/font/" cp -r /usr/share/groff/current/tmac "$WORKDIR/initrd/usr/share/groff/current/" cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/current/" fi p="$WORKDIR/initrd/usr/bin/uftpd" if [ ! -s "$p" ] then ( cd "$WORKDIR/uftp/" make NO_ENCRYPTION=1 DESTDIR="../initrd" install ) ( set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi p="$WORKDIR/kernel/tools/perf/perf" if [ "x$KERNEL_TOOLS" == "xy" -a ! -s "$p" ] then ( cp -a "$p" "$WORKDIR/initrd/sbin/" set +x mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') ) fi if [ ! -s "$WORKDIR/initrd/usr/share/misc/pci.ids" ] then [ -s "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL" [ -s "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL" mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc" cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/" cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/" fi if [ ! -d "$WORKDIR/initrd/lib/firmware" ] then [ -s "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL" dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/" find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v fi cat > "$WORKDIR/initrd/etc/init.d/funcs" <<"EOF" rescue_shell() { echo "Something went wrong. Dropping to a shell." setsid cttyhack /bin/busybox sh sync umount /dev /sys /proc #umount /dev/pts /dev /sys /proc poweroff -d1 -f } tty_prog() { tty=/dev/tty$1; shift while true do echo "(re)spawning $* on $tty" >$tty setsid sh -c "exec $* <$tty >$tty 2>&1" sleep 2 done } network_up() { ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface do echo ip link set dev $iface up ip link set dev $iface up done sleep 8 # PHY link det. + IPv6 DAD & Autoconf } machine_info() { for k in system-manufacturer system-product-name \ baseboard-manufacturer baseboard-product-name \ bios-version bios-release-date do echo $k: $(dmidecode -s $k) done grep -F MemTotal: /proc/meminfo lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p' ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p' lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/' } network_show() { ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p' } start_uftpd() { # FIXME : -I is there as a bug workaround "address already in use" uftpd -q -B 2097152 -x2 -F /run/uftpd.csv -L /run/uftpd.log -t -D /run/uftpd -M ff02::42 -I 2/6 [ $? -eq 0 ] && netstat -nlu | grep -q ':1044 ' && echo -e '\033[42m' ----- UFTPD ready ----- '\033[0m' } EOF cat > "$WORKDIR/initrd/etc/inittab" <<"EOF" # Custom init scripts ::sysinit:/etc/init.d/rcS # Executes as root what is received via uftpd (no security at all) ::respawn:/bin/sinkdo /run/uftpd /run/sink # Standard things follow ::ctrlaltdel:/sbin/reboot ::shutdown:/sbin/swapoff -a ::shutdown:/bin/umount -a -r ::restart:/sbin/init tty2::askfirst:/bin/sh tty3::askfirst:/bin/sh tty4::askfirst:/bin/sh EOF cat > "$WORKDIR/initrd/etc/init.d/rcS" <<"EOF" #!/bin/busybox sh echo "rcS script started" # Load helper functions . /etc/init.d/funcs # Trace execution set -v # Setup links for programs supported by busybox /bin/busybox --install -s || rescue_shell # Mount pseudo-filesystems mount -t proc none /proc || rescue_shell mount -t sysfs none /sys || rescue_shell mount -t devtmpfs -o size=1m none /dev || rescue_shell ln -s /proc/self/fd/2 /dev/stderr #mkdir /dev/pts || rescue_shell #mount -t devpts none /dev/pts || rescue_shell # Allow using most of the RAM for rootfs mount -o remount -o size=80% / # Load keyboard map loadkmap < /etc/keys.bmap network_up efibootmgr -v machine_info network_show start_uftpd EOF chmod +x "$WORKDIR/initrd/etc/init.d/rcS" cat > "$WORKDIR/initrd/bin/sinkcat" <<"EOF" #!/bin/busybox sh if [ $# -ne 1 ] then cat < Concatenate then delete files as soon they appear in sink-dir to stdout. If multiple files are found in sink, the first in alphabetical order is choosen. must not exists, this program must create it (avoiding mistakes). Dropping an empty file in dir-sink will clean exit this program. EOT exit 1 fi SINKDIR=$1 mkdir "$SINKDIR" && cd "$SINKDIR" if [ $? -ne 0 ] then echo "Cannot mkdir/chdir to '$SINKDIR'" >&2 exit 2 fi while true do f=$(ls | grep -v '~' | head -n1) if [ -n "$f" ] then if [ -f "$f" -a -r "$f" ] then size=$(stat -c'%s' -- "$f") # Do the actual work on the following line cat -- "$f" && rm -- "$f" >&2 # Normal exit condition if [ $size -eq 0 ] then cd / && rmdir -- "$SINKDIR" >&2 exit 0 fi else echo "'$SINKDIR/$f' is not a readable file" >&2 exit 3 fi fi sleep 1 done EOF chmod +x "$WORKDIR/initrd/bin/sinkcat" cat > "$WORKDIR/initrd/bin/sinkdo" <<"EOF" #!/bin/busybox sh if [ $# -ne 2 ] then cat < EOT exit 1 fi LANDDIR=$1 SINKDIR=$2 cd "$LANDDIR" || exit 2 curtask="(start)" while true do f=$(ls | grep -v '~' | head -n1) if [ -z "$f" ] then sleep 1 continue fi if [ ! -f "$f" -o ! -r "$f" ] then echo "'$LANDDIR/$f' is not a readable file" >&2 exit 2 fi size=$(stat -c'%s' -- "$f") task=${f:0:2} if [ "$curtask" == "$task" ] then # Next file of an already started task mv -- "$f" "$SINKDIR/" else # Switch to the next task if [ -d "$SINKDIR" ] then # Inform sinkcat about end-of-data touch -- "$SINKDIR/zz" wait res=$? # Error code from "./$f" # sinkcat always rmdir "$SINKDIR" on normal exit if [ $res -ne 0 -o -d "$SINKDIR" ] then echo "Task $task has ran into troubles" >&2 exit 3 fi fi echo "Switching from task $curtask to $task" >&2 curtask=$task if [ "$task" == "99" ] then # Normal exit condition rm -- "$f" >&2 echo "All tasks completed sucessfully" >&2 exit 0 else # Start a new task chmod +x -- "$f" # XXX Checks on $f (is a script ?) sinkcat $SINKDIR | "./$f" & while [ ! -d "$SINKDIR" ]; do sleep 1; done rm -- "$f" >&2 fi fi done EOF chmod +x "$WORKDIR/initrd/bin/sinkdo" # Kernel build (with embed initramfs) # ####################################### ( cd "$WORKDIR/kernel" # Workaround : some kernel version forget to update embed initramfs in certain cases [ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz nproc=$(nproc --all) make -j ${nproc:-4} ) # Copy / run result EFI file # ############################## # Workaround : direct kernel boot with libvirt/virt-manager do a chown on BOOTX64.EFI and cp won't overwrite [ -f "$OUTDIR/BOOTX64.EFI" ] && rm -f "$OUTDIR/BOOTX64.EFI" cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI" if [ -n "$OUTUSB" -a -b "$OUTUSB" ] then mkdir -p "$WORKDIR/mountpoint" mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB" $ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint" $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD umount "$WORKDIR/mountpoint" fi [ "x$DEVEL_MODE" == "xy" ] && qemu-system-x86_64 -M q35 -m 256 -kernel "$OUTDIR/BOOTX64.EFI" -enable-kvm -serial stdio -append "console=ttyAMA0 console=ttyS0"