path: root/make-boot-image.sh

#!/bin/bash -xe
PS4='+ ($LINENO) '

# Config #
##########
WORKDIR=./work
DLDIR=./downloads
OUTDIR=./out
#OUTUSB=/dev/sdb1
DEVEL_MODE=y
ROOTCMD=sudo
WGET="wget" #"wget --no-check-certificate"
KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.tar.xz
KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py
KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch
NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.162_all.udeb
UFTP_TARBALL_URL=http://downloads.sourceforge.net/project/uftp-multicast/source-tar/uftp-4.9.3.tar.gz
BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/busybox-x86_64
PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids
USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids

# Utilities #
#############
# From https://landley.net/writing/rootfs-programming.html
# Its first argument is the new directory, and the rest of its arguments are executables to copy.
function mkchroot
{
  [ $# -lt 2 ] && return 0
  dest=$1
  shift
  for i in "$@"
  do
    # Get an absolute path for the file
    p=$i
    [ "${p:0:1}" == "/" ] || p=$(which $i) || true
    if [ ! -e "$p" ]
    then echo "mkchoot not found: $i"
         return 1
    fi
    # Skip files that already exist at target.
    [ -f "$dest/$p" ] && continue
    # Create destination path
    d=$(echo "$p" | grep -o '.*/') &&
    mkdir -p "$dest/$d" &&
    # Copy file
    echo + cp --dereference --preserve=mode "$p" "$dest/$p" &&
    cp --dereference --preserve=mode "$p" "$dest/$p" &&
    # Recursively copy shared libraries' shared libraries.
    mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $?
  done
}

# Environement and dependencies #
#################################
codename=$(lsb_release -sc || true)
if [ "x$codename" != "xjessie" ]
then	cat >&2 <<EOT
This script is tested only on Debian 8 (aka jessie).
The fastest way to have the right environnment is :
 * download debian live http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-8.5.0-amd64-standard.iso
 * burn it or copy it on a USB stick, alternatively launch a VM with it
 * run this script from there
EOT
	exit 1
fi

[ -d "$WORKDIR" ] || mkdir "$WORKDIR"
[ -d "$DLDIR" ] || mkdir "$DLDIR"
[ -d "$OUTDIR" ] || mkdir "$OUTDIR"

if [ ! -e "$WORKDIR/apt-done" ]
then	$ROOTCMD apt-get update
	# Dependencies of this script (assuming default debian install or live)
	$ROOTCMD apt-get install wget libncurses5-dev coreutils
	# Dependencies for kernel building
	$ROOTCMD apt-get build-dep linux-source
	# Dependancies for kernel tools
	[ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \
		libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev
	# Optionnally qemu to run the result for santity checking
	[ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86
	# Dependencies to put into the initrd
	$ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace
	$ROOTCMD apt-get install partclone sfdisk udpcast gdisk efibootmgr tcpdump
	> "$WORKDIR/apt-done"
fi

# Kernel build setup #
######################
kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL)
[ -f "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL"
if [ ! -d "$WORKDIR/kernel" ]
then	mkdir "$WORKDIR/kernel"
	tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel"
fi

if [ ! -d "$WORKDIR/kernel/scripts/Kconfiglib" ]
then
	[ -f "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL"
	[ -f "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL"
	mkdir "$WORKDIR/kernel/scripts/Kconfiglib"
	cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py"
	patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch"
fi

cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT"
#!/usr/bin/env python
import kconfiglib
import sys

def sset(sym, value=None):
    if not sym.is_modifiable():
        print("%s is not modifiable at all"%(sym.get_name()))
        return True
    if value is None and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]:
        value = sym.get_upper_bound()
    old_value = sym.get_value()
    if old_value == value:
        return False
    print("CONFIG_%s=%s [was: %s]"%(sym.get_name(),value,old_value))
    sym.set_user_value(value)
    return True

debug = '--debug' in sys.argv;
conf = kconfiglib.Config(sys.argv[1])
conf.load_config('.config')
support_xz = conf['KERNEL_XZ'] is not None
i = 0
more_work = True
while more_work and i < 10:
    more_work = False
    i += 1
    print("Kconfiglib/customize.py pass %i"%i)

    for sym in conf.get_symbols():
        name = sym.get_name()

        # Default hostname is (none) and could make FreeBSD's dhcpd complain because unallowed '()'
        if name in ['DEFAULT_HOSTNAME']:
            more_work = sset(sym, 'eficast') or more_work

        # Embed initrd in the EFI bootable kernel
        if name in ['INITRAMFS_SOURCE']:
            more_work = sset(sym, '../initrd/') or more_work

        # Make kernel directly loadable by EFI, add USB3, Dell flash
        if name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']:
            more_work = sset(sym) or more_work

        # Support soft RAID (linux) and hard RAID (some cards)
        if name in ['DM_RAID', 'SCSI_LOWLEVEL', 'MEGARAID_SAS', 'MEGARAID_NEWGEN']:
            more_work = sset(sym) or more_work

        # If --debug passed as arg, make kernel aware of virtual drivers (used for testing eficast on qemu/kvm)
        if debug and name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']:
            more_work = sset(sym) or more_work

        # Disable thing that are unneeded or annoying for the purpose of disk cloning
        if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NETWORK_FILESYSTEMS', 'KEYS', 'SECURITY', 'VIRTUALIZATION']:
            more_work = sset(sym, 'n') or more_work

        # Compress everything with XZ if available (slower, smaller)
        if support_xz:
            if name in ['KERNEL_XZ']:
                more_work = sset(sym, 'y') or more_work
            if name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']:
                more_work = sset(sym, 'n') or more_work

        # Following generic actions are meant for features, not choices
        if not sym.is_choice_symbol():

            # Build all available net/ethernet drivers
            if sym.is_modifiable() and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ] \
            and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]:
                more_work = sset(sym) or more_work

            # Try to get everything in kernel, not as a module
            if sym.get_value() == 'm' and sym.get_upper_bound() == 'y':
                more_work = sset(sym, 'y') or more_work

if i == 10:
    print("ERROR : can't set some of kernel config symbols after 10 passes")
    sys.exit(1)
else:
    sys.exit( conf.write_config(".config") )
EOT
chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py"

# Kernel prepare + make tools #
###############################
(
	cd "$WORKDIR/kernel"
	if [ ! -f .config ]
	then	make defconfig
		if [ "x$DEVEL_MODE" == "xy" ]
		then extra="SCRIPT_ARG=--debug"
		else extra=""
		fi
		make scriptconfig SCRIPT=scripts/Kconfiglib/customize.py $extra
	fi
)

p="$WORKDIR/kernel/tools/perf/perf"
if [ "x$DEVEL_MODE" == "xy" -a ! -f "$p" ]
then	(
		cd "$WORKDIR/kernel"
		make tools/perf
	)
fi

# Build additionnal tools from source #
#######################################
uftp_tarball=$DLDIR/$(basename "$UFTP_TARBALL_URL")
[ -f "$uftp_tarball" ] || $WGET -O "$uftp_tarball" "$UFTP_TARBALL_URL"
if [ ! -d "$WORKDIR/uftp" ]
then	mkdir "$WORKDIR/uftp"
	tar xf "$uftp_tarball" --strip-components=1 -C "$WORKDIR/uftp"
fi

if [ ! -f "$WORKDIR/uftp/uftpd" ]
then	(
		cd "$WORKDIR/uftp/"
		make NO_ENCRYPTION=1
	)
fi

# Initial Ram Disk building (embed in kernel) #
###############################################
if [ ! -d "$WORKDIR/initrd" ]
then	mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt,root,proc,root,sbin,sys,run/lock,run/uftpd,tmp,var/log}
	$ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/"
	$ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock"
	ln -s "/proc/mounts" "$WORKDIR/initrd/etc/mtab"
	ln -s "../run" "$WORKDIR/initrd/var/run"
	ln -s "../run/lock" "$WORKDIR/initrd/var/lock"
fi

if [ ! -f "$WORKDIR/initrd/bin/busybox" ]
then	[ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL"
	cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox"
	chmod +x "$WORKDIR/initrd/bin/busybox"
	ln -s /bin/busybox "$WORKDIR/initrd/init"
fi

if [ ! -f "$WORKDIR/initrd/etc/keys.bmap" ]
then	# When using sudo with password auth, ask and cache pass first
	$ROOTCMD true
	# The following compound command will suck at asking pass
	$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap"
fi

if [ ! -f "$WORKDIR/initrd/usr/sbin/partclone.restore" ]
then	(
		set +x
		PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin"
		# Diagnostic tools
		mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump
		# Manpage display
		mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl
		# Cloning tools
		mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr sfdisk gdisk sgdisk udp-receiver
		# Some dyn-loaded libraries (ldd will not display them)
		mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
		cp -ar /lib/terminfo "$WORKDIR/initrd/lib/"
	)
fi

if [ "x$DEVEL_MODE" == "xy" ]
then	(
		p="$WORKDIR/kernel/tools/perf/perf"
		cp -a "$p" "$WORKDIR/initrd/sbin/"
		set +x
		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
	)
fi

if [ ! -d "$WORKDIR/initrd/usr/man" ]
then	mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/1.22.2/font "$WORKDIR/initrd/etc/groff/"
	cp /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/"
	cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/"
	cp -r /usr/share/groff/1.22.2/font/devascii "$WORKDIR/initrd/usr/share/groff/1.22.2/font/"
	cp -r /usr/share/groff/1.22.2/tmac "$WORKDIR/initrd/usr/share/groff/1.22.2/"
	cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/"
fi

p="$WORKDIR/initrd/bin/uftp"
if [ ! -f "$p" ]
then	(
		cd "$WORKDIR/uftp/"
		make DESTDIR="../initrd" install
	)
	(
		set +x
		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
	)
fi

p="$WORKDIR/kernel/tools/perf/perf"
if [ "x$KERNEL_TOOLS" == "xy" -a ! -f "$p" ]
then	(
		cp -a "$p" "$WORKDIR/initrd/sbin/"
		set +x
		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
	)
fi

if [ ! -d "$WORKDIR/initrd/var/lib" ]
then	[ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL"
	[ -f "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL"
	mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc"
	cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/"
	cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/"
fi

if [ ! -d "$WORKDIR/initrd/lib/firmware" ]
then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL"
	dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/"
	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
fi

cat > "$WORKDIR/initrd/etc/init.d/funcs" <<"EOF"
rescue_shell() {
	echo "Something went wrong. Dropping to a shell."
	setsid cttyhack /bin/busybox sh
	sync
	umount /dev /sys /proc
	#umount /dev/pts /dev /sys /proc
	poweroff -d1 -f
}

tty_prog() {
	tty=/dev/tty$1; shift
	while true
	do
		echo "(re)spawning $* on $tty" >$tty
		setsid sh -c "exec $* <$tty >$tty 2>&1"
		sleep 2
	done
}

network_up() {
	ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface
	do
		echo ip link set dev $iface up
		ip link set dev $iface up
	done
	sleep 8 # PHY link det. + IPv6 DAD & Autoconf
}

machine_info() {
	for k in system-manufacturer system-product-name \
		baseboard-manufacturer baseboard-product-name \
		bios-version bios-release-date
	do
		echo $k: $(dmidecode -s $k)
	done
	grep -F MemTotal: /proc/meminfo
	lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p'
	ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p'
	lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/'
}

network_show() {
	ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p'
}

start_uftpd() {
	# FIXME : -I is there as a bug workaround "address already in use"
	uftpd -q -B 2097152 -x2 -F /run/uftpd.csv -L /run/uftpd.log -t -D /run/uftpd -M ff02::42 -I 2/6
	[ $? -eq 0 ] && netstat -nlu | grep -q ':1044 ' && echo -e '\033[42m' ----- UFTPD ready ----- '\033[0m'
}
EOF

cat > "$WORKDIR/initrd/etc/inittab" <<"EOF"
# Custom init scripts
::sysinit:/etc/init.d/rcS
# Executes as root what is received via uftpd (no security at all)
::respawn:/bin/sinkdo /run/uftpd /run/sink
# Standard things follow
::ctrlaltdel:/sbin/reboot
::shutdown:/sbin/swapoff -a
::shutdown:/bin/umount -a -r
::restart:/sbin/init
tty2::askfirst:/bin/sh
tty3::askfirst:/bin/sh
tty4::askfirst:/bin/sh
EOF

cat > "$WORKDIR/initrd/etc/init.d/rcS" <<"EOF"
#!/bin/busybox sh
echo "rcS script started"
# Load helper functions
. /etc/init.d/funcs
# Trace execution
set -v

# Setup links for programs supported by busybox
/bin/busybox --install -s || rescue_shell

# Mount pseudo-filesystems
mount -t proc none /proc || rescue_shell
mount -t sysfs none /sys || rescue_shell
mount -t devtmpfs -o size=1m none /dev || rescue_shell
ln -s /proc/self/fd/2 /dev/stderr
#mkdir /dev/pts || rescue_shell
#mount -t devpts none /dev/pts || rescue_shell

# Allow using most of the RAM for rootfs
mount -o remount -o size=80% /

# Load keyboard map
loadkmap < /etc/keys.bmap

network_up
efibootmgr -v
machine_info
network_show
start_uftpd
EOF
chmod +x "$WORKDIR/initrd/etc/init.d/rcS"

cat > "$WORKDIR/initrd/bin/sinkcat" <<"EOF"
#!/bin/busybox sh

if [ $# -ne 1 ]
then	cat <<EOT
Usage: $(basename $0) <sink-dir>
  Concatenate then delete files as soon they appear in sink-dir to stdout.
  If multiple files are found in sink, the first in alphabetical order is choosen.
  <sink-dir> must not exists, this program must create it (avoiding mistakes).
  Dropping an empty file in dir-sink will clean exit this program.
EOT
	exit 1
fi

SINKDIR=$1
mkdir "$SINKDIR" && cd "$SINKDIR"
if [ $? -ne 0 ]
then	echo "Cannot mkdir/chdir to '$SINKDIR'" >&2
	exit 2
fi

while true
do
	f=$(ls | grep -v '~' | head -n1)
	if [ -n "$f" ]
	then	if [ -f "$f" -a -r "$f" ]
		then	size=$(stat -c'%s' -- "$f")
			# Do the actual work on the following line
			cat -- "$f" && rm -- "$f" >&2
			# Normal exit condition
			if [ $size -eq 0 ]
			then	cd / && rmdir -- "$SINKDIR" >&2
				exit 0
			fi
		else	echo "'$SINKDIR/$f' is not a readable file" >&2
			exit 3
		fi
	fi
	sleep 1
done
EOF
chmod +x "$WORKDIR/initrd/bin/sinkcat"

cat > "$WORKDIR/initrd/bin/sinkdo" <<"EOF"
#!/bin/busybox sh

if [ $# -ne 2 ]
then	cat <<EOT
Usage: $(basename $0) <land-dir> <sink-dir>
EOT
	exit 1
fi

LANDDIR=$1
SINKDIR=$2
cd "$LANDDIR" || exit 2

curtask="(start)"
while true
do
	f=$(ls | grep -v '~' | head -n1)
	if [ -z "$f" ]
	then	sleep 1
		continue
	fi
	if [ ! -f "$f" -o ! -r "$f" ]
	then	echo "'$LANDDIR/$f' is not a readable file" >&2
		exit 2
	fi
	size=$(stat -c'%s' -- "$f")
	task=${f:0:2}
	if [ "$curtask" == "$task" ]
	then	# Next file of an already started task
		mv -- "$f" "$SINKDIR/"
	else	# Switch to the next task
		if [ -d "$SINKDIR" ]
		then	# Inform sinkcat about end-of-data
			touch -- "$SINKDIR/zz"
			wait
			res=$? # Error code from "./$f"
			# sinkcat always rmdir "$SINKDIR" on normal exit
			if [ $res -ne 0 -o -d "$SINKDIR" ]
			then	echo "Task $task has ran into troubles" >&2
				exit 3
			fi
		fi
		echo "Switching from task $curtask to $task" >&2
		curtask=$task
		if [ "$task" == "99" ]
		then	# Normal exit condition
			rm -- "$f" >&2
			echo "All tasks completed sucessfully" >&2
			exit 0
		else	# Start a new task
			chmod +x -- "$f" # XXX Checks on $f (is a script ?) 
			sinkcat $SINKDIR | "./$f" &
			while [ ! -d "$SINKDIR" ]; do sleep 1; done
			rm -- "$f" >&2
		fi
	fi
done
EOF
chmod +x "$WORKDIR/initrd/bin/sinkdo"

# Kernel build (with embed initramfs) #
#######################################
(
	cd "$WORKDIR/kernel"
	# Workaround : some kernel version forget to update embed initramfs in certain cases
	[ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz
	nproc=$(nproc --all)
	nproc=${nproc:-4}
	make -j $((nproc+1))
)

# Copy / run result EFI file #
##############################
[ -f "$OUTDIR/BOOTX64.EFI" ] && rm -f "$OUTDIR/BOOTX64.EFI"
cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI"

if [ -n "$OUTUSB" -a -b "$OUTUSB" ]
then	[ -d "$WORKDIR/mountpoint" ] || mkdir "$WORKDIR/mountpoint"
	mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB"
	$ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint"
	[ -d "$WORKDIR/mountpoint/BOOT/EFI" ] || $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT"
	$ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT"
	$ROOTCMD umount "$WORKDIR/mountpoint"
fi

[ "x$DEVEL_MODE" == "xy" ] && qemu-system-x86_64 -M q35 -m 256 -kernel "$OUTDIR/BOOTX64.EFI" -enable-kvm -serial stdio -append "console=ttyAMA0 console=ttyS0"