path: root/make-boot-image.sh

#!/bin/bash -xe
# Config #
##########
WORKDIR=./work
DLDIR=./downloads
OUTDIR=./out
OUTUSB=/dev/sdb1
KERNEL_TOOLS=y
RUN_QEMU=y
ROOTCMD=sudo
WGET="wget --no-check-certificate"
KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.tar.xz
KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py
KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch
NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.162_all.udeb
BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/busybox-x86_64
PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids
USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids

# Utilities #
#############
# From https://landley.net/writing/rootfs-programming.html
# Its first argument is the new directory, and the rest of its arguments are executables to copy.
function mkchroot
{
  [ $# -lt 2 ] && return 0
  dest=$1
  shift
  for i in "$@"
  do
    # Get an absolute path for the file
    p=$i
    [ "${p:0:1}" == "/" ] || p=$(which $i) || true
    if [ ! -e "$p" ]
    then echo "mkchoot not found: $i"
         return 1
    fi
    # Skip files that already exist at target.
    [ -f "$dest/$p" ] && continue
    # Create destination path
    d=$(echo "$p" | grep -o '.*/') &&
    mkdir -p "$dest/$d" &&
    # Copy file
    echo ${PS4}cp --dereference --preserve=mode "$p" "$dest/$p" &&
    cp --dereference --preserve=mode "$p" "$dest/$p" &&
    # Recursively copy shared libraries' shared libraries.
    mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $?
  done
}

# Environement and dependencies #
#################################
codename=$(lsb_release -sc || true)
if [ "x$codename" != "xjessie" ]
then	cat >&2 <<EOT
This script is tested only on Debian 8 (aka jessie).
The fastest way to have the right environnment is :
 * download debian live http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-8.5.0-amd64-standard.iso
 * burn it or copy it on a USB stick, alternatively launch a VM with it
 * run this script from there
EOT
	exit 1
fi

[ -d "$WORKDIR" ] || mkdir "$WORKDIR"
[ -d "$DLDIR" ] || mkdir "$DLDIR"
[ -d "$OUTDIR" ] || mkdir "$OUTDIR"

if [ ! -e "$WORKDIR/apt-done" ]
then	$ROOTCMD apt-get update
	# Dependencies of this script (assuming default debian install or live)
	$ROOTCMD apt-get install wget libncurses5-dev
	# Dependencies for kernel building
	$ROOTCMD apt-get build-dep linux-source
	# Dependancies for kernel tools
	[ "x$KERNEL_TOOLS" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \
		libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev
	# Optionnally qemu to run the result for santity checking
	[ "x$RUN_QEMU" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86
	# Dependencies to put into the initrd
	$ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace
	$ROOTCMD apt-get install partclone udpcast gdisk efibootmgr tcpdump
	> "$WORKDIR/apt-done"
fi

# Kernel build setup #
######################
kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL)
[ -f "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL"
if [ ! -d "$WORKDIR/kernel" ]
then	mkdir "$WORKDIR/kernel"
	tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel"
fi

if [ ! -d "$WORKDIR/kernel/scripts/Kconfiglib" ]
then
	[ -f "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL"
	[ -f "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL"
	mkdir "$WORKDIR/kernel/scripts/Kconfiglib"
	cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py"
	patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch"
fi

cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT"
#!/usr/bin/env python
import kconfiglib
import sys

def sset(sym, value=None):
    if not sym.is_modifiable():
        print("%s is not modifiable at all"%(sym.get_name()))
        return True
    if value is None and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]:
        value = sym.get_upper_bound()
    old_value = sym.get_value()
    if old_value == value:
        return False
    print("CONFIG_%s=%s [was: %s]"%(sym.get_name(),value,old_value))
    sym.set_user_value(value)
    return True

conf = kconfiglib.Config(sys.argv[1])
conf.load_config('.config')
support_xz = conf['KERNEL_XZ'] is not None
i = 0
more_work = True
while more_work and i < 10:
    more_work = False
    i += 1
    print("Kconfiglib/customize.py pass %i"%i)

    for sym in conf.get_symbols():
        name = sym.get_name()
        if name in ['DEFAULT_HOSTNAME']:
		# default is (none) and could make FreeBSD's dhcpd complain because unallowed '()'
            more_work = sset(sym, 'eficast') or more_work

        if name in ['INITRAMFS_SOURCE']:
            # embed initrd in the EFI bootable kernel
            more_work = sset(sym, '../initrd/') or more_work

        if name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']:
            # Make kernel directly loadable by EFI, add USB3, Dell flash...
            more_work = sset(sym) or more_work

        if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NETWORK_FILESYSTEMS', 'KEYS', 'SECURITY', 'VIRTUALIZATION']:
            more_work = sset(sym, 'n') or more_work

        if support_xz:
            # Compress everything with XZ if available (slower, smaller)
            if name in ['KERNEL_XZ']:
                more_work = sset(sym, 'y') or more_work
            if name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']:
                more_work = sset(sym, 'n') or more_work

        # Following generic actions are meant for features, not choices
        if not sym.is_choice_symbol():

            # Build all available net/ethernet drivers
            if sym.is_modifiable() and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ] \
            and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]:
                more_work = sset(sym) or more_work

            # Try to get everything in kernel, not as a module
            if sym.get_value() == 'm' and sym.get_upper_bound() == 'y':
                more_work = sset(sym, 'y') or more_work

if i == 10:
    print("ERROR : can't set some of kernel config symbols after 10 passes")
    sys.exit(1)
else:
    sys.exit( conf.write_config(".config") )
EOT
chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py"

# Kernel prepare + make tools #
###############################
(
	cd "$WORKDIR/kernel"
	if [ ! -f .config ]
	then	make defconfig
		make scriptconfig SCRIPT=scripts/Kconfiglib/customize.py
	fi
)

if [ "x$KERNEL_TOOLS" == "xy" ]
then	(
		cd "$WORKDIR/kernel"
		make tools/perf
	)
fi

# Initial Ram Disk building (embed in kernel) #
###############################################
if [ ! -d "$WORKDIR/initrd" ]
then	mkdir -p "$WORKDIR/initrd/"{bin,dev,etc,mnt,root,proc,root,sbin,sys,run/lock,tmp,var}
	$ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/"
	$ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock"
	ln -s "../run" "$WORKDIR/initrd/var/run"
	ln -s "../run/lock" "$WORKDIR/initrd/var/lock"
fi

if [ ! -f "$WORKDIR/initrd/bin/busybox" ]
then	[ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL"
	cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox"
	chmod +x "$WORKDIR/initrd/bin/busybox"
fi

if [ ! -f "$WORKDIR/initrd/etc/keys.bmap" ]
then	$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap"
fi

(
	set +x
	PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin"
	# Diagnostic tools
	mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump
	# Manpage display
	mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl
	# Cloning tools
	mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr gdisk udp-receiver
	# Some dyn-loaded libraries (ldd will not display them)
	mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
)

if [ "x$KERNEL_TOOLS" == "xy" ]
then	(
		p="$WORKDIR/kernel/tools/perf/perf"
		cp -a "$p" "$WORKDIR/initrd/sbin/"
		set +x
		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
	)
fi

if [ ! -d "$WORKDIR/initrd/usr/man" ]
then	mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/1.22.2/font "$WORKDIR/initrd/etc/groff/"
	cp /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/"
	cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/"
	cp -r /usr/share/groff/1.22.2/font/devascii "$WORKDIR/initrd/usr/share/groff/1.22.2/font/"
	cp -r /usr/share/groff/1.22.2/tmac "$WORKDIR/initrd/usr/share/groff/1.22.2/"
	cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/"
fi

if [ ! -d "$WORKDIR/initrd/var/lib" ]
then	[ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL"
	[ -f "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL"
	mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc"
	cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/"
	cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/"
fi

if [ ! -d "$WORKDIR/initrd/lib/firmware" ]
then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL"
	dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/"
	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
fi

cat > "$WORKDIR/initrd/funcs" <<"EOT"
rescue_shell() {
	echo "Something went wrong. Dropping to a shell."
	setsid cttyhack sh
	sync
	umount /dev /sys /proc
	poweroff -d1 -f
}

tty_prog() {
	tty=/dev/tty$1; shift
	while true
	do
		echo "(re)spawning $@ on $tty" >$tty
		setsid sh -c "exec $@ <$tty >$tty 2>&1"
		sleep 2
	done
}

network_up() {
	ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface
	do
		echo ip link set dev $iface up
		ip link set dev $iface up
	done
}

machine_info() {
	for k in system-manufacturer system-product-name \
		baseboard-manufacturer baseboard-product-name \
		bios-version bios-release-date
	do
		echo $k: $(dmidecode -s $k)
	done
	grep -F MemTotal: /proc/meminfo
	lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p'
	ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p'
	lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/'
}
EOT

cat > "$WORKDIR/initrd/init" <<"EOT"
#!/bin/busybox sh
echo "Initrd started"
# Load helper functions
. /funcs
# Trace execution
set -v

# Setup links for programs supported by busybox
/bin/busybox --install -s

# Mount pseudo-filesystems
mount -t proc none /proc || rescue_shell
mount -t sysfs none /sys || rescue_shell
mount -t devtmpfs none /dev || rescue_shell

# Load keyboard map
loadkmap < /etc/keys.bmap || rescue_shell

# Start some debug shells (background)
for i in 2 3 4 5 6; do tty_prog $i sh & done

# Activate networking interfaces
network_up
sleep 8 # PHY link detection + IPv6 Duplicate Address Detection & Autoconf

# Dump EFIvars if available
efibootmgr -v

# Machine basic informations
machine_info

# You could press Alt+F2 to have a shell. Remote control on telnet port (TCP 23)
nc -ll -p 23 -e /bin/sh -i

# Network IP adresses
ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p'

EOT
chmod +x "$WORKDIR/initrd/init"

# Kernel build (with embed initramfs) #
#######################################
(
	cd "$WORKDIR/kernel"
	# Workaround : some kernel version forget to update embed initramfs in certain cases
	[ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz
	nproc=$(nproc --all)
	nproc=${nproc:-4}
	make -j ${nproc+1}
)

# Copy / run result EFI file #
##############################
cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI"

if [ -n "$OUTUSB" -a -b "$OUTUSB" ]
then	[ -d "$WORKDIR/mountpoint" ] || mkdir "$WORKDIR/mountpoint"
	mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB"
	$ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint"
	[ -d "$WORKDIR/mountpoint/BOOT/EFI" ] || $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT"
	$ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT"
	$ROOTCMD umount "$WORKDIR/mountpoint"
fi

[ "x$RUN_QEMU" == "xy" ] && qemu-system-x86_64 -M q35 -m 256 -kernel "$OUTDIR/BOOTX64.EFI"