Initial import : make-boot-image.sh operationnal. Dumb mcastseed.

1 files changed, 331 insertions, 0 deletions

diff --git a/make-boot-image.sh b/make-boot-image.sh
new file mode 100755
index 0000000..032398d
--- /dev/null
+++ b/make-boot-image.sh
@@ -0,0 +1,331 @@
+#!/bin/bash -xe
+
+# Config #
+##########
+WORKDIR=/tmp/work
+DLDIR=./downloads
+OUTDIR=./out
+OUTUSB=/dev/sdb1
+RUN_QEMU=y
+ROOTCMD=sudo
+WGET="wget --no-check-certificate"
+KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.tar.xz
+KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py
+KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch
+NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.157_all.udeb
+BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/busybox-x86_64
+PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids
+USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids
+
+# Utilities #
+#############
+# From https://landley.net/writing/rootfs-programming.html
+# Its first argument is the new directory, and the rest of its arguments are executables to copy.
+function mkchroot
+{
+  [ $# -lt 2 ] && return 0
+  dest=$1
+  shift
+  for i in "$@"
+  do
+    # Get an absolute path for the file
+    p=$i
+    [ "${p:0:1}" == "/" ] || p=$(which $i) || true
+    if [ ! -e "$p" ]
+    then echo "mkchoot not found: $i"
+         return 1
+    fi
+    # Skip files that already exist at target.
+    [ -f "$dest/$p" ] && continue
+    # Create destination path
+    d=$(echo "$p" | grep -o '.*/') &&
+    mkdir -p "$dest/$d" &&
+    # Copy file
+    echo ${PS4}cp --dereference --preserve=mode "$p" "$dest/$p" &&
+    cp --dereference --preserve=mode "$p" "$dest/$p" &&
+    # Recursively copy shared libraries' shared libraries.
+    mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $?
+  done
+}
+
+
+# Environement and dependencies #
+#################################
+[ -d "$WORKDIR" ] || mkdir "$WORKDIR"
+[ -d "$DLDIR" ] || mkdir "$DLDIR"
+[ -d "$OUTDIR" ] || mkdir "$OUTDIR"
+
+if [ ! -e "$WORKDIR/apt-done" ]
+then	$ROOTCMD apt-get update
+	# Dependencies of this script
+	$ROOTCMD apt-get build-dep linux-source
+	$ROOTCMD apt-get install wget libncurses5-dev 
+	# Optionnally qemu to make some santity checks
+	[ "x$RUN_QEMU" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86
+	# Dependencies to put into the initrd
+	$ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop atop
+	$ROOTCMD apt-get install partclone udpcast gdisk efibootmgr tcpdump
+	> "$WORKDIR/apt-done"
+fi
+
+# Initial Ram Disk building (embed in kernel) #
+###############################################
+if [ ! -d "$WORKDIR/initrd" ]
+then	mkdir -p "$WORKDIR/initrd/"{bin,dev,etc,mnt,root,proc,root,sbin,sys,run/lock,tmp,var}
+	$ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/"
+	$ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock"
+	ln -s "../run" "$WORKDIR/initrd/var/run"
+	ln -s "../run/lock" "$WORKDIR/initrd/var/lock"
+fi
+
+if [ ! -f "$WORKDIR/initrd/bin/busybox" ]
+then	[ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL"
+	cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox"
+	chmod +x "$WORKDIR/initrd/bin/busybox"
+fi
+
+if [ ! -f "$WORKDIR/initrd/etc/keys.bmap" ]
+then	$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap"
+fi
+
+(
+	set +x
+	PATH="/usr/sbin:/usr/bin:/sbin:/bin"
+	# Diagnostic tools
+	mkchroot "$WORKDIR/initrd" atop dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump
+	# Manpage display
+	mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl
+	# Cloning tools
+	mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr gdisk udp-receiver
+	# Some dyn-loaded libraries (ldd will not display them)
+	mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
+)
+
+#if [ ! -L "$WORKDIR/initrd/lib/x86_64-linux-gnu/tls/x86_64" ]
+#then	mkdir -p "$WORKDIR/initrd/lib/x86_64-linux-gnu/tls"
+#	ln -s "../.." "$WORKDIR/initrd/lib/x86_64-linux-gnu/tls/x86_64"
+#fi
+
+if [ ! -d "$WORKDIR/initrd/usr/man" ]
+then	mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/1.22.2/font "$WORKDIR/initrd/etc/groff"
+	cp /usr/share/man/man1/{atop.1,iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/"
+	cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/"
+	cp -r /usr/share/groff/1.22.2/font/devascii "$WORKDIR/initrd/usr/share/groff/1.22.2/font/"
+	cp -r /usr/share/groff/1.22.2/tmac "$WORKDIR/initrd/usr/share/groff/1.22.2/"
+	cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/"
+fi
+
+if [ ! -d "$WORKDIR/initrd/var/lib" ]
+then	[ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL"
+	[ -f "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL"
+	mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc"
+	cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/"
+	cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/"
+fi
+
+if [ ! -d "$WORKDIR/initrd/lib/firmware" ]
+then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL"
+	dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/"
+	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
+fi
+
+cat > "$WORKDIR/initrd/funcs" <<"EOT"
+rescue_shell() {
+	echo "Something went wrong. Dropping to a shell."
+	setsid cttyhack sh
+	sync
+	umount /dev /sys /proc
+	poweroff -d1 -f
+}
+
+tty_prog() {
+	tty=/dev/tty$1; shift
+	while true
+	do
+		echo "(re)spawning $@ on $tty" >$tty
+		setsid sh -c "exec $@ <$tty >$tty 2>&1"
+		sleep 2
+	done
+}
+
+network_up() {
+	ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface
+	do
+		echo ip link set dev $iface up
+		ip link set dev $iface up
+	done
+}
+
+machine_info() {
+	for k in system-manufacturer system-product-name \
+		baseboard-manufacturer baseboard-product-name \
+		bios-version bios-release-date
+	do
+		echo $k: $(dmidecode -s $k)
+	done
+	grep -F MemTotal: /proc/meminfo
+	lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p'
+	ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p'
+	lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/'
+}
+EOT
+
+cat > "$WORKDIR/initrd/init" <<"EOT"
+#!/bin/busybox sh
+echo "Initrd started"
+# Load helper functions
+. /funcs
+# Trace execution
+set -v
+
+# Setup links for programs supported by busybox
+/bin/busybox --install -s
+
+# Mount pseudo-filesystems
+mount -t proc none /proc || rescue_shell
+mount -t sysfs none /sys || rescue_shell
+mount -t devtmpfs none /dev || rescue_shell
+
+# Load keyboard map
+loadkmap < /etc/keys.bmap || rescue_shell
+
+# Start some debug shells (background)
+for i in 2 3 4 5 6; do tty_prog $i sh & done
+
+# Activate networking interfaces
+network_up
+sleep 8 # PHY link detection + IPv6 Duplicate Address Detection & Autoconf
+
+# Dump EFIvars if available
+efibootmgr -v
+
+# Machine basic informations
+machine_info
+
+# You could press Alt+F2 to have a shell. Remote control on telnet port (TCP 23)
+nc -ll -p 23 -e /bin/sh -i
+
+# Network IP adresses
+ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p'
+
+EOT
+chmod +x "$WORKDIR/initrd/init"
+
+
+# Kernel build setup #
+######################
+kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL)
+[ -f "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL"
+if [ ! -d "$WORKDIR/kernel" ]
+then	mkdir "$WORKDIR/kernel"
+	tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel"
+fi
+
+if [ ! -d "$WORKDIR/kernel/scripts/Kconfiglib" ]
+then	
+	[ -f "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL"
+	[ -f "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL"
+	mkdir "$WORKDIR/kernel/scripts/Kconfiglib"
+	cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py"
+	patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch"
+fi
+
+cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT"
+#!/usr/bin/env python
+import kconfiglib
+import sys
+
+def sset(sym, value=None):
+    if not sym.is_modifiable():
+        print("%s is not modifiable at all"%(sym.get_name()))
+        return True
+    if value is None and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]:
+        value = sym.get_upper_bound()
+    old_value = sym.get_value()
+    if old_value == value:
+        return False
+    print("CONFIG_%s=%s [was: %s]"%(sym.get_name(),value,old_value))
+    sym.set_user_value(value)
+    return True
+
+conf = kconfiglib.Config(sys.argv[1])
+conf.load_config('.config')
+support_xz = conf['KERNEL_XZ'] is not None
+i = 0
+more_work = True
+while more_work and i < 10:
+    more_work = False
+    i += 1
+    print("Kconfiglib/customize.py pass %i"%i)
+
+    for sym in conf.get_symbols():
+        name = sym.get_name()
+        if name in ['DEFAULT_HOSTNAME']:
+		# default is (none) and could make FreeBSD's dhcpd complain because unallowed '()' 
+            more_work = sset(sym, 'eficast') or more_work
+
+        if name in ['INITRAMFS_SOURCE']:
+            # embed initrd in the EFI bootable kernel
+            more_work = sset(sym, '../initrd/') or more_work
+
+        if name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']:
+            # Make kernel directly loadable by EFI, add USB3, Dell flash...
+            more_work = sset(sym) or more_work
+    
+        if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NETWORK_FILESYSTEMS', 'KEYS', 'SECURITY', 'VIRTUALIZATION']:
+            more_work = sset(sym, 'n') or more_work
+
+        if support_xz:
+            # Compress everything with XZ if available (slower, smaller)
+            if name in ['KERNEL_XZ']:
+                more_work = sset(sym, 'y') or more_work
+            if name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']:
+                more_work = sset(sym, 'n') or more_work
+
+        # Following generic actions are meant for features, not choices
+        if not sym.is_choice_symbol():
+
+            # Build all available net/ethernet drivers
+            if sym.is_modifiable() and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ] \
+            and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]:
+                more_work = sset(sym) or more_work
+
+            # Try to get everything in kernel, not as a module
+            if sym.get_value() == 'm' and sym.get_upper_bound() == 'y':
+                more_work = sset(sym, 'y') or more_work
+
+if i == 10:
+    print("ERROR : can't set some of kernel config symbols after 10 passes")
+    sys.exit(1)
+else:
+    sys.exit( conf.write_config(".config") )
+EOT
+chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py"
+
+# Kernel build kernel #
+#######################
+(
+	cd "$WORKDIR/kernel"
+	if [ ! -f .config ]
+	then	make defconfig
+		make scriptconfig SCRIPT=scripts/Kconfiglib/customize.py
+	fi
+	# Workaround : some kernel version forget to update embed initramfs in certain cases
+	[ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz
+	make -j8
+)
+# Copy / run result EFI file #
+##############################
+cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI"
+
+if [ -n "$OUTUSB" -a -b "$OUTUSB" ]
+then	[ -d "$WORKDIR/mountpoint" ] || mkdir "$WORKDIR/mountpoint"
+	mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB"
+	$ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint"
+	[ -d "$WORKDIR/mountpoint/BOOT/EFI" ] || $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT"
+	$ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT"
+	$ROOTCMD umount "$WORKDIR/mountpoint"
+fi
+
+[ "x$RUN_QEMU" == "xy" ] && qemu-system-x86_64 -kernel "$OUTDIR/BOOTX64.EFI"
+