The following will be removed
- clang
- relinking sources
- Install OpenBSD normally. I put it in a single partition for processing. Only base.tgz install (and bsd/bsd.mp) are needed.
- If this is a local VM, possibly enable root SSH so that commands can be pasted into terminal more easily.
- Boot into the new system with the cd inserted, login as root
- Notes:
- Adjust the version if needed.
- the p in
tar xzpfis important to retain file permissions - setuid/setgid binaries, some special directories, and some files owned by daemon users are not owned by root.
- The script will move all man pages to the man package, and all compiler related stuff to the compiler packages.
- Kernel relink will be removed as it may not even run on an embedded system, and isn't a major impact to security in this situation.
mkdir -p /tmp/cdroot
mount -t cd9660 /dev/cd0a /mnt/cdrom
#umount /mnt/cdrom
cd /tmp/cdroot
cp -r /mnt/cdrom/* /tmp/cdroot/
ARCH=amd64
VERSION=78
VERSIOND=7.8
CDPATH=/tmp/cdroot/$VERSIOND/$ARCH
mkdir -p "/tmp/base${VERSION}"
mkdir -p "/tmp/comp${VERSION}"
mkdir -p "/tmp/man${VERSION}"
mkdir -p "/tmp/xbase${VERSION}"
# mkdir -p "/tmp/xfont${VERSION}"
mkdir -p "/tmp/xserv${VERSION}"
mkdir -p "/tmp/xshare${VERSION}"
tar xzpf "${CDPATH}/base${VERSION}.tgz" -C "/tmp/base${VERSION}"
tar xzpf "${CDPATH}/comp${VERSION}.tgz" -C "/tmp/comp${VERSION}"
tar xzpf "${CDPATH}/man${VERSION}.tgz" -C "/tmp/man${VERSION}"
tar xzpf "${CDPATH}/xbase${VERSION}.tgz" -C "/tmp/xbase${VERSION}"
tar xzpf "${CDPATH}/xserv${VERSION}.tgz" -C "/tmp/xserv${VERSION}"
tar xzpf "${CDPATH}/xshare${VERSION}.tgz" -C "/tmp/xshare${VERSION}"
# base78.tgz
cd "/tmp/base${VERSION}"
du -sh .
touch etc/no_relink etc/no_reorder_kernel
rm -rf usr/share/relink
cp -r usr/share/man/* /tmp/man78/usr/share/man/
rm -rf usr/share/man/*
cp -r usr/share/info/* /tmp/man78/usr/share/info/
rm -rf usr/share/info
rm -rf usr/games/*
cp -r usr/share/doc/* /tmp/man78/usr/share/doc/
rm -rf usr/share/doc
# add notes to root profile to remind user how the system is set up.
cat << 'EOF' > root/.profile
Use mount-rw.sh and mount-ro.sh to control mounting of /.
Use enable_perl to set up perl before running pkg commands.
To make the system have read-only root, use /usr/bin/prep-read-only.sh
EOF
#perl - breaks pkg_add
# rm usr/bin/perl
# rm usr/libdata/perl*
#perl - store in a gzip and extract to /tmp when needed
tar cpf /tmp/perl5.tar -C usr/libdata/perl5 .
gzip -6 -o usr/libdata/perl5.tar.gz /tmp/perl5.tar
rm -rf usr/libdata/perl5
ln -s /tmp/perl5 usr/libdata/perl5
cat << 'EOF' > usr/bin/enable-perl
#!/bin/sh
mkdir -p /tmp/perl5
tar xzf /usr/libdata/perl5.tar.gz -C /tmp/perl5
EOF
chmod +x usr/bin/enable-perl
# readonly scripts
# /usr/bin/mount-rw.sh
cat > usr/bin/mount-rw.sh <<'EOF'
#!/bin/sh
# make / read-write
mount -uw /
EOF
# /usr/bin/mount-ro.sh
cat > usr/bin/mount-ro.sh <<'EOF'
#!/bin/sh
# make / read-only
sync
mount -ur /
EOF
# /usr/bin/update-var.sh
cat > usr/bin/update-var.sh <<'EOF'
#!/bin/sh
# copy running /var to /cfg/var
# do this after pkg_add (and some config changes)
mount -uw /
cp -Rp /var/ /cfg/
# cp -Rp /var/{db,cron,spool,mail,run,www} /cfg/var/
sync
mount -ur /
EOF
# /usr/bin/update-dev.sh
cat > usr/bin/update-dev.sh <<'EOF'
#!/bin/sh
# make all dev files, may be necessary after kernel update.
mount -uw /
cd /cfg/dev
sh MAKEDEV all
sync
mount -ur /
EOF
chmod 755 usr/bin/mount-rw.sh usr/bin/mount-ro.sh usr/bin/update-var.sh usr/bin/update-dev.sh
# /usr/bin/prep-read-only.sh
cat > usr/bin/prep-read-only.sh << 'EOF'
# remount root rw just in case
mount -uw / 2>/dev/null || true
# ensure target dir exists
mkdir -p /usr/bin
umask 022
#mount-rw just in case
mount -ur / 2>/dev/null || true
mkdir /var/etc-rw
cp /etc/random.seed /var/etc-rw
rm /etc/random.seed
ln -s /var/etc-rw/random.seed /etc/random.seed
rcctl stop resolvd
cp /etc/resolv.conf /var/etc-rw
cp /etc/resolv.conf.tail /var/etc-rw
rm /etc/resolv.conf
rm /etc/resolv.conf.tail
ln -s /var/etc-rw/resolv.conf /etc/resolv.conf
ln -s /var/etc-rw/resolv.conf.tail /etc/resolv.conf.tail
cp /etc/hosts /var/etc-rw
rm /etc/hosts
ln -s /var/etc-rw/hosts /etc/hosts
rcctl start resolvd
mkdir /cfg
cp -Rp /var /cfg/
cp -Rp /dev /cfg/
echo '/usr/bin/update-var.sh' >> /etc/rc.shutdown
echo "swap /tmp mfs rw,-s=256m 0 0" >> /etc/fstab
echo "swap /var mfs rw,-P=/cfg/var,-s=128m 0 0" >> /etc/fstab
echo "swap /dev mfs rw,-P=/cfg/dev,-s=16m 0 0" >> /etc/fstab
echo "edit /etc/fstab and change / from rw to ro to finish."
EOF
#build
mv usr/bin/c++ /tmp/comp78/usr/bin
mv usr/bin/cc /tmp/comp78/usr/bin
mv usr/bin/clang /tmp/comp78/usr/bin
mv usr/bin/clang++ /tmp/comp78/usr/bin
mv usr/bin/clang-cpp /tmp/comp78/usr/bin
mv usr/bin/ld /tmp/comp78/usr/bin
mv usr/bin/ld.lld /tmp/comp78/usr/bin
mv usr/libexec/cpp /tmp/comp78/usr/bin
mv usr/lib/libLLVM* /tmp/comp78/usr/bin
mv usr/lib/*.a /tmp/comp78/usr/bin
mv usr/lib/*.o /tmp/comp78/usr/bin
cp -r usr/share/mk/ /tmp/comp78/usr/share/mk/
rm -rf usr/share/mk
# rm -rf etc/firmware
mv usr/share/misc /tmp/comp78/usr/share/misc
mkdir -p /tmp/man78/usr/bin
mv usr/bin/man /tmp/man78/usr/bin/man
mv usr/bin/mandoc /tmp/man78/usr/bin/mandoc
mkdir -p /tmp/man78/etc/examples/
mv etc/examples/man.conf /tmp/man78/etc/examples/man.conf
find usr/lib -type f -name 'lib*.so.*' -exec /usr/bin/strip -s {} +
du -sh .
# xbase
cd "/tmp/xbase${VERSION}"
du -sh .
find usr/X11R6/lib -type f -name 'lib*.so.*' -exec /usr/bin/strip -s {} +
mkdir -p /tmp/comp78/usr/X11R6/include/
cp -r usr/X11R6/include/ /tmp/comp78/usr/X11R6/include/
rm -rf usr/X11R6/include/*
mkdir -p /tmp/man78/usr/X11R6/man/
cp -r usr/X11R6/man/ /tmp/man78/usr/X11R6/man/
rm -rf usr/X11R6/man/*
find usr/X11R6/bin -type f -exec /usr/bin/strip -s {} +
du -sh .
# xserv
cd "/tmp/xserv${VERSION}"
du -sh .
find usr/X11R6/lib/modules -type f -name 'lib*.so' -exec /usr/bin/strip -s {} +
# Saves a couple hundred KB and messess symlinks
# find usr/X11R6/lib/modules/dri -type f -name '*.so' -exec /usr/bin/strip -s {} +
find usr/X11R6/lib/modules/drivers -type f -name '*.so' -exec /usr/bin/strip -s {} +
find usr/X11R6/lib/modules/extensions -type f -name 'lib*.so' -exec /usr/bin/strip -s {} +
find usr/X11R6/lib/modules/input -type f -name '*.so' -exec /usr/bin/strip -s {} +
mkdir -p /tmp/man78/usr/X11R6/man/
cp -r usr/X11R6/man/ /tmp/man78/usr/X11R6/man/
rm -rf usr/X11R6/man/*
du -sh .
# xshare
cd "/tmp/xshare${VERSION}"
du -sh .
mkdir -p /tmp/man78/usr/X11R6/share/doc
cp -r usr/X11R6/share/doc/ /tmp/man78/usr/X11R6/share/doc
rm -rf usr/X11R6/share/doc/*
mkdir -p /tmp/man78/usr/X11R6/man
cp -r usr/X11R6/man/ /tmp/man78/usr/X11R6/man
rm -rf usr/X11R6/man/*
mkdir -p /tmp/comp78/usr/X11R6/include
cp -r usr/X11R6/include/ /tmp/comp78/usr/X11R6/include
rm -rf usr/X11R6/include/*
du -sh .
# place cd stuff.
export GZIP=-9
rm "${CDPATH}/base${VERSION}.tgz"
tar czpf "${CDPATH}/base${VERSION}.tgz" -C "/tmp/base${VERSION}" .
rm "${CDPATH}/xbase${VERSION}.tgz"
tar czpf "${CDPATH}/xbase${VERSION}.tgz" -C "/tmp/xbase${VERSION}" .
rm "${CDPATH}/xserv${VERSION}.tgz"
tar czpf "${CDPATH}/xserv${VERSION}.tgz" -C "/tmp/xserv${VERSION}" .
rm "${CDPATH}/xshare${VERSION}.tgz"
tar czpf "${CDPATH}/xshare${VERSION}.tgz" -C "/tmp/xshare${VERSION}" .
#rm "${CDPATH}/xfont${VERSION}.tgz"
#tar czpf "${CDPATH}/xfont${VERSION}.tgz" -C "/tmp/xfont${VERSION}" .
rm "${CDPATH}/man${VERSION}.tgz"
tar czpf "${CDPATH}/man${VERSION}.tgz" -C "/tmp/man${VERSION}" .
rm "${CDPATH}/comp${VERSION}.tgz"
tar czpf "${CDPATH}/comp${VERSION}.tgz" -C "/tmp/comp${VERSION}" .
mkhybrid \
-r -T -l \
-A "OpenBSD #{VERSIOND} ${ARCH} small" \
-V "OPENBSD_${VERSION}" \
-b "${VERSIOND}/${ARCH}/cdbr" \
-c boot.catalog \
-o "/tmp/install${VERSION}-${ARCH}-custom.iso" \
/tmp/cdroot
More compression - this didn't help, it made base78.tgz go from 50M to 49.6M and took MUCH longer:
GZIP=-9 tar czpf /tmp/cdroot/7.8/amd64/base78.tgz -C /tmp/base78 .
The CD is 389MB if you include the now vastly larger (~180MB) compiler package. It's below 300MB with the original compiler package, but the actual compiler is missing.
The installed system with just base and the four X11 packages is 442MB on AMD64.
The install system with just base is 223MB (two kernels which are 31MB each but either the single processor or multiproccessor kernel could be deleted if needed). The disk image compressed with 7-zip to under 41MB.
Before compressing an image after deleting things:
dd if=/dev/zero of=/EMPTY bs=1M
rm /EMPTY
Perl is needed to install packages but can be removed if not needed after doing so. With perl and the single processor kernel removed, disk usage was 138MB. The disk compressed to 28.1MB with 7-zip.
On i386: just base and a kernel: 175MB (7-zip Compressed 45MB) base + x: 393MB (7-zip Compressed 92MB)
later: Make a better version of this where we store perl in an xz and extract it to tmp. Note xz package has no dependencies, but we should add xz and liblzma to the base system to facilitate this. Could also do it with gzip if we don't want to add anymore binaries, but xz+liblzma is relatively small.
On a running system:
cd /tmp
tar cpf /tmp/perl5.tar -C /usr/libdata/perl5 .
gzip -9 -o perl5.tar.gz perl5.tar
xz -zk perl5.tar
xz -zk9 perl5.tar
Uncompressed: 49.4M Default Gzip: 12.1M Max Gzip (7zip): 11.1M Default xz: 7.5M Max xz: 7.19M [this uses a lot of memory, so not worth it] Max bzip2 (7zip): 9.35M Max brotli: 7.61M Max zpaq: 5.41M [this takes forever to (de)compress. OpenBSD port: zpaqfranz-62.5 https://openbsd.app/path/archivers/zpaqfranz - has no dependencies]
echo 'swap /tmp mfs rw,nodev,nosuid,-s=300m 0 0' >> /etc/fstab
Reboot into single user (shutdown now)and do -
rm -rf /tmp
umount -f /tmp
mkdir -p /tmp
chmod 1777 /tmp
mount /tmp
reboot
cd /tmp
tar cpf /tmp/perl5.tar -C /usr/libdata/perl5 .
gzip -6 -o /usr/libdata/perl5.tar.gz /tmp/perl5.tar
rm -rf /usr/libdata/perl5
ln -s /tmp/perl5 /usr/libdata/perl5
cat << 'EOF' > /usr/bin/enable-perl
#!/bin/sh
mkdir -p /tmp/perl5
tar xzf /usr/libdata/perl5.tar.gz -C /tmp/perl5
EOF
chmod +x /usr/bin/enable-perl
Run enable perl before running pkg_add, etc.
This doesn't work quite correctly due to pkg_add/etc's shbangs not working with the shell script.
mv /usr/bin/perl /usr/bin/perl2
cat << 'EOF' > /usr/bin/perl
#!/bin/sh
PERLDIR="/tmp/perl5"
ARCHIVE="/usr/libdata/perl5.tar.gz"
REALPERL="/usr/bin/perl2"
# If perl isn't yet extracted, do it
if [ ! -d "$PERLDIR" ]; then
mkdir -p "$PERLDIR" || exit 1
tar xzf "$ARCHIVE" -C /tmp/perl5 || exit 1
fi
# Exec real perl with original args
exec "$REALPERL" "$@"
EOF
chmod 755 /usr/bin/perl
Script 2 - fix expansion
#!/bin/sh
PERLDIR="/tmp/perl5"
ARCHIVE="/usr/libdata/perl5.tar.gz"
REALPERL="/usr/bin/perl2"
# Ensure perl tree is extracted
if [ ! -d "$PERLDIR" ]; then
mkdir -p "$PERLDIR" || exit 1
tar xzf "$ARCHIVE" -C /tmp || exit 1
fi
# Detect whether invoked via shebang:
# argv[1] = script, unless options were in shebang
SCRIPT=""
ARGS=""
for arg in "$@"; do
case "$arg" in
-*)
ARGS="$ARGS $arg"
;;
*)
SCRIPT="$arg"
shift
ARGS="$ARGS $@"
break
;;
esac
shift
done
# If no script was found, just exec normally (interactive perl, -e, etc)
if [ -z "$SCRIPT" ]; then
exec "$REALPERL" "$@"
fi
# Execute the real perl with exact ordering:
exec "$REALPERL" $ARGS "$SCRIPT"
dd if=/dev/zero of=/EMPTY bs=1M
rm /EMPTY
- We can write stubs or setup scripts to replace various random things in the system that are large / not used often into archives and just expand them as needed.
- Obviously, this needs to be documented.
- If the main concern is minimal disk (but we can use more RAM/swap), we can extract various system components at boot time.
For building out the durable servers, use a custom partition layout.
- If FreeBSD or OpenBSD image is available from the host, deploy this, then install custom ISO on top of it to make sure bsd optimizations in KVM are used. Then zero the disk so snapshots compress well (make sure you choose the right disk)
sysctl hw.disknames
dd if=/dev/zero of=/dev/rsd0c
- Create a / that's 1-4GB
- Create a swap that's 2-4GB to facilitate the memory file systems getting full
- Create a /data partition (4.2BSD) that fills the rest of the disk.
You can grow data after applying the image to a larger disk:
umount /data
disklabel -E sd0
- Print the current layout
p - Delete the old
/datapartition entry:d d(if it's partition d) - Recreate d with a larger size beginning at the same offset as before, ending later to consume more space:
a d-> start sector = old start (you can copy fromp)- size = * to use the rest of the available space, or specify size.
- Write and quit:
w, thenq. - Grow the filesystem (FFS):
growfs /dev/rsd0d
Mount /data again.
mount /data
- Set up doas so we can set everything up easily using SSH.
- Set the installurl to lysator as they keep ancient packages.
# echo "permit persist keepenv user" | tee -a /etc/doas.conf
echo "permit persist keepenv :wheel" | tee -a /etc/doas.conf
usermod -G wheel user
echo "https://ftp.lysator.liu.se/pub/OpenBSD/" > /etc/installurl
dd if=/dev/zero of=/EMPTY bs=1M
rm /EMPTY
dd if=/dev/zero of=/data/EMPTY bs=1M
rm /data/EMPTY
This is necessary to minimize the size of the snapshot since we've installed on top of another OS.
sysctl hw.disklabels
cd /dev
sh MAKEDEV sd0
disklabel sd0
dd if=/dev/zero of=/dev/rsd0b bs=1m
mkdir -p /mnt/sd0a
mkdir -p /mnt/sd0c
mount /dev/sd0a /mnt/sd0a
mount /dev/sd0d /mnt/sd0d
dd if=/dev/zero of=/mnt/sd0a/EMPTY bs=1M
rm /mnt/sd0a/EMPTY
dd if=/dev/zero of=/mnt/sd0d/EMPTY bs=1M
rm /mnt/sd0d/EMPTY
/bin/sh /usr/bin/prep-read-only.sh
vi /etc/fstab
Make the partitions read-only by changing rw to ro in /etc/fstab for / and /data, reboot to make sure it works, and take a snapshot.
Add a user named chicken.
useradd -m -G wheel -s /bin/ksh chicken
passwd chicken
(as root)
mount-rw.sh
mv /home /data/home
ln -s /data/home /home
mount-ro.sh
Note any user who's home is moved will need to log out and back in.