Commit d48c89ae authored by Markus Frosch's avatar Markus Frosch 📣
Browse files

Add mkimage helper

Based on Docker example scripts.
parent 6539bb66
Pipeline #3166 failed with stage
in 1 minute and 50 seconds
......@@ -5,10 +5,6 @@ image: docker:latest
services:
- docker:dind
variables:
DOCKER_REGISTRY: ${CI_REGISTRY}
DOCKER_IMAGE_PREFIX: ${CI_PROJECT_PATH}/
before_script:
- docker info
- docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
......@@ -17,6 +13,6 @@ before_script:
build:
stage: build
tags:
- docker-build-arm
- docker
script:
- make all push
FROM debian:stretch-slim
FROM ubuntu:bionic
RUN apt-get update \
&& apt-get install -y debootstrap xz-utils tar \
&& apt-get install -y qemu-user-static debootstrap xz-utils tar \
&& rm -rf /var/lib/apt/lists/*
COPY debootstrap.sh /usr/local/sbin/
COPY script-raspbian /usr/share/debootstrap/scripts/raspbian
COPY qemu-arm-static /usr/bin/qemu-arm-static
COPY mkimage.sh /
COPY debootstrap /mkimage/
CMD ["debootstrap.sh"]
CMD ["/mkimage.sh"]
IMAGE_PREFIX := ${DOCKER_IMAGE_PREFIX}
ifeq ($(IMAGE_PREFIX),)
IMAGE_PREFIX := icinga/raspbian-base
ifeq ($(CI_REGISTRY),)
CI_REGISTRY := registry.icinga.com
endif
REGISTRY := ${DOCKER_REGISTRY}
ifneq ($(REGISTRY),)
IMAGE_PREFIX := $(REGISTRY)/$(IMAGE_PREFIX)
ifeq ($(CI_PROJECT_PATH),)
CI_PROJECT_PATH := build-docker/raspbian-base
endif
FROM := $(shell grep FROM Dockerfile | cut -d" " -f2)
IMAGE := $(IMAGE_PREFIX)debootstrap
ifeq ($(CI_COMMIT_REF_NAME),)
CI_COMMIT_REF_NAME := mkimage
endif
FROM := $(shell grep FROM Dockerfile | cut -d' ' -f2)
IMAGE := $(CI_REGISTRY)/$(CI_PROJECT_PATH)/$(CI_COMMIT_REF_NAME)
.PHONY: all clean build
.PHONY: all clean build scripts
all: pull build
......@@ -20,11 +22,15 @@ pull:
docker pull "$(FROM)"
build:
cp -av /usr/bin/qemu-arm-static .
docker build --cache-from "$(IMAGE)" --tag "$(IMAGE)" .
push:
docker push "$(IMAGE)"
scripts:
curl -LsS https://raw.githubusercontent.com/docker/docker/master/contrib/mkimage.sh -o mkimage.sh
chmod +x mkimage.sh
curl -LsS https://raw.githubusercontent.com/docker/docker/master/contrib/mkimage/debootstrap -o debootstrap
clean:
if (docker inspect --type image "$(IMAGE)" >/dev/null 2>&1); then docker rmi "$(IMAGE)"; fi
#!/usr/bin/env bash
set -e
mkimgdeb="$(basename "$0")"
mkimg="$(dirname "$0").sh"
usage() {
echo >&2 "usage: $mkimgdeb rootfsDir suite [debootstrap-args]"
echo >&2 " note: $mkimgdeb meant to be used from $mkimg"
exit 1
}
rootfsDir="$1"
if [ -z "$rootfsDir" ]; then
echo >&2 "error: rootfsDir is missing"
echo >&2
usage
fi
shift
# we have to do a little fancy footwork to make sure "rootfsDir" becomes the second non-option argument to debootstrap
before=()
while [ $# -gt 0 ] && [[ "$1" == -* ]]; do
before+=( "$1" )
shift
done
suite="$1"
if [ -z "$suite" ]; then
echo >&2 "error: suite is missing"
echo >&2
usage
fi
shift
# get path to "chroot" in our current PATH
chrootPath="$(type -P chroot || :)"
if [ -z "$chrootPath" ]; then
echo >&2 "error: chroot not found. Are you root?"
echo >&2
usage
fi
rootfs_chroot() {
# "chroot" doesn't set PATH, so we need to set it explicitly to something our new debootstrap chroot can use appropriately!
# set PATH and chroot away!
PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
"$chrootPath" "$rootfsDir" "$@"
}
# allow for DEBOOTSTRAP=qemu-debootstrap ./mkimage.sh ...
: ${DEBOOTSTRAP:=debootstrap}
(
set -x
$DEBOOTSTRAP "${before[@]}" "$suite" "$rootfsDir" "$@"
)
# now for some Docker-specific tweaks
# prevent init scripts from running during install/update
echo >&2 "+ echo exit 101 > '$rootfsDir/usr/sbin/policy-rc.d'"
cat > "$rootfsDir/usr/sbin/policy-rc.d" <<-'EOF'
#!/bin/sh
# For most Docker users, "apt-get install" only happens during "docker build",
# where starting services doesn't work and often fails in humorous ways. This
# prevents those failures by stopping the services from attempting to start.
exit 101
EOF
chmod +x "$rootfsDir/usr/sbin/policy-rc.d"
# prevent upstart scripts from running during install/update
(
set -x
rootfs_chroot dpkg-divert --local --rename --add /sbin/initctl
cp -a "$rootfsDir/usr/sbin/policy-rc.d" "$rootfsDir/sbin/initctl"
sed -i 's/^exit.*/exit 0/' "$rootfsDir/sbin/initctl"
)
# shrink a little, since apt makes us cache-fat (wheezy: ~157.5MB vs ~120MB)
( set -x; rootfs_chroot apt-get clean )
# this file is one APT creates to make sure we don't "autoremove" our currently
# in-use kernel, which doesn't really apply to debootstraps/Docker images that
# don't even have kernels installed
rm -f "$rootfsDir/etc/apt/apt.conf.d/01autoremove-kernels"
# Ubuntu 10.04 sucks... :)
if strings "$rootfsDir/usr/bin/dpkg" | grep -q unsafe-io; then
# force dpkg not to call sync() after package extraction (speeding up installs)
echo >&2 "+ echo force-unsafe-io > '$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup'"
cat > "$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF'
# For most Docker users, package installs happen during "docker build", which
# doesn't survive power loss and gets restarted clean afterwards anyhow, so
# this minor tweak gives us a nice speedup (much nicer on spinning disks,
# obviously).
force-unsafe-io
EOF
fi
if [ -d "$rootfsDir/etc/apt/apt.conf.d" ]; then
# _keep_ us lean by effectively running "apt-get clean" after every install
aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";'
echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-clean'"
cat > "$rootfsDir/etc/apt/apt.conf.d/docker-clean" <<-EOF
# Since for most Docker users, package installs happen in "docker build" steps,
# they essentially become individual layers due to the way Docker handles
# layering, especially using CoW filesystems. What this means for us is that
# the caches that APT keeps end up just wasting space in those layers, making
# our layers unnecessarily large (especially since we'll normally never use
# these caches again and will instead just "docker build" again and make a brand
# new image).
# Ideally, these would just be invoking "apt-get clean", but in our testing,
# that ended up being cyclic and we got stuck on APT's lock, so we get this fun
# creation that's essentially just "apt-get clean".
DPkg::Post-Invoke { ${aptGetClean} };
APT::Update::Post-Invoke { ${aptGetClean} };
Dir::Cache::pkgcache "";
Dir::Cache::srcpkgcache "";
# Note that we do realize this isn't the ideal way to do this, and are always
# open to better suggestions (https://github.com/docker/docker/issues).
EOF
# remove apt-cache translations for fast "apt-get update"
echo >&2 "+ echo Acquire::Languages 'none' > '$rootfsDir/etc/apt/apt.conf.d/docker-no-languages'"
cat > "$rootfsDir/etc/apt/apt.conf.d/docker-no-languages" <<-'EOF'
# In Docker, we don't often need the "Translations" files, so we're just wasting
# time and space by downloading them, and this inhibits that. For users that do
# need them, it's a simple matter to delete this file and "apt-get update". :)
Acquire::Languages "none";
EOF
echo >&2 "+ echo Acquire::GzipIndexes 'true' > '$rootfsDir/etc/apt/apt.conf.d/docker-gzip-indexes'"
cat > "$rootfsDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF'
# Since Docker users using "RUN apt-get update && apt-get install -y ..." in
# their Dockerfiles don't go delete the lists files afterwards, we want them to
# be as small as possible on-disk, so we explicitly request "gz" versions and
# tell Apt to keep them gzipped on-disk.
# For comparison, an "apt-get update" layer without this on a pristine
# "debian:wheezy" base image was "29.88 MB", where with this it was only
# "8.273 MB".
Acquire::GzipIndexes "true";
Acquire::CompressionTypes::Order:: "gz";
EOF
# update "autoremove" configuration to be aggressive about removing suggests deps that weren't manually installed
echo >&2 "+ echo Apt::AutoRemove::SuggestsImportant 'false' > '$rootfsDir/etc/apt/apt.conf.d/docker-autoremove-suggests'"
cat > "$rootfsDir/etc/apt/apt.conf.d/docker-autoremove-suggests" <<-'EOF'
# Since Docker users are looking for the smallest possible final images, the
# following emerges as a very common pattern:
# RUN apt-get update \
# && apt-get install -y <packages> \
# && <do some compilation work> \
# && apt-get purge -y --auto-remove <packages>
# By default, APT will actually _keep_ packages installed via Recommends or
# Depends if another package Suggests them, even and including if the package
# that originally caused them to be installed is removed. Setting this to
# "false" ensures that APT is appropriately aggressive about removing the
# packages it added.
# https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant
Apt::AutoRemove::SuggestsImportant "false";
EOF
fi
if [ -z "$DONT_TOUCH_SOURCES_LIST" ]; then
# tweak sources.list, where appropriate
lsbDist=
if [ -z "$lsbDist" -a -r "$rootfsDir/etc/os-release" ]; then
lsbDist="$(. "$rootfsDir/etc/os-release" && echo "$ID")"
fi
if [ -z "$lsbDist" -a -r "$rootfsDir/etc/lsb-release" ]; then
lsbDist="$(. "$rootfsDir/etc/lsb-release" && echo "$DISTRIB_ID")"
fi
if [ -z "$lsbDist" -a -r "$rootfsDir/etc/debian_version" ]; then
lsbDist='Debian'
fi
# normalize to lowercase for easier matching
lsbDist="$(echo "$lsbDist" | tr '[:upper:]' '[:lower:]')"
case "$lsbDist" in
debian)
# updates and security!
if curl -o /dev/null -s --head --location --fail "http://security.debian.org/dists/$suite/updates/main/binary-$(rootfs_chroot dpkg --print-architecture)/Packages.gz"; then
(
set -x
sed -i "
p;
s/ $suite / ${suite}-updates /
" "$rootfsDir/etc/apt/sources.list"
echo "deb http://security.debian.org $suite/updates main" >> "$rootfsDir/etc/apt/sources.list"
)
fi
;;
ubuntu)
# add the updates and security repositories
(
set -x
sed -i "
p;
s/ $suite / ${suite}-updates /; p;
s/ $suite-updates / ${suite}-security /
" "$rootfsDir/etc/apt/sources.list"
)
;;
tanglu)
# add the updates repository
if [ "$suite" != 'devel' ]; then
(
set -x
sed -i "
p;
s/ $suite / ${suite}-updates /
" "$rootfsDir/etc/apt/sources.list"
)
fi
;;
steamos)
# add contrib and non-free if "main" is the only component
(
set -x
sed -i "s/ $suite main$/ $suite main contrib non-free/" "$rootfsDir/etc/apt/sources.list"
)
;;
esac
fi
(
set -x
# make sure we're fully up-to-date
rootfs_chroot sh -xc 'apt-get update && apt-get dist-upgrade -y'
# delete all the apt list files since they're big and get stale quickly
rm -rf "$rootfsDir/var/lib/apt/lists"/*
# this forces "apt-get update" in dependent images, which is also good
mkdir "$rootfsDir/var/lib/apt/lists/partial" # Lucid... "E: Lists directory /var/lib/apt/lists/partial is missing."
)
#!/bin/sh
: "${DIST:=stretch}"
: "${VARIANT:=minbase}"
: "${ARCH:=armhf}"
: "${TARGET:=/image}"
: "${GPGKEY:=A0DA38D0D76E8B5D638872819165938D90FDDD2E}"
: "${KEYSERVER:=keyserver.ubuntu.com}"
: "${MIRROR:=http://mirrordirector.raspbian.org/raspbian}"
: "${SCRIPT:=/usr/share/debootstrap/scripts/raspbian}"
: "${TARBALL:=rootfs.tar.xz}"
set -exu
gpg --no-tty --keyserver "$KEYSERVER" --recv-key "$GPGKEY"
umount -R "${TARGET:?}/proc" || true
umount -R "${TARGET:?}/sys" || true
umount -R "${TARGET:?}/dev" || true
rm -rf "${TARGET:?}"
rm -f "${TARBALL}"
if ! debootstrap \
--variant="$VARIANT" \
--arch="$ARCH" \
--keyring="$HOME"/.gnupg/pubring.kbx \
"$@" \
"$DIST" \
"$TARGET" \
"$MIRROR" \
"$SCRIPT"
then
tail -n 100 "${TARGET:?}"/debootstrap/debootstrap.log
exit 1
fi
umount -R "${TARGET:?}/proc" || true
umount -R "${TARGET:?}/sys" || true
umount -R "${TARGET:?}/dev" || true
# /var/cache/apt/pkgcache.bin /var/cache/apt/srcpkgcache.bin
rm -rf "${TARGET:?}"/var/cache/apt/*pkgcache.bin
rm -rf "${TARGET:?}"/var/lib/apt/lists/*
rm -rf "${TARGET:?}"/var/cache/apt/archives/*
tar -C "${TARGET:?}" -Jcvf "${TARBALL}" .
#!/usr/bin/env bash
set -e
mkimg="$(basename "$0")"
usage() {
echo >&2 "usage: $mkimg [-d dir] [-t tag] [--compression algo| --no-compression] script [script-args]"
echo >&2 " ie: $mkimg -t someuser/debian debootstrap --variant=minbase jessie"
echo >&2 " $mkimg -t someuser/ubuntu debootstrap --include=ubuntu-minimal --components=main,universe trusty"
echo >&2 " $mkimg -t someuser/busybox busybox-static"
echo >&2 " $mkimg -t someuser/centos:5 rinse --distribution centos-5"
echo >&2 " $mkimg -t someuser/mageia:4 mageia-urpmi --version=4"
echo >&2 " $mkimg -t someuser/mageia:4 mageia-urpmi --version=4 --mirror=http://somemirror/"
exit 1
}
scriptDir="$(dirname "$(readlink -f "$BASH_SOURCE")")/mkimage"
os=
os=$(uname -o)
optTemp=$(getopt --options '+d:t:c:hC' --longoptions 'dir:,tag:,compression:,no-compression,help' --name "$mkimg" -- "$@")
eval set -- "$optTemp"
unset optTemp
dir=
tag=
compression="auto"
while true; do
case "$1" in
-d|--dir) dir="$2" ; shift 2 ;;
-t|--tag) tag="$2" ; shift 2 ;;
--compression) compression="$2" ; shift 2 ;;
--no-compression) compression="none" ; shift 1 ;;
-h|--help) usage ;;
--) shift ; break ;;
esac
done
script="$1"
[ "$script" ] || usage
shift
if [ "$compression" == 'auto' ] || [ -z "$compression" ]
then
compression='xz'
fi
[ "$compression" == 'none' ] && compression=''
if [ ! -x "$scriptDir/$script" ]; then
echo >&2 "error: $script does not exist or is not executable"
echo >&2 " see $scriptDir for possible scripts"
exit 1
fi
# don't mistake common scripts like .febootstrap-minimize as image-creators
if [[ "$script" == .* ]]; then
echo >&2 "error: $script is a script helper, not a script"
echo >&2 " see $scriptDir for possible scripts"
exit 1
fi
delDir=
if [ -z "$dir" ]; then
dir="$(mktemp -d ${TMPDIR:-/var/tmp}/docker-mkimage.XXXXXXXXXX)"
delDir=1
fi
rootfsDir="$dir/rootfs"
( set -x; mkdir -p "$rootfsDir" )
# pass all remaining arguments to $script
"$scriptDir/$script" "$rootfsDir" "$@"
# Docker mounts tmpfs at /dev and procfs at /proc so we can remove them
rm -rf "$rootfsDir/dev" "$rootfsDir/proc"
mkdir -p "$rootfsDir/dev" "$rootfsDir/proc"
# make sure /etc/resolv.conf has something useful in it
mkdir -p "$rootfsDir/etc"
cat > "$rootfsDir/etc/resolv.conf" <<'EOF'
nameserver 8.8.8.8
nameserver 8.8.4.4
EOF
tarFile="$dir/rootfs.tar${compression:+.$compression}"
touch "$tarFile"
(
set -x
tar --numeric-owner --create --auto-compress --file "$tarFile" --directory "$rootfsDir" --transform='s,^./,,' .
)
echo >&2 "+ cat > '$dir/Dockerfile'"
cat > "$dir/Dockerfile" <<EOF
FROM scratch
ADD $(basename "$tarFile") /
EOF
# if our generated image has a decent shell, let's set a default command
for shell in /bin/bash /usr/bin/fish /usr/bin/zsh /bin/sh; do
if [ -x "$rootfsDir/$shell" ]; then
( set -x; echo 'CMD ["'"$shell"'"]' >> "$dir/Dockerfile" )
break
fi
done
( set -x; rm -rf "$rootfsDir" )
if [ "$tag" ]; then
( set -x; docker build -t "$tag" "$dir" )
elif [ "$delDir" ]; then
# if we didn't specify a tag and we're going to delete our dir, let's just build an untagged image so that we did _something_
( set -x; docker build "$dir" )
fi
if [ "$delDir" ]; then
( set -x; rm -rf "$dir" )
fi
mirror_style release
download_style apt
finddebs_style from-indices
variants - buildd fakechroot minbase
keyring /usr/share/keyrings/debian-archive-keyring.gpg
ignore_packages="udev dmsetup systemd mountall pinentry-curses procps"
if doing_variant fakechroot; then
test "$FAKECHROOT" = "true" || error 1 FAKECHROOTREQ "This variant requires fakechroot environment to be started"
fi
case $ARCH in
alpha|ia64) LIBC="libc6.1" ;;
kfreebsd-*) LIBC="libc0.1" ;;
hurd-*) LIBC="libc0.3" ;;
*) LIBC="libc6" ;;
esac
work_out_debs () {
required="$(get_debs Priority: required)"
# remove some packages for Docker
required="$(without "$required" "$ignore_packages")"
if doing_variant - || doing_variant fakechroot; then
#required="$required $(get_debs Priority: important)"
# ^^ should be getting debconf here somehow maybe
base="$(get_debs Priority: important)"
elif doing_variant buildd; then
base="apt build-essential"
elif doing_variant minbase; then
base="apt"
fi
if doing_variant fakechroot; then
# ldd.fake needs binutils
required="$required binutils"
fi
case $MIRRORS in
https://*)
base="$base apt-transport-https ca-certificates"
;;
esac
}
first_stage_install () {
case "$CODENAME" in
etch|etch-m68k|jessie|jessie-kfreebsd|lenny|squeeze|wheezy) ;;
*)
EXTRACT_DEB_TAR_OPTIONS="$EXTRACT_DEB_TAR_OPTIONS -k"
setup_merged_usr
;;
esac
extract $required
mkdir -p "$TARGET/var/lib/dpkg"
: >"$TARGET/var/lib/dpkg/status"
: >"$TARGET/var/lib/dpkg/available"
setup_etc
if [ ! -e "$TARGET/etc/fstab" ]; then
echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab"
chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab"
fi
setup_devices
}
second_stage_install () {
setup_dynamic_devices
if [ -x /usr/bin/qemu-arm-static ]; then
cp -a /usr/bin/qemu-arm-static "$TARGET/usr/bin/qemu-arm-static"
else
echo "No qemu-arm-static found!" >&2
exit 1
fi
x_feign_install () {
local pkg="$1"
local deb="$(debfor $pkg)"
local ver="$(in_target dpkg-deb -f "$deb" Version)"
mkdir -p "$TARGET/var/lib/dpkg/info"
echo \
"Package: $pkg
Version: $ver
Maintainer: unknown
Status: install ok installed" >> "$TARGET/var/lib/dpkg/status"
touch "$TARGET/var/lib/dpkg/info/${pkg}.list"
}
x_feign_install dpkg
x_core_install () {
smallyes '' | in_target dpkg --force-depends --install $(debfor "$@")
}
p () {
baseprog="$(($baseprog + ${1:-1}))"
}
if doing_variant fakechroot; then
setup_proc_fakechroot
else
setup_proc
in_target /sbin/ldconfig
fi
DEBIAN_FRONTEND=noninteractive
DEBCONF_NONINTERACTIVE_SEEN=true
export DEBIAN_FRONTEND DEBCONF_NONINTERACTIVE_SEEN
baseprog=0
bases=7
p; progress $baseprog $bases INSTCORE "Installing core packages" #1
info INSTCORE "Installing core packages..."
p; progress $baseprog $bases INSTCORE "Installing core packages" #2
ln -sf mawk "$TARGET/usr/bin/awk"
x_core_install base-passwd
x_core_install base-files
p; progress $baseprog $bases INSTCORE "Installing core packages" #3
x_core_install dpkg
if [ ! -e "$TARGET/etc/localtime" ]; then
ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime"
fi
if doing_variant fakechroot; then
install_fakechroot_tools