From c6ed03f331541d4cba5dfdceb321949debf945b8 Mon Sep 17 00:00:00 2001 From: Kroese Date: Wed, 12 Jun 2024 22:36:39 +0200 Subject: [PATCH] feat: Support download of disk images (#118) --- Dockerfile | 3 + readme.md | 24 ++-- src/config.sh | 3 +- src/disk.sh | 35 +++--- src/install.sh | 327 ++++++++++++++++++++++++++++++++++++++++++------- src/reset.sh | 2 + 6 files changed, 328 insertions(+), 66 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b6de84..020734d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ RUN set -eu && \ apt-get --no-install-recommends -y install \ tini \ wget \ + 7zip \ + fdisk \ nginx \ procps \ seabios \ @@ -19,6 +21,7 @@ RUN set -eu && \ iproute2 \ apt-utils \ dnsmasq \ + xz-utils \ net-tools \ qemu-utils \ genisoimage \ diff --git a/readme.md b/readme.md index 736923d..ea07274 100644 --- a/readme.md +++ b/readme.md @@ -18,13 +18,15 @@ Docker container for running ARM-based virtual machines using QEMU, for devices - Create VM's which behave just like normal containers - - Manage them using all your existing tools (like Portainer) and configure them in a language (YAML) you are already familiar with + - Manage them using all your existing tools (like Portainer) - - Reduces the learning curve and eliminates the need for a dedicated Proxmox or ESXi server + - Configure them in a language (YAML) you are already familiar with - - Web-based viewer to control the machine directly from your browser + - Web-based viewer to control the machine directly from your browser - - High-performance QEMU options (like KVM acceleration, kernel-mode networking, IO threading, etc.) to achieve near-native speed + - Supports `.iso`, `.img`, `.qcow2`, `.vhd`, `.vhdx`, `.vdi`, `.vmdk` and `.raw` disk formats + + - High-performance options (like KVM acceleration, kernel-mode networking, IO threading, etc.) to achieve near-native speed *Note: for KVM acceleration you need a Linux-based operating system, as it's not available on MacOS unfortunately.* @@ -66,7 +68,7 @@ kubectl apply -f kubernetes.yml Very simple! These are the steps: - - Set the `BOOT` environment variable to the URL of an ISO image you want to install. + - Set the `BOOT` environment variable to the URL of any [disk image](https://github.com/qemus/qemu-docker#what-image-formats-are-supported) you want to install. - Start the container and connect to [port 8006](http://localhost:8006) using your web browser. @@ -96,16 +98,16 @@ kubectl apply -f kubernetes.yml This can also be used to resize the existing disk to a larger capacity without any data loss. -* ### How do I boot a local ISO? +* ### How do I boot a local image? - You can use a local file directly, and skip the download altogether, by binding it in your compose file in this way: + You can use a local image file directly, and skip the download altogether, by binding it in your compose file: ```yaml volumes: - /home/user/example.iso:/boot.iso ``` - Replace the example path `/home/user/example.iso` with the filename of the desired ISO file, the value of `BOOT` will be ignored in this case. + This way you can supply a `boot.iso`, `boot.img` or `boot.qcow2` file. The URL of the `BOOT` variable will be ignored in this case. * ### How do I boot Windows? @@ -257,6 +259,12 @@ kubectl apply -f kubernetes.yml ARGUMENTS: "-device usb-tablet" ``` +* ### What image formats are supported? + + You can set the `BOOT` URL to any `.iso`, `.img`, `.raw`, `.qcow2`, `.vhd`, `.vhdx`, `.vdi` or `.vmdk` file. + + It will even automaticly extract compressed images, like `.img.gz`, `.qcow2.xz`, `.iso.zip` and many more! + ## Stars 🌟 [![Stars](https://starchart.cc/qemus/qemu-arm.svg?variant=adaptive)](https://starchart.cc/qemus/qemu-arm) diff --git a/src/config.sh b/src/config.sh index 83898bc..e043754 100644 --- a/src/config.sh +++ b/src/config.sh @@ -4,12 +4,13 @@ set -Eeuo pipefail : "${SERIAL:="mon:stdio"}" : "${USB:="qemu-xhci,id=xhci"}" : "${MONITOR:="telnet:localhost:7100,server,nowait,nodelay"}" +: "${SMP:="$CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1"}" DEF_OPTS="-nodefaults" SERIAL_OPTS="-serial $SERIAL" +CPU_OPTS="-cpu $CPU_FLAGS -smp $SMP" USB_OPTS="-device $USB -device usb-kbd -device usb-tablet" RAM_OPTS=$(echo "-m ${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') -CPU_OPTS="-cpu $CPU_FLAGS -smp $CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1" MON_OPTS="-monitor $MONITOR -name $PROCESS,process=$PROCESS,debug-threads=on" MAC_OPTS="-machine type=${MACHINE},secure=${SECURE},dump-guest-core=off${KVM_OPTS}" DEV_OPTS="-object rng-random,id=objrng0,filename=/dev/urandom" diff --git a/src/disk.sh b/src/disk.sh index bbb2877..2d02992 100644 --- a/src/disk.sh +++ b/src/disk.sh @@ -274,10 +274,10 @@ convertDisk() { isCow "$FS" && DISK_PARAM+=",nocow=on" if [[ "$DST_FMT" != "raw" ]]; then - if [[ "$ALLOCATE" == [Nn]* ]]; then - CONV_FLAGS+=" -c" - fi - [ -n "$DISK_FLAGS" ] && DISK_PARAM+=",$DISK_FLAGS" + if [[ "$ALLOCATE" == [Nn]* ]]; then + CONV_FLAGS+=" -c" + fi + [ -n "$DISK_FLAGS" ] && DISK_PARAM+=",$DISK_FLAGS" fi # shellcheck disable=SC2086 @@ -287,13 +287,13 @@ convertDisk() { fi if [[ "$DST_FMT" == "raw" ]]; then - if [[ "$ALLOCATE" != [Nn]* ]]; then - # Work around qemu-img bug - CUR_SIZE=$(stat -c%s "$TMP_FILE") - if ! fallocate -l "$CUR_SIZE" "$TMP_FILE"; then - error "Failed to allocate $CUR_SIZE bytes for $DISK_DESC image $TMP_FILE" - fi + if [[ "$ALLOCATE" != [Nn]* ]]; then + # Work around qemu-img bug + CUR_SIZE=$(stat -c%s "$TMP_FILE") + if ! fallocate -l "$CUR_SIZE" "$TMP_FILE"; then + error "Failed to allocate $CUR_SIZE bytes for $DISK_DESC image $TMP_FILE" fi + fi fi rm -f "$SOURCE_FILE" @@ -524,6 +524,7 @@ addDevice () { html "Initializing disks..." [ -z "${DISK_OPTS:-}" ] && DISK_OPTS="" +[ -z "${DISK_NAME:-}" ] && DISK_NAME="data" [ -z "${DISK_TYPE:-}" ] && DISK_TYPE="scsi" case "${DISK_TYPE,,}" in @@ -564,10 +565,16 @@ if [ -f "$DRIVERS" ] && [ -s "$DRIVERS" ]; then DISK_OPTS+=$(addMedia "$DRIVERS" "$FALLBACK" "1" "" "0x6") fi -DISK1_FILE="$STORAGE/data" -DISK2_FILE="/storage2/data2" -DISK3_FILE="/storage3/data3" -DISK4_FILE="/storage4/data4" +DISK1_FILE="/boot" +if [ ! -f "$DISK1_FILE.img" ] || [ ! -s "$DISK1_FILE.img" ]; then + if [ ! -f "$DISK1_FILE.qcow2" ] || [ ! -s "$DISK1_FILE.qcow2" ]; then + DISK1_FILE="$STORAGE/${DISK_NAME}" + fi +fi + +DISK2_FILE="/storage2/${DISK_NAME}2" +DISK3_FILE="/storage3/${DISK_NAME}3" +DISK4_FILE="/storage4/${DISK_NAME}4" if [ -z "$DISK_FMT" ]; then if [ -f "$DISK1_FILE.qcow2" ]; then diff --git a/src/install.sh b/src/install.sh index f480f8d..bf98147 100644 --- a/src/install.sh +++ b/src/install.sh @@ -1,76 +1,317 @@ #!/usr/bin/env bash set -Eeuo pipefail -detect () { +detectType() { + local dir="" local file="$1" + [ ! -f "$file" ] && return 1 [ ! -s "$file" ] && return 1 - dir=$(isoinfo -f -i "$file") + case "${file,,}" in + *".iso" ) - if [ -z "${BOOT_MODE:-}" ]; then - # Automaticly detect UEFI-compatible ISO's - dir=$(isoinfo -f -i "$file") - dir=$(echo "${dir^^}" | grep "^/EFI") - [ -n "$dir" ] && BOOT_MODE="uefi" - fi + BOOT="$file" + [ -n "${BOOT_MODE:-}" ] && return 0 + + # Automaticly detect UEFI-compatible ISO's + dir=$(isoinfo -f -i "$file") + [ -z "$dir" ] && error "Failed to read ISO file, invalid format!" && BOOT="" && return 1 + + dir=$(echo "${dir^^}" | grep "^/EFI") + [ -n "$dir" ] && BOOT_MODE="uefi" + ;; + + *".img" ) + + DISK_NAME=$(basename "$file") + DISK_NAME="${DISK_NAME%.*}" + [ -n "${BOOT_MODE:-}" ] && return 0 + + # Automaticly detect UEFI-compatible images + dir=$(sfdisk -l "$file") + [ -z "$dir" ] && error "Failed to read IMG file, invalid format!" && DISK_NAME="" && return 1 + + dir=$(echo "${dir^^}" | grep "EFI SYSTEM") + [ -n "$dir" ] && BOOT_MODE="uefi" + ;; + + *".qcow2" ) + + DISK_NAME=$(basename "$file") + DISK_NAME="${DISK_NAME%.*}" + [ -n "${BOOT_MODE:-}" ] && return 0 + + # TODO: Detect boot mode from partition table in image + BOOT_MODE="uefi" + ;; + + * ) + return 1 ;; + esac - BOOT="$file" return 0 } -file=$(find / -maxdepth 1 -type f -iname boot.iso | head -n 1) -[ ! -s "$file" ] && file=$(find "$STORAGE" -maxdepth 1 -type f -iname boot.iso | head -n 1) -detect "$file" && return 0 +downloadFile() { + + local url="$1" + local base="$2" + local msg rc total progress + + local dest="$STORAGE/$base.tmp" + rm -f "$dest" + + # Check if running with interactive TTY or redirected to docker log + if [ -t 1 ]; then + progress="--progress=bar:noscroll" + else + progress="--progress=dot:giga" + fi + + msg="Downloading image" + info "Downloading $base..." + html "$msg..." + + /run/progress.sh "$dest" "0" "$msg ([P])..." & + + { wget "$url" -O "$dest" -q --timeout=30 --show-progress "$progress"; rc=$?; } || : + + fKill "progress.sh" + + if (( rc == 0 )) && [ -f "$dest" ]; then + total=$(stat -c%s "$dest") + if [ "$total" -lt 100000 ]; then + error "Invalid image file: is only $total bytes?" && return 1 + fi + html "Download finished successfully..." + mv -f "$dest" "$STORAGE/$base" + return 0 + fi + + msg="Failed to download $url" + (( rc == 3 )) && error "$msg , cannot write file (disk full?)" && return 1 + (( rc == 4 )) && error "$msg , network failure!" && return 1 + (( rc == 8 )) && error "$msg , server issued an error response!" && return 1 + + error "$msg , reason: $rc" + return 1 +} + +convertImage() { + + local source_file=$1 + local source_fmt=$2 + local dst_file=$3 + local dst_fmt=$4 + local dir base fs fa cur_size src_size space disk_param + + [ -f "$dst_file" ] && error "Conversion failed, destination file $dst_file already exists?" && return 1 + [ ! -f "$source_file" ] && error "Conversion failed, source file $source_file does not exists?" && return 1 + + if [[ "$source_fmt" == "raw" ]] && [[ "$dst_fmt" == "raw" ]]; then + mv -f "$source_file" "$dst_file" + return 0 + fi + + local tmp_file="$dst_file.tmp" + dir=$(dirname "$tmp_file") + + rm -f "$tmp_file" + + if [ -n "$ALLOCATE" ] && [[ "$ALLOCATE" != [Nn]* ]]; then + + # Check free diskspace + src_size=$(qemu-img info "$source_file" -f "$source_fmt" | grep '^virtual size: ' | sed 's/.*(\(.*\) bytes)/\1/') + space=$(df --output=avail -B 1 "$dir" | tail -n 1) + + if (( src_size > space )); then + local space_gb=$(( (space + 1073741823)/1073741824 )) + error "Not enough free space to convert image in $dir, it has only $space_gb GB available..." && return 1 + fi + fi + + base=$(basename "$source_file") + info "Converting $base..." + html "Converting image..." + + local conv_flags="-p" + + if [ -z "$ALLOCATE" ] || [[ "$ALLOCATE" == [Nn]* ]]; then + disk_param="preallocation=off" + else + disk_param="preallocation=falloc" + fi + + fs=$(stat -f -c %T "$dir") + [[ "${fs,,}" == "btrfs" ]] && disk_param+=",nocow=on" + + if [[ "$dst_fmt" != "raw" ]]; then + if [ -z "$ALLOCATE" ] || [[ "$ALLOCATE" == [Nn]* ]]; then + conv_flags+=" -c" + fi + [ -n "${DISK_FLAGS:-}" ] && disk_param+=",$DISK_FLAGS" + fi + + # shellcheck disable=SC2086 + if ! qemu-img convert -f "$source_fmt" $conv_flags -o "$disk_param" -O "$dst_fmt" -- "$source_file" "$tmp_file"; then + rm -f "$tmp_file" + error "Failed to convert image in $dir, is there enough space available?" && return 1 + fi + + if [[ "$dst_fmt" == "raw" ]]; then + if [ -n "$ALLOCATE" ] && [[ "$ALLOCATE" != [Nn]* ]]; then + # Work around qemu-img bug + cur_size=$(stat -c%s "$tmp_file") + if ! fallocate -l "$cur_size" "$tmp_file"; then + error "Failed to allocate $cur_size bytes for image!" + fi + fi + fi + + rm -f "$source_file" + mv "$tmp_file" "$dst_file" + + if [[ "${fs,,}" == "btrfs" ]]; then + fa=$(lsattr "$dst_file") + if [[ "$fa" != *"C"* ]]; then + error "Failed to disable COW for image on ${fs^^} filesystem!" + fi + fi + + html "Conversion completed..." + + return 0 +} + +findFile() { + + local ext="$1" + local file + + file=$(find / -maxdepth 1 -type f -iname "boot.$ext" | head -n 1) + [ ! -s "$file" ] && file=$(find "$STORAGE" -maxdepth 1 -type f -iname "boot.$ext" | head -n 1) + detectType "$file" && return 0 + + return 1 +} + +findFile "iso" && return 0 +findFile "img" && return 0 +findFile "qcow2" && return 0 if [ -z "$BOOT" ] || [[ "$BOOT" == *"example.com/image.iso" ]]; then hasDisk && return 0 - error "No boot disk specified, set BOOT= to the URL of an ISO file." && exit 64 + error "No boot disk specified, set BOOT= to the URL of a disk image file." && exit 64 fi -base=$(basename "$BOOT") -detect "$STORAGE/$base" && return 0 - base=$(basename "${BOOT%%\?*}") : "${base//+/ }"; printf -v base '%b' "${_//%/\\x}" base=$(echo "$base" | sed -e 's/[^A-Za-z0-9._-]/_/g') -detect "$STORAGE/$base" && return 0 -TMP="$STORAGE/${base%.*}.tmp" -rm -f "$TMP" +case "${base,,}" in -# Check if running with interactive TTY or redirected to docker log -if [ -t 1 ]; then - progress="--progress=bar:noscroll" -else - progress="--progress=dot:giga" + *".iso" | *".img" | *".qcow2" ) + + detectType "$STORAGE/$base" && return 0 ;; + + *".raw" | *".vdi" | *".vmdk" | *".vhd" | *".vhdx" ) + + detectType "$STORAGE/${base%.*}.img" && return 0 + detectType "$STORAGE/${base%.*}.qcow2" && return 0 ;; + + *".gz" | *".gzip" | *".xz" | *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" ) + + case "${base%.*}" in + *".iso" | *".img" | *".qcow2" ) + + detectType "$STORAGE/${base%.*}" && return 0 ;; + + *".raw" | *".vdi" | *".vmdk" | *".vhd" | *".vhdx" ) + + find="${base%.*}" + + detectType "$STORAGE/${find%.*}.img" && return 0 + detectType "$STORAGE/${find%.*}.qcow2" && return 0 ;; + + esac ;; + + * ) + error "Unknown file format, extension \".${base/*./}\" is not recognized!" && exit 33 ;; +esac + +if ! downloadFile "$BOOT" "$base"; then + rm -f "$STORAGE/$base.tmp" && exit 60 fi -msg="Downloading $base" -info "$msg..." && html "$msg..." +case "${base,,}" in + *".gz" | *".gzip" | *".xz" | *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" ) + info "Extracting $base..." + html "Extracting image..." ;; +esac -/run/progress.sh "$TMP" "" "$msg ([P])..." & -{ wget "$BOOT" -O "$TMP" -q --timeout=30 --show-progress "$progress"; rc=$?; } || : +case "${base,,}" in + *".gz" | *".gzip" ) -fKill "progress.sh" + gzip -dc "$STORAGE/$base" > "$STORAGE/${base%.*}" + rm -f "$STORAGE/$base" + base="${base%.*}" -msg="Failed to download $BOOT" -(( rc == 3 )) && error "$msg , cannot write file (disk full?)" && exit 60 -(( rc == 4 )) && error "$msg , network failure!" && exit 60 -(( rc == 8 )) && error "$msg , server issued an error response!" && exit 60 -(( rc != 0 )) && error "$msg , reason: $rc" && exit 60 -[ ! -s "$TMP" ] && error "$msg" && exit 61 + ;; + *".xz" ) -html "Download finished successfully..." + xz -dc "$STORAGE/$base" > "$STORAGE/${base%.*}" + rm -f "$STORAGE/$base" + base="${base%.*}" -size=$(stat -c%s "$TMP") + ;; + *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" ) -if ((size<100000)); then - error "Invalid ISO file: Size is smaller than 100 KB" && exit 62 -fi + tmp="$STORAGE/extract" + rm -rf "$tmp" + mkdir -p "$tmp" + 7z x "$STORAGE/$base" -o"$tmp" > /dev/null -mv -f "$TMP" "$STORAGE/$base" -! detect "$STORAGE/$base" && exit 63 + rm -f "$STORAGE/$base" + base="${base%.*}" -return 0 + if [ ! -s "$tmp/$base" ]; then + rm -rf "$tmp" + error "Cannot find file \"${base}\" in .${BOOT/*./} archive!" && exit 32 + fi + + mv "$tmp/$base" "$STORAGE/$base" + rm -rf "$tmp" + + ;; +esac + +case "${base,,}" in + *".iso" | *".img" | *".qcow2" ) + detectType "$STORAGE/$base" && return 0 + error "Cannot read file \"${base}\"" && exit 63 ;; +esac + +target_ext="img" +target_fmt="${DISK_FMT:-}" +[ -z "$target_fmt" ] && target_fmt="raw" +[[ "$target_fmt" != "raw" ]] && target_ext="qcow2" + +case "${base,,}" in + *".raw" ) source_fmt="raw" ;; + *".vdi" ) source_fmt="vdi" ;; + *".vhd" ) source_fmt="vhd" ;; + *".vmdk" ) source_fmt="vmdk" ;; + *".vhdx" ) source_fmt="vhdx" ;; + * ) + error "Unknown file format, extension \".${base/*./}\" is not recognized!" && exit 33 ;; +esac + +dst="$STORAGE/${base%.*}.$target_ext" + +! convertImage "$STORAGE/$base" "$source_fmt" "$dst" "$target_fmt" && exit 35 + +base=$(basename "$dst") +detectType "$STORAGE/$base" && return 0 +error "Cannot read file \"${base}\"" && exit 36 diff --git a/src/reset.sh b/src/reset.sh index 11a3f84..545ec57 100644 --- a/src/reset.sh +++ b/src/reset.sh @@ -196,6 +196,8 @@ hasDisk() { [ -b "/disk1" ] && return 0 [ -b "/dev/disk1" ] && return 0 + [ -s "/boot.img" ] && return 0 + [ -s "/boot.qcow2" ] && return 0 [ -b "${DEVICE:-}" ] && return 0 [ -s "$STORAGE/data.img" ] && return 0 [ -s "$STORAGE/data.qcow2" ] && return 0