distro-tester: allow non virtiofs kernels to be tested

- this change allows GKE kernels to be tested

tests/distro-tester/00-config

@@ -1,2 +1,2 @@
 LXD_STORAGE="/home/rafaeldtinoco/space/lxd/virtual-machines"
-IMAGES="vm-stream8 vm-stream9 vm-fedora34 vm-fedora35 vm-fedora36 vm-focal vm-focalhwe vm-jammy"
+IMAGES="vm-stream8 vm-stream9 vm-fedora34 vm-fedora35 vm-fedora36 vm-focal vm-focalhwe vm-jammy gkekernel1.21.11"

tests/distro-tester/01-copy-images.sh

@@ -5,7 +5,11 @@
 [[ ! -d ./images-full ]] && mkdir ./images-full
 
 for image in $IMAGES; do
-  cp ${LXD_STORAGE}/$image/root.img ./images-full/${image/vm-/}
+  if [[ -f ${LXD_STORAGE}/$image/root.img ]]; then
+    cp ${LXD_STORAGE}/$image/root.img ./images-full/${image/vm-/}
+  else
+    echo "skipping non-existent $image..."
+  fi
 done
 
 chown -R $(whoami): ./images*

tests/distro-tester/02-extract-images.sh

@@ -53,11 +53,11 @@ for image in $IMAGES; do
   mount $un_ext4 $tempdir
   mount $un_fat $tempdir/boot/efi
 
-  rsync -av --delete $tempdir/ $destdir/
+  #rsync -av --delete $tempdir/ $destdir/
   #rsync -av $tempdir/ $destdir/
 
   # fstab fix
-  echo "/dev/vda / ext4 errors=remount-ro 0 1" > $destdir/etc/fstab
+  echo "/dev/sda / ext4 errors=remount-ro 0 1" > $destdir/etc/fstab
 
   # qemu entrypoint execution at ttyS0
   mkdir -p $destdir/etc/systemd/system/[email protected]/

tests/distro-tester/03-run-qemu.sh

@@ -6,6 +6,10 @@
 
 command -v qemu-system-x86_64 || exit 1
 command -v qemu-img || exit 1
+command -v truncate || exit 1
+command -v mount || exit 1
+command -v rsync || exit 1
+command -v stat || exit 1
 
 image=$1
 tracee=$2
@@ -15,9 +19,6 @@ isnoncore=$5
 cpus=$6 # optional
 mem=$7 # optional
 
-qemu-img info images/$image | grep -q raw && format="raw"
-qemu-img info images/$image | grep -q qcow2 && format="qcow2"
-
 error_syntax() {
   echo ""
   echo -n "ERROR: "
@@ -28,10 +29,40 @@ error_syntax() {
   exit 1
 }
 
-if [[ ! -f images/$image ]]; then
+# check where the image is coming from (if inside container)
+
+if [[ -f ./kernels/$image.vmlinuz ]]; then
+  vmlinuz=./kernels/$image.vmlinuz
+  initrd=./kernels/$image.initrd
+else
+  vmlinuz=./kernels-copy/$image.vmlinuz
+  initrd=./kernels-copy/$image.initrd
+fi
+
+if [[ -f ./images-copy/$image ]]; then
+  image=./images-copy/$image
+else
+  image=./images/$image
+fi
+
+qemu-img info $image | grep -q raw && format="raw"
+qemu-img info $image | grep -q qcow2 && format="qcow2"
+
+# check if kernel needs initrd
+
+cmd_initrd=""
+[[ -f $initrd ]] && cmd_initrd="-initrd $initrd"
+
+# regular checks
+
+if [[ ! -f $image ]]; then
   error_syntax "image file $image does not exist"
 fi
 
+if [[ ! -f $vmlinuz ]]; then
+  error_syntax "vmlinuz file $vmlinuz does not exist"
+fi
+
 if [[ ! -d $tracee && ! -f $tracee/go.mod ]]; then
   error_syntax "invalid tracee dir: $tracee"
 fi
@@ -58,48 +89,75 @@ if [[ $mem -ne 2 && $mem -ne 4 && $mem -ne 6 && $mem -ne 8 ]]; then
   error_syntax "should provide amount of mem"
 fi
 
-mount -t tmpfs -o rw,nosuid,nodev,inode64 tmpfs /dev/shm
-
-rm -f "/tmp/vhostqemu-$image"
-
-/usr/lib/qemu/virtiofsd \
-  -o cache=always \
-  -o no_flock \
-  -o log_level=err \
-  -o no_posix_lock \
-  -o sandbox=chroot \
-  -o no_writeback \
-  -o no_xattr \
-  -o no_allow_direct_io \
-  -o source=$tracee \
-  -o allow_root \
-  --socket-path="/tmp/vhostqemu-$image" &
-
-if [[ -f ./kernels/$image.vmlinuz ]]; then
-  vmlinuz="./kernels/$image.vmlinuz"
-  initrd="./kernels/$image.initrd"
-else
-  vmlinuz="./kernels-copy/$image.vmlinuz"
-  initrd="./kernels-copy/$image.initrd"
-fi
-
+# create tracee source directory filesystem (as a 2nd disk)
+# NOTE: idea here is to avoid using virtiofs and/or p9 filesystems
+
+tempfile=$(mktemp)
+tempdir=$(mktemp -d)
+truncate -s 300M $tempfile
+mkfs.ext4 $tempfile
+
+mount $tempfile $tempdir
+rm -rf $tempdir/load+found
+
+rsync -avz \
+  $tracee/ \
+  --exclude=3rdparty/btfhub/* \
+  --exclude=3rdparty/btfhub-archive/* \
+  --exclude=tests/distro* \
+  $tempdir/
+
+ouid=$(stat -c %u $tracee/LICENSE)
+ogid=$(stat -c %g $tracee/LICENSE)
+
+umount $tempdir
+rmdir $tempdir
+
+# kernel cmdline
+cmd_kernel=$cmd_kernel"root=/dev/sda "
+cmd_kernel=$cmd_kernel"console=ttyS0 "
+cmd_kernel=$cmd_kernel"testname=$testname "
+cmd_kernel=$cmd_kernel"isnoncore=$isnoncore "
+cmd_kernel=$cmd_kernel"selinux=0 "
+cmd_kernel=$cmd_kernel"apparmor=0 "
+cmd_kernel=$cmd_kernel"systemd.unified_cgroup_hierarchy=false "
+cmd_kernel=$cmd_kernel"net.ifnames=0"
+
+# qemu cmdline
 qemu-system-x86_64 \
   -name guest=$image \
   -machine accel=$kvmaccel \
   --cpu max --smp $cpus -m ${mem}G \
-  -object memory-backend-file,id=mem,size=${mem}G,mem-path=/dev/shm,share=on \
-  -numa node,nodeid=0,memdev=mem \
   -rtc base=utc,clock=vm,driftfix=none \
   -boot c \
   -display none \
   -serial stdio \
   -kernel $vmlinuz \
-  -initrd $initrd \
-  -append "root=/dev/vda console=ttyS0 testname=$testname isnoncore=$isnoncore selinux=0 apparmor=0 systemd.unified_cgroup_hierarchy=false net.ifnames=0" \
+  $cmd_initrd \
+  -append "$cmd_kernel" \
   -netdev user,id=mynet,net=192.168.76.0/24,dhcpstart=192.168.76.9 \
   -device virtio-net-pci,netdev=mynet \
-  -drive file="./images/$image",if=virtio,format=$format \
-  -chardev socket,id=char0,path="/tmp/vhostqemu-$image" \
-  -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=/tracee
+  -device virtio-scsi-pci,id=scsi \
+  -device scsi-hd,drive=hd0 \
+  -drive if=none,id=hd0,file=$image,format=$format,index=0 \
+  -device scsi-hd,drive=hd1 \
+  -drive if=none,id=hd1,file=$tempfile,format=raw,index=1
+
+tempdir=$(mktemp -d)
+mount $tempfile $tempdir
+chown -R $ouid:$ogid $tempdir/
+
+# clean up tracee source directory filesystem
+
+rsync -avz --delete \
+  $tempdir/ \
+  --exclude=3rdparty/btfhub/* \
+  --exclude=3rdparty/btfhub-archive/* \
+  --exclude=tests/distro* \
+  $tracee/
+
+umount $tempdir
+rmdir $tempdir
+rm -rf $tempfile
 
 # vi:syntax=sh:expandtab:smarttab:tabstop=2:shiftwidth=2:softtabstop=2

tests/distro-tester/05-create-docker.sh

@@ -6,7 +6,9 @@ for image in $IMAGES; do
   image_name=${image/vm-/}
 
   cp ./kernels/$image_name.vmlinuz ./kernels-copy/$image_name.vmlinuz
-  cp ./kernels/$image_name.initrd ./kernels-copy/$image_name.initrd
+  if [[ -f ./kernels/$image_name.initrd ]]; then
+    cp ./kernels/$image_name.initrd ./kernels-copy/$image_name.initrd
+  fi
 
   cp ./images/$image_name ./images-copy/$image_name
 
@@ -15,9 +17,9 @@ for image in $IMAGES; do
     -t rafaeldtinoco/tracee-distro-tester:$image_name \
     .
 
-  rm ./kernels-copy/$image_name.vmlinuz
-  rm ./kernels-copy/$image_name.initrd
-  rm ./images-copy/$image_name
+  rm -f ./kernels-copy/$image_name.vmlinuz
+  rm -f ./kernels-copy/$image_name.initrd
+  rm -f ./images-copy/$image_name
 done
 
 # vi:syntax=sh:expandtab:smarttab:tabstop=2:shiftwidth=2:softtabstop=2

tests/distro-tester/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:impish
+FROM ubuntu:jammy
 
 ARG IMAGE
 
@@ -7,6 +7,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get dist-upgrade -y && \
     apt-get install -y --no-install-recommends coreutils findutils && \
     apt-get install -y --no-install-recommends bash vim curl rsync git && \
+    apt-get install -y --no-install-recommends iproute2 openssh-client && \
     apt-get install -y --no-install-recommends ssl-cert ca-certificates && \
     apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils
 
@@ -19,8 +20,7 @@ RUN mkdir -p /tracee && \
     echo "export PS1=\"\u@\h \w $ \"" > /root/.bashrc && \
     echo "alias ls=\"ls --color\"" >> /root/.bashrc && \
     ln -s /root/.bashrc /root/.profile && \
-    git config --global --add safe.directory /tracee && \
-    ln -s /tester/images-copy/$IMAGE /tester/images/$IMAGE
+    git config --global --add safe.directory /tracee
 
 ENV IMAGE=$IMAGE
 

tests/distro-tester/files/docker-entrypoint.sh

@@ -36,6 +36,16 @@ if [[ ! -f /tracee/go.mod ]]; then
   error_exit "/tracee doesn't seem to be tracee source directory"
 fi
 
+# create loop devices if running in LXD guest
+
+for seq in $(echo {150..170}); do
+  if [[ ! -f /dev/loop$seq ]]; then
+    mknod -m 660 /dev/loop$seq b 7 $seq
+  fi
+done
+
+# run qemu
+
 cd /tester
 
 ./03-run-qemu.sh $image_name /tracee $test_name $kvm_accel $non_core $cpus $mem | tee /tmp/qemu.log

tests/distro-tester/files/qemu-entrypoint.sh

@@ -24,7 +24,7 @@ beginhook() {
   mkdir -p /tracee
   dmesg --console-off
   trap cleanup EXIT
-  mount -t virtiofs /tracee /tracee
+  mount /dev/sdb /tracee
 }
 
 ## main
@@ -51,7 +51,7 @@ info "GO: $(go version)"
 info "pulling aquasec/tracee-tester:latest docker image"
 docker image pull aquasec/tracee-tester:latest
 
-git config --global --add safe.directory /tracee
+git config --global --add safe.directory "*"
 
 rm -rf /tmp/tracee/*
 cd /tracee