Add uefi image support for yardstick

Add support for downloading and building an uefi single part image.
Keep three part image support if needed.
Keep modifications required for arm64, when using vivid.
Install and use parted required for GPT partitioning.

Change-Id: I1a9a20701e2fe1f6f644f2c687c5da2e09b69d89
Signed-off-by: Catalina Focsa <catalina.focsa@enea.com>

diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh
index 6f950ec..e5a7ae0 100755 (executable)
--- a/tests/ci/load_images.sh
+++ b/tests/ci/load_images.sh
@@ -15,6 +15,12 @@ set -e
 YARD_IMG_ARCH=amd64
 export YARD_IMG_ARCH
 
+HW_FW_TYPE=""
+if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
+    HW_FW_TYPE=uefi
+fi
+export HW_FW_TYPE
+
 if ! grep -q "Defaults env_keep += \"YARD_IMG_ARCH\"" "/etc/sudoers"; then
     sudo echo "Defaults env_keep += \"YARD_IMG_ARCH YARDSTICK_REPO_DIR\"" >> /etc/sudoers
 fi
@@ -25,10 +31,10 @@ if [ "$INSTALLER_TYPE" == "fuel" ]; then
 fi
 
 UCA_HOST="cloud-images.ubuntu.com"
-if [ $YARD_IMG_ARCH = "arm64" ]; then
-    export VIVID_IMG_URL="http://${UCA_HOST}/vivid/current/vivid-server-cloudimg-arm64.tar.gz"
-    if ! grep -q "Defaults env_keep += \"VIVID_IMG_URL\"" "/etc/sudoers"; then
-        sudo echo "Defaults env_keep += \"VIVID_IMG_URL\"" >> /etc/sudoers
+if [ "${YARD_IMG_ARCH}"= "arm64" ]; then
+    export CLOUD_IMG_URL="http://${UCA_HOST}/${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}.tar.gz"
+    if ! grep -q "Defaults env_keep += \"CLOUD_IMG_URL\"" "/etc/sudoers"; then
+        sudo echo "Defaults env_keep += \"CLOUD_IMG_URL\"" >> /etc/sudoers
     fi
 fi
 
@@ -65,24 +71,24 @@ load_yardstick_image()
     echo
     echo "========== Loading yardstick cloud image =========="
     EXTRA_PARAMS=""
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
-        VIVID_IMAGE="/tmp/vivid-server-cloudimg-arm64.tar.gz"
-        VIVID_KERNEL="/tmp/vivid-server-cloudimg-arm64-vmlinuz-generic"
+    if [[ "${YARD_IMG_ARCH}" = "arm64" && "${YARD_IMG_AKI}" = "true" ]]; then
+        CLOUD_IMAGE="/tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}.tar.gz"
+        CLOUD_KERNEL="/tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}-vmlinuz-generic"
         cd /tmp
-        if [ ! -f $VIVID_IMAGE ]; then
-            wget $VIVID_IMG_URL
+        if [ ! -f "${CLOUD_IMAGE}" ]; then
+            wget $CLOUD_IMG_URL
         fi
-        if [ ! -f $VIVID_KERNEL ]; then
-            tar zxf $VIVID_IMAGE $(basename $VIVID_KERNEL)
+        if [ ! -f "${CLOUD_KERNEL}" ]; then
+            tar zxf $CLOUD_IMAGE $(basename $CLOUD_KERNEL)
         fi
-        create_vivid_kernel=$(openstack image create \
+        create_kernel=$(openstack image create \
                 --public \
                 --disk-format qcow2 \
                 --container-format bare \
-                --file $VIVID_KERNEL \
-                yardstick-vivid-kernel)
+                --file $CLOUD_KERNEL \
+                yardstick-${release}-kernel)
 
-        GLANCE_KERNEL_ID=$(echo "$create_vivid_kernel" | grep " id " | awk '{print $(NF-1)}')
+        GLANCE_KERNEL_ID=$(echo "$create_kernel" | grep " id " | awk '{print $(NF-1)}')
         if [ -z "$GLANCE_KERNEL_ID" ]; then
             echo 'Failed uploading kernel to cloud'.
             exit 1
@@ -92,7 +98,7 @@ load_yardstick_image()
 
         EXTRA_PARAMS="--property kernel_id=$GLANCE_KERNEL_ID --property os_command_line=\"$command_line\""
 
-        rm -f $VIVID_KERNEL $VIVID_IMAGE
+        rm -f $CLOUD_KERNEL $CLOUD_IMAGE
         cd $YARDSTICK_REPO_DIR
     fi
 
@@ -101,6 +107,10 @@ load_yardstick_image()
         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
     fi
 
+    if [[ -n "${HW_FW_TYPE}" ]]; then
+        EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_firmware_type=${HW_FW_TYPE}"
+    fi
+
     if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then
         output=$(eval openstack image create \
             --public \
@@ -223,7 +233,7 @@ main()
 
     build_yardstick_image
     load_yardstick_image
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
+    if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
         sed -i 's/image: cirros-0.3.3/image: TestVM/g' tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml \
         samples/ping.yaml
         #We have overlapping IP with the real network

diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh
index c0ae774..ce320a2 100755 (executable)
--- a/tools/ubuntu-server-cloudimg-modify.sh
+++ b/tools/ubuntu-server-cloudimg-modify.sh
@@ -25,8 +25,8 @@ fi
 
 # iperf3 only available for trusty in backports
 if [ grep -q trusty /etc/apt/sources.list ]; then
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
-        echo "deb [arch=arm64] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list
+    if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
+        echo "deb [arch=${YARD_IMG_ARCH}] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list
     else
         echo "deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list
     fi
@@ -46,11 +46,11 @@ chpasswd: { expire: False }
 ssh_pwauth: True
 EOF
 apt-get update
-if [ $YARD_IMG_ARCH = "arm64" ]; then
-apt-get install -y \
-    linux-headers-$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) \
-    unzip
-#resize root parition (/dev/vdb1) It is supposed to be default but the image is booted differently for arm64
+if [[ "${YARD_IMG_ARCH}" = "arm64" && "$release" = "vivid" ]]; then
+    apt-get install -y \
+        linux-headers-$(echo $CLOUD_KERNEL_VERSION | cut -d'-' -f3,4,5) \
+        unzip
+    #resize root partition (/dev/vdb1) It is supposed to be default but the image is booted differently for arm64
 cat <<EOF >/etc/cloud/cloud.cfg.d/15_growpart.cfg
 #cloud-config
 bootcmd:
@@ -76,7 +76,7 @@ apt-get install -y \
     stress \
     sysstat
 
-if [ $YARD_IMG_ARCH = "arm64" ]; then
+if [[ "${YARD_IMG_ARCH}" = "arm64" && "$release" = "vivid" ]]; then
     wget https://github.com/kdlucas/byte-unixbench/archive/master.zip
     unzip master.zip && rm master.zip
     mkdir /opt/tempT
@@ -88,7 +88,7 @@ else
 fi
 make --directory /opt/tempT/UnixBench/
 
-if [ $YARD_IMG_ARCH = "arm64" ]; then
+if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
     wget https://github.com/beefyamoeba5/ramspeed/archive/master.zip
     unzip master.zip && rm master.zip
     mkdir /opt/tempT/RAMspeed
@@ -100,7 +100,7 @@ cd /opt/tempT/RAMspeed/ramspeed-2.6.0
 mkdir temp
 bash build.sh
 
-if [ $YARD_IMG_ARCH = "arm64" ]; then
+if [[ "${YARD_IMG_ARCH}" = "arm64" && "$release" = "vivid" ]]; then
     wget https://github.com/beefyamoeba5/cachestat/archive/master.zip
     unzip master.zip && rm master.zip
     mv cachestat-master/cachestat /opt/tempT

diff --git a/tools/yardstick-img-modify b/tools/yardstick-img-modify
index 68ce6e2..da8e1c9 100755 (executable)
--- a/tools/yardstick-img-modify
+++ b/tools/yardstick-img-modify
@@ -23,7 +23,6 @@
 #
 # TODO: image resize is needed if the base image is too small
 #
-
 set -e
 set -x
 
@@ -42,15 +41,23 @@ mountdir="/mnt/yardstick"
 workspace=${WORKSPACE:-"/tmp/workspace/yardstick"}
 host=${HOST:-"cloud-images.ubuntu.com"}
 release=${RELEASE:-"xenial"}
-image_path="${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}-disk1.img"
+boot_mode="disk1"
+if [[ "${YARD_IMG_ARCH}" = "arm64" ]]; then
+    boot_mode="uefi1"
+fi
+
+image_path="${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}-${boot_mode}.img"
 image_url=${IMAGE_URL:-"https://${host}/${image_path}"}
 md5sums_path="${release}/current/MD5SUMS"
 md5sums_url=${MD5SUMS_URL:-"https://${host}/${md5sums_path}"}
 
 imgfile="${workspace}/yardstick-image.img"
-raw_imgfile="${workspace}/yardstick-${release}-server.raw"
+raw_imgfile_basename="yardstick-${release}-server.raw"
+raw_imgfile="${workspace}/${raw_imgfile_basename}"
 filename=$(basename $image_url)
 
+apt-get install -y parted
+
 # download and checksum base image, conditionally if local copy is outdated
 download() {
     test -d $workspace || mkdir -p $workspace
@@ -69,24 +76,25 @@ download() {
         [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i
     done
 
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
+    if [[ "${YARD_IMG_ARCH}" = "arm64"  && "$release" = "vivid" ]]; then
         cd /tmp
-        if [ ! -f /tmp/vivid-server-cloudimg-arm64-kernel-info.txt ]; then
-            wget http://cloud-images.ubuntu.com/vivid/current/vivid-server-cloudimg-arm64-kernel-info.txt
+        if [ ! -f /tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}-kernel-info.txt ]; then
+            wget http://${host}/${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}-kernel-info.txt
         fi
-        export VIVID_KERNEL_VERSION=$(cut -d$'\t' -f4 vivid-server-cloudimg-arm64-kernel-info.txt)
-        mkdir -p /tmp/vivid-modules
-        if [ ! -f "/tmp/vivid-server-cloudimg-arm64.tar.gz" ]; then
-            wget $VIVID_IMG_URL
+        export CLOUD_KERNEL_VERSION=$(cut -d$'\t' -f4 ${release}-server-cloudimg-${YARD_IMG_ARCH}-kernel-info.txt)
+
+        mkdir -p /tmp/${release}-modules
+        if [ ! -f "/tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}.tar.gz" ]; then
+            wget $CLOUD_IMG_URL
         fi
-        if [ ! -f "/tmp/vivid-server-cloudimg-arm64.img" ]; then
-            tar zxvf vivid-server-cloudimg-arm64.tar.gz vivid-server-cloudimg-arm64.img
+        if [ ! -f "/tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}.img" ]; then
+            tar zxvf ${release}-server-cloudimg-${YARD_IMG_ARCH}.tar.gz ${release}-server-cloudimg-${YARD_IMG_ARCH}.img
         fi
-        mkdir -p /mnt/vivid
-        mount /tmp/vivid-server-cloudimg-arm64.img /mnt/vivid
-        cp -r /mnt/vivid/lib/modules/$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) /tmp/vivid-modules
-        umount /mnt/vivid
-        rm /tmp/vivid-server-cloudimg-arm64.img
+        mkdir -p /mnt/${release}
+        mount /tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}.img /mnt/${release}
+        cp -r /mnt/${release}/lib/modules/$(echo $CLOUD_KERNEL_VERSION) /tmp/${release}-modules
+        umount /mnt/${release}
+        rm /tmp/${release}-server-cloudimg-${YARD_IMG_ARCH}.img
         cd $workspace
     fi
     qemu-img convert $filename $raw_imgfile
@@ -96,27 +104,32 @@ download() {
 # mount image
 setup() {
     # qemu-img resize $raw_imgfile +5GB
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
-        echo -e "d\nn\np\n1\n\n\nw" | fdisk $raw_imgfile
+    if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
+        echo -e "d\nn\np\n1\n\n\nw" | parted -l $raw_imgfile
     fi
     mkdir -p $mountdir
 
-    loopdevice=$(kpartx -l $raw_imgfile | head -1 | cut -f1 -d ' ')
+    #kpartx fails with image paths longer than 63 characters
+    #try shortest relative path to image as temporary workaround
+    cd ${workspace}
+    loopdevice=$(kpartx -l $raw_imgfile_basename | head -1 | cut -f1 -d ' ')
+
+    kpartx -av $raw_imgfile_basename
+    cd -
 
-    kpartx -av $raw_imgfile
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
+    if [[ "${YARD_IMG_ARCH}" = "arm64" && "$release" = "vivid" ]]; then
         e2fsck -p -f /dev/mapper/$loopdevice
         resize2fs /dev/mapper/$loopdevice
     fi
     # for trouble shooting
     sleep 2
     dmsetup ls
-    fdisk -l /dev/${loopdevice:0:5} || true
+    parted -l /dev/${loopdevice:0:5} || true
     mount /dev/mapper/$loopdevice $mountdir
     mount -t proc none $mountdir/proc
 
-    if [ $YARD_IMG_ARCH = "arm64" ]; then
-        cp -r /tmp/vivid-modules/$(echo $VIVID_KERNEL_VERSION | cut -d'-' -f3,4,5) "$mountdir/lib/modules"
+    if [[ "${YARD_IMG_ARCH}" = "arm64" && "$release" = "vivid" ]]; then
+        cp -r /tmp/${release}-modules/$(echo $CLOUD_KERNEL_VERSION) "$mountdir/lib/modules"
         cp $(which "qemu-aarch64-static") "$mountdir/usr/bin"
     fi
     cp $cmd $mountdir/$(basename $cmd)
@@ -151,9 +164,9 @@ cleanup() {
     # designed to be idempotent
     mount | grep $mountdir/proc && umount $mountdir/proc
     mount | grep $mountdir && umount $mountdir
-    mount | grep "/mnt/vivid" && umount "/mnt/vivid"
+    mount | grep "/mnt/${release}" && umount "/mnt/${release}"
 
-    if [ -f $raw_imgfile ]; then
+    if [ -f "${raw_imgfile}" ]; then
         #kpartx -dv $raw_imgfile sometimes failed, we should checked it agein.
         #if [ -z "$(kpartx -l $raw_imgfile | grep 'loop deleted')" ]; then
         #    kpartx -dv $raw_imgfile