From: Jiang, Yunhong Date: Tue, 23 Aug 2016 22:17:42 +0000 (+0000) Subject: Merge "OPNFV KVM4NFV CICD: Scripts for creating Rpms & Debians for Kernel, Qemu" X-Git-Tag: danube.1.0~51 X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=76c2ed6659d79ee83c8fc4f00e955268779db826;hp=2b76a9cbd6c127109286c8705f3347415eb26753;p=kvmfornfv.git Merge "OPNFV KVM4NFV CICD: Scripts for creating Rpms & Debians for Kernel, Qemu" --- diff --git a/ci/README b/ci/README new file mode 100644 index 000000000..d4da16561 --- /dev/null +++ b/ci/README @@ -0,0 +1,47 @@ +The "ci" folder consists of- + +build.sh --> builds rpm packages (if -p package option is passed as "centos" +or in default case) by calling build_rpm subfolder files ; builds debian +packages (if -p package option is passed as "ubuntu") by calling build_deb +subfolder files; builds both debian and rpm packages (if -p package option is +passed as "both"). + +build_deb --> contains the files required to build debian backages for kernel +and qemu, inside ubuntu docker. + +build_rpm --> contains the files required to build rpm backages for kernel and +qemu, inside centos docker. + +envs --> contains the host and guest setup+configuration files. + + +The actual Tree structure is as below- + +|-- build_deb +| |-- build_debs_docker.sh +| |-- build_debs.sh +| |-- Dockerfile +| |-- kernel_deb_build.sh +| |-- mkcontrol.sh +| `-- qemu_deb_build.sh +|-- build_rpm +| |-- build_rpms_docker.sh +| |-- build_rpms.sh +| |-- Dockerfile +| |-- kernel_rpm_build.sh +| |-- mkspec +| |-- mkversion +| |-- qemu_rpm_build.sh +|-- build.sh +|-- envs +| |-- create-rt-tests-rpm.sh +| |-- guest-cmd.sh +| |-- guest-modify.sh +| |-- guest-setup0.sh +| |-- guest-setup1.sh +| |-- host-config +| |-- host-run-qemu.sh +| |-- host-setup0.sh +| |-- host-setup1.sh +| `-- rt-tests.patch +`-- README diff --git a/ci/build.sh b/ci/build.sh index d27b353bb..064c48a8b 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,50 +1,93 @@ #!/bin/bash -kernel_src_dir=kernel -rpmbuild_dir=/tmp/kvmfornfv_rpmbuild.$$ -artifact_dir=${rpmbuild_dir}/RPMS/x86_64 -config_file="${kernel_src_dir}/arch/x86/configs/opnfv.config" -output_dir="$1" - -usage () { - echo "usage: ${0} output_dir" - exit 1 +# +# Common parameter parsing for kvmfornfv scripts +# +function usage() { + echo "" + echo "Usage --> $0 [-p package_type] [-o output_dir] [-h]" + echo " package_type : centos/ubuntu/both ; default is centos" + echo " output_dir : stores rpm and debian packages" + echo " -h : Help section" + echo "" } -if [[ -z "$@" ]]; then - usage -fi - -if [ ! -d ${output_dir} -o ! -w ${output_dir} ] ; then - echo "${0}: Output directory '${output_dir}' does not exist or cannot be written" - exit 1 -fi - -if [ ! -d ${kernel_src_dir} ] ; then - echo "${0}: Directory '${kernel_src_dir}' does not exist, run this script from the root of kvmfornfv source tree" - exit 1 -fi +output_dir="" +type="" -if [ ! -f ${config_file} ] ; then - echo "${0}: ${config_file} does not exist" - exit 1 -fi +function build_package() { + choice=$1 -echo -echo "Build" -echo + case "$choice" in + "centos") + echo "Build $choice Rpms" + cd ci/build_rpm + ./build_rpms.sh + cd $WORKSPACE + ;; + "ubuntu") + echo "Build $choice Debians" + cd ci/build_deb + ./build_debs.sh + cd $WORKSPACE + ;; + "both") + echo "Build $choice Debians and Rpms" + cd ci/build_deb + ./build_debs.sh + cd ../build_rpm + ./build_rpms.sh + cd $WORKSPACE + ;; + *) + echo "Invalid package option" + usage + exit 1 + ;; + esac +} -cp -f ${config_file} "${kernel_src_dir}/.config" +## --- Parse command line arguments / parameters --- +while getopts ":o:p:h" option; do + case $option in + p) # package + type=$OPTARG + ;; + o) # output_dir + output_dir=$OPTARG + ;; + :) + echo "Option -$OPTARG requires an argument." + usage + exit 1 + ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option: $OPTARG." + usage + exit 1 + ;; + ?) + echo "[WARNING] Unknown parameters!!!" + echo "Using default values for package generation & output" + esac +done -# Make timestamp part of version string for automated kernel boot verification -date "+-%y%m%d%H%M" > "${kernel_src_dir}/localversion-zzz" +if [[ -z "$type" ]] +then + type='centos' +fi -( cd ${kernel_src_dir}; make RPMOPTS="--define '_topdir ${rpmbuild_dir}'" rpm-pkg ) -if [ ${?} -ne 0 ] ; then - echo "${0}: Kernel build failed" - rm -rf ${rpmbuild_dir} - exit 1 +if [[ -z "$output_dir" ]] +then + output_dir=$WORKSPACE/build_output fi -cp -f ${artifact_dir}/* ${output_dir} +echo "" +echo "Building for $type package in $output_dir" +echo "" -rm -rf ${rpmbuild_dir} +mkdir -p $output_dir +build_package $type diff --git a/ci/build_deb/Dockerfile b/ci/build_deb/Dockerfile new file mode 100644 index 000000000..676afcf67 --- /dev/null +++ b/ci/build_deb/Dockerfile @@ -0,0 +1,22 @@ +#!/bin/bash + +FROM ubuntu:14.04.3 +RUN apt-get update && apt-get install -y \ + git \ + fakeroot \ + build-essential \ + ncurses-dev xz-utils \ + kernel-package \ + bc \ + autoconf \ + python \ + pkg-config \ + zlibc \ + zlib1g \ + zlib1g-dev \ + libglib2.0-dev \ + libtool \ + flex \ + bison +RUN echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo +RUN chmod 0440 /etc/sudoers.d/open-sudo diff --git a/ci/build_deb/build_debs.sh b/ci/build_deb/build_debs.sh new file mode 100755 index 000000000..2fcd3629b --- /dev/null +++ b/ci/build_deb/build_debs.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eux +BUILD_FOR=${BUILD_FOR:-ubuntu} + +function build_deb_pkg { + case $1 in + ubuntu) + sudo docker build -t kvm_deb . + sudo docker run -v $WORKSPACE:/opt/kvmfornfv -t kvm_deb \ + /opt/kvmfornfv/ci/build_deb/build_debs_docker.sh + ;; + *) echo "Not supported system"; exit 1;; + esac +} + +for system in $BUILD_FOR +do + build_deb_pkg $system +done diff --git a/ci/build_deb/build_debs_docker.sh b/ci/build_deb/build_debs_docker.sh new file mode 100755 index 000000000..3fd35ff54 --- /dev/null +++ b/ci/build_deb/build_debs_docker.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +tmp_deb_build_dir=/root/kvmfornfv +deb_build_dir=/opt/kvmfornfv +tmp_deb_output_dir=$tmp_deb_build_dir/build_output +deb_output_dir=$deb_build_dir/build_output +cp -r $deb_build_dir $tmp_deb_build_dir + +# Build qemu debian packages +cd $tmp_deb_build_dir/qemu +make clean +./configure +make +cd $tmp_deb_build_dir +./ci/build_deb/qemu_deb_build.sh build_output + +# Build kernel debian packages +./ci/build_deb/kernel_deb_build.sh build_output + +# Move Kernel and Qemu Debian builds from tmp_output_dir to output_dir +mv $tmp_deb_output_dir/qemu-* $deb_output_dir +mv $tmp_deb_output_dir/linux-* $deb_output_dir diff --git a/ci/build_deb/kernel_deb_build.sh b/ci/build_deb/kernel_deb_build.sh new file mode 100755 index 000000000..824960e7f --- /dev/null +++ b/ci/build_deb/kernel_deb_build.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +SRC=/root +kernel_src_dir=kernel +config_file="arch/x86/configs/opnfv.config" +VERSION="1.0.OPNFV" +output_dir="$1" + +usage () { + echo "usage: ${0} output_dir" + exit 1 +} + +if [[ -z "$@" ]]; then + usage +fi + +if [ ! -d ${output_dir} -o ! -w ${output_dir} ] ; then + echo "${0}: Output directory '${output_dir}' does not exist or cannot be written" + exit 1 +fi + +if [ ! -d ${kernel_src_dir} ] ; then + echo "${0}: Directory '${kernel_src_dir}' does not exist, run this script from the root of kvmfornfv source tree" + exit 1 +fi + +quirks() { +# +# Apply out of tree patches +# +for i in $SRC/kvmfornfv/patches/$1/*.patch +do + if [ -f "$i" ] + then + echo "Applying: $i" + patch -p1 <$i + fi +done +} + +quirks kernel + +cd kernel + +if [ ! -f ${config_file} ] ; then + echo "${0}: ${config_file} does not exist" + exit 1 +fi + +# Workaround build bug on Ubuntu 14.04 +cat < arch/x86/boot/install.sh +#!/bin/sh +cp -a -- "\$2" "\$4/vmlinuz-\$1" +EOF + +# Configure the kernel +cp $config_file .config + +make oldconfig control) + mv $scripts_dir/control $debbuild_dir/qemu-$VERSION/DEBIAN/control + +#building the qemu debian with control file developed. + dpkg-deb --build $debbuild_dir/qemu-$VERSION + if [ ${?} -ne 0 ] ; then + echo "${0}: qemu build failed" + exit 1 + fi +} + +if [ ! -d ${debbuild_dir} ] ; then + echo "creating debbuild directory" + mkdir -p $debbuild_dir +fi + +qemu_deb_build +latest_qemu_build=`ls -rt $debbuild_dir | tail -1` +cp $debbuild_dir/$latest_qemu_build build_output diff --git a/ci/build_qemu_rpm_deb/qemu_build.sh b/ci/build_qemu_rpm_deb/qemu_build.sh deleted file mode 100644 index a8863c3ca..000000000 --- a/ci/build_qemu_rpm_deb/qemu_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -qemu_src_dir=qemu -workspace=/root -debbuild_dir=$workspace/debbuild -rpmbuild_dir=$workspace/rpmbuild -artifact_rpms=$rpmbuild_dir/RPMS -artifact_dir=$artifact_rpms/x86_64 -scripts_dir=ci/build_qemu_rpm_deb -output_dir="$1" -VERSION=`grep -m 1 "VERSION" ${qemu_src_dir}/config-host.mak | cut -d= -f2-` - -usage () { - echo "usage: ${0} output_dir" - exit 1 -} - -if [[ -z "$@" ]]; then - usage -fi - -if [ ! -d ${output_dir} -o ! -w ${output_dir} ] ; then - echo "${0}: Output directory '${output_dir}' does not exist or cannot be written" - exit 1 -fi - -if [ ! -d ${qemu_src_dir} ] ; then - echo "${0}: Directory '${qemu_src_dir}' does not exist, run this script from the root of kvmfornfv source tree" - exit 1 -fi - -echo -echo "Build" -echo diff --git a/ci/build_qemu_rpm_deb/qemu_deb_build.sh b/ci/build_qemu_rpm_deb/qemu_deb_build.sh deleted file mode 100755 index 7a830183d..000000000 --- a/ci/build_qemu_rpm_deb/qemu_deb_build.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -#Build process for generating qemu debain file. - -source ci/build_qemu_rpm_deb/qemu_build.sh -qemu_deb_build() { - sudo mkdir -p $debbuild_dir/qemu-$VERSION - sudo cp -r $qemu_src_dir $debbuild_dir/qemu-$VERSION - sudo mkdir -p $debbuild_dir/qemu-$VERSION/DEBIAN - sudo touch control - -#creating control file for debian build. - (cd ${scripts_dir}; sudo ./mkcontrol.sh $VERSION > control) - sudo mv $scripts_dir/control $debbuild_dir/qemu-$VERSION/DEBIAN/control - -#building the qemu debian with control file developed. - sudo dpkg-deb --build $debbuild_dir/qemu-$VERSION - if [ ${?} -ne 0 ] ; then - echo "${0}: qemu build failed" - exit 1 - fi -} - -if [ ! -d ${debbuild_dir} ] ; then - echo "creating debbuild directory" - sudo mkdir -p $debbuild_dir -fi - -qemu_deb_build -latest_qemu_build=`sudo ls -rt $debbuild_dir | tail -1` -sudo cp $debbuild_dir/$latest_qemu_build build_output diff --git a/ci/build_qemu_rpm_deb/qemu_rpm_build.sh b/ci/build_qemu_rpm_deb/qemu_rpm_build.sh deleted file mode 100755 index a52ee0f4a..000000000 --- a/ci/build_qemu_rpm_deb/qemu_rpm_build.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#Build process for Generating qemu rpm. - -source ci/build_qemu_rpm_deb/qemu_build.sh -qemu_rpm_build() { - sudo cp -r ${qemu_src_dir} ${qemu_src_dir}-$VERSION - sudo tar -zcvf ${qemu_src_dir}-$VERSION.tar.gz ${qemu_src_dir}-$VERSION - sudo mv ${qemu_src_dir}-$VERSION.tar.gz ${rpmbuild_dir}/SOURCES/ - -#create a spec file for rpm creation. - (cd ${scripts_dir}; ./mkspec $VERSION > qemu.spec) - sudo cp ${scripts_dir}/qemu.spec ${rpmbuild_dir}/SPECS/ - -#build the qemu rpm with spec file developed - sudo rpmbuild -ba ${rpmbuild_dir}/SPECS/qemu.spec - if [ ${?} -ne 0 ] ; then - echo "${0}: qemu build failed" - exit 1 - fi - sudo rm -rf ${qemu_src_dir}-$VERSION - sudo rm -rf ${rpmbuild_dir}/SOURCES/${qemu_src_dir}-$VERSION.tar.gz -} - -if [ ! -d ${rpmbuild_dir} ] ; then - sudo yum install rpm-build -y - mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} - sudo mv rpmbuild $workspace -fi - -qemu_rpm_build -latest_qemu_build=`ls -rt $artifact_dir | grep qemu-2.6* | tail -1` -sudo cp $artifact_dir/$latest_qemu_build build_output diff --git a/ci/build_rpm/Dockerfile b/ci/build_rpm/Dockerfile new file mode 100644 index 000000000..6d6d3ba35 --- /dev/null +++ b/ci/build_rpm/Dockerfile @@ -0,0 +1,14 @@ +#!/bin/bash + +FROM centos +RUN yum -y update && yum -y install \ + git \ + gcc \ + gcc-c++ \ + zlib-devel \ + gtk2-devel \ + make \ + gettext \ + bc \ + rpm \ + rpm-build diff --git a/ci/build_rpm/build_rpms.sh b/ci/build_rpm/build_rpms.sh new file mode 100755 index 000000000..40cae6ccf --- /dev/null +++ b/ci/build_rpm/build_rpms.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eux +BUILD_FOR=${BUILD_FOR:-centos} + +function build_rpm_pkg { + case $1 in + centos) + sudo docker build -t kvm_rpm . + sudo docker run -v $WORKSPACE:/opt/kvmfornfv -t kvm_rpm \ + /opt/kvmfornfv/ci/build_rpm/build_rpms_docker.sh + ;; + *) echo "Not supported system"; exit 1;; + esac +} + +for system in $BUILD_FOR +do + build_rpm_pkg $system +done diff --git a/ci/build_rpm/build_rpms_docker.sh b/ci/build_rpm/build_rpms_docker.sh new file mode 100755 index 000000000..2670163e4 --- /dev/null +++ b/ci/build_rpm/build_rpms_docker.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +tmp_rpm_build_dir=/root/kvmfornfv +rpm_build_dir=/opt/kvmfornfv +tmp_rpm_output_dir=$tmp_rpm_build_dir/build_output +rpm_output_dir=$rpm_build_dir/build_output +cp -r $rpm_build_dir $tmp_rpm_build_dir + +# Build qemu rpm packages +cd $tmp_rpm_build_dir/qemu +make clean +./configure +make +cd $tmp_rpm_build_dir +./ci/build_rpm/qemu_rpm_build.sh build_output + +# Build kernel rpm packages +./ci/build_rpm/kernel_rpm_build.sh build_output + +# Move Kernel and Qemu Rpm builds from tmp_output_dir to output_dir +mv $tmp_rpm_output_dir/qemu-* $rpm_output_dir +mv $tmp_rpm_output_dir/kernel-* $rpm_output_dir diff --git a/ci/build_rpm/kernel_rpm_build.sh b/ci/build_rpm/kernel_rpm_build.sh new file mode 100755 index 000000000..fa6383eb8 --- /dev/null +++ b/ci/build_rpm/kernel_rpm_build.sh @@ -0,0 +1,53 @@ +#!/bin/bash +kernel_src_dir=kernel +rpmbuild_dir=/tmp/kvmfornfv_rpmbuild.$$ +artifact_dir=${rpmbuild_dir}/RPMS/x86_64 +mkdir -p $artifact_dir +config_file="${kernel_src_dir}/arch/x86/configs/opnfv.config" +output_dir="$1" + +usage () { + echo "usage: ${0} output_dir" + exit 1 +} + +if [[ -z "$@" ]]; then + usage +fi + +if [ ! -d ${output_dir} -o ! -w ${output_dir} ] ; then + echo "${0}: Output directory '${output_dir}' does not exist or cannot \ + be written" + exit 1 +fi + +if [ ! -d ${kernel_src_dir} ] ; then + echo "${0}: Directory '${kernel_src_dir}' does not exist, run this script \ + from the root of kvmfornfv source tree" + exit 1 +fi + +if [ ! -f ${config_file} ] ; then + echo "${0}: ${config_file} does not exist" + exit 1 +fi + +echo +echo "Build" +echo + +cp -f ${config_file} "${kernel_src_dir}/.config" + +# Make timestamp part of version string for automated kernel boot verification +date "+-%y%m%d%H%M" > "${kernel_src_dir}/localversion-zzz" + +(cd ${kernel_src_dir}; make RPMOPTS="--define '_topdir ${rpmbuild_dir}'" rpm-pkg) +if [ ${?} -ne 0 ] ; then + echo "${0}: Kernel build failed" + rm -rf ${rpmbuild_dir} + exit 1 +fi + +cp -f ${artifact_dir}/* ${output_dir} + +rm -rf ${rpmbuild_dir} diff --git a/ci/build_qemu_rpm_deb/mkspec b/ci/build_rpm/mkspec similarity index 93% rename from ci/build_qemu_rpm_deb/mkspec rename to ci/build_rpm/mkspec index 0b75a181b..4aa542325 100755 --- a/ci/build_qemu_rpm_deb/mkspec +++ b/ci/build_rpm/mkspec @@ -14,7 +14,7 @@ echo "Summary: The Linux qemu" echo "Version: $__QEMURELEASE" # we need to determine the NEXT version number # rpm -q will agree -echo "Release: `sudo sh mkversion`" +echo "Release: `sh mkversion`" echo "License: GPLv2+ and LGPLv2+ and BSD" echo "Group: Development/Tools" echo "Vendor: The Linux Community" @@ -36,6 +36,7 @@ echo "rm -rf %{buildroot}" echo "%files" echo "%dir" echo "/usr/local/share/qemu" +echo "/usr/local/share/locale/*" echo "%doc" echo "/usr/local/bin/ivshmem*" echo "/usr/local/bin/qemu*" diff --git a/ci/build_qemu_rpm_deb/mkversion b/ci/build_rpm/mkversion similarity index 84% rename from ci/build_qemu_rpm_deb/mkversion rename to ci/build_rpm/mkversion index fa4e585b9..9668da7e0 100755 --- a/ci/build_qemu_rpm_deb/mkversion +++ b/ci/build_rpm/mkversion @@ -1,7 +1,7 @@ if [ ! -f .version ] then touch .version - sudo chmod 777 .version + chmod 777 .version echo 1 > .version echo 1 else diff --git a/ci/build_rpm/qemu_rpm_build.sh b/ci/build_rpm/qemu_rpm_build.sh new file mode 100755 index 000000000..1e87fbdd6 --- /dev/null +++ b/ci/build_rpm/qemu_rpm_build.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +qemu_src_dir=qemu +workspace=/root +rpmbuild_dir=$workspace/rpmbuild +artifact_rpms=$rpmbuild_dir/RPMS +artifact_dir=$artifact_rpms/x86_64 +scripts_dir=ci/build_rpm +output_dir="$1" +VERSION=`grep -m 1 "VERSION" ${qemu_src_dir}/config-host.mak | cut -d= -f2-` + +usage () { + echo "usage: ${0} output_dir" + exit 1 +} + +if [[ -z "$@" ]]; then + usage +fi + +if [ ! -d ${output_dir} -o ! -w ${output_dir} ] ; then + echo "${0}: Output directory '${output_dir}' does not exist or cannot \ + be written" + exit 1 +fi + +if [ ! -d ${qemu_src_dir} ] ; then + echo "${0}: Directory '${qemu_src_dir}' does not exist, run this script \ + from the root of kvmfornfv source tree" + exit 1 +fi + +echo +echo "Build" +echo + +qemu_rpm_build() { + cp -r ${qemu_src_dir} ${qemu_src_dir}-$VERSION + tar -zcvf ${qemu_src_dir}-$VERSION.tar.gz ${qemu_src_dir}-$VERSION + mv ${qemu_src_dir}-$VERSION.tar.gz ${rpmbuild_dir}/SOURCES/ + + #create a spec file for rpm creation. + (cd ${scripts_dir}; ./mkspec $VERSION > qemu.spec) + cp ${scripts_dir}/qemu.spec ${rpmbuild_dir}/SPECS/ + + #build the qemu rpm with spec file developed + rpmbuild -ba ${rpmbuild_dir}/SPECS/qemu.spec + if [ ${?} -ne 0 ] ; then + echo "${0}: qemu build failed" + exit 1 + fi + rm -rf ${qemu_src_dir}-$VERSION + rm -rf ${rpmbuild_dir}/SOURCES/${qemu_src_dir}-$VERSION.tar.gz +} + +if [ ! -d ${rpmbuild_dir} ] ; then + yum install rpm-build -y + mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + mv rpmbuild $workspace +fi + +qemu_rpm_build +latest_qemu_build=`ls -rt $artifact_dir | grep qemu-2.6* | tail -1` +cp $artifact_dir/$latest_qemu_build build_output