From 4c54b37aca48867c694539692a3ada76be39f4d6 Mon Sep 17 00:00:00 2001 From: Thomas F Herbert Date: Sat, 23 Jan 2016 19:06:55 -0500 Subject: [PATCH] Introduce ability to accept command arguments to build with DPDK or Linux kernel. Specify patches to apply, build kernel module RPM and build with a special version of the kernel. Also instack_ovs.sh creates a VM for building and initial testing of RPMs and copies scripts to VMs build_ovs_rpm.sh builds the RPM according to command line options. test_ovs_rpm.sh tests the RPM by installing the RPM and running some ovs commands to create a bridge and dump flows. Also add two scripts generally to be executed in VM to do actual building and Testing. This 2nd series adds the following: Use fedora spec file for kmod builds. Added Explanations to configuration options. Fixes in response to review comments. Rename script for clarity and change references: buildtestovs.sh -> BuildTestOVS.sh Fix nocheck option, print default options, fix kmod option, fix test of kmod. For command line options execute build/buildtestovs.sh -h Change-Id: I415ca39afab27482b1cb473d392f48b36c8e0745 Signed-off-by: Thomas F Herbert --- build/BuildAndTestOVS.sh | 145 +++++++++++++++++++ build/build_ovs_rpm.sh | 130 +++++++++++++++++ build/config | 25 +++- build/instack_ovs.sh | 361 +++++++++++++++++++++++++++++++++++++++++++++++ build/test_ovs_rpm.sh | 81 +++++++++++ ci/buildovs.sh | 119 +++++----------- ci/clean.sh | 43 ++++++ 7 files changed, 815 insertions(+), 89 deletions(-) create mode 100755 build/BuildAndTestOVS.sh create mode 100755 build/build_ovs_rpm.sh create mode 100755 build/instack_ovs.sh create mode 100755 build/test_ovs_rpm.sh create mode 100755 ci/clean.sh diff --git a/build/BuildAndTestOVS.sh b/build/BuildAndTestOVS.sh new file mode 100755 index 0000000..ff2495c --- /dev/null +++ b/build/BuildAndTestOVS.sh @@ -0,0 +1,145 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Red Hat Inc. and others. +# therbert@redhat.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e + +echo "===============================" +echo executing $0 $@ + +usage() { + echo "$0 -a -d -g -h\ + -i -p -t -u -v -- Specify major release if special kernel is required\ + The default kernel is Centos 7.2 kernel after upgrade.\ + -d -- Specify dpdk build.\ + The default is to build ovs for linux kernel data path.\ + -g -- OVS release tag or branch to build such as 2.4.\ + The default is master.\ + -h print this message\ + -i -- Specify minor release if special kernel is required.\ + The default kernel is Centos 7.2 kernel after upgrade.\ + -p -- Specify url to patches if required for ovs rpm.\ + -t -- Test rpm.\ + -u -- path to OVS repo if using fork for patch.\ + The default is https://github.com/openvswitch/ovs.git\ + -v -- Set verbose mode." +} + +while getopts "a:dg:hi:p:tu:v" opt; do + case "$opt" in + a) + kernel_major=${OPTARG} + ;; + d) + DPDK="yes" + ;; + g) + TAG=${OPTARG} + ;; + h) + usage + exit 1 + ;; + i) + kernel_minor=${OPTARG} + ;; + p) + OVS_PATCH=${OPTARG} + ;; + t) + TESTRPM="yes" + ;; + u) + OVS_REPO_URL=${OPTARG} + ;; + v) + verbose="yes" + ;; + esac +done + +if [ -z $TAG ]; then + TAG=master +fi + +if [ -z $OVS_REPO_URL ]; then + OVS_REPO_URL=https://github.com/openvswitch/ovs.git +fi + +if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then + kernel_version=$kernel_major.$kernel_minor + echo =================== + echo Will install kernel version: major is $kernel_major and minor is $kernel_minor + echo =================== +else + echo Will use default kernel in ovs test vm +fi + +if [ -z ${WORKSPACE+1} ]; then + # We are not being run by Jenkins. + export WORKSPACE=$HOME/opnfv/ovsnfv + mkdir -p opnfv + cd opnfv + git clone https://git.opnfv.org/ovsnfv +fi + +export BUILD_BASE=$WORKSPACE/build + + + +if [ ! -d $BUILD_BASE ] +then + mkdir -p $BUILD_BASE +fi + +if [ ! -f $BUILD_BASE/config ]; then + touch $BUILD_BASE/config +fi + +export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE +source $BUILD_BASE/config + +cd $BUILD_BASE +export TOPDIR=$BUILD_BASE + +# build variables + +export TMP_RELEASE_DIR=$TOPDIR/release +export CACHE_DIR=$TOPDIR/cache +export TMPDIR=$TOPDIR/scratch +export RPMDIR=$TOPDIR/rpmbuild + + +mkdir -p $RPMDIR/RPMS +mkdir -p $RPMDIR/SOURCES +mkdir -p $RPMDIR/SPECS +mkdir -p $RPMDIR/SRPMS + + +if [ ! -z $TESTRPM ]; then + # Spawn VM to do the testing. + if [ ! -z $kernel_version ]; then + instack_ovs.sh -a $kernel_major -g $TAG -i $kernel_minor -p $OVS_PATCH -t -u $OVS_REPO_URL + else + instack_ovs.sh -g $TAG -p $OVS_PATCH -t -u $OVS_REPO_URL + fi +else + # Run build locally. + build_ovs_rpm.sh -d -g -p $OVS_PATCH -u $OVS_REPO_URL + cp $HOME/rpmbuild/RPMS/* $TMP_RELEASE_DIR +fi + +echo "--------------------------------------------------" +echo "Build OVS RPM from upstream git $OVS_REPO_URL version $TAG" +if [ ! -z $OVS_PATCH ]; then + echo "Apply patches from: $OVS_PATCH" +fi +echo + +exit 0 diff --git a/build/build_ovs_rpm.sh b/build/build_ovs_rpm.sh new file mode 100755 index 0000000..60b92e6 --- /dev/null +++ b/build/build_ovs_rpm.sh @@ -0,0 +1,130 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Red Hat Inc. and others. +# therbert@redhat.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e +declare -i CNT + +echo "===============================" +echo executing $0 $@ +echo executing on machine `uname -a` + +usage() { + echo run BuildAndTestOVS -h for help +} + +while getopts "cdg:hkp:u:v" opt; do + case "$opt" in + c) + setnocheck="--without check" + ;; + d) + DPDK="yes" + ;; + g) + TAG=${OPTARG} + ;; + h|\?) + usage + exit 1 + ;; + k) + kmod="yes" + ;; + p) + OVS_PATCH=${OPTARG} + ;; + u) + OVS_REPO_URL=${OPTARG} + ;; + v) + verbose="yes" + ;; + esac +done + +HOME=`pwd` +TOPDIR=$HOME +TMPDIR=$TOPDIR/ovsrpm + +if [ -d $TMPDIR ] +then + rm -rf $TMPDIR +fi + +sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \ + kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \ + libtool + +VERSION=2.3.90 +os_type=fedora +kernel_version=$(uname -a | awk '{print $3}') + +mkdir -p $TMPDIR + +cd $TMPDIR + +mkdir -p $HOME/rpmbuild/RPMS +mkdir -p $HOME/rpmbuild/SOURCES +mkdir -p $HOME/rpmbuild/SPECS +mkdir -p $HOME/rpmbuild/SRPMS + +RPMDIR=$HOME/rpmbuild + + +echo "---------------------" +echo "Clone git repo $OVS_REPO_URL and checkout branch or tag $TAG" +echo +git clone $OVS_REPO_URL + +cd ovs +echo "--------------------" +echo "Checkout OVS $TAG" +echo +if [[ ! "$TAG" =~ "master" ]]; then + git checkout $TAG +fi +if [[ ! "$OVS_PATCH" =~ "no" ]]; then + echo "Apply patches from $OVS_PATCH" +fi +./boot.sh +if [ ! -z $DPDK ]; then + ./configure --with-dpdk +else + ./configure --with-linux=/lib/modules/`uname -r`/build +fi +echo "--------------------" +echo "Make OVS $TAG" +echo +make + +if [[ "$TAG" =~ "master" ]]; then + v=$($TMPDIR/ovs/utilities/ovs-vsctl --version | head -1 | cut -d' ' -f4) + export VERSION=$v +else + export VERSION=${TAG:1} +fi + +echo making RPM for Open vswitch version $VERSION +make dist + +echo cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES +cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES + +if [ ! -z $kmod ]; then + echo "Building kernel module..." + rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec +echo " Kernel RPM built!" +fi + +echo "Building User Space..." +rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch.spec + +cp $RPMDIR/RPMS/x86_64/*.rpm $HOME + +exit 0 diff --git a/build/config b/build/config index 337e07d..a197871 100644 --- a/build/config +++ b/build/config @@ -1,5 +1,26 @@ # config file to be populated with configuration parameters for build. -# OVS TAG +# OVS configuration. +# +# OVS TAG or revision to build. The default is master which causes an RPM to be +# built from the top of current master. The tag uses the current ovs naming and +# release convention. This option can be overriden in the command line. +# export OVSTAG=master -#export NOCHECK=true +# +# when NOCHECK is yes, the ovs rpm is build with the --without check option which +# Disables running "make check" as the RPM is built. +# +export NOCHECK=yes +# +# URL to external URL for patches to be applied for Open vSwitch. +# +export PATCH=none +# +# DPDK configuration. +# +export DPDK_PATCH=none +# +# When KMOD is set, the Linux kernel module is built and tested. +# +export KMOD=yes diff --git a/build/instack_ovs.sh b/build/instack_ovs.sh new file mode 100755 index 0000000..c4378d9 --- /dev/null +++ b/build/instack_ovs.sh @@ -0,0 +1,361 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Red Hat Inc. and others. +# therbert@redhat.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e +declare -i CNT + +echo "===============================" +echo executing $0 $@ +echo path is $PATH + +usage() { + echo run BuildAndTestOVS -h for help +} + +while getopts "a:dg:hi:p:tu:v" opt; do + case "$opt" in + a) + kernel_major=${OPTARG} + ;; + d) + DPDK="yes" + ;; + g) + TAG=${OPTARG} + ;; + h|\?) + usage + exit 1 + ;; + i) + kernel_minor=${OPTARG} + ;; + p) + OVS_PATCH=${OPTARG} + ;; + t) + TESTRPM="yes" + ;; + u) + OVS_REPO_URL=${OPTARG} + ;; + v) + verbose="yes" + ;; + esac +done +# +# Default Config options +# +echo =============================================== +echo Default Configuration Options. +echo =============================================== +echo option NOCHECK is set to $NOCHECK +echo DPDK Patch URL is set to $DPDK_PATCH +echo Build and Test OVS Kernel Module is set to $KMOD +echo =============================================== +if [[ $NOCHECK =~ "yes" ]]; then + setnocheck="-c" +fi +if [[ $KMOD =~ "yes" ]]; then + setkmod="-k" +fi + + + +if [ -z $OVS_REPO_URL ]; then + OVS_REPO_URL=https://github.com/openvswitch/ovs.git +fi + +if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then + kernel_version=$kernel_major.$kernel_minor + echo =================== + echo Will install kernel version: major is $kernel_major and minor is $kernel_minor + echo =================== +else + echo Will use default kernel in ovs test vm +fi + +if [ -z ${WORKSPACE+1} ]; then + # We are not being run by Jenkins. + export WORKSPACE=$HOME/opnfv/ovsnfv + mkdir -p opnfv + cd opnfv + git clone https://git.opnfv.org/ovsnfv +fi + +export BUILD_BASE=$WORKSPACE/build + +if [ ! -d $BUILD_BASE ] +then + mkdir -p $BUILD_BASE +fi + +if [ ! -f $BUILD_BASE/config ]; then + touch $BUILD_BASE/config +fi + +export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE +source $BUILD_BASE/config + +cd $BUILD_BASE +export TOPDIR=$BUILD_BASE + +export TMP_RELEASE_DIR=$TOPDIR/release +if [ ! -d $TMP_RELEASE_DIR ]; then + mkdir -p $TMP_RELEASE_DIR +fi + +export CACHE_DIR=$TOPDIR/cache +if [ ! -d $CACHE_DIR ]; then + mkdir -p $CACHE_DIR +fi +export TMPDIR=$TOPDIR/scratch +if [ ! -d $SCRATCH_DIR ]; then + mkdir -p $SCRATCH_DIR +fi + +rdo_images_uri=https://ci.centos.org/artifacts/rdo/images/liberty/delorean/stable + +vm_index=4 +RDO_RELEASE=liberty +SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null) +OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network" + +# check for dependancy packages +for i in rpm-build createrepo libguestfs-tools python-docutils bsdtar; do + if ! rpm -q $i > /dev/null; then + sudo yum install -y $i + fi +done + +# RDO Manager expects a stack user to exist, this checks for one +# and creates it if you are root +if ! id stack > /dev/null; then + sudo useradd stack; + sudo echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack + sudo echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack + sudo chmod 0440 /etc/sudoers.d/stack + echo 'Added user stack' +fi + +# ensure that I can ssh as the stack user +if ! sudo grep "$(cat ~/.ssh/id_rsa.pub)" /home/stack/.ssh/authorized_keys; then + if ! sudo ls -d /home/stack/.ssh/ ; then + sudo mkdir /home/stack/.ssh + sudo chown stack:stack /home/stack/.ssh + sudo chmod 700 /home/stack/.ssh + fi + USER=$(whoami) sudo sh -c "cat ~$USER/.ssh/id_rsa.pub >> /home/stack/.ssh/authorized_keys" + sudo chown stack:stack /home/stack/.ssh/authorized_keys +fi + +# clean up stack user previously build instack disk images +ssh -T ${SSH_OPTIONS[@]} stack@localhost "rm -f instack*.qcow2" + +# Yum repo setup for building the undercloud +if ! rpm -q rdo-release > /dev/null && [ "$1" != "-master" ]; then + sudo yum -y install yum-plugin-priorities + sudo yum-config-manager --disable openstack-${RDO_RELEASE} + sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo + sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo + sudo rm -f /etc/yum.repos.d/delorean-current.repo +elif [ "$1" == "-master" ]; then + sudo yum -y install yum-plugin-priorities + sudo yum-config-manager --disable openstack-${RDO_RELEASE} + sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo + sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo + sudo rm -f /etc/yum.repos.d/delorean-current.repo +fi + +# ensure the undercloud package is installed so we can build the undercloud +if ! rpm -q instack-undercloud > /dev/null; then + sudo yum install -y python-tripleoclient +fi + +# ensure openvswitch is installed +if ! rpm -q openvswitch > /dev/null; then + sudo yum install -y openvswitch +fi + +# ensure libvirt is installed +if ! rpm -q libvirt-daemon-kvm > /dev/null; then + sudo yum install -y libvirt-daemon-kvm +fi + +# clean this up incase it's there +sudo rm -f /tmp/instack.answers + +# ensure that no previous undercloud VMs are running +clean.sh +# and rebuild the bare undercloud VMs +ssh -T ${SSH_OPTIONS[@]} stack@localhost < /dev/null && [ $CNT -gt 0 ]; do + echo -n "." + sleep 3 + CNT=CNT-1 +done +# TODO fail if CNT=0 + +# yum update undercloud and reboot. +ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" < /dev/null; then + yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + fi + + yum -y install yum-plugin-priorities + curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo + curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo + + cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys + chown stack:stack /home/stack/.ssh/authorized_keys +EOI +# +# If using special kernel version, install on undercloud vm. +# +if [ ! -z $kernel_version ]; then + echo "Install special kernel version $kernel_version on undercloud" + ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" < /dev/null || echo -n '' +virsh undefine instack --remove-all-storage 2> /dev/null || echo -n '' +virsh vol-delete instack.qcow2 --pool default 2> /dev/null +rm -f /var/lib/libvirt/images/instack.qcow2 2> /dev/null + +# Clean off baremetal VMs in case they exist +for i in $(seq 0 $vm_index); do + virsh destroy baremetalbrbm_brbm1_$i 2> /dev/null || echo -n '' + virsh undefine baremetalbrbm_brbm1_$i --remove-all-storage 2> /dev/null || echo -n '' + virsh vol-delete baremetalbrbm_brbm1_${i}.qcow2 --pool default 2> /dev/null + rm -f /var/lib/libvirt/images/baremetalbrbm_brbm1_${i}.qcow2 2> /dev/null +done + +# Clean off brbm bridges +virsh net-destroy brbm 2> /dev/null +virsh net-undefine brbm 2> /dev/null +vs-vsctl del-br brbm 2> /dev/null + +virsh net-destroy brbm1 2> /dev/null +virsh net-undefine brbm1 2> /dev/null +vs-vsctl del-br brbm1 2> /dev/null + +# clean pub keys from root's auth keys +sed -i '/stack@instack.localdomain/d' /root/.ssh/authorized_keys +sed -i '/virtual-power-key/d' /root/.ssh/authorized_keys + + +echo "Cleanup Completed" -- 2.16.6