From: Jiang, Yunhong Date: Thu, 18 May 2017 16:34:00 +0000 (+0000) Subject: Merge "Optimizing the code and uploading ftrace logs to artifacts" X-Git-Tag: opnfv-5.0.0~14 X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?p=kvmfornfv.git;a=commitdiff_plain;h=c4d68bc45aa6aa8b7fe39bce206cb17c2cb2d365;hp=156570aeb2bd567890da31e4efecbe48d7129b8b Merge "Optimizing the code and uploading ftrace logs to artifacts" --- diff --git a/ci/apex.conf b/ci/apex.conf index 03e5830bc..03d2b791a 100644 --- a/ci/apex.conf +++ b/ci/apex.conf @@ -1,2 +1,2 @@ -branch=stable/danube -commit_id=977317100f3b52caafb091ad5924e5dd81982433 +branch=master +commit_id=4bfeded9ee7ddc1cf05f6bd2c388894a7d736291 diff --git a/ci/build.sh b/ci/build.sh index ef06a716c..a52055940 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -119,6 +119,8 @@ then output_dir=$WORKSPACE/build_output fi +job_type=`echo $JOB_NAME | cut -d '-' -f 2` + echo "" echo "Building for $type package in $output_dir" echo "" @@ -139,3 +141,18 @@ if [ ${apex_build_flag} -eq 1 ];then variable=`ls kvmfornfv-* | grep "devel" | awk -F "_" '{print $3}' | awk -F "." '{print $1}'` rename "s/${variable}/centos/" kvmfornfv-* fi + +# Uploading rpms only for daily job +if [ $job_type == "verify" ]; then + if [ $type == "centos" ]; then + echo "Removing kernel-debuginfo rpm from output_dir" + rm -f ${output_dir}/kernel-debug* + echo "Checking packages in output_dir" + ls -lrth ${output_dir} + else + echo "Removing debug debian from output_dir" + rm -f ${output_dir}/*dbg* + echo "Checking packages in output_dir" + ls -lrth ${output_dir} + fi +fi \ No newline at end of file diff --git a/ci/envs/cyclictest.sh b/ci/envs/cyclictest.sh index 920a46b93..d1ae5c7fa 100755 --- a/ci/envs/cyclictest.sh +++ b/ci/envs/cyclictest.sh @@ -11,6 +11,7 @@ testName=$2 #idle_idle/stress_idle HOST_IP=$( getHostIP ) pod_config='/opt/scripts/pod.yaml' cyclictest_context_file='/opt/kvmfornfv_cyclictest_'${testName}'.yaml' +yardstick_prefix='/root/yardstick/yardstick/benchmark/scenarios/compute' # yardstick teardown path if [ ! -f ${pod_config} ] ; then echo "file ${pod_config} not found" @@ -22,6 +23,12 @@ if [ ! -f ${cyclictest_context_file} ] ; then exit 1 fi +#As yardstick executes only *.bash scripts, copy the post-execute script as .bash script +cp /opt/scripts/disable_trace.sh ${yardstick_prefix}/disable_trace.bash + +#Execution of the post-execute script copied requires re-installation of yardstick +( cd /root/yardstick ; python setup.py install ) + #setting up of image for launching guest vm. ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ root@$HOST_IP "cp /root/images/guest1.qcow2 /root/" diff --git a/ci/envs/guest-setup1.sh b/ci/envs/guest-setup1.sh index 678baa43b..6d790a080 100755 --- a/ci/envs/guest-setup1.sh +++ b/ci/envs/guest-setup1.sh @@ -24,3 +24,6 @@ set_irq_affinity # Disable RT throttling echo -1 > /proc/sys/kernel/sched_rt_period_us echo -1 > /proc/sys/kernel/sched_rt_runtime_us + +# Disable the timer migration +echo 0 > /proc/sys/kernel/timer_migration diff --git a/ci/kernel_build.sh b/ci/kernel_build.sh index 91e8b7765..6071ca760 100755 --- a/ci/kernel_build.sh +++ b/ci/kernel_build.sh @@ -50,7 +50,7 @@ EOF # Build the kernel debs make-kpkg clean -fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers +fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers kernel_debug -j$(nproc) make mv /root/kvmfornfv/linux-* /root/kvmfornfv/build_output } diff --git a/ci/kernel_debug.sh b/ci/kernel_debug.sh new file mode 100755 index 000000000..5ccf1af76 --- /dev/null +++ b/ci/kernel_debug.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +tmpdir=$1 #/tmp/kvmfornfv_rpmbuild.1/BUILD/kernel-4.4.50_rt62nfv +OBJCOPY=objcopy +if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then +for module in $(find $tmpdir/lib/modules/ -name *.ko -printf '%P\n'); do + module=lib/modules/$module + mkdir -p $(dirname $tmpdir/usr/lib/debug/$module) + # only keep debug symbols in the debug file + $OBJCOPY --only-keep-debug $tmpdir/$module $tmpdir/usr/lib/debug/$module + # strip original module from debug symbols + $OBJCOPY --strip-debug $tmpdir/$module + # then add a link to those + $OBJCOPY --add-gnu-debuglink=$tmpdir/usr/lib/debug/$module $tmpdir/$module + done +fi \ No newline at end of file diff --git a/docker_image_build/Dockerfile b/docker_image_build/Dockerfile index 6af1176f4..f292bf2f5 100644 --- a/docker_image_build/Dockerfile +++ b/docker_image_build/Dockerfile @@ -4,7 +4,7 @@ #kvmfornfv1 image has all dependency packages installed. # FROM kvmfornfv1:latest -RUN cd /root && git clone https://gerrit.opnfv.org/gerrit/p/yardstick.git -b stable/danube +RUN cd /root && git clone https://gerrit.opnfv.org/gerrit/p/yardstick.git RUN sed -i -e "s/3600/9000/g" /root/yardstick/yardstick/ssh.py RUN cd /root/yardstick && python setup.py install WORKDIR /root diff --git a/fuel-plugin/build_kvm.sh b/fuel-plugin/build_kvm.sh index cc1df2791..40220b7b0 100755 --- a/fuel-plugin/build_kvm.sh +++ b/fuel-plugin/build_kvm.sh @@ -46,7 +46,7 @@ make oldconfig vlan_cnt) { - DP(NETIF_MSG_IFUP, "No need to re-configure vlan filters\n"); - return 0; - } - + /* Configure all non-configured entries */ list_for_each_entry(vlan, &bp->vlan_reg, link) { - /* Prepare for cleanup in case of errors */ - if (rc) { - vlan->hw = false; - continue; - } - - if (!vlan->hw) + if (vlan->hw) continue; - DP(NETIF_MSG_IFUP, "Re-configuring vlan 0x%04x\n", vlan->vid); + if (bp->vlan_cnt >= bp->vlan_credit) + return -ENOBUFS; rc = __bnx2x_vlan_configure_vid(bp, vlan->vid, true); if (rc) { - BNX2X_ERR("Unable to configure VLAN %d\n", vlan->vid); - vlan->hw = false; - rc = -EINVAL; - continue; + BNX2X_ERR("Unable to config VLAN %d\n", vlan->vid); + return rc; } + + DP(NETIF_MSG_IFUP, "HW configured for VLAN %d\n", vlan->vid); + vlan->hw = true; + bp->vlan_cnt++; } - return rc; + return 0; +} + +static void bnx2x_vlan_configure(struct bnx2x *bp, bool set_rx_mode) +{ + bool need_accept_any_vlan; + + need_accept_any_vlan = !!bnx2x_vlan_configure_vid_list(bp); + + if (bp->accept_any_vlan != need_accept_any_vlan) { + bp->accept_any_vlan = need_accept_any_vlan; + DP(NETIF_MSG_IFUP, "Accept all VLAN %s\n", + bp->accept_any_vlan ? "raised" : "cleared"); + if (set_rx_mode) { + if (IS_PF(bp)) + bnx2x_set_rx_mode_inner(bp); + else + bnx2x_vfpf_storm_rx_mode(bp); + } + } +} + +int bnx2x_vlan_reconfigure_vid(struct bnx2x *bp) +{ + struct bnx2x_vlan_entry *vlan; + + /* The hw forgot all entries after reload */ + list_for_each_entry(vlan, &bp->vlan_reg, link) + vlan->hw = false; + bp->vlan_cnt = 0; + + /* Don't set rx mode here. Our caller will do it. */ + bnx2x_vlan_configure(bp, false); + + return 0; } static int bnx2x_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) { struct bnx2x *bp = netdev_priv(dev); struct bnx2x_vlan_entry *vlan; - bool hw = false; - int rc = 0; - - if (!netif_running(bp->dev)) { - DP(NETIF_MSG_IFUP, - "Ignoring VLAN configuration the interface is down\n"); - return -EFAULT; - } DP(NETIF_MSG_IFUP, "Adding VLAN %d\n", vid); @@ -12883,93 +12902,47 @@ static int bnx2x_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) if (!vlan) return -ENOMEM; - bp->vlan_cnt++; - if (bp->vlan_cnt > bp->vlan_credit && !bp->accept_any_vlan) { - DP(NETIF_MSG_IFUP, "Accept all VLAN raised\n"); - bp->accept_any_vlan = true; - if (IS_PF(bp)) - bnx2x_set_rx_mode_inner(bp); - else - bnx2x_vfpf_storm_rx_mode(bp); - } else if (bp->vlan_cnt <= bp->vlan_credit) { - rc = __bnx2x_vlan_configure_vid(bp, vid, true); - hw = true; - } - vlan->vid = vid; - vlan->hw = hw; + vlan->hw = false; + list_add_tail(&vlan->link, &bp->vlan_reg); - if (!rc) { - list_add(&vlan->link, &bp->vlan_reg); - } else { - bp->vlan_cnt--; - kfree(vlan); - } - - DP(NETIF_MSG_IFUP, "Adding VLAN result %d\n", rc); + if (netif_running(dev)) + bnx2x_vlan_configure(bp, true); - return rc; + return 0; } static int bnx2x_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) { struct bnx2x *bp = netdev_priv(dev); struct bnx2x_vlan_entry *vlan; + bool found = false; int rc = 0; - if (!netif_running(bp->dev)) { - DP(NETIF_MSG_IFUP, - "Ignoring VLAN configuration the interface is down\n"); - return -EFAULT; - } - DP(NETIF_MSG_IFUP, "Removing VLAN %d\n", vid); - if (!bp->vlan_cnt) { - BNX2X_ERR("Unable to kill VLAN %d\n", vid); - return -EINVAL; - } - list_for_each_entry(vlan, &bp->vlan_reg, link) - if (vlan->vid == vid) + if (vlan->vid == vid) { + found = true; break; + } - if (vlan->vid != vid) { + if (!found) { BNX2X_ERR("Unable to kill VLAN %d - not found\n", vid); return -EINVAL; } - if (vlan->hw) + if (netif_running(dev) && vlan->hw) { rc = __bnx2x_vlan_configure_vid(bp, vid, false); + DP(NETIF_MSG_IFUP, "HW deconfigured for VLAN %d\n", vid); + bp->vlan_cnt--; + } list_del(&vlan->link); kfree(vlan); - bp->vlan_cnt--; - - if (bp->vlan_cnt <= bp->vlan_credit && bp->accept_any_vlan) { - /* Configure all non-configured entries */ - list_for_each_entry(vlan, &bp->vlan_reg, link) { - if (vlan->hw) - continue; - - rc = __bnx2x_vlan_configure_vid(bp, vlan->vid, true); - if (rc) { - BNX2X_ERR("Unable to config VLAN %d\n", - vlan->vid); - continue; - } - DP(NETIF_MSG_IFUP, "HW configured for VLAN %d\n", - vlan->vid); - vlan->hw = true; - } - DP(NETIF_MSG_IFUP, "Accept all VLAN Removed\n"); - bp->accept_any_vlan = false; - if (IS_PF(bp)) - bnx2x_set_rx_mode_inner(bp); - else - bnx2x_vfpf_storm_rx_mode(bp); - } + if (netif_running(dev)) + bnx2x_vlan_configure(bp, true); DP(NETIF_MSG_IFUP, "Removing VLAN result %d\n", rc); diff --git a/kernel/include/linux/intel-iommu.h b/kernel/include/linux/intel-iommu.h index d49e26c6c..23e129ef6 100644 --- a/kernel/include/linux/intel-iommu.h +++ b/kernel/include/linux/intel-iommu.h @@ -153,8 +153,8 @@ static inline void dmar_writeq(void __iomem *addr, u64 val) #define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60) #define DMA_TLB_DSI_FLUSH (((u64)2) << 60) #define DMA_TLB_PSI_FLUSH (((u64)3) << 60) -#define DMA_TLB_IIRG(type) ((type >> 60) & 7) -#define DMA_TLB_IAIG(val) (((val) >> 57) & 7) +#define DMA_TLB_IIRG(type) ((type >> 60) & 3) +#define DMA_TLB_IAIG(val) (((val) >> 57) & 3) #define DMA_TLB_READ_DRAIN (((u64)1) << 49) #define DMA_TLB_WRITE_DRAIN (((u64)1) << 48) #define DMA_TLB_DID(id) (((u64)((id) & 0xffff)) << 32) @@ -164,9 +164,9 @@ static inline void dmar_writeq(void __iomem *addr, u64 val) /* INVALID_DESC */ #define DMA_CCMD_INVL_GRANU_OFFSET 61 -#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 3) -#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 3) -#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 3) +#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 4) +#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 4) +#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 4) #define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7) #define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6) #define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16))) @@ -316,8 +316,8 @@ enum { #define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) #define QI_DEV_EIOTLB_GLOB(g) ((u64)g) #define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32) -#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32) -#define QI_DEV_EIOTLB_QDEP(qd) (((qd) & 0x1f) << 16) +#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) +#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) #define QI_DEV_EIOTLB_MAX_INVS 32 #define QI_PGRP_IDX(idx) (((u64)(idx)) << 55) diff --git a/kernel/scripts/package/Makefile b/kernel/scripts/package/Makefile index 493e22635..a7ed59597 100644 --- a/kernel/scripts/package/Makefile +++ b/kernel/scripts/package/Makefile @@ -52,7 +52,7 @@ rpm-pkg rpm: FORCE $(call cmd,src_tar,$(KERNELPATH),kernel.spec) $(CONFIG_SHELL) $(srctree)/scripts/mkversion > $(objtree)/.tmp_version mv -f $(objtree)/.tmp_version $(objtree)/.version - rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz + QA_SKIP_BUILD_ROOT=1 rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz rm $(KERNELPATH).tar.gz kernel.spec # binrpm-pkg diff --git a/kernel/scripts/package/mkspec b/kernel/scripts/package/mkspec index fe44d68e9..0f96d4f8d 100755 --- a/kernel/scripts/package/mkspec +++ b/kernel/scripts/package/mkspec @@ -39,10 +39,9 @@ if ! $PREBUILT; then echo "Source: kernel-$__KERNELRELEASE.tar.gz" fi +echo "%undefine _missing_build_ids_terminate_build" echo "BuildRoot: %{_tmppath}/%{name}-%{PACKAGE_VERSION}-root" echo "Provides: $PROVIDES" -echo "%define __spec_install_post /usr/lib/rpm/brp-compress || :" -echo "%define debug_package %{nil}" echo "" echo "%description" echo "The Linux Kernel, the operating system core itself" @@ -81,6 +80,8 @@ echo "make clean && make %{?_smp_mflags}" echo "" fi +echo "" + echo "%install" echo 'KBUILD_IMAGE=$(make image_name)' echo "%ifarch ia64" @@ -91,6 +92,7 @@ echo "%endif" echo 'mkdir -p $RPM_BUILD_ROOT'"/lib/firmware/$KERNELRELEASE" echo 'INSTALL_MOD_PATH=$RPM_BUILD_ROOT make %{?_smp_mflags} KBUILD_SRC= mod-fw= modules_install' +echo '/opt/kvmfornfv/ci/kernel_debug.sh $RPM_BUILD_ROOT' echo 'INSTALL_FW_PATH=$RPM_BUILD_ROOT'"/lib/firmware/$KERNELRELEASE" echo 'make INSTALL_FW_PATH=$INSTALL_FW_PATH' firmware_install echo "%ifarch ia64" @@ -110,15 +112,15 @@ echo 'cp System.map $RPM_BUILD_ROOT'"/boot/System.map-$KERNELRELEASE" echo 'cp .config $RPM_BUILD_ROOT'"/boot/config-$KERNELRELEASE" -echo "%ifnarch ppc64" -echo 'bzip2 -9 --keep vmlinux' -echo 'mv vmlinux.bz2 $RPM_BUILD_ROOT'"/boot/vmlinux-$KERNELRELEASE.bz2" -echo "%endif" +#echo "%ifnarch ppc64" +#echo 'bzip2 -9 --keep vmlinux' +#echo 'mv vmlinux.bz2 $RPM_BUILD_ROOT'"/boot/vmlinux-$KERNELRELEASE.bz2" +#echo "%endif" if ! $PREBUILT; then echo 'rm -f $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE/{build,source}" echo "mkdir -p "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE" -echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*vmlinux* --exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude=firmware --exclude .config.old --exclude .missing-syscalls.d\"" +echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*.o --exclude=*vmlinux* --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude=firmware --exclude .config.old --exclude .missing-syscalls.d\"" echo "tar "'$EXCLUDES'" -cf- . | (cd "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE;tar xvf -)" echo 'cd $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE" echo "ln -sf /usr/src/kernels/$KERNELRELEASE build" diff --git a/tests/kvmfornfv_cyclictest_hostenv_guestenv.yaml b/tests/kvmfornfv_cyclictest_hostenv_guestenv.yaml index 0a35e95a3..f716c8a01 100644 --- a/tests/kvmfornfv_cyclictest_hostenv_guestenv.yaml +++ b/tests/kvmfornfv_cyclictest_hostenv_guestenv.yaml @@ -5,7 +5,7 @@ # For this sample just like running the command below on the test vm and # getting latencies info back to the yardstick. # -# sudo bash cyclictest -a 1 -i 1000 -p 99 -l 1000 -t 1 -h 90 -m -n -q +# sudo bash cyclictest -a 1 -b 1000 -i 1000 -p 99 -l 1000 -t 1 -h 90 -m -n -q # schema: "yardstick:task:0.1" @@ -15,6 +15,7 @@ scenarios: type: Cyclictest options: affinity: 1 + breaktrace: 1000 interval: 1000 priority: 99 loops: 600000 @@ -27,9 +28,9 @@ scenarios: interval: 1 tc: "kvmfornfv_cyclictest-node-context" sla: - max_min_latency: 100 - max_avg_latency: 600000 - max_max_latency: 1200000 + max_min_latency: 50 + max_avg_latency: 100 + max_max_latency: 1000 action: monitor setup_options: rpm_dir: "/opt/rpm" @@ -50,3 +51,10 @@ context: type: Node name: LF file: /opt/scripts/pod.yaml + env: + type: script + prefix: yardstick.benchmark.scenarios.compute + teardown: + - + kvm: + script: disable_trace.bash