Merge "Optimizing the code and uploading ftrace logs to artifacts"
authorJiang, Yunhong <yunhong.jiang@intel.com>
Thu, 18 May 2017 16:34:00 +0000 (16:34 +0000)
committerGerrit Code Review <gerrit@opnfv.org>
Thu, 18 May 2017 16:34:00 +0000 (16:34 +0000)
14 files changed:
ci/apex.conf
ci/build.sh
ci/envs/cyclictest.sh
ci/envs/guest-setup1.sh
ci/kernel_build.sh
ci/kernel_debug.sh [new file with mode: 0755]
docker_image_build/Dockerfile
fuel-plugin/build_kvm.sh
kernel/arch/x86/configs/opnfv.config
kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
kernel/include/linux/intel-iommu.h
kernel/scripts/package/Makefile
kernel/scripts/package/mkspec
tests/kvmfornfv_cyclictest_hostenv_guestenv.yaml

index 03e5830..03d2b79 100644 (file)
@@ -1,2 +1,2 @@
-branch=stable/danube
-commit_id=977317100f3b52caafb091ad5924e5dd81982433
+branch=master
+commit_id=4bfeded9ee7ddc1cf05f6bd2c388894a7d736291
index ef06a71..a520559 100755 (executable)
@@ -119,6 +119,8 @@ then
     output_dir=$WORKSPACE/build_output
 fi
 
+job_type=`echo $JOB_NAME | cut -d '-' -f 2`
+
 echo ""
 echo "Building for $type package in $output_dir"
 echo ""
@@ -139,3 +141,18 @@ if [ ${apex_build_flag} -eq 1 ];then
     variable=`ls kvmfornfv-* | grep "devel" | awk -F "_" '{print $3}' | awk -F "." '{print $1}'`
     rename "s/${variable}/centos/" kvmfornfv-*
 fi
+
+# Uploading rpms only for daily job
+if [ $job_type == "verify" ]; then
+   if [ $type == "centos" ]; then
+      echo "Removing kernel-debuginfo rpm from output_dir"
+      rm -f ${output_dir}/kernel-debug*
+      echo "Checking packages in output_dir"
+      ls -lrth ${output_dir}
+   else
+     echo "Removing debug debian from output_dir"
+     rm -f ${output_dir}/*dbg*
+     echo "Checking packages in output_dir"
+     ls -lrth ${output_dir}
+   fi
+fi
\ No newline at end of file
index 920a46b..d1ae5c7 100755 (executable)
@@ -11,6 +11,7 @@ testName=$2 #idle_idle/stress_idle
 HOST_IP=$( getHostIP )
 pod_config='/opt/scripts/pod.yaml'
 cyclictest_context_file='/opt/kvmfornfv_cyclictest_'${testName}'.yaml'
+yardstick_prefix='/root/yardstick/yardstick/benchmark/scenarios/compute' # yardstick teardown path
 
 if [ ! -f ${pod_config} ] ; then
     echo "file ${pod_config} not found"
@@ -22,6 +23,12 @@ if [ ! -f ${cyclictest_context_file} ] ; then
     exit 1
 fi
 
+#As yardstick executes only *.bash scripts, copy the post-execute script as .bash script
+cp /opt/scripts/disable_trace.sh ${yardstick_prefix}/disable_trace.bash
+
+#Execution of the post-execute script copied requires re-installation of yardstick
+( cd /root/yardstick ; python setup.py install )
+
 #setting up of image for launching guest vm.
 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
 root@$HOST_IP "cp /root/images/guest1.qcow2 /root/"
index 678baa4..6d790a0 100755 (executable)
@@ -24,3 +24,6 @@ set_irq_affinity
 # Disable RT throttling
 echo -1 > /proc/sys/kernel/sched_rt_period_us
 echo -1 > /proc/sys/kernel/sched_rt_runtime_us
+
+# Disable the timer migration
+echo 0 > /proc/sys/kernel/timer_migration
index 91e8b77..6071ca7 100755 (executable)
@@ -50,7 +50,7 @@ EOF
 
 # Build the kernel debs
 make-kpkg clean
-fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers
+fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers kernel_debug -j$(nproc)
 make
 mv /root/kvmfornfv/linux-* /root/kvmfornfv/build_output
 }
diff --git a/ci/kernel_debug.sh b/ci/kernel_debug.sh
new file mode 100755 (executable)
index 0000000..5ccf1af
--- /dev/null
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+tmpdir=$1 #/tmp/kvmfornfv_rpmbuild.1/BUILD/kernel-4.4.50_rt62nfv
+OBJCOPY=objcopy
+if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then
+for module in $(find $tmpdir/lib/modules/ -name *.ko -printf '%P\n'); do
+                        module=lib/modules/$module
+                        mkdir -p $(dirname $tmpdir/usr/lib/debug/$module)
+                        # only keep debug symbols in the debug file
+                        $OBJCOPY --only-keep-debug $tmpdir/$module $tmpdir/usr/lib/debug/$module
+                        # strip original module from debug symbols
+                        $OBJCOPY --strip-debug $tmpdir/$module
+                        # then add a link to those
+                        $OBJCOPY --add-gnu-debuglink=$tmpdir/usr/lib/debug/$module $tmpdir/$module
+                done
+fi
\ No newline at end of file
index 6af1176..f292bf2 100644 (file)
@@ -4,7 +4,7 @@
 #kvmfornfv1 image has all dependency packages installed.
 #
 FROM kvmfornfv1:latest
-RUN cd /root && git clone https://gerrit.opnfv.org/gerrit/p/yardstick.git -b stable/danube
+RUN cd /root && git clone https://gerrit.opnfv.org/gerrit/p/yardstick.git
 RUN sed -i -e "s/3600/9000/g" /root/yardstick/yardstick/ssh.py
 RUN cd /root/yardstick && python setup.py install
 WORKDIR /root
index cc1df27..40220b7 100755 (executable)
@@ -46,7 +46,7 @@ make oldconfig </dev/null
 # Build the kernel debs
 make-kpkg clean
 
-fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers
+fakeroot make-kpkg --initrd --revision=$VERSION kernel_image kernel_headers -j$(nproc)
 
 cp $SRC/kvmfornfv/linux-headers*.deb /kvmfornfv/.
 cp $SRC/kvmfornfv/linux-image*.deb /kvmfornfv/.
index b8ec4a1..a3233ee 100644 (file)
@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 4.4.6 Kernel Configuration
+# Linux/x86 4.4.50 Kernel Configuration
 #
 CONFIG_64BIT=y
 CONFIG_X86_64=y
@@ -370,10 +370,12 @@ CONFIG_FREEZER=y
 CONFIG_ZONE_DMA=y
 CONFIG_SMP=y
 CONFIG_X86_FEATURE_NAMES=y
-# CONFIG_X86_X2APIC is not set
+CONFIG_X86_X2APIC=y
 CONFIG_X86_MPPARSE=y
 CONFIG_X86_EXTENDED_PLATFORM=y
+# CONFIG_X86_NUMACHIP is not set
 # CONFIG_X86_VSMP is not set
+# CONFIG_X86_UV is not set
 # CONFIG_X86_GOLDFISH is not set
 CONFIG_X86_INTEL_LPSS=y
 CONFIG_X86_AMD_PLATFORM_DEVICE=y
@@ -1948,7 +1950,7 @@ CONFIG_HMC6352=m
 CONFIG_DS1682=m
 CONFIG_TI_DAC7512=m
 CONFIG_VMWARE_BALLOON=m
-CONFIG_BMP085=y
+CONFIG_BMP085=m
 CONFIG_BMP085_I2C=m
 CONFIG_BMP085_SPI=m
 CONFIG_USB_SWITCH_FSA9480=m
@@ -6933,7 +6935,7 @@ CONFIG_AMD_IOMMU_V2=m
 CONFIG_DMAR_TABLE=y
 CONFIG_INTEL_IOMMU=y
 CONFIG_INTEL_IOMMU_SVM=y
-CONFIG_INTEL_IOMMU_DEFAULT_ON=y
+# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set
 CONFIG_INTEL_IOMMU_FLOPPY_WA=y
 CONFIG_IRQ_REMAP=y
 
index 1c81238..ea6478f 100644 (file)
@@ -12830,52 +12830,71 @@ static int __bnx2x_vlan_configure_vid(struct bnx2x *bp, u16 vid, bool add)
        return rc;
 }
 
-int bnx2x_vlan_reconfigure_vid(struct bnx2x *bp)
+static int bnx2x_vlan_configure_vid_list(struct bnx2x *bp)
 {
        struct bnx2x_vlan_entry *vlan;
        int rc = 0;
 
-       if (!bp->vlan_cnt) {
-               DP(NETIF_MSG_IFUP, "No need to re-configure vlan filters\n");
-               return 0;
-       }
-
+       /* Configure all non-configured entries */
        list_for_each_entry(vlan, &bp->vlan_reg, link) {
-               /* Prepare for cleanup in case of errors */
-               if (rc) {
-                       vlan->hw = false;
-                       continue;
-               }
-
-               if (!vlan->hw)
+               if (vlan->hw)
                        continue;
 
-               DP(NETIF_MSG_IFUP, "Re-configuring vlan 0x%04x\n", vlan->vid);
+               if (bp->vlan_cnt >= bp->vlan_credit)
+                       return -ENOBUFS;
 
                rc = __bnx2x_vlan_configure_vid(bp, vlan->vid, true);
                if (rc) {
-                       BNX2X_ERR("Unable to configure VLAN %d\n", vlan->vid);
-                       vlan->hw = false;
-                       rc = -EINVAL;
-                       continue;
+                       BNX2X_ERR("Unable to config VLAN %d\n", vlan->vid);
+                       return rc;
                }
+
+               DP(NETIF_MSG_IFUP, "HW configured for VLAN %d\n", vlan->vid);
+               vlan->hw = true;
+               bp->vlan_cnt++;
        }
 
-       return rc;
+       return 0;
+}
+
+static void bnx2x_vlan_configure(struct bnx2x *bp, bool set_rx_mode)
+{
+       bool need_accept_any_vlan;
+
+       need_accept_any_vlan = !!bnx2x_vlan_configure_vid_list(bp);
+
+       if (bp->accept_any_vlan != need_accept_any_vlan) {
+               bp->accept_any_vlan = need_accept_any_vlan;
+               DP(NETIF_MSG_IFUP, "Accept all VLAN %s\n",
+                  bp->accept_any_vlan ? "raised" : "cleared");
+               if (set_rx_mode) {
+                       if (IS_PF(bp))
+                               bnx2x_set_rx_mode_inner(bp);
+                       else
+                               bnx2x_vfpf_storm_rx_mode(bp);
+               }
+       }
+}
+
+int bnx2x_vlan_reconfigure_vid(struct bnx2x *bp)
+{
+       struct bnx2x_vlan_entry *vlan;
+
+       /* The hw forgot all entries after reload */
+       list_for_each_entry(vlan, &bp->vlan_reg, link)
+               vlan->hw = false;
+       bp->vlan_cnt = 0;
+
+       /* Don't set rx mode here. Our caller will do it. */
+       bnx2x_vlan_configure(bp, false);
+
+       return 0;
 }
 
 static int bnx2x_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
 {
        struct bnx2x *bp = netdev_priv(dev);
        struct bnx2x_vlan_entry *vlan;
-       bool hw = false;
-       int rc = 0;
-
-       if (!netif_running(bp->dev)) {
-               DP(NETIF_MSG_IFUP,
-                  "Ignoring VLAN configuration the interface is down\n");
-               return -EFAULT;
-       }
 
        DP(NETIF_MSG_IFUP, "Adding VLAN %d\n", vid);
 
@@ -12883,93 +12902,47 @@ static int bnx2x_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
        if (!vlan)
                return -ENOMEM;
 
-       bp->vlan_cnt++;
-       if (bp->vlan_cnt > bp->vlan_credit && !bp->accept_any_vlan) {
-               DP(NETIF_MSG_IFUP, "Accept all VLAN raised\n");
-               bp->accept_any_vlan = true;
-               if (IS_PF(bp))
-                       bnx2x_set_rx_mode_inner(bp);
-               else
-                       bnx2x_vfpf_storm_rx_mode(bp);
-       } else if (bp->vlan_cnt <= bp->vlan_credit) {
-               rc = __bnx2x_vlan_configure_vid(bp, vid, true);
-               hw = true;
-       }
-
        vlan->vid = vid;
-       vlan->hw = hw;
+       vlan->hw = false;
+       list_add_tail(&vlan->link, &bp->vlan_reg);
 
-       if (!rc) {
-               list_add(&vlan->link, &bp->vlan_reg);
-       } else {
-               bp->vlan_cnt--;
-               kfree(vlan);
-       }
-
-       DP(NETIF_MSG_IFUP, "Adding VLAN result %d\n", rc);
+       if (netif_running(dev))
+               bnx2x_vlan_configure(bp, true);
 
-       return rc;
+       return 0;
 }
 
 static int bnx2x_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
 {
        struct bnx2x *bp = netdev_priv(dev);
        struct bnx2x_vlan_entry *vlan;
+       bool found = false;
        int rc = 0;
 
-       if (!netif_running(bp->dev)) {
-               DP(NETIF_MSG_IFUP,
-                  "Ignoring VLAN configuration the interface is down\n");
-               return -EFAULT;
-       }
-
        DP(NETIF_MSG_IFUP, "Removing VLAN %d\n", vid);
 
-       if (!bp->vlan_cnt) {
-               BNX2X_ERR("Unable to kill VLAN %d\n", vid);
-               return -EINVAL;
-       }
-
        list_for_each_entry(vlan, &bp->vlan_reg, link)
-               if (vlan->vid == vid)
+               if (vlan->vid == vid) {
+                       found = true;
                        break;
+               }
 
-       if (vlan->vid != vid) {
+       if (!found) {
                BNX2X_ERR("Unable to kill VLAN %d - not found\n", vid);
                return -EINVAL;
        }
 
-       if (vlan->hw)
+       if (netif_running(dev) && vlan->hw) {
                rc = __bnx2x_vlan_configure_vid(bp, vid, false);
+               DP(NETIF_MSG_IFUP, "HW deconfigured for VLAN %d\n", vid);
+               bp->vlan_cnt--;
+       }
 
        list_del(&vlan->link);
        kfree(vlan);
 
-       bp->vlan_cnt--;
-
-       if (bp->vlan_cnt <= bp->vlan_credit && bp->accept_any_vlan) {
-               /* Configure all non-configured entries */
-               list_for_each_entry(vlan, &bp->vlan_reg, link) {
-                       if (vlan->hw)
-                               continue;
-
-                       rc = __bnx2x_vlan_configure_vid(bp, vlan->vid, true);
-                       if (rc) {
-                               BNX2X_ERR("Unable to config VLAN %d\n",
-                                         vlan->vid);
-                               continue;
-                       }
-                       DP(NETIF_MSG_IFUP, "HW configured for VLAN %d\n",
-                          vlan->vid);
-                       vlan->hw = true;
-               }
-               DP(NETIF_MSG_IFUP, "Accept all VLAN Removed\n");
-               bp->accept_any_vlan = false;
-               if (IS_PF(bp))
-                       bnx2x_set_rx_mode_inner(bp);
-               else
-                       bnx2x_vfpf_storm_rx_mode(bp);
-       }
+       if (netif_running(dev))
+               bnx2x_vlan_configure(bp, true);
 
        DP(NETIF_MSG_IFUP, "Removing VLAN result %d\n", rc);
 
index d49e26c..23e129e 100644 (file)
@@ -153,8 +153,8 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
 #define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
 #define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
 #define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
-#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
-#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
+#define DMA_TLB_IIRG(type) ((type >> 60) & 3)
+#define DMA_TLB_IAIG(val) (((val) >> 57) & 3)
 #define DMA_TLB_READ_DRAIN (((u64)1) << 49)
 #define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
 #define DMA_TLB_DID(id)        (((u64)((id) & 0xffff)) << 32)
@@ -164,9 +164,9 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
 
 /* INVALID_DESC */
 #define DMA_CCMD_INVL_GRANU_OFFSET  61
-#define DMA_ID_TLB_GLOBAL_FLUSH        (((u64)1) << 3)
-#define DMA_ID_TLB_DSI_FLUSH   (((u64)2) << 3)
-#define DMA_ID_TLB_PSI_FLUSH   (((u64)3) << 3)
+#define DMA_ID_TLB_GLOBAL_FLUSH        (((u64)1) << 4)
+#define DMA_ID_TLB_DSI_FLUSH   (((u64)2) << 4)
+#define DMA_ID_TLB_PSI_FLUSH   (((u64)3) << 4)
 #define DMA_ID_TLB_READ_DRAIN  (((u64)1) << 7)
 #define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6)
 #define DMA_ID_TLB_DID(id)     (((u64)((id & 0xffff) << 16)))
@@ -316,8 +316,8 @@ enum {
 #define QI_DEV_EIOTLB_SIZE     (((u64)1) << 11)
 #define QI_DEV_EIOTLB_GLOB(g)  ((u64)g)
 #define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32)
-#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
-#define QI_DEV_EIOTLB_QDEP(qd) (((qd) & 0x1f) << 16)
+#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16)
+#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4)
 #define QI_DEV_EIOTLB_MAX_INVS 32
 
 #define QI_PGRP_IDX(idx)       (((u64)(idx)) << 55)
index 493e226..a7ed595 100644 (file)
@@ -52,7 +52,7 @@ rpm-pkg rpm: FORCE
        $(call cmd,src_tar,$(KERNELPATH),kernel.spec)
        $(CONFIG_SHELL) $(srctree)/scripts/mkversion > $(objtree)/.tmp_version
        mv -f $(objtree)/.tmp_version $(objtree)/.version
-       rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz
+       QA_SKIP_BUILD_ROOT=1 rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz
        rm $(KERNELPATH).tar.gz kernel.spec
 
 # binrpm-pkg
index fe44d68..0f96d4f 100755 (executable)
@@ -39,10 +39,9 @@ if ! $PREBUILT; then
 echo "Source: kernel-$__KERNELRELEASE.tar.gz"
 fi
 
+echo "%undefine _missing_build_ids_terminate_build"
 echo "BuildRoot: %{_tmppath}/%{name}-%{PACKAGE_VERSION}-root"
 echo "Provides: $PROVIDES"
-echo "%define __spec_install_post /usr/lib/rpm/brp-compress || :"
-echo "%define debug_package %{nil}"
 echo ""
 echo "%description"
 echo "The Linux Kernel, the operating system core itself"
@@ -81,6 +80,8 @@ echo "make clean && make %{?_smp_mflags}"
 echo ""
 fi
 
+echo ""
+
 echo "%install"
 echo 'KBUILD_IMAGE=$(make image_name)'
 echo "%ifarch ia64"
@@ -91,6 +92,7 @@ echo "%endif"
 echo 'mkdir -p $RPM_BUILD_ROOT'"/lib/firmware/$KERNELRELEASE"
 
 echo 'INSTALL_MOD_PATH=$RPM_BUILD_ROOT make %{?_smp_mflags} KBUILD_SRC= mod-fw= modules_install'
+echo '/opt/kvmfornfv/ci/kernel_debug.sh $RPM_BUILD_ROOT'
 echo 'INSTALL_FW_PATH=$RPM_BUILD_ROOT'"/lib/firmware/$KERNELRELEASE"
 echo 'make INSTALL_FW_PATH=$INSTALL_FW_PATH' firmware_install
 echo "%ifarch ia64"
@@ -110,15 +112,15 @@ echo 'cp System.map $RPM_BUILD_ROOT'"/boot/System.map-$KERNELRELEASE"
 
 echo 'cp .config $RPM_BUILD_ROOT'"/boot/config-$KERNELRELEASE"
 
-echo "%ifnarch ppc64"
-echo 'bzip2 -9 --keep vmlinux'
-echo 'mv vmlinux.bz2 $RPM_BUILD_ROOT'"/boot/vmlinux-$KERNELRELEASE.bz2"
-echo "%endif"
+#echo "%ifnarch ppc64"
+#echo 'bzip2 -9 --keep vmlinux'
+#echo 'mv vmlinux.bz2 $RPM_BUILD_ROOT'"/boot/vmlinux-$KERNELRELEASE.bz2"
+#echo "%endif"
 
 if ! $PREBUILT; then
 echo 'rm -f $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE/{build,source}"
 echo "mkdir -p "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE"
-echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*vmlinux* --exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude=firmware --exclude .config.old --exclude .missing-syscalls.d\""
+echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*.o --exclude=*vmlinux* --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude=firmware --exclude .config.old --exclude .missing-syscalls.d\""
 echo "tar "'$EXCLUDES'" -cf- . | (cd "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE;tar xvf -)"
 echo 'cd $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE"
 echo "ln -sf /usr/src/kernels/$KERNELRELEASE build"
index 0a35e95..f716c8a 100644 (file)
@@ -5,7 +5,7 @@
 # For this sample just like running the command below on the test vm and
 # getting latencies info back to the yardstick.
 #
-# sudo bash cyclictest -a 1 -i 1000 -p 99 -l 1000 -t 1 -h 90 -m -n -q
+# sudo bash cyclictest -a 1 -b 1000 -i 1000 -p 99 -l 1000 -t 1 -h 90 -m -n -q
 #
 
 schema: "yardstick:task:0.1"
@@ -15,6 +15,7 @@ scenarios:
   type: Cyclictest
   options:
     affinity: 1
+    breaktrace: 1000
     interval: 1000
     priority: 99
     loops: 600000
@@ -27,9 +28,9 @@ scenarios:
     interval: 1
   tc: "kvmfornfv_cyclictest-node-context"
   sla:
-    max_min_latency: 100
-    max_avg_latency: 600000
-    max_max_latency: 1200000
+    max_min_latency: 50
+    max_avg_latency: 100
+    max_max_latency: 1000
     action: monitor
   setup_options:
     rpm_dir: "/opt/rpm"
@@ -50,3 +51,10 @@ context:
   type: Node
   name: LF
   file: /opt/scripts/pod.yaml
+  env:
+     type: script
+     prefix:  yardstick.benchmark.scenarios.compute
+     teardown:
+     -
+       kvm:
+         script: disable_trace.bash