Merge "Updates to Caching"

diff --combined build/overcloud-opendaylight.sh
index f7f8958,49c5f21..4f81a30
--- 1/build/overcloud-opendaylight.sh
--- 2/build/overcloud-opendaylight.sh
+++ b/build/overcloud-opendaylight.sh
@@@ -52,20 -52,8 +52,19 @@@ enabled=
  gpgcheck=0
  EOF
  
 +#BGPVPN Repo
 +cat > /tmp/bgpvpn.repo << EOF
 +[bgpvpn]
 +name=bgpvpn
 +baseurl=https://trunk.rdoproject.org/centos7/consistent/
 +includepkgs=python2-networking-bgpvpn
 +enabled=1
 +gpgcheck=0
 +priority=1
 +EOF
 +
  # SDNVPN - Copy tunnel setup script
- wget https://raw.githubusercontent.com/openstack/fuel-plugin-opendaylight/brahmaputra-sr2/deployment_scripts/puppet/modules/opendaylight/templates/setup_TEPs.py
- 
+ curl -fO https://raw.githubusercontent.com/openstack/fuel-plugin-opendaylight/brahmaputra-sr2/deployment_scripts/puppet/modules/opendaylight/templates/setup_TEPs.py
  
  # install ODL packages
  # install Jolokia for ODL HA
@@@ -79,8 -67,7 +78,8 @@@ LIBGUESTFS_BACKEND=direct virt-customiz
      --run-command "yum install --downloadonly --downloaddir=/root/master/ opendaylight" \
      --upload /tmp/opendaylight.repo:/etc/yum.repos.d/opendaylight.repo \
      --install opendaylight,python-networking-odl \
 -    --install https://github.com/michaeltchapman/networking_rpm/raw/master/openstack-neutron-bgpvpn-2015.2-1.el7.centos.noarch.rpm \
 +    --upload /tmp/bgpvpn.repo:/etc/yum.repos.d/bgpvpn.repo \
 +    --install python-networking-bgpvpn \
      --run-command "wget https://github.com/rhuss/jolokia/releases/download/v1.3.3/jolokia-1.3.3-bin.tar.gz -O /tmp/jolokia-1.3.3-bin.tar.gz" \
      --run-command "tar -xvf /tmp/jolokia-1.3.3-bin.tar.gz -C /opt/opendaylight/system/org" \
      --install honeycomb \

diff --combined ci/build.sh
index 1b66d55,4238818..0536cf9
--- 1/ci/build.sh
--- 2/ci/build.sh
+++ b/ci/build.sh
@@@ -18,7 -18,7 +18,7 @@@ $0 Builds the Apex OPNFV Deployment Too
  usage: $0 [ -c cache_dir ] -r release_name [ --iso | --rpms ]
  
  OPTIONS:
-   -c cache destination - directory of cached files, defaults to ./cache
+   -c cache destination - destination to save tarball of cache
    -r release name/version of the build result
    --iso build the iso (implies RPMs too)
    --rpms build the rpms
@@@ -36,7 -36,6 +36,7 @@@ CACHE_DIR="cache
  CACHE_NAME="apex-cache"
  MAKE_TARGETS="images"
  REQUIRED_PKGS="rpm-build python-docutils"
 +RELEASE_RPM=""
  
  parse_cmdline() {
    while [ "${1:0:1}" = "-" ]
@@@ -46,7 -45,7 +46,7 @@@
                  display_usage
                  exit 0
              ;;
-         -c|--cache-dir)
+         -c|--cache-dest)
                  CACHE_DEST=${2}
                  shift 2
              ;;
@@@ -64,16 -63,16 +64,21 @@@
                  echo "Buiding opnfv-apex RPMs"
                  shift 1
              ;;
 +        --release-rpm )
 +                RELEASE_RPM=" release-rpm"
 +                echo "Buiding opnfv-apex RPMs"
 +                shift 1
 +            ;;
          --debug )
                  debug="TRUE"
                  echo "Enable debug output"
                  shift 1
              ;;
+         --build-cache )
+                 MAKE_TARGETS=""
+                 echo "Building Cache"
+                 shift 1
+             ;;
          *)
                  display_usage
                  exit 1
@@@ -89,9 -88,6 +94,9 @@@ run_make() 
  
  parse_cmdline "$@"
  
 +# Add release rpm to make targets if defined
 +MAKE_TARGETS+=$RELEASE_RPM
 +
  # Install build dependencies
  for pkg in $REQUIRED_PKGS; do
    if ! rpm -q $pkg > /dev/null; then
@@@ -105,20 -101,35 +110,35 @@@ don
  if [ -n "$RELEASE" ]; then MAKE_ARGS+="RELEASE=$RELEASE "; fi
  
  # Get the Old Cache
- if [ -n "$CACHE_DEST" ]; then
+ if [[ -n "$CACHE_DEST" && -n "$MAKE_TARGETS" ]]; then
      echo "Retrieving Cache"
      if [ -f $CACHE_DEST/${CACHE_NAME}.tgz ]; then
          echo "Cache found at ${CACHE_DEST}/${CACHE_NAME}.tgz"
          rm -rf $BUILD_BASE/$CACHE_DIR
          echo "Unpacking Cache to $BUILD_BASE"
          tar -xvzf $CACHE_DEST/${CACHE_NAME}.tgz -C ${BUILD_BASE}
+         if [ -f $BUILD_BASE/.cache ]; then
+             echo "Rebuilding .cache file"
+             if [ ! -d $BUILD_BASE/$CACHE_DIR ]; then
+                 mkdir $BUILD_BASE/$CACHE_DIR
+             fi
+             for i in $(ls $BUILD_BASE/$CACHE_DIR); do
+                 grep $i $BUILD_BASE/.cache >> $BUILD_BASE/$CACHE_DIR/.cache
+             done
+         fi
          echo "Cache contents after unpack:"
          ls -l $BUILD_BASE/$CACHE_DIR
-     elif [ ! -d $BUILD_BASE/$CACHE_DIR ]; then
-         mkdir $BUILD_BASE/$CACHE_DIR
+     else
+         echo "No Cache Found"
      fi
  fi
  
+ # Ensure the build cache dir exists
+ if [ ! -d "$BUILD_BASE/$CACHE_DIR" ]; then
+     echo "Creating Build Cache Directory"
+     mkdir $BUILD_BASE/$CACHE_DIR
+ fi
+ 
  # Conditionally execute RPM build checks if the specs change and target is not rpm or iso
  if [[ "$MAKE_TARGETS" == "images" ]]; then
      commit_file_list=$(git show --pretty="format:" --name-status)
@@@ -132,9 -143,6 +152,9 @@@
          if [[ $commit_file_list == *build/rpm_specs/opnfv-apex-undercloud.spec* ]]; then
              MAKE_TARGETS+=" undercloud-rpm-check"
          fi
 +        if [[ $commit_file_list == *build/rpm_specs/opnfv-apex-release.spec* ]]; then
 +            MAKE_TARGETS+=" release-rpm-check"
 +        fi
          if [[ $commit_file_list == *build/rpm_specs/opnfv-apex-common.spec* ]]; then
              MAKE_TARGETS+=" common-rpm-check"
          fi
@@@ -166,8 -174,17 +186,17 @@@ echo "Build Complete
  # Build new Cache
  if [ -n "$CACHE_DEST" ]; then
      echo "Building Cache"
+     ls -lh $BUILD_BASE/$CACHE_DIR/
+     # ensure the destination exists
      if [ ! -d $CACHE_DEST ]; then mkdir -p $CACHE_DEST; fi
-     tar --atime-preserve --dereference -C $BUILD_BASE -caf $BUILD_BASE/${CACHE_NAME}.tgz $CACHE_DIR -C ${CACHE_DEST}/
+     # ensure a sub cache dir exists to mirror the build base for extraction
+     if [ ! -d $BUILD_BASE/$CACHE_DIR/$CACHE_DIR/ ]; then mkdir -p $BUILD_BASE/$CACHE_DIR/$CACHE_DIR/; fi
+     # move directly cached files to cache dir for future extraction
+     for i in $(cat $BUILD_BASE/$CACHE_DIR/.cache | awk '{ print $2 }'); do
+         if [ -f $i ]; then mv $i $BUILD_BASE/$CACHE_DIR/$CACHE_DIR/; fi
+     done
+     # roll the cache tarball
+     tar --atime-preserve --dereference -C ${BUILD_BASE}/$CACHE_DIR -caf $CACHE_DEST/${CACHE_NAME}.tgz .
      if [ -f "${CACHE_DEST}/${CACHE_NAME}.tgz" ]; then
        echo "Cache Build Complete"
      else