*.pyc
+.cache*
.project
.pydevproject
ci/config/
--- /dev/null
+[submodule "fuel-main"]
+ path = build/f_repos/sub/fuel-main
+ url = https://github.com/openstack/fuel-main.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-library"]
+ path = build/f_repos/sub/fuel-library
+ url = https://github.com/openstack/fuel-library.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-web"]
+ path = build/f_repos/sub/fuel-web
+ url = https://github.com/openstack/fuel-web.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "python-fuelclient"]
+ path = build/f_repos/sub/python-fuelclient
+ url = https://github.com/openstack/python-fuelclient.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-agent"]
+ path = build/f_repos/sub/fuel-agent
+ url = https://github.com/openstack/fuel-agent.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-nailgun-agent"]
+ path = build/f_repos/sub/fuel-nailgun-agent
+ url = https://github.com/openstack/fuel-nailgun-agent.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-astute"]
+ path = build/f_repos/sub/fuel-astute
+ url = https://github.com/openstack/fuel-astute.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-ostf"]
+ path = build/f_repos/sub/fuel-ostf
+ url = https://github.com/openstack/fuel-ostf.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-mirror"]
+ path = build/f_repos/sub/fuel-mirror
+ url = https://github.com/openstack/fuel-mirror.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-menu"]
+ path = build/f_repos/sub/fuel-menu
+ url = https://github.com/openstack/fuel-menu.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "shotgun"]
+ path = build/f_repos/sub/shotgun
+ url = https://github.com/openstack/shotgun.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "network-checker"]
+ path = build/f_repos/sub/network-checker
+ url = https://github.com/openstack/network-checker.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-upgrade"]
+ path = build/f_repos/sub/fuel-upgrade
+ url = https://github.com/openstack/fuel-upgrade.git
+ branch = stable/mitaka
+ ignore = all
+[submodule "fuel-ui"]
+ path = build/f_repos/sub/fuel-ui
+ url = https://github.com/openstack/fuel-ui.git
+ branch = stable/mitaka
+ ignore = all
export MOSVERSION = 9.0
export ISOSRC = file:$(shell pwd)/fuel-$(MOSVERSION).iso
export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
-export PRODNO = "OPNFV_FUEL"
+export PRODNO ?= "OPNFV_FUEL"
export REVSTATE = "P0000"
export USER ?= $(shell whoami)
export BUILD_DATE = $(shell date --utc +%Y-%m-%d:%H:%M)
-export OPNFV_GIT_SHA = $(shell git rev-parse HEAD)
+export OPNFV_GIT_SHA ?= $(shell git rev-parse HEAD)
# Store in /etc/fuel_build_id on fuel master
export BUILD_ID := $(PRODNO)_$(BUILD_DATE)_$(OPNFV_GIT_SHA)
# END of variables to customize
#############################################################################
-# Fuel-main destination path and fuel-* submodule patching, for full list check:
-# https://github.com/openstack/fuel-main/blob/stable/mitaka/repos.mk#L32-L44
FUEL_MAIN_DIR := /tmp/fuel-main
-FUEL_PATCHES := $(shell find $(BUILD_BASE)/patch-repos -name '*.patch' | sort)
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
$(ISOCACHE):
# Clone Fuel to non-persistent location and build
sudo rm -rf $(FUEL_MAIN_DIR)
+ $(MAKE) -C f_repos -f Makefile release
git clone $(FUEL_MAIN_REPO) $(FUEL_MAIN_DIR)
- git -C $(FUEL_MAIN_DIR) checkout $(FUEL_MAIN_TAG)
- @echo "fuel" `git -C $(FUEL_MAIN_DIR) rev-parse HEAD` >> $(VERSION_FILE)
+ # Save upstream ref, checkout OPNFV tag (upstream ref + patches)
+ @echo "fuel" `git -C $(FUEL_MAIN_DIR) \
+ rev-parse $(F_OPNFV_TAG)-root` >> $(VERSION_FILE)
+ git -C $(FUEL_MAIN_DIR) checkout $(F_OPNFV_TAG)
# Remove Docker optimizations, otherwise multistrap will fail during
# Fuel build.
sudo rm -f /etc/apt/apt.conf.d/docker*
sudo docker info
# fuel-main Makefiles do not like `make -C`
cd $(FUEL_MAIN_DIR) && make repos
- $(REPOINFO) -r $(FUEL_MAIN_DIR) > gitinfo_fuel.txt
- # OPNFV patches at Fuel build time
- # Need to be commited in order for them to be considered by the Fuel
- # build system
- $(foreach patch,$(FUEL_PATCHES),git \
- -C $(subst $(BUILD_BASE)/patch-repos,$(FUEL_MAIN_DIR),$(dir $(patch))) \
- am --whitespace=nowarn --committer-date-is-author-date $(patch) || \
- (echo 'Error: Failed patching Fuel repos!' ; exit 1);)
+ cp f_repos/.cachefuelinfo gitinfo_fuel.txt
# Repeat build up to three times
sudo -E ./fuel_build_loop
.PHONY: clean $(SUBCLEAN)
clean: $(SUBCLEAN)
+ $(MAKE) -C f_repos -f Makefile clean
$(MAKE) -C patch-packages -f Makefile clean
@rm -f *.iso
@rm -Rf release
.PHONY: deepclean
deepclean: clean clean-cache
- make -C docker clean
+ $(MAKE) -C f_repos -f Makefile deepclean
+ $(MAKE) -C docker clean
docker rmi opnfv.org/ubuntu-builder:14.04 &>/dev/null || exit 0
docker rmi opnfv.org/ubuntu-builder:latest &>/dev/null || exit 0
docker rmi ubuntu:14.04 &>/dev/null || exit 0
#############################################################################
# Create a unique hash to be used for getting and putting cache, based on:
-# - The commit ID of the full Fuel repo structre
+# - The commit ID of the full Fuel repo structure
# - The contents of all local Fuel patches
+# - Makefile, config and sensitive build scripts fingerprints
+# NOTE: Patching is deterministic, so we can fingerprint (submodule root commit
+# info + OPNFV patches) at once, after patches-import.
+# NOTE: When git submodule remote tracking is active, `git submodule status`
+# will point to the latest commits (remote/branch/HEAD) + OPNFV patches.
.cacheid:
- git clone $(FUEL_MAIN_REPO) $(FUEL_MAIN_DIR)
- git -C $(FUEL_MAIN_DIR) checkout $(FUEL_MAIN_TAG)
- make -C $(FUEL_MAIN_DIR) repos
- $(REPOINFO) -r $(FUEL_MAIN_DIR) > .cachedata
- $(foreach patch,$(FUEL_PATCHES),sha1sum $(patch) >> .cachedata;)
+ $(MAKE) -C f_repos -f Makefile get-cache release
+ git submodule status | cut -c2-41 > .cachedata
sha1sum fuel_build_loop >> .cachedata
sha1sum config.mk >> .cachedata
sha1sum Makefile >> .cachedata
$(CACHETOOL) getbiweek >> .cachedata
cat .cachedata | $(CACHETOOL) getid > .cacheid
- # Not removing fuel-main as it is re-used in build
# Clean local data related to caching - called prior to ordinary build
.PHONY: clean-cache
clean-cache: $(SUBCLEANCACHE)
+ $(MAKE) -C f_repos -f Makefile clean-cache
rm -f .cachedata .cacheid
# Try to download cache - called prior to ordinary build
##############################################################################
-# Copyright (c) 2015 Ericsson AB and others.
+# Copyright (c) 2015,2016 Ericsson AB and others.
# stefan.k.berg@ericsson.com
# jonas.bjurel@ericsson.com
# All rights reserved. This program and the accompanying materials
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-FUEL_MAIN_REPO := https://github.com/openstack/fuel-main
-FUEL_MAIN_TAG := 9.0.1
-MOS_VERSION = 9.0
+# fuel-main tag checked out from upstream `fuel-main` repo before patching it
+FUEL_MAIN_TAG = 9.0.1
+MOS_VERSION = 9.0
OPENSTACK_VERSION = mitaka-9.0
-# Pinning down exact Fuel repo versions for Fuel 9.0.1
-export FUELLIB_COMMIT?=e283b62750d9e26355981b3ad3be7c880944ae0f
-export NAILGUN_COMMIT?=e2b85bafb68c348f25cb7cceda81edc668ba2e64
-export PYTHON_FUELCLIENT_COMMIT?=67d8c693a670d27c239d5d175f3ea2a0512c498c
-export FUEL_AGENT_COMMIT?=7ffbf39caf5845bd82b8ce20a7766cf24aa803fb
-export FUEL_NAILGUN_AGENT_COMMIT?=46fa0db0f8944f9e67699d281d462678aaf4db26
-export ASTUTE_COMMIT?=390b257240d49cc5e94ed5c4fcd940b5f2f6ec64
-export OSTF_COMMIT?=f09c98ff7cc71ee612b2450f68a19f2f9c64345a
-export FUEL_MIRROR_COMMIT?=d1ef06b530ce2149230953bb3810a88ecaff870c
-export FUELMENU_COMMIT?=0ed9e206ed1c6271121d3acf52a6bf757411286b
-export SHOTGUN_COMMIT?=781a8cfa0b6eb290e730429fe2792f2b6f5e0c11
-export NETWORKCHECKER_COMMIT?=fcb47dd095a76288aacf924de574e39709e1f3ca
-export FUELUPGRADE_COMMIT?=c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb
-export FUEL_UI_COMMIT?=90de7ef4477230cb7335453ed26ed4306ca6f04f
-
-# for the patches applying purposes
+# FIXME(alav): Disable remote tracking for now, stick to submodule commits
+FUEL_TRACK_REMOTES =
+
+##############################################################################
+# Fuel components pinning / remote tracking; use submodules from f_repos
+##############################################################################
+
+# git submodule & patch locations for Fuel components
+F_GIT_ROOT := $(shell git rev-parse --show-toplevel)
+F_GIT_DIR := $(shell git rev-parse --git-dir)
+F_SUBMOD_DIR := ${F_GIT_ROOT}/build/f_repos/sub
+F_PATCH_DIR := ${F_GIT_ROOT}/build/f_repos/patch
+F_OPNFV_TAG := ${FUEL_MAIN_TAG}-opnfv
+
+# fuel-main repo location used by main Makefile ISO building, use submodule
+FUEL_MAIN_REPO := ${F_SUBMOD_DIR}/fuel-main
+
+export FUELLIB_REPO?=${F_SUBMOD_DIR}/fuel-library
+export NAILGUN_REPO?=${F_SUBMOD_DIR}/fuel-web
+export PYTHON_FUELCLIENT_REPO?=${F_SUBMOD_DIR}/python-fuelclient
+export FUEL_AGENT_REPO?=${F_SUBMOD_DIR}/fuel-agent
+export FUEL_NAILGUN_AGENT_REPO?=${F_SUBMOD_DIR}/fuel-nailgun-agent
+export ASTUTE_REPO?=${F_SUBMOD_DIR}/fuel-astute
+export OSTF_REPO?=${F_SUBMOD_DIR}/fuel-ostf
+export FUEL_MIRROR_REPO?=${F_SUBMOD_DIR}/fuel-mirror
+export FUELMENU_REPO?=${F_SUBMOD_DIR}/fuel-menu
+export SHOTGUN_REPO?=${F_SUBMOD_DIR}/shotgun
+export NETWORKCHECKER_REPO?=${F_SUBMOD_DIR}/network-checker
+export FUELUPGRADE_REPO?=${F_SUBMOD_DIR}/fuel-upgrade
+export FUEL_UI_REPO?=${F_SUBMOD_DIR}/fuel-ui
+
+# OPNFV tags are automatically applied by `make -C f_repos patches-import`
+export FUELLIB_COMMIT?=${F_OPNFV_TAG}
+export NAILGUN_COMMIT?=${F_OPNFV_TAG}
+export PYTHON_FUELCLIENT_COMMIT?=${F_OPNFV_TAG}
+export FUEL_AGENT_COMMIT?=${F_OPNFV_TAG}
+export FUEL_NAILGUN_AGENT_COMMIT?=${F_OPNFV_TAG}
+export ASTUTE_COMMIT?=${F_OPNFV_TAG}
+export OSTF_COMMIT?=${F_OPNFV_TAG}
+export FUEL_MIRROR_COMMIT?=${F_OPNFV_TAG}
+export FUELMENU_COMMIT?=${F_OPNFV_TAG}
+export SHOTGUN_COMMIT?=${F_OPNFV_TAG}
+export NETWORKCHECKER_COMMIT?=${F_OPNFV_TAG}
+export FUELUPGRADE_COMMIT?=${F_OPNFV_TAG}
+export FUEL_UI_COMMIT?=${F_OPNFV_TAG}
+
+# for the patches applying purposes (empty git config in docker build container)
export GIT_COMMITTER_NAME?=Fuel OPNFV
export GIT_COMMITTER_EMAIL?=fuel@opnfv.org
DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
DOCKER_TAG := docker-latest
-
-.PHONY: get-fuel-repo
-get-fuel-repo:
- @echo $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)
-
-e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE \
-e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT \
-e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL \
- -e LATEST_TARGET_UBUNTU -e UBUNTU_ARCH \
+ -e LATEST_TARGET_UBUNTU -e UBUNTU_ARCH -e OPNFV_GIT_SHA \
-u $USER_ID:$GROUP_ID -w $PWD \
-v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT"
TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2)
include ../../config.mk
-include config.mk
export MOS_VERSION
export OPENSTACK_VERSION
sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml
rm -Rf nailgun
sudo mkdir -p /var/www/nailgun
- git clone $(FUEL_MIRROR_REPO)
- cd fuel-mirror && git checkout -q $(FUEL_MIRROR_COMMIT)
+ ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror
sudo pip install -U -r ./fuel-mirror/requirements.txt
sudo pip install ./fuel-mirror
sudo pip install ./fuel-mirror/contrib/fuel_mirror
sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu
sudo chmod -R 755 /var/www/nailgun
cp -Rp /var/www/nailgun .
- # On the end we want to have ubuntu repository in mirrors/ubuntu directory
+ # In the end we want to have ubuntu repository in mirrors/ubuntu directory
-if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \
mkdir -p nailgun/mirrors/ubuntu;\
mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\
.cacheid:
date +"Repocache %G%V" > .cachedata
sha1sum Makefile >> .cachedata
- sha1sum config.mk >> .cachedata
$(CACHETOOL) packages >> .cachedata
echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata
cat .cachedata | $(CACHETOOL) getid > .cacheid
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 Ericsson AB and others.
-# mskalski@mirantis.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror
-# Point to the commit where 302 redirects are handled
-# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674
-FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11
#!/bin/bash
-##############################################################################
-# Copyright (c) 2015,2016 Ericsson AB and others.
-# mskalski@mirantis.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-# Try to choose close ubuntu mirror from mirrors.txt, but "whitewash" this
-# against the full repo list to removed mirrors not up-to-date.
-
-# Some Ubuntu mirrors seem less reliable for this type of mirroring -
-# as they are discoved they can be added to the blacklist below in order
-# for them not to be considered.
-BLACKLIST="mirror.clibre.uqam.ca"
-
-#NOTE: For now the mirror selection is disabled due to issues not yet
-# understood/resolved.
-#for url in $((curl -s https://launchpad.net/ubuntu/+archivemirrors | \
-# grep -P -B8 "statusUP|statusSIX" | \
-# grep -o -P "(f|ht)tp.*\"" | \
-# sed 's/"$//' | sort | uniq; \
-# curl -s http://mirrors.ubuntu.com/mirrors.txt | sort | uniq) | \
-# sort | uniq -d)
-#do
-# host=$(echo $url | cut -d'/' -f3)
-# echo ${BLACKLIST} | grep -q ${host} && continue
-# if curl -s -o /dev/null --head --fail "$url"; then
-# echo $url
-# exit 0
-# else
-# continue
-# fi
-#done
-
-# If no suitable local mirror can be found,
-# the default archive is returned instead.
-echo "http://archive.ubuntu.com/ubuntu/"
+cleanup() {
+ rm -f $TMPFILE
+}
+
+debugmsg() {
+ test -n "$DEBUG" && echo "$@" >&2
+}
+
+# Check mirror's integrity
+check_mirror () {
+ mirror=$1
+ status=0
+ for packdir in dists/trusty-updates/main/binary-amd64 \
+ dists/trusty-updates/restricted/binary-amd64 \
+ dists/trusty-updates/universe/binary-amd64 \
+ dists/trusty-updates/multiverse/binary-amd64 \
+ dists/trusty-security/main/binary-amd64 \
+ dists/trusty-security/restricted/binary-amd64 \
+ dists/trusty-security/universe/binary-amd64 \
+ dists/trusty-security/multiverse/binary-amd64 \
+ dists/trusty-proposed/main/binary-amd64 \
+ dists/trusty-proposed/restricted/binary-amd64 \
+ dists/trusty-proposed/universe/binary-amd64 \
+ dists/trusty-proposed/multiverse/binary-amd64 \
+ dists/trusty/main/binary-amd64 \
+ dists/trusty/restricted/binary-amd64 \
+ dists/trusty/universe/binary-amd64 \
+ dists/trusty/multiverse/binary-amd64 \
+ dists/trusty-backports/main/binary-amd64 \
+ dists/trusty-backports/restricted/binary-amd64 \
+ dists/trusty-backports/universe/binary-amd64 \
+ dists/trusty-backports/multiverse/binary-amd64
+ do
+ for packfile in Release Packages.gz
+ do
+ if [ $status -ne 1 ]; then
+ curl --output /dev/null --silent --head --fail \
+ $mirror/$packdir/$packfile
+ if [ $? -ne 0 ]; then
+ debugmsg "$mirror: Faulty (at least missing $packdir/$packfile)"
+ status=1
+ fi
+ fi
+ done
+ done
+ return $status
+}
+
+if [ "$1" == "-d" ]; then
+ DEBUG=1
+fi
+
+# Hardcode for testing purposes
+DEBUG=1
+
+TMPFILE=$(mktemp /tmp/mirrorsXXXXX)A
+trap cleanup exit
+
+# Generated a list of mirrors considered as "up"
+curl -s https://launchpad.net/ubuntu/+archivemirrors | \
+ grep -P -B8 "statusUP|statusSIX" | \
+ grep -o -P "(f|ht)tp.*\"" | \
+ sed 's/"$//' | sort | uniq > $TMPFILE
+
+# Iterate over "close" mirror, check that they are considered up
+# and sane.
+for url in $(curl -s http://mirrors.ubuntu.com/mirrors.txt)
+do
+ grep -q $url $TMPFILE || debugmsg "$url Faulty (detected by Ubuntu)"
+ if [ -z $BESTURL ]; then
+ if grep -q $url $TMPFILE && check_mirror $url; then
+ debugmsg "$url: OK (setting as primary URL)"
+ BESTURL=$url
+ test -z "$DEBUG" && break
+ fi
+ else
+ grep -q $url $TMPFILE && check_mirror $url && debugmsg "$url: OK"
+ fi
+done
+
+echo "$BESTURL"
--- /dev/null
+##############################################################################
+# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+SHELL = /bin/bash
+REPOINFO = $(shell readlink -e ../repo_info.sh)
+FREPODIR = $(shell pwd)
+FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch')
+
+include ../config.mk
+
+# NOTE: Mechanism overview is presented in ./README.md.
+
+# Submodule consistent states:
+# - NOT initialized (submodule trees are not populated at all);
+# - initialized, bound to saved commits;
+# - initialized, tracking remote origin (only for FUEL_TRACK_REMOTES);
+# - patched (local patches are applied);
+
+# In order to keep things sort of separate, we should only pass up (to main
+# Makefile) the fully-pactched repos, and gather any fingerprinting info here.
+
+# Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways:
+# - pinned down to tag objects (e.g. "9.0.1")
+# - tracking upstream remote HEAD on a stable or master branch
+# FIXME(alav): Should we support mixed cases? (e.g. pin down only fuel-main)
+
+# To enable remote tracking, set the following var to any non-empty string.
+# Leaving this var empty will bind each git submodule to its saved commit.
+# NOTE: For non-stable branches, unless overriden, always track remotes
+FUEL_TRACK_REMOTES ?= $(shell git symbolic-ref --short HEAD | grep -v stable)
+
+.PHONY: all
+all: release
+
+##############################################################################
+# git submodule operations - to be used stand-alone or from parent Makefile
+##############################################################################
+
+# Fetch & update git submodules, checkout remote HEAD or saved commit
+# Also gather fingerprints for parent gitinfo_fuel.txt.
+.PHONY: sub
+sub: .cachefuelinfo
+
+.cachefuelinfo:
+ @if [ -n "${FUEL_TRACK_REMOTES}" ]; then \
+ git submodule update --init --remote; \
+ else \
+ git submodule update --init; \
+ fi
+ @rm -f $@
+ @git submodule -q foreach '${REPOINFO} . >> ${FREPODIR}/$@'
+
+# Generate patches from submodules
+.PHONY: patches-export
+patches-export: sub
+ @git submodule -q foreach ' \
+ SUB_DIR=${F_PATCH_DIR}/$$name; \
+ git tag | awk "!/root/ && /${F_OPNFV_TAG}-fuel/" | while read F_TAG; do \
+ SUB_FEATURE=`dirname $${F_TAG#${F_OPNFV_TAG}-fuel/}`; \
+ echo "`tput setaf 2`-- exporting $$name ($$F_TAG)`tput sgr0`"; \
+ mkdir -p $$SUB_DIR/$${SUB_FEATURE} && \
+ git format-patch --no-signature --ignore-space-at-eol \
+ -o $$SUB_DIR/$$SUB_FEATURE -N $$F_TAG-root..$$F_TAG; \
+ done'
+ @sed -i -e '1d' -e 's/[[:space:]]*$$//' ${FPATCHES}
+
+# Apply patches from patch/* to respective submodules
+# We rely on `make sub` and/or `make clean` to checkout correct base
+.PHONY: patches-import
+patches-import: sub .cachepatched
+
+.cachepatched: ${FPATCHES}
+ @$(MAKE) clean
+ @git submodule -q foreach ' \
+ SUB_DIR=${F_PATCH_DIR}/$$name; mkdir -p $$SUB_DIR && \
+ git tag ${F_OPNFV_TAG}-root && \
+ git checkout -q -b opnfv-fuel && \
+ find $$SUB_DIR -type d | sort | while read p_dir; do \
+ SUB_PATCHES=$$(ls $$p_dir/*.patch 2>/dev/null); \
+ if [ -n "$$SUB_PATCHES" ]; then \
+ SUB_FEATURE=$${p_dir#$$SUB_DIR}; \
+ SUB_TAG=${F_OPNFV_TAG}-fuel$$SUB_FEATURE/patch; \
+ echo "`tput setaf 2`-- patching $$name ($$SUB_TAG)`tput sgr0`";\
+ git tag $$SUB_TAG-root && git am -3 --whitespace=nowarn \
+ --committer-date-is-author-date $$SUB_PATCHES && \
+ git tag $$SUB_TAG || exit 1; \
+ fi \
+ done && \
+ git tag ${F_OPNFV_TAG}'
+ @touch $@
+
+# Clean any changes made to submodules, checkout upstream Fuel root commit
+.PHONY: clean
+clean:
+ @git submodule -q foreach ' \
+ git am -q --abort > /dev/null 2>&1; \
+ git checkout -q ${F_OPNFV_TAG}-root > /dev/null 2>&1; \
+ git branch -q -D opnfv-fuel > /dev/null 2>&1; \
+ git tag | grep ${F_OPNFV_TAG} | xargs git tag -d > /dev/null 2>&1; \
+ git reset -q --hard HEAD; \
+ git clean -xdff'
+ @rm -f .cachepatched
+
+.PHONY: deepclean
+deepclean: clean clean-cache
+ @git submodule deinit -f .
+ @rm -f .cache*
+
+.PHONY: release
+release: sub
+ # Store artifact in cache straight away if caching is enabled
+ # (no .cacheid will be present unless this is a cached build)
+ test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
+ # NOTE: Patches are not included in cache
+ $(MAKE) -f Makefile patches-import
+
+##############################################################################
+# Cache operations - only used when building through ci/build.sh
+##############################################################################
+
+# NOTE: Current method of collecting submodule refs requires submodules to be
+# NOT initialized <OR> NO patches applied.
+# NOTE: Querying `git submodule status` from parent will show the patched tree.
+# Create a unique hash to be used for getting and putting cache, based on:
+# - git submodule SHAs, collected with `git submodule status`
+# - The contents of this Makefile
+.cacheid:
+ @$(MAKE) clean
+ sha1sum Makefile > .cachedata
+ git submodule status | cut -c2-41 >> .cachedata
+ cat .cachedata | $(CACHETOOL) getid > .cacheid
+
+# Clean local data related to caching - called prior to ordinary build
+.PHONY: clean-cache
+clean-cache: clean
+ rm -f .cachedata .cacheid
+
+# Try to download cache - called prior to ordinary build
+.PHONY: get-cache
+get-cache: .cacheid
+ @if $(CACHETOOL) check $(shell cat .cacheid); then \
+ $(MAKE) clean && \
+ $(CACHETOOL) get $(shell cat .cacheid) | \
+ tar xf - -C ${F_GIT_DIR}; \
+ else \
+ echo "No cache item found for $(shell cat .cacheid)" ;\
+ exit 0;\
+ fi
+
+# Store cache if not already stored - called after ordinary build
+.PHONY: put-cache
+put-cache: .cacheid
+ @if ! $(CACHETOOL) check $(shell cat .cacheid); then \
+ $(MAKE) clean && \
+ tar cf - -C ${F_GIT_DIR} modules | \
+ $(CACHETOOL) put $(shell cat .cacheid); \
+ fi
--- /dev/null
+Fuel@OPNFV submodule fetching and patching
+==========================================
+
+This directory holds submodule fetching/patching scripts, intended for
+working with upstream Fuel components (fuel-library, ... , fuel-ui) in
+developing/applying OPNFV patches (backports, custom fixes etc.).
+
+The scripts should be friendly to the following 2 use-cases:
+ - development work: easily cloning, binding repos to specific commits,
+ remote tracking, patch development etc.;
+ - to provide parent build scripts an easy method of tracking upstream
+ references and applying OPNFV patches on top;
+
+Also, we need to support at least the following modes of operations:
+ - submodule bind - each submodule patches will be based on the commit ID
+ saved in the .gitmodules config file;
+ - remote tracking - each submodule will sync with the upstream remote
+ and patches will be applied on top of <sub_remote>/<sub_branch>/HEAD;
+
+Workflow (development)
+----------------------
+The standard development workflow should look as follows:
+
+1. Decide whether remote tracking should be active or not:
+ NOTE: Setting the following var to any non-empty str enables remote track.
+ NOTE: Leaving unset will enable remote track for anything but stable branch.
+
+ $ export FUEL_TRACK_REMOTES=""
+
+2. All Fuel sub-projects are registered as submodules. To initialize them, call:
+ If remote tracking is active, upstream remote is queried and latest remote
+ branch HEAD is fetched. Otherwise, checkout commit IDs from .gitmodules.
+
+ $ make sub
+
+3. Apply patches from `patches/<sub-project>/*` to respective submodules via:
+
+ $ make patches-import
+
+ This will result in creation of:
+ - a tag called `${FUEL_MAIN_TAG}-opnfv-root` at the same commit as Fuel@OPNFV
+ upstream reference (bound to git submodule OR tracking remote HEAD);
+ - a new branch `opnfv-fuel` which will hold all the OPNFV patches,
+ each patch is applied on this new branch with `git-am`;
+ - a tag called `${FUEL_MAIN_TAG}-opnfv` at `opnfv-fuel/HEAD`;
+
+4. Modify sub-projects for whatever you need.
+ Commit your changes when you want them taken into account in the build.
+
+5. Re-create patches via:
+
+ $ make patches-export
+
+ Each commit on `opnfv-fuel` branch of each subproject will be
+ exported to `patches/subproject/` via `git format-patch`.
+
+ NOTE: Only commit (-f) submodules when you need to bump upstream ref.
+ NOTE: DO NOT commit patched submodules!
+
+6. Clean workbench branches and tags with:
+
+ $ make clean
+
+7. De-initialize submodules and force a clean clone with:
+
+ $ make deepclean
+
+Workflow (ISO build)
+--------------------
+Parent build scripts require this mechanism to do some fingerprinting,
+so here is the intended flow for all artifacts to be generated right:
+
+1. (Optional) Cached submodules might be fetched from build cache.
+
+2. Submodules are updated
+ We also dump each submodule's git info using repo_info.sh, since
+ we want to collect git refs before patching (i.e. upstream refs).
+
+3. Make target `release` is built
+ This will make sure the modules are in a clean state, put them in cache,
+ then apply the patches.
+
+4. fuel-main's `${FUEL_MAIN_TAG}-opnfv-root` tag is used to determine VERSION info
+ It will accommodate both bound tags and remote tracking references.
+
+Sub-project maintenance
+-----------------------
+1. Adding a new submodule
+ If you need to add another subproject, you can do it with `git submodule`.
+ Make sure that you specify branch (with `-b`), short name (with `--name`)
+ and point it to `upstream/*` directory, i.e.:
+
+ $ git submodule -b stable/mitaka add --name fuel-web \
+ https://github.com/openstack/fuel-web.git upstream/fuel-web
-From 38315ee3945a5444de13918eb29e7771eb9927c7 Mon Sep 17 00:00:00 2001
From: Michael Polenchuk <mpolenchuk@mirantis.com>
Date: Thu, 8 Sep 2016 19:25:14 +0300
-Subject: [PATCH 1/1] Reduce ceilometer memory usage
+Subject: [PATCH] Reduce ceilometer memory usage
Change-Id: I8b1a97bd710c859a3543d1aed8226313f35f4f10
---
- .../manifests/ceilometer/controller.pp | 7 +++++++
+ .../puppet/openstack_tasks/manifests/ceilometer/controller.pp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp
+ }
}
}
---
-1.7.9.5
-
-From ba542dde34743b276547b7ca3c6591feb9000877 Mon Sep 17 00:00:00 2001
From: iberezovskiy <iberezovskiy@mirantis.com>
Date: Mon, 19 Sep 2016 12:35:05 +0300
Subject: [PATCH] Disable token revoke to increase keystone performance
Change-Id: I4df816369093908ad1eac372f38c92155efbe8e0
Closes-bug: #1625077
---
- .../openstack_tasks/manifests/keystone/keystone.pp | 2 ++
- tests/noop/spec/hosts/keystone/keystone_spec.rb | 5 +++++
+ deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp | 2 ++
+ tests/noop/spec/hosts/keystone/keystone_spec.rb | 5 +++++
2 files changed, 7 insertions(+)
diff --git a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp
+ # Set revoke_by_id to false according to LP #1625077
+ revoke_by_id => false,
}
-
+
Package<| title == 'keystone'|> ~> Service<| title == 'keystone'|>
diff --git a/tests/noop/spec/hosts/keystone/keystone_spec.rb b/tests/noop/spec/hosts/keystone/keystone_spec.rb
index b29e691..3c0f847 100644
@@ -176,6 +176,11 @@ describe manifest do
should contain_class('keystone').with('sync_db' => primary_controller)
end
-
+
+ it 'should declare keystone class with revoke_by_id set to false' do
+ # Set revoke_by_id to false according to LP #1625077
+ should contain_class('keystone').with('revoke_by_id' => false)
it 'should configure keystone with paramters' do
should contain_keystone_config('token/caching').with(:value => 'false')
should contain_keystone_config('cache/enabled').with(:value => 'true')
---
-1.7.9.5
-
From: Fuel OPNFV <fuel@opnfv.org>
Date: Mon, 13 Jun 2016 22:23:57 +0200
-Subject: OPNFV: Additions to bootstrap_admin_node.sh
+Subject: [PATCH] OPNFV: Additions to bootstrap_admin_node.sh
---
+ iso/bootstrap_admin_node.sh | 39 +++++++++++++++++++++++++++++++--------
+ 1 file changed, 31 insertions(+), 8 deletions(-)
+
diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
-index 3197c91..e035145 100755
+index 3197c91..db3123d 100755
--- a/iso/bootstrap_admin_node.sh
+++ b/iso/bootstrap_admin_node.sh
@@ -339,8 +339,22 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail
if [ ! -f "${ASTUTE_YAML}" ]; then
echo ${fuelmenu_fail_message}
-@@ -377,7 +392,5 @@ if [ ! -f /etc/fuel_build_id ]; then
+@@ -377,9 +392,7 @@ if [ ! -f /etc/fuel_build_id ]; then
+ ln -s ${wwwdir}/${OPENSTACK_VERSION}/ubuntu ${wwwdir}/ubuntu
+ fi
-# Enable sshd
-systemctl enable sshd
if [ "`get_bootstrap_skip`" = "False" ]; then
build_ubuntu_bootstrap bs_status || true
else
-@@ -571,8 +571,8 @@
+@@ -540,8 +563,8 @@ else
fi
#Check if repo is accessible
From: Fuel OPNFV <fuel@opnfv.org>
Date: Mon, 13 Jun 2016 22:23:57 +0200
-Subject: OPNFV: showmenu=yes in isolinux.cfg
+Subject: [PATCH] OPNFV: showmenu=yes in isolinux.cfg
---
+ iso/isolinux/isolinux.cfg | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
diff --git a/iso/isolinux/isolinux.cfg b/iso/isolinux/isolinux.cfg
index c6b1ed9..77a4b18 100644
--- a/iso/isolinux/isolinux.cfg
From: Stanislaw Kardach <stanislaw.kardach@cavium.com>
Date: Thu, 25 Feb 2016 13:31:19 +0100
-Subject: repo mirror: Allow multi-arch local mirrors
+Subject: [PATCH] repo mirror: Allow multi-arch local mirrors
This patch allows specifying multiple architectures via UBUNTU_ARCH in
form of a list of space separated architectures. The first architecture
Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com>
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
+ config.mk | 2 +-
mirror/ubuntu/module.mk | 2 +-
sandbox.mk | 2 +-
- config.mk | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
+diff --git a/config.mk b/config.mk
+index 74ee039..45a3b30 100644
+--- a/config.mk
++++ b/config.mk
+@@ -49,7 +49,7 @@ UBUNTU_MINOR:=04
+ UBUNTU_RELEASE_NUMBER:=$(UBUNTU_MAJOR).$(UBUNTU_MINOR)
+ UBUNTU_KERNEL_FLAVOR?=lts-trusty
+ UBUNTU_NETBOOT_FLAVOR?=netboot
+-UBUNTU_ARCH:=amd64
++UBUNTU_ARCH?=amd64
+ UBUNTU_IMAGE_RELEASE:=$(UBUNTU_MAJOR)$(UBUNTU_MINOR)
+ SEPARATE_IMAGES?=/boot,ext2 /,ext4
+
diff --git a/mirror/ubuntu/module.mk b/mirror/ubuntu/module.mk
index 7a9466e..fe1ada2 100644
--- a/mirror/ubuntu/module.mk
if [ -e $(SANDBOX_UBUNTU)/etc/resolv.conf ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf.orig; fi
sudo cp /etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf
if [ -e $(SANDBOX_UBUNTU)/etc/hosts ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/hosts $(SANDBOX_UBUNTU)/etc/hosts.orig; fi
-diff --git a/config.mk b/config.mk
-index 74ee039..45a3b30 100644
---- a/config.mk
-+++ b/config.mk
-@@ -49,7 +49,7 @@ UBUNTU_MINOR:=04
- UBUNTU_RELEASE_NUMBER:=$(UBUNTU_MAJOR).$(UBUNTU_MINOR)
- UBUNTU_KERNEL_FLAVOR?=lts-trusty
- UBUNTU_NETBOOT_FLAVOR?=netboot
--UBUNTU_ARCH:=amd64
-+UBUNTU_ARCH?=amd64
- UBUNTU_IMAGE_RELEASE:=$(UBUNTU_MAJOR)$(UBUNTU_MINOR)
- SEPARATE_IMAGES?=/boot,ext2 /,ext4
-
--- /dev/null
+From: Bulat Gaifullin <bgaifullin@mirantis.com>
+Date: Fri, 24 Jun 2016 15:52:41 +0200
+Subject: [PATCH] Fixed handling http redirects
+
+Override method 'redirect_request' for patching
+new request, that has been created on handling
+http redirect.
+
+Change-Id: I40db406e2377bebec1113639b91a0b5262e2e9ad
+Closes-Bug: 1593674
+(cherry picked from commit 192a3d9f8f993afb12c5108dd9339c6688c23e11)
+---
+ packetary/library/connections.py | 21 ++++++++++++++++++++-
+ packetary/tests/test_connections.py | 19 +++++++++++++++++++
+ 2 files changed, 39 insertions(+), 1 deletion(-)
+
+diff --git a/packetary/library/connections.py b/packetary/library/connections.py
+index 36a7a84..49b6c9b 100644
+--- a/packetary/library/connections.py
++++ b/packetary/library/connections.py
+@@ -93,9 +93,23 @@ class ResumableResponse(StreamWrapper):
+ self.stream = response.stream
+
+
+-class RetryHandler(urllib.BaseHandler):
++class RetryHandler(urllib.HTTPRedirectHandler):
+ """urllib Handler to add ability for retrying on server errors."""
+
++ def redirect_request(self, req, fp, code, msg, headers, newurl):
++ new_req = urllib.HTTPRedirectHandler.redirect_request(
++ self, req, fp, code, msg, headers, newurl
++ )
++ if new_req is not None:
++ # We use class assignment for casting new request to type
++ # RetryableRequest
++ new_req.__class__ = RetryableRequest
++ new_req.retries_left = req.retries_left
++ new_req.offset = req.offset
++ new_req.start_time = req.start_time
++ new_req.retry_interval = req.retry_interval
++ return new_req
++
+ @staticmethod
+ def http_request(request):
+ """Initialises http request.
+@@ -118,6 +132,11 @@ class RetryHandler(urllib.BaseHandler):
+ :return: ResumableResponse if success otherwise same response
+ """
+ code, msg = response.getcode(), response.msg
++
++ if 300 <= code < 400:
++ # the redirect group, pass to next handler as is
++ return response
++
+ # the server should response partial content if range is specified
+ if request.offset > 0 and code != 206:
+ raise RangeError(msg)
+diff --git a/packetary/tests/test_connections.py b/packetary/tests/test_connections.py
+index a2621c8..c80b03d 100644
+--- a/packetary/tests/test_connections.py
++++ b/packetary/tests/test_connections.py
+@@ -268,6 +268,25 @@ class TestRetryHandler(base.TestCase):
+ self.handler.http_response(request, response_mock)
+ self.handler.parent.open.assert_called_once_with(request)
+
++ @mock.patch(
++ 'packetary.library.connections.urllib.'
++ 'HTTPRedirectHandler.redirect_request'
++ )
++ def test_redirect_request(self, redirect_mock, _):
++ redirect_mock.return_value = connections.urllib.Request(
++ 'http://localhost/'
++ )
++ req = mock.MagicMock(retries_left=10, retry_interval=5, offset=100)
++ new_req = self.handler.redirect_request(req, -1, 301, "", {}, "")
++ self.assertIsInstance(new_req, connections.RetryableRequest)
++ self.assertEqual(req.retries_left, new_req.retries_left)
++ self.assertEqual(req.retry_interval, new_req.retry_interval)
++ self.assertEqual(req.offset, new_req.offset)
++ redirect_mock.return_value = None
++ self.assertIsNone(
++ self.handler.redirect_request(req, -1, 301, "", {}, "")
++ )
++
+
+ class TestResumeableResponse(base.TestCase):
+ def setUp(self):
index 74fa509..d08a4fc 100644
--- a/nailgun/nailgun/fixtures/openstack.yaml
+++ b/nailgun/nailgun/fixtures/openstack.yaml
-@@ -2215,3 +2215,3 @@
+@@ -2213,7 +2213,7 @@
+ "15ad:07b0", "8086:15a5", "1137:0043", "1137:0071", "14e4:168a",
+ "14e4:16a9", "14e4:164f", "14e4:168e", "14e4:16af", "14e4:163d",
"14e4:163f", "14e4:168d", "14e4:16a1", "14e4:16a2", "14e4:16ad",
- "14e4:16ae", "14e4:163e", "14e4:16a4"
+ "14e4:16ae", "14e4:163e", "14e4:16a4", "8086:10f8"
]
---
-1.9.1
-
+ - pk: 3
+ extend: *ubuntu_release
--- /dev/null
+Subproject commit 7ffbf39caf5845bd82b8ce20a7766cf24aa803fb
--- /dev/null
+Subproject commit 390b257240d49cc5e94ed5c4fcd940b5f2f6ec64
--- /dev/null
+Subproject commit e283b62750d9e26355981b3ad3be7c880944ae0f
--- /dev/null
+Subproject commit d6a22557d132c592b18c6bac90f5f4b8d1aa3ad7
--- /dev/null
+Subproject commit 0ed9e206ed1c6271121d3acf52a6bf757411286b
--- /dev/null
+Subproject commit d1ef06b530ce2149230953bb3810a88ecaff870c
--- /dev/null
+Subproject commit 46fa0db0f8944f9e67699d281d462678aaf4db26
--- /dev/null
+Subproject commit f09c98ff7cc71ee612b2450f68a19f2f9c64345a
--- /dev/null
+Subproject commit 90de7ef4477230cb7335453ed26ed4306ca6f04f
--- /dev/null
+Subproject commit c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb
--- /dev/null
+Subproject commit e2b85bafb68c348f25cb7cceda81edc668ba2e64
--- /dev/null
+Subproject commit fcb47dd095a76288aacf924de574e39709e1f3ca
--- /dev/null
+Subproject commit 67d8c693a670d27c239d5d175f3ea2a0512c498c
--- /dev/null
+Subproject commit 781a8cfa0b6eb290e730429fe2792f2b6f5e0c11
find . -name TRANS.TBL -exec rm {} \;
rm -rf rr_moved
+ if [[ -z "$OPNFV_GIT_SHA" ]]; then
+ OPNFV_GIT_SHA=$(git rev-parse --verify HEAD)
+ fi
+
mkisofs --quiet -r -V "$VOLUMEID" -publisher "$PUBLISHER" \
- -p `git rev-parse --verify HEAD` -J -R -b isolinux/isolinux.bin \
+ -p "$OPNFV_GIT_SHA" -J -R -b isolinux/isolinux.bin \
-no-emul-boot \
-boot-load-size 4 -boot-info-table \
--hide-rr-moved \
log('Deleting VM %s with disks %s' % (vm_name, disk_files))
exec_cmd('virsh destroy %s' % vm_name, False)
exec_cmd('virsh undefine --managed-save --remove-all-storage %s' % vm_name, False)
+ for file in disk_files:
+ delete(file)
def overwrite_xml(self, vm_xml, vm_definition_overwrite):
if not vm_definition_overwrite:
Code Review / fuel.git / commitdiff