From f595c31b29147efb1410756b491ff7bf48d256ec Mon Sep 17 00:00:00 2001
From: Michal Skalski <mskalski@mirantis.com>
Date: Thu, 28 Jan 2016 15:28:07 +0100
Subject: [PATCH] Create local mirrors for offline installation
- switch to fuel-mirror [1]
- plugin developers can create packages.yaml
file with names of required packages
- use custom configuration for fuel-bootstrap
which always point to local repositories
- delay process of bootstrap image creation until
mirrors will be ready
[1] https://github.com/openstack/fuel-mirror
Change-Id: Ie4a47c8cc13d9ded8c6b5fe48cdcbfeb1f6277a4
Signed-off-by: Michal Skalski <mskalski@mirantis.com>
---
build/Makefile | 2 +-
build/config.mk | 2 +
build/docker/Dockerfile | 2 +-
build/docker/runcontext | 2 +-
build/f_isoroot/Makefile | 8 +--
.../f_isoroot/f_bootstrap/bootstrap_admin_node.sh | 19 +++---
.../f_bootstrap/post-scripts/03_install_repo.sh | 5 ++
build/f_isoroot/f_odlpluginbuild/packages.yaml | 3 +
.../f_odlpluginbuild/requirements-deb.txt | 1 -
build/f_isoroot/f_repobuild/Makefile | 48 ++++++--------
.../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 75 ++++++++++++++++++++++
build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 50 +++++++++++++++
12 files changed, 173 insertions(+), 44 deletions(-)
create mode 100644 build/f_isoroot/f_odlpluginbuild/packages.yaml
delete mode 100644 build/f_isoroot/f_odlpluginbuild/requirements-deb.txt
create mode 100644 build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
diff --git a/build/Makefile b/build/Makefile
index 3d7dd634d..ab60f005f 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -53,7 +53,7 @@ export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py)
export MIRROR_MOS_UBUNTU := $(shell echo "$(LATEST_MIRROR_ID_URL)" | cut -d'/' -f3)
export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/8.0.target.txt" | head -1)
-export MIRROR_MOS_UBUNTU_ROOT := "/mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU)"
+export MIRROR_MOS_UBUNTU_ROOT := /mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU)
export LATEST_TARGET_CENTOS := $(shell curl -sSf "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/os.target.txt" | head -1)
export MIRROR_FUEL := "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/$(LATEST_TARGET_CENTOS)/x86_64"
diff --git a/build/config.mk b/build/config.mk
index 6b2f6a30e..eca426773 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -10,6 +10,8 @@
FUEL_MAIN_REPO := https://github.com/openstack/fuel-main
FUEL_MAIN_TAG = stable/8.0
+MOS_VERSION = 8.0
+OPENSTACK_VERSION = liberty-8.0
DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
DOCKER_TAG := docker-latest
diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index 4ac51476e..b7ef6d3a7 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -25,7 +25,7 @@ RUN echo "Defaults env_keep += \"ftp_proxy http_proxy https_proxy no_proxy RSYNC
# Keeping PWD is needed to build as root
RUN echo "Defaults env_keep += \"PWD\"" > /etc/sudoers.d/keep-pwd
# Keeping variables for ISO build
-RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL\"" > /etc/sudoers.d/keep-mos
+RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL LATEST_TARGET_UBUNTU\"" > /etc/sudoers.d/keep-mos
RUN chmod 0440 /etc/sudoers.d/open-sudo
RUN chmod 0440 /etc/sudoers.d/keep-proxies
RUN chmod 0440 /etc/sudoers.d/keep-pwd
diff --git a/build/docker/runcontext b/build/docker/runcontext
index 2df765868..9f077764e 100755
--- a/build/docker/runcontext
+++ b/build/docker/runcontext
@@ -111,7 +111,7 @@ if [ -n "$CACHEBASE" ]; then
fi
fi
-RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT"
+RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -e LATEST_TARGET_UBUNTU -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT"
# Passing "debug" puts up an interactive bash shell
if [ "$1" == "debug" ]; then
diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile
index 21d47f4f3..ee27adb49 100644
--- a/build/f_isoroot/Makefile
+++ b/build/f_isoroot/Makefile
@@ -10,6 +10,7 @@
# Add Fuel plugin build targets here
PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild
+export PLUGINS
#NON_8-0_REBASED_PLUGINS = f_bgpvpn-pluginbuild
# If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins
@@ -17,12 +18,11 @@ PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsper
# Temporarilu disabling f_repobuild that currently have some issues on
# Fuel 8.
ifdef BUILD_FUEL_PLUGINS
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS)
+export BUILD_FUEL_PLUGINS
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
$(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS))
else
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS)
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
endif
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
index a5ca56c64..b096a7812 100755
--- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
+++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
@@ -414,6 +414,16 @@ rmdir /var/log/remote && ln -s /var/log/docker-logs/remote /var/log/remote
dockerctl check || fail
bash /etc/rc.local
+### OPNFV addition BEGIN
+shopt -s nullglob
+for script in /opt/opnfv/bootstrap/post.d/*.sh
+do
+ echo "Post script: $script" >> /root/post.log 2>&1
+ $script >> /root/post.log 2>&1
+done
+shopt -u nullglob
+### OPNFV addition END
+
if [ "`get_bootstrap_flavor`" = "ubuntu" ]; then
if [ "`get_bootstrap_skip`" = "False" ]; then
build_ubuntu_bootstrap bs_status || true
@@ -426,15 +436,6 @@ else
bs_status=3
fi
-### OPNFV addition BEGIN
-shopt -s nullglob
-for script in /opt/opnfv/bootstrap/post.d/*.sh
-do
- echo "Post script: $script" >> /root/post.log 2>&1
- $script >> /root/post.log 2>&1
-done
-shopt -u nullglob
-### OPNFV addition END
# Enable updates repository
cat > /etc/yum.repos.d/mos${FUEL_RELEASE}-updates.repo << EOF
diff --git a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
index 427a55add..8dad89397 100755
--- a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
+++ b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
@@ -16,4 +16,9 @@ if [ $? -ne 0 ]; then
echo "Error removing /opt/opnfv/nailgun directory!"
exit 1
fi
+mv /opt/opnfv/fuel_bootstrap_cli.yaml /etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml
+if [ $? -ne 0 ]; then
+ echo "Error moving bootstrap image configuration!"
+ exit 1
+fi
echo "Done installing pre-build repo"
diff --git a/build/f_isoroot/f_odlpluginbuild/packages.yaml b/build/f_isoroot/f_odlpluginbuild/packages.yaml
new file mode 100644
index 000000000..7ef537f31
--- /dev/null
+++ b/build/f_isoroot/f_odlpluginbuild/packages.yaml
@@ -0,0 +1,3 @@
+packages:
+ - "openjdk-7-jre-headless"
+ - "acct"
diff --git a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt b/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt
deleted file mode 100644
index f0d06681e..000000000
--- a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt
+++ /dev/null
@@ -1 +0,0 @@
-openjdk-7-jre-headless
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
index 1fc703031..966da3634 100644
--- a/build/f_isoroot/f_repobuild/Makefile
+++ b/build/f_isoroot/f_repobuild/Makefile
@@ -10,54 +10,48 @@
SHELL := /bin/bash
TOP := $(shell pwd)
-RSYNC_HOST := $(shell ./select_ubuntu_repo.sh)
-DEPENDENCIES_FILE := /etc/fuel-createmirror/requirements-deb.txt
+UBUNTU_DIR := $(shell dirname $(MIRROR_UBUNTU_ROOT))
+include ../../config.mk
+
+export MOS_VERSION
+export OPENSTACK_VERSION
.PHONY: all
all: nailgun
nailgun:
-# If RSYNC_CONNECT_PROG is set we need to do some magic to copy the
-# keys. Make sure to have the username set in the SSH_CONNECT_PROG
-# as well!
- @if [ -n "${RSYNC_CONNECT_PROG}" -a ! -d /root/.ssh ]; then \
- sudo mkdir -p /root/.ssh; \
- test -d ${HOME}/.ssh && sudo find ${HOME}/.ssh -maxdepth 1 -type f -exec cp {} /root/.ssh \; ; \
- sudo bash -c "echo StrictHostKeyChecking=no > /root/.ssh/config"; \
- sudo chmod 700 /root/.ssh; \
- fi
- sudo apt-get install -y rsync python python-yaml dpkg-dev openssl
- rm -rf tmpiso tmpdir
- mkdir tmpiso
- fuseiso ${ISOCACHE} tmpiso
- cp tmpiso/ubuntu/pool/main/f/fuel-createmirror/fuel-createmirror_*.deb .
- fusermount -u tmpiso
- rm -rf tmpiso
- sudo dpkg -i fuel-createmirror_*.deb
- sudo sed -i 's/DOCKER_MODE=true/DOCKER_MODE=false/' /etc/fuel-createmirror/common.cfg
- sudo sed -i 's/DEBUG="no"/DEBUG="yes"/' /etc/fuel-createmirror/ubuntu.cfg
- sudo sed -i "s/MIRROR_UBUNTU_HOST=\".*\"/MIRROR_UBUNTU_HOST=\"$(RSYNC_HOST)\"/" /etc/fuel-createmirror/common.cfg
- # Add dependencies from plugins. With awk help try add only unique packages.
- sudo bash -c 'find ../ -name requirements-deb.txt -print0 | xargs -0 -I% awk '\''NR == FNR {first[$$0];next}; ! ($$0 in first)'\'' "${DEPENDENCIES_FILE}" "%" >> "${DEPENDENCIES_FILE}"'
+ sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml
rm -Rf nailgun
- sudo mkdir -p /var/www
- sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror
+ sudo mkdir -p /var/www/nailgun
+ git clone -b $(FUEL_MAIN_TAG) https://github.com/openstack/fuel-mirror
+ sudo pip install -U -r ./fuel-mirror/requirements.txt
+ sudo pip install ./fuel-mirror
+ sudo pip install ./fuel-mirror/contrib/fuel_mirror
+ ./opnfv_mirror_conf.py
+ sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group mos ubuntu --pattern=ubuntu
sudo chmod -R 755 /var/www/nailgun
cp -Rp /var/www/nailgun .
+ mv nailgun/mirrors$(MIRROR_MOS_UBUNTU_ROOT) nailgun/mirrors/mos-repos/ubuntu/$(MOS_VERSION)
+ rmdir nailgun/mirrors/mos-repos/ubuntu/snapshots
+ if [ "$(UBUNTU_DIR)" != "/" ]; then \
+ mv nailgun/mirrors$(UBUNTU_DIR)/ubuntu naligun/mirros/ubuntu;\
+ rm -rf nailgun/mirrors$(UBUNTU_DIR);\
+ fi
# Store artifact in cache straight away if caching is enabled
# (no .cacheid will be present unless this is a cached build)
test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
.PHONY: clean
clean:
- @rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb
+ @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror
.PHONY: release
release:nailgun
@rm -Rf ../release/opnfv/nailgun
@mkdir -p ../release/opnfv
@cp -Rp nailgun ../release/opnfv/nailgun
+ @cp fuel_bootstrap_cli.yaml ../release/opnfv/
#############################################################################
# Cache operations - only used when building through ci/build.sh
diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
new file mode 100644
index 000000000..28f6427f7
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
@@ -0,0 +1,75 @@
+---
+ root_ssh_authorized_file: /root/.ssh/id_rsa.pub
+ extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8"
+ extra_dirs:
+ - /usr/share/fuel_bootstrap_cli/files/trusty
+ output_dir: /tmp/
+ kernel_flavor: linux-image-generic-lts-trusty
+ packages:
+ - vim
+ - mc
+ - openssh-client
+ - openssh-server
+ - ntp
+ - mcollective
+ - nailgun-agent
+ - nailgun-mcagents
+ - network-checker
+ - fuel-agent
+ - ubuntu-minimal
+ - live-boot
+ - live-boot-initramfs-tools
+ - wget
+ - linux-firmware
+ - linux-firmware-nonfree
+ - linux-headers-generic
+ - hpsa-dkms
+ - i40e-dkms
+ - xz-utils
+ - squashfs-tools
+ - msmtp-mta
+ bootstrap_images_dir: /var/www/nailgun/bootstraps
+ active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap
+ flavor: ubuntu
+ http_proxy: ""
+ https_proxy: ""
+ repos:
+ - name: ubuntu
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty
+ type: deb
+ - name: ubuntu-updates
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty-updates
+ type: deb
+ - name: ubuntu-security
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty-security
+ type: deb
+ - name: mos
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/ubuntu/x86_64"
+ priority: 1050
+ suite: mos8.0
+ type: deb
+ - name: mos-updates
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+ priority: 1050
+ suite: mos8.0-updates
+ type: deb
+ - name: mos-security
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+ priority: 1050
+ suite: mos8.0-security
+ type: deb
+ skip_default_img_build: true
+ direct_repo_addresses:
+ - "127.0.0.1"
diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
new file mode 100755
index 000000000..9abd0242b
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import yaml
+
+current_snapshot = os.environ["LATEST_TARGET_UBUNTU"]
+mos_version = os.environ['MOS_VERSION']
+openstack_version = os.environ['OPENSTACK_VERSION']
+mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU']
+mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT']
+mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL']
+if os.environ.get('BUILD_FUEL_PLUGINS'):
+ plugins = os.environ['BUILD_FUEL_PLUGINS']
+else:
+ plugins = os.environ['PLUGINS']
+
+
+configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read()
+conf = yaml.load(configuration_file)
+conf['pattern_dir'] = '.'
+conf['openstack_version'] = openstack_version
+conf['mos_version'] = mos_version
+
+with open('opnfv-config.yaml', 'w') as outfile:
+ outfile.write( yaml.dump(conf, default_flow_style=False) )
+
+pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
+pattern = yaml.load(pattern_file)
+pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
+pattern['ubuntu_baseurl'] = mirror_ubuntu
+for group in pattern['groups']['mos']:
+ group['uri'] = pattern['mos_baseurl']
+for group in pattern['groups']['ubuntu']:
+ group['uri'] = pattern['ubuntu_baseurl']
+
+for plugin in plugins.split():
+ path = "../{}/packages.yaml".format(plugin)
+ if os.path.isfile(path):
+ f = open(path).read()
+ plugin_yaml = yaml.load(f)
+ plugin_set = set(plugin_yaml['packages'])
+ main_set = set(pattern['packages'])
+ new_packages = plugin_set - main_set
+ print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages))
+ pattern['packages'] = pattern['packages'] + list(new_packages)
+
+pattern['requirements']['ubuntu'] = pattern['packages']
+
+with open('ubuntu.yaml', 'w') as outfile:
+ outfile.write( yaml.safe_dump(pattern, default_flow_style=False) )
--
2.16.6