From f595c31b29147efb1410756b491ff7bf48d256ec Mon Sep 17 00:00:00 2001 From: Michal Skalski Date: Thu, 28 Jan 2016 15:28:07 +0100 Subject: [PATCH] Create local mirrors for offline installation - switch to fuel-mirror [1] - plugin developers can create packages.yaml file with names of required packages - use custom configuration for fuel-bootstrap which always point to local repositories - delay process of bootstrap image creation until mirrors will be ready [1] https://github.com/openstack/fuel-mirror Change-Id: Ie4a47c8cc13d9ded8c6b5fe48cdcbfeb1f6277a4 Signed-off-by: Michal Skalski --- build/Makefile | 2 +- build/config.mk | 2 + build/docker/Dockerfile | 2 +- build/docker/runcontext | 2 +- build/f_isoroot/Makefile | 8 +-- .../f_isoroot/f_bootstrap/bootstrap_admin_node.sh | 19 +++--- .../f_bootstrap/post-scripts/03_install_repo.sh | 5 ++ build/f_isoroot/f_odlpluginbuild/packages.yaml | 3 + .../f_odlpluginbuild/requirements-deb.txt | 1 - build/f_isoroot/f_repobuild/Makefile | 48 ++++++-------- .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 75 ++++++++++++++++++++++ build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 50 +++++++++++++++ 12 files changed, 173 insertions(+), 44 deletions(-) create mode 100644 build/f_isoroot/f_odlpluginbuild/packages.yaml delete mode 100644 build/f_isoroot/f_odlpluginbuild/requirements-deb.txt create mode 100644 build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py diff --git a/build/Makefile b/build/Makefile index 3d7dd634d..ab60f005f 100644 --- a/build/Makefile +++ b/build/Makefile @@ -53,7 +53,7 @@ export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py) export MIRROR_MOS_UBUNTU := $(shell echo "$(LATEST_MIRROR_ID_URL)" | cut -d'/' -f3) export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/8.0.target.txt" | head -1) -export MIRROR_MOS_UBUNTU_ROOT := "/mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU)" +export MIRROR_MOS_UBUNTU_ROOT := /mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU) export LATEST_TARGET_CENTOS := $(shell curl -sSf "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/os.target.txt" | head -1) export MIRROR_FUEL := "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/$(LATEST_TARGET_CENTOS)/x86_64" diff --git a/build/config.mk b/build/config.mk index 6b2f6a30e..eca426773 100644 --- a/build/config.mk +++ b/build/config.mk @@ -10,6 +10,8 @@ FUEL_MAIN_REPO := https://github.com/openstack/fuel-main FUEL_MAIN_TAG = stable/8.0 +MOS_VERSION = 8.0 +OPENSTACK_VERSION = liberty-8.0 DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64 DOCKER_TAG := docker-latest diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 4ac51476e..b7ef6d3a7 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -25,7 +25,7 @@ RUN echo "Defaults env_keep += \"ftp_proxy http_proxy https_proxy no_proxy RSYNC # Keeping PWD is needed to build as root RUN echo "Defaults env_keep += \"PWD\"" > /etc/sudoers.d/keep-pwd # Keeping variables for ISO build -RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL\"" > /etc/sudoers.d/keep-mos +RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL LATEST_TARGET_UBUNTU\"" > /etc/sudoers.d/keep-mos RUN chmod 0440 /etc/sudoers.d/open-sudo RUN chmod 0440 /etc/sudoers.d/keep-proxies RUN chmod 0440 /etc/sudoers.d/keep-pwd diff --git a/build/docker/runcontext b/build/docker/runcontext index 2df765868..9f077764e 100755 --- a/build/docker/runcontext +++ b/build/docker/runcontext @@ -111,7 +111,7 @@ if [ -n "$CACHEBASE" ]; then fi fi -RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT" +RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -e LATEST_TARGET_UBUNTU -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT" # Passing "debug" puts up an interactive bash shell if [ "$1" == "debug" ]; then diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile index 21d47f4f3..ee27adb49 100644 --- a/build/f_isoroot/Makefile +++ b/build/f_isoroot/Makefile @@ -10,6 +10,7 @@ # Add Fuel plugin build targets here PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild +export PLUGINS #NON_8-0_REBASED_PLUGINS = f_bgpvpn-pluginbuild # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins @@ -17,12 +18,11 @@ PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsper # Temporarilu disabling f_repobuild that currently have some issues on # Fuel 8. ifdef BUILD_FUEL_PLUGINS -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS) +export BUILD_FUEL_PLUGINS +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) $(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) else -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS) +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) endif SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh index a5ca56c64..b096a7812 100755 --- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh +++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh @@ -414,6 +414,16 @@ rmdir /var/log/remote && ln -s /var/log/docker-logs/remote /var/log/remote dockerctl check || fail bash /etc/rc.local +### OPNFV addition BEGIN +shopt -s nullglob +for script in /opt/opnfv/bootstrap/post.d/*.sh +do + echo "Post script: $script" >> /root/post.log 2>&1 + $script >> /root/post.log 2>&1 +done +shopt -u nullglob +### OPNFV addition END + if [ "`get_bootstrap_flavor`" = "ubuntu" ]; then if [ "`get_bootstrap_skip`" = "False" ]; then build_ubuntu_bootstrap bs_status || true @@ -426,15 +436,6 @@ else bs_status=3 fi -### OPNFV addition BEGIN -shopt -s nullglob -for script in /opt/opnfv/bootstrap/post.d/*.sh -do - echo "Post script: $script" >> /root/post.log 2>&1 - $script >> /root/post.log 2>&1 -done -shopt -u nullglob -### OPNFV addition END # Enable updates repository cat > /etc/yum.repos.d/mos${FUEL_RELEASE}-updates.repo << EOF diff --git a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh index 427a55add..8dad89397 100755 --- a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh +++ b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh @@ -16,4 +16,9 @@ if [ $? -ne 0 ]; then echo "Error removing /opt/opnfv/nailgun directory!" exit 1 fi +mv /opt/opnfv/fuel_bootstrap_cli.yaml /etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml +if [ $? -ne 0 ]; then + echo "Error moving bootstrap image configuration!" + exit 1 +fi echo "Done installing pre-build repo" diff --git a/build/f_isoroot/f_odlpluginbuild/packages.yaml b/build/f_isoroot/f_odlpluginbuild/packages.yaml new file mode 100644 index 000000000..7ef537f31 --- /dev/null +++ b/build/f_isoroot/f_odlpluginbuild/packages.yaml @@ -0,0 +1,3 @@ +packages: + - "openjdk-7-jre-headless" + - "acct" diff --git a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt b/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt deleted file mode 100644 index f0d06681e..000000000 --- a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt +++ /dev/null @@ -1 +0,0 @@ -openjdk-7-jre-headless diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile index 1fc703031..966da3634 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -10,54 +10,48 @@ SHELL := /bin/bash TOP := $(shell pwd) -RSYNC_HOST := $(shell ./select_ubuntu_repo.sh) -DEPENDENCIES_FILE := /etc/fuel-createmirror/requirements-deb.txt +UBUNTU_DIR := $(shell dirname $(MIRROR_UBUNTU_ROOT)) +include ../../config.mk + +export MOS_VERSION +export OPENSTACK_VERSION .PHONY: all all: nailgun nailgun: -# If RSYNC_CONNECT_PROG is set we need to do some magic to copy the -# keys. Make sure to have the username set in the SSH_CONNECT_PROG -# as well! - @if [ -n "${RSYNC_CONNECT_PROG}" -a ! -d /root/.ssh ]; then \ - sudo mkdir -p /root/.ssh; \ - test -d ${HOME}/.ssh && sudo find ${HOME}/.ssh -maxdepth 1 -type f -exec cp {} /root/.ssh \; ; \ - sudo bash -c "echo StrictHostKeyChecking=no > /root/.ssh/config"; \ - sudo chmod 700 /root/.ssh; \ - fi - sudo apt-get install -y rsync python python-yaml dpkg-dev openssl - rm -rf tmpiso tmpdir - mkdir tmpiso - fuseiso ${ISOCACHE} tmpiso - cp tmpiso/ubuntu/pool/main/f/fuel-createmirror/fuel-createmirror_*.deb . - fusermount -u tmpiso - rm -rf tmpiso - sudo dpkg -i fuel-createmirror_*.deb - sudo sed -i 's/DOCKER_MODE=true/DOCKER_MODE=false/' /etc/fuel-createmirror/common.cfg - sudo sed -i 's/DEBUG="no"/DEBUG="yes"/' /etc/fuel-createmirror/ubuntu.cfg - sudo sed -i "s/MIRROR_UBUNTU_HOST=\".*\"/MIRROR_UBUNTU_HOST=\"$(RSYNC_HOST)\"/" /etc/fuel-createmirror/common.cfg - # Add dependencies from plugins. With awk help try add only unique packages. - sudo bash -c 'find ../ -name requirements-deb.txt -print0 | xargs -0 -I% awk '\''NR == FNR {first[$$0];next}; ! ($$0 in first)'\'' "${DEPENDENCIES_FILE}" "%" >> "${DEPENDENCIES_FILE}"' + sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml rm -Rf nailgun - sudo mkdir -p /var/www - sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror + sudo mkdir -p /var/www/nailgun + git clone -b $(FUEL_MAIN_TAG) https://github.com/openstack/fuel-mirror + sudo pip install -U -r ./fuel-mirror/requirements.txt + sudo pip install ./fuel-mirror + sudo pip install ./fuel-mirror/contrib/fuel_mirror + ./opnfv_mirror_conf.py + sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group mos ubuntu --pattern=ubuntu sudo chmod -R 755 /var/www/nailgun cp -Rp /var/www/nailgun . + mv nailgun/mirrors$(MIRROR_MOS_UBUNTU_ROOT) nailgun/mirrors/mos-repos/ubuntu/$(MOS_VERSION) + rmdir nailgun/mirrors/mos-repos/ubuntu/snapshots + if [ "$(UBUNTU_DIR)" != "/" ]; then \ + mv nailgun/mirrors$(UBUNTU_DIR)/ubuntu naligun/mirros/ubuntu;\ + rm -rf nailgun/mirrors$(UBUNTU_DIR);\ + fi # Store artifact in cache straight away if caching is enabled # (no .cacheid will be present unless this is a cached build) test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 .PHONY: clean clean: - @rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb + @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror .PHONY: release release:nailgun @rm -Rf ../release/opnfv/nailgun @mkdir -p ../release/opnfv @cp -Rp nailgun ../release/opnfv/nailgun + @cp fuel_bootstrap_cli.yaml ../release/opnfv/ ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml new file mode 100644 index 000000000..28f6427f7 --- /dev/null +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml @@ -0,0 +1,75 @@ +--- + root_ssh_authorized_file: /root/.ssh/id_rsa.pub + extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" + extra_dirs: + - /usr/share/fuel_bootstrap_cli/files/trusty + output_dir: /tmp/ + kernel_flavor: linux-image-generic-lts-trusty + packages: + - vim + - mc + - openssh-client + - openssh-server + - ntp + - mcollective + - nailgun-agent + - nailgun-mcagents + - network-checker + - fuel-agent + - ubuntu-minimal + - live-boot + - live-boot-initramfs-tools + - wget + - linux-firmware + - linux-firmware-nonfree + - linux-headers-generic + - hpsa-dkms + - i40e-dkms + - xz-utils + - squashfs-tools + - msmtp-mta + bootstrap_images_dir: /var/www/nailgun/bootstraps + active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap + flavor: ubuntu + http_proxy: "" + https_proxy: "" + repos: + - name: ubuntu + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty + type: deb + - name: ubuntu-updates + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-updates + type: deb + - name: ubuntu-security + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-security + type: deb + - name: mos + section: "main restricted" + uri: "http://127.0.0.1:8080/ubuntu/x86_64" + priority: 1050 + suite: mos8.0 + type: deb + - name: mos-updates + section: "main restricted" + uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0" + priority: 1050 + suite: mos8.0-updates + type: deb + - name: mos-security + section: "main restricted" + uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0" + priority: 1050 + suite: mos8.0-security + type: deb + skip_default_img_build: true + direct_repo_addresses: + - "127.0.0.1" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py new file mode 100755 index 000000000..9abd0242b --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import os +import yaml + +current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] +mos_version = os.environ['MOS_VERSION'] +openstack_version = os.environ['OPENSTACK_VERSION'] +mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] +mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] +mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] +if os.environ.get('BUILD_FUEL_PLUGINS'): + plugins = os.environ['BUILD_FUEL_PLUGINS'] +else: + plugins = os.environ['PLUGINS'] + + +configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() +conf = yaml.load(configuration_file) +conf['pattern_dir'] = '.' +conf['openstack_version'] = openstack_version +conf['mos_version'] = mos_version + +with open('opnfv-config.yaml', 'w') as outfile: + outfile.write( yaml.dump(conf, default_flow_style=False) ) + +pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() +pattern = yaml.load(pattern_file) +pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) +pattern['ubuntu_baseurl'] = mirror_ubuntu +for group in pattern['groups']['mos']: + group['uri'] = pattern['mos_baseurl'] +for group in pattern['groups']['ubuntu']: + group['uri'] = pattern['ubuntu_baseurl'] + +for plugin in plugins.split(): + path = "../{}/packages.yaml".format(plugin) + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) + plugin_set = set(plugin_yaml['packages']) + main_set = set(pattern['packages']) + new_packages = plugin_set - main_set + print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) + pattern['packages'] = pattern['packages'] + list(new_packages) + +pattern['requirements']['ubuntu'] = pattern['packages'] + +with open('ubuntu.yaml', 'w') as outfile: + outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) -- 2.16.6