This patch adds capability to deploy kubernetes cluster instead of openstack.
Kubernetes will be deployed using kubespray and is run after TripleO bookstraps
overcloud nodes.
JIRA: APEX-574
Change-Id: If9c171620c933a052b719e7112a50e22bbab667f
Signed-off-by: Feng Pan <fpan@redhat.com>
Signed-off-by: Zenghui Shi <zshi@redhat.com>
}
DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \
'/tripleomaster/'
+KUBESPRAY_URL = 'https://github.com/kubernetes-incubator/kubespray.git'
Executes ansible playbook and checks for errors
:param ansible_vars: dictionary of variables to inject into ansible run
:param playbook: playbook to execute
+ :param host: inventory file or string of target hosts
+ :param user: remote user to run ansible tasks
:param tmp_dir: temp directory to store ansible command
:param dry_run: Do not actually apply changes
:return: None
"""
logging.info("Executing ansible playbook: {}".format(playbook))
- inv_host = "{},".format(host)
+ if not os.path.isfile(host):
+ inv_host = "{},".format(host)
+ else:
+ inv_host = host
if host == 'localhost':
conn_type = 'local'
else:
##############################################################################
import argparse
+import git
import json
import logging
import os
import shutil
import sys
import tempfile
+import yaml
import apex.virtual.configure_vm as vm_lib
import apex.virtual.utils as virt_utils
# Parse all settings
deploy_settings = DeploySettings(args.deploy_settings_file)
logging.info("Deploy settings are:\n {}".format(pprint.pformat(
- deploy_settings)))
+ deploy_settings)))
net_settings = NetworkSettings(args.network_settings_file)
logging.info("Network settings are:\n {}".format(pprint.pformat(
- net_settings)))
+ net_settings)))
os_version = deploy_settings['deploy_options']['os_version']
net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
net_env = NetworkEnvironment(net_settings, net_env_file,
'deploy_overcloud.yml')
virt_env = 'virtual-environment.yaml'
bm_env = 'baremetal-environment.yaml'
- for p_env in virt_env, bm_env:
+ k8s_env = 'kubernetes-environment.yaml'
+ for p_env in virt_env, bm_env, k8s_env:
shutil.copyfile(os.path.join(args.deploy_dir, p_env),
os.path.join(APEX_TEMP_DIR, p_env))
deploy_vars['os_version'] = os_version
deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
+ deploy_vars['vim'] = ds_opts['vim']
for dns_server in net_settings['dns_servers']:
deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
dns_server)
else:
deploy_vars['overcloudrc_files'] = ['overcloudrc']
- post_undercloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
+ post_undercloud = os.path.join(args.lib_dir,
+ constants.ANSIBLE_PATH,
'post_deploy_undercloud.yml')
- logging.info("Executing post deploy configuration undercloud playbook")
+ logging.info("Executing post deploy configuration undercloud "
+ "playbook")
try:
- utils.run_ansible(deploy_vars, post_undercloud, host=undercloud.ip,
- user='stack', tmp_dir=APEX_TEMP_DIR)
+ utils.run_ansible(deploy_vars, post_undercloud,
+ host=undercloud.ip, user='stack',
+ tmp_dir=APEX_TEMP_DIR)
logging.info("Post Deploy Undercloud Configuration Complete")
except Exception:
logging.error("Post Deploy Undercloud Configuration failed. "
"Please check log")
raise
+
+ # Deploy kubernetes if enabled
+ # (TODO)zshi move handling of kubernetes deployment
+ # to its own deployment class
+ if deploy_vars['vim'] == 'k8s':
+ # clone kubespray repo
+ git.Repo.clone_from(constants.KUBESPRAY_URL,
+ os.path.join(APEX_TEMP_DIR, 'kubespray'))
+ shutil.copytree(
+ os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+ 'sample'),
+ os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+ 'apex'))
+ k8s_node_inventory = {
+ 'all':
+ {'hosts': {},
+ 'children': {
+ 'k8s-cluster': {
+ 'children': {
+ 'kube-master': {
+ 'hosts': {}
+ },
+ 'kube-node': {
+ 'hosts': {}
+ }
+ }
+ },
+ 'etcd': {
+ 'hosts': {}
+ }
+ }
+ }
+ }
+ for node, ip in deploy_vars['overcloud_nodes'].items():
+ k8s_node_inventory['all']['hosts'][node] = {
+ 'ansible_become': True,
+ 'ansible_ssh_host': ip,
+ 'ansible_become_user': 'root',
+ 'ip': ip
+ }
+ if 'controller' in node:
+ k8s_node_inventory['all']['children']['k8s-cluster'][
+ 'children']['kube-master']['hosts'][node] = None
+ k8s_node_inventory['all']['children']['etcd'][
+ 'hosts'][node] = None
+ elif 'compute' in node:
+ k8s_node_inventory['all']['children']['k8s-cluster'][
+ 'children']['kube-node']['hosts'][node] = None
+
+ kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray')
+ with open(os.path.join(kubespray_dir, 'inventory', 'apex',
+ 'apex.yaml'), 'w') as invfile:
+ yaml.dump(k8s_node_inventory, invfile,
+ default_flow_style=False)
+ k8s_deploy_vars = {}
+ # Add kubespray ansible control variables in k8s_deploy_vars,
+ # example: 'kube_network_plugin': 'flannel'
+ k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml')
+ k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory',
+ 'apex', 'apex.yaml')
+
+ k8s_remove_pkgs = os.path.join(args.lib_dir,
+ constants.ANSIBLE_PATH,
+ 'k8s_remove_pkgs.yml')
+ try:
+ logging.debug("Removing any existing overcloud docker "
+ "packages")
+ utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs,
+ host=k8s_deploy_inv_file,
+ user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+ logging.info("k8s Deploy Remove Existing Docker Related "
+ "Packages Complete")
+ except Exception:
+ logging.error("k8s Deploy Remove Existing Docker Related "
+ "Packages failed. Please check log")
+ raise
+
+ try:
+ utils.run_ansible(k8s_deploy_vars, k8s_deploy,
+ host=k8s_deploy_inv_file,
+ user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+ logging.info("k8s Deploy Overcloud Configuration Complete")
+ except Exception:
+ logging.error("k8s Deploy Overcloud Configuration failed."
+ "Please check log")
+ raise
+
# Post deploy overcloud node configuration
# TODO(trozet): just parse all ds_opts as deploy vars one time
deploy_vars['sfc'] = ds_opts['sfc']
if ds_opts['sriov']:
prep_sriov_env(ds, tmp_dir)
+ # Check for 'k8s' here intentionally, as we may support other values
+ # such as openstack/openshift for 'vim' option.
+ if ds_opts['vim'] == 'k8s':
+ deploy_options.append('kubernetes-environment.yaml')
+
if virtual:
deploy_options.append('virtual-environment.yaml')
else:
'l2gw',
'sriov',
'containers',
- 'ceph_device']
+ 'ceph_device',
+ 'vim']
OPT_DEPLOY_SETTINGS = ['performance',
'vsperf',
elif req_set == 'os_version':
self['deploy_options'][req_set] = \
constants.DEFAULT_OS_VERSION
+ elif req_set == 'vim':
+ self['deploy_options'][req_set] = 'openstack'
else:
self['deploy_options'][req_set] = False
elif req_set == 'odl_version' and self['deploy_options'][
def test_run_ansible(self):
playbook = 'apex/tests/playbooks/test_playbook.yaml'
+ extra_vars = [{'testvar1': 'value1', 'testvar2': 'value2'}]
assert_equal(utils.run_ansible(None, os.path.join(playbook),
dry_run=True), None)
+ assert_equal(utils.run_ansible(extra_vars, os.path.join(playbook),
+ dry_run=True, host='1.1.1.1'), None)
def test_failed_run_ansible(self):
playbook = 'apex/tests/playbooks/test_failed_playbook.yaml'
'dataplane': 'ovs',
'sfc': False,
'vpn': False,
+ 'vim': 'openstack',
'yardstick': 'test',
'os_version': DEFAULT_OS_VERSION,
'containers': False}}
'dataplane': 'ovs',
'sfc': False,
'vpn': False,
+ 'vim': 'openstack',
'yardstick': 'test',
'os_version': DEFAULT_OS_VERSION,
'containers': False}}
'dataplane': 'ovs',
'sfc': False,
'vpn': False,
+ 'vim': 'openstack',
'yardstick': 'test',
'os_version': DEFAULT_OS_VERSION,
'containers': True}}
args.virt_compute_ram = 16
args.virt_default_ram = 10
main()
- mock_oc_deploy.prep_image.assert_called
+ mock_oc_deploy.prep_image.assert_called()
# TODO(trozet) add assertions here with arguments for functions in
# deploy main
+
+ @patch('apex.deploy.uc_builder')
+ @patch('apex.deploy.network_data.create_network_data')
+ @patch('apex.deploy.shutil')
+ @patch('apex.deploy.git')
+ @patch('apex.deploy.oc_deploy')
+ @patch('apex.deploy.uc_lib')
+ @patch('apex.deploy.build_vms')
+ @patch('apex.deploy.Inventory')
+ @patch('apex.deploy.virt_utils')
+ @patch('apex.deploy.oc_cfg')
+ @patch('apex.deploy.parsers')
+ @patch('apex.deploy.utils')
+ @patch('apex.deploy.NetworkEnvironment')
+ @patch('apex.deploy.NetworkSettings')
+ @patch('apex.deploy.DeploySettings')
+ @patch('apex.deploy.os')
+ @patch('apex.deploy.json')
+ @patch('apex.deploy.jumphost')
+ @patch('apex.deploy.validate_cross_settings')
+ @patch('apex.deploy.validate_deploy_args')
+ @patch('apex.deploy.create_deploy_parser')
+ @patch('builtins.open', a_mock_open, create=True)
+ def test_main_k8s(self, mock_parser, mock_val_args, mock_cross_sets,
+ mock_jumphost, mock_json, mock_os,
+ mock_deploy_sets, mock_net_sets, mock_net_env,
+ mock_utils, mock_parsers, mock_oc_cfg,
+ mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
+ mock_oc_deploy, mock_git, mock_shutil,
+ mock_network_data, mock_uc_builder):
+ net_sets_dict = {'networks': MagicMock(),
+ 'dns_servers': 'test'}
+ ds_opts_dict = {'global_params': MagicMock(),
+ 'deploy_options': {'gluon': False,
+ 'congress': True,
+ 'sdn_controller': False,
+ 'dataplane': 'ovs',
+ 'sfc': False,
+ 'vpn': False,
+ 'vim': 'k8s',
+ 'yardstick': 'test',
+ 'os_version': DEFAULT_OS_VERSION,
+ 'containers': False}}
+ args = mock_parser.return_value.parse_args.return_value
+ args.virtual = False
+ args.quickstart = False
+ args.debug = False
+ args.upstream = False
+ net_sets = mock_net_sets.return_value
+ net_sets.enabled_network_list = ['external']
+ net_sets.__getitem__.side_effect = net_sets_dict.__getitem__
+ net_sets.__contains__.side_effect = net_sets_dict.__contains__
+ deploy_sets = mock_deploy_sets.return_value
+ deploy_sets.__getitem__.side_effect = ds_opts_dict.__getitem__
+ deploy_sets.__contains__.side_effect = ds_opts_dict.__contains__
+ mock_parsers.parse_nova_output.return_value = {'testnode1': 'test'}
+ main()
'containers': False,
'barometer': True,
'ceph': False,
- 'sriov': False
+ 'sriov': False,
+ 'vim': 'openstack'
},
'global_params': MagicMock()}
'ceph': True,
'sdn_controller': 'opendaylight',
'sriov': False,
- 'os_version': 'queens'
+ 'os_version': 'queens',
+ 'vim': 'openstack'
},
'global_params': MagicMock()}
--- /dev/null
+---
+resource_registry:
+ OS::TripleO::Services::Docker: OS::Heat::None
+
+
+parameter_defaults:
+ ControllerServices:
+ - OS::TripleO::Services::Kernel
+ - OS::TripleO::Services::Ntp
+ - OS::TripleO::Services::Snmp
+ - OS::TripleO::Services::Timezone
+ - OS::TripleO::Services::TripleoPackages
+ # - OS::TripleO::Services::TripleoFirewall
+ - OS::TripleO::Services::Sshd
+ # - OS::TripleO::Services::OpenShift::Master
+ # - OS::TripleO::Services::OpenShift::Worker
+ ComputeServices:
+ - OS::TripleO::Services::Kernel
+ - OS::TripleO::Services::Ntp
+ - OS::TripleO::Services::Snmp
+ - OS::TripleO::Services::Timezone
+ - OS::TripleO::Services::TripleoPackages
+ # - OS::TripleO::Services::TripleoFirewall
+ - OS::TripleO::Services::Sshd
+ # - OS::TripleO::Services::OpenShift::Worker
+ Debug: true
%{_sysconfdir}/opnfv-apex/network_settings_csit.yaml
%{_sysconfdir}/opnfv-apex/network_settings_vlans.yaml
%{_sysconfdir}/opnfv-apex/network_settings_v6.yaml
+%{_sysconfdir}/opnfv-apex/k8s-nosdn-nofeature-noha.yaml
%doc %{_docdir}/opnfv/LICENSE.rst
%doc %{_docdir}/opnfv/installation-instructions.html
%doc %{_docdir}/opnfv/release-notes.rst
%doc %{_docdir}/opnfv/inventory.yaml.example
%changelog
+* Wed Jun 20 2018 Zenghui Shi <zshi@redhat.com> - 7.0-3
+ Adds Kubernetes deployment scenario
* Fri Jun 15 2018 Tim Rozet <trozet@redhat.com> - 7.0-2
Adds missing HA deploy settings for Queens
* Fri May 25 2018 Tim Rozet <trozet@redhat.com> - 7.0-1
# Set dovetail option to install dovetail
# dovetail: false
+
+ # Whether the nodes are deployed as openstack, kubernetes or openshift nodes
+ # Defaults to openstack.
+ # Possible values are openstack, k8s, openshift
+ vim: 'openstack'
--- /dev/null
+---
+global_params:
+ ha_enabled: false
+ patches:
+ undercloud:
+ - change-id: Ib8ff69a4bc869de21ad838b3bc6c38a8676036c6
+ project: openstack/tripleo-heat-templates
+
+deploy_options:
+ sdn_controller: false
+ tacker: false
+ congress: false
+ sfc: false
+ vpn: false
+ ceph: false
+ vim: 'k8s'
--- /dev/null
+.. _k8s-nosdn-nofeature-noha:
+
+.. This work is licensed under a Creative Commons Attribution 4.0 International Licence.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) <optionally add copywriters name>
+
+=================================================
+k8s-nosdn-nofeature-noha overview and description
+=================================================
+
+.. toctree::
+ :numbered:
+ :maxdepth: 4
+
+ k8s-nosdn-nofeature-noha.rst
--- /dev/null
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) <optionally add copywriters name>
+
+This document provides scenario level details for Gambia 1.0 of
+Kubernetes deployment with no SDN controller, no extra features
+and no High Availability enabled.
+
+============
+Introduction
+============
+
+This scenario is used primarily to validate and deploy a Kubernetes
+deployment without any NFV features or SDN controller enabled.
+
+Scenario components and composition
+===================================
+
+This scenario deploys a Kubernetes cluster on bare metal or virtual
+environment with a single master node. TripleO is used to bootstrap
+all the nodes and set up basic services like SSH. An undercloud VM
+used similarly to Openstack deployments, however no Openstack services
+(Nova, Neutron, Keystone, etc) will be deployed to the nodes. After
+TripleO successfully executes all the bootstrapping tasks, Kubespray
+is run (using ansible) to deploy Kubernetes cluster on the nodes.
+
+
+Scenario usage overview
+=======================
+
+Simply deploy this scenario by using the k8s-nosdn-nofeature-noha.yaml deploy
+settings file.
+
+Limitations, Issues and Workarounds
+===================================
+
+None
+
+References
+==========
+
+For more information on the OPNFV Gambia release, please visit
+http://www.opnfv.org/gambia
+
- python-lxml
- libvirt-python
- libguestfs-tools
+ - python-netaddr
+ - python2-pip
+ - pip:
+ name: ansible-modules-hashivault,hvac,Jinja2
+ state: latest
+ executable: pip2
- sysctl:
name: net.ipv4.ip_forward
state: present
- deploy_command
- virtual-environment.yaml
- baremetal-environment.yaml
+ - kubernetes-environment.yaml
- "{{ apex_env_file }}"
- name: Copy network data to undercloud
copy:
become: yes
- name: Show Keystone output
shell: "{{ overcloudrc }} && {{ item }}"
- when: debug
+ when: debug and vim == 'openstack'
with_items:
- openstack endpoint list
- openstack service list
--- /dev/null
+---
+- hosts: all
+ tasks:
+ - name: Removing existing docker related packages
+ shell: "rpm -e --nodeps docker docker-common docker-client container-selinux"
- name: Configure external network
shell: "{{ overcloudrc }} && {{ item }}"
with_items: "{{ external_network_cmds }}"
+ when: vim == 'openstack'
- name: Configure gluon networks
shell: "{{ overcloudrc }} && {{ item }}"
when: gluon
- name: Register OS Region
shell: "{{ overcloudrc }} && openstack endpoint list -c Region -f json"
register: region
- become: yes
- become_user: stack
+ when: vim == 'openstack'
- name: Write Region into overcloudrc
lineinfile:
line: "export OS_REGION_NAME={{(region.stdout|from_json)[1]['Region']}}"
become: yes
become_user: stack
with_items: "{{ overcloudrc_files }}"
+ when: vim == 'openstack'
- name: Create congress datasources
shell: "{{ overcloudrc }} && openstack congress datasource create {{ item }}"
become: yes
build/network-environment.yaml
build/opnfv-environment.yaml
build/upstream-environment.yaml
+ build/kubernetes-environment.yaml
build/nics-template.yaml.jinja2
build/csit-environment.yaml
build/virtual-environment.yaml
Code Review / apex.git / commitdiff