Added support for Docker Hub credentials (CI)

When credentials are available on the Ansible host (/opt/kuberef/docker_config)
they will be added to the K8s nodes.

Also includes a minor change to increase the resources in the Kuberef VM

Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: I9acb0df97d2b423449ee57ae3d8241b2d9567ba2
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72768
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>

diff --git a/functions.sh b/functions.sh
index 6d8e923..ccc8bc7 100755 (executable)
--- a/functions.sh
+++ b/functions.sh
@@ -161,6 +161,7 @@ get_vm_ip() {
 # Copy files needed by Infra engine & BMRA in the jumphost VM
 copy_files_jump() {
     vm_ip="$(get_vm_ip)"
+    docker_config="/opt/kuberef/docker_config"
     scp -r -o StrictHostKeyChecking=no \
     "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \
     "$USERNAME@${vm_ip}:$PROJECT_ROOT"
@@ -169,6 +170,10 @@ copy_files_jump() {
         ~/.ssh/id_rsa \
         "$USERNAME@${vm_ip}:.ssh/id_rsa"
     fi
+    if [ -f "$docker_config" ]; then
+        scp -r -o StrictHostKeyChecking=no \
+        "$docker_config" "$USERNAME@${vm_ip}:$PROJECT_ROOT"
+    fi
 }
 
 # Host Provisioning
@@ -229,6 +234,14 @@ if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
     git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v21.03 ${PROJECT_ROOT}/container-experience-kits/
     cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/
 fi
+if [ -f "${PROJECT_ROOT}/docker_config" ]; then
+    cp ${PROJECT_ROOT}/docker_config \
+        ${PROJECT_ROOT}/${INSTALLER}/dockerhub_credentials/vars/main.yml
+    cp -r ${PROJECT_ROOT}/${INSTALLER}/dockerhub_credentials \
+        ${PROJECT_ROOT}/container-experience-kits/roles/
+    cp ${PROJECT_ROOT}/${INSTALLER}/patched_k8s.yml \
+        ${PROJECT_ROOT}/container-experience-kits/playbooks/k8s/k8s.yml
+fi
 cp ${PROJECT_ROOT}/${INSTALLER}/{inventory.ini,ansible.cfg} \
     ${PROJECT_ROOT}/container-experience-kits/
 cp ${PROJECT_ROOT}/${INSTALLER}/{all.yml,kube-node.yml} \

diff --git a/playbooks/roles/jump-vm/tasks/main.yaml b/playbooks/roles/jump-vm/tasks/main.yaml
index 9c556da..b6ed840 100644 (file)
--- a/playbooks/roles/jump-vm/tasks/main.yaml
+++ b/playbooks/roles/jump-vm/tasks/main.yaml
@@ -92,7 +92,7 @@
 
 - name: define jump VM
   command: "virt-install --connect qemu:///system --name {{ jumphost.name }} \
-              --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \
+              --ram 8192 --vcpus=8 --os-type linux --os-variant ubuntu16.04 \
               --disk path={{ workspace }}/kuberef-jump.qcow2,format=qcow2 \
               --disk {{ workspace }}/kuberef-jump-cidata.iso,device=cdrom \
               --network network=default,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}' \

diff --git a/sw_config/bmra/dockerhub_credentials/tasks/main.yml b/sw_config/bmra/dockerhub_credentials/tasks/main.yml
new file mode 100644 (file)
index 0000000..6531df8
--- /dev/null
+++ b/sw_config/bmra/dockerhub_credentials/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- name: Add Docker hub credentials on target nodes
+  docker_login:
+    username: "{{ dhub_user }}"
+    password: "{{ dhub_pass }}"
+    email: "{{ dhub_email }}"
+    registry_url: "{{ dhub_reg_url }}"
+  no_log: True

diff --git a/sw_config/bmra/dockerhub_credentials/vars/.gitignore b/sw_config/bmra/dockerhub_credentials/vars/.gitignore
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml
new file mode 100644 (file)
index 0000000..52239b0
--- /dev/null
+++ b/sw_config/bmra/patched_k8s.yml
@@ -0,0 +1,166 @@
+##
+##   Copyright (c) 2020-2021 Intel Corporation.
+##
+##   Licensed under the Apache License, Version 2.0 (the "License");
+##   you may not use this file except in compliance with the License.
+##   You may obtain a copy of the License at
+##
+##       http://www.apache.org/licenses/LICENSE-2.0
+##
+##   Unless required by applicable law or agreed to in writing, software
+##   distributed under the License is distributed on an "AS IS" BASIS,
+##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+##   See the License for the specific language governing permissions and
+##   limitations under the License.
+##
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks: []
+  roles:
+    - { role: kubespray_install }
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- hosts: k8s-cluster
+  tasks: []
+  roles:
+    - role: cluster_defaults
+    - role: kubespray_target_setup
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- hosts: all
+  gather_facts: false
+  tasks:
+    - name: prepare additional kubespray facts
+      set_fact:
+        kubelet_node_custom_flags_prepare: >-
+          {%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%}
+            --reserved-cpus={{ native_cpu_manager_reserved_cpus }}
+          {%- endif -%}
+        enable_admission_plugins_prepare: >-
+          [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}]
+        kube_config_dir: /etc/kubernetes
+    - name: set kube_cert_dir
+      set_fact:
+        kube_cert_dir: "{{ kube_config_dir }}/ssl"
+        kube_csr_dir: "{{ kube_config_dir }}/csr"
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- name: run kubespray
+  import_playbook: kubespray/cluster.yml
+  vars:
+    kubeadm_enabled: true
+    multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf
+    docker_iptables_enabled: true
+    docker_dns_servers_strict: false
+    override_system_hostname: false
+    docker_version: '19.03'
+    kube_proxy_mode: iptables
+    enable_nodelocaldns: false
+    system_reserved: true
+    dashboard_enabled: true
+    system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}"
+    kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}"
+    kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}"
+    kube_api_anonymous_auth: true
+    kube_feature_gates:
+      - CPUManager=true # feature gate can be enabled by default, default policy is none in Kubernetes
+      - TopologyManager={{ topology_manager_enabled | default(true) }}
+      - RotateKubeletServerCertificate=true
+    # Kubernetes cluster hardening
+    kubernetes_audit: true
+    audit_log_maxbackups: 10
+    kube_controller_manager_bind_address: 127.0.0.1
+    kube_scheduler_bind_address: 127.0.0.1
+    kube_proxy_healthz_bind_address: 127.0.0.1
+    kube_proxy_metrics_bind_address: 127.0.0.1
+    kube_read_only_port: 0
+    kube_override_hostname: ""
+    kube_kubeadm_apiserver_extra_args:
+      service-account-lookup: true
+      service-account-key-file: "{{ kube_cert_dir }}/sa.key"
+      admission-control-config-file: "{{ kube_config_dir }}/admission-control/config.yaml"
+    kube_kubeadm_scheduler_extra_args:
+      address: 127.0.0.1
+      profiling: false
+    kube_kubeadm_controller_extra_args:
+      address: 127.0.0.1
+      service-account-private-key-file: "{{ kube_cert_dir }}/sa.key"
+    kubelet_config_extra_args:
+      protectKernelDefaults: true
+      cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}"
+      topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}"
+      eventRecordQPS: 0
+    kube_apiserver_request_timeout: 60s
+    kube_apiserver_enable_admission_plugins: "{{ enable_admission_plugins_prepare | from_yaml }}"
+    podsecuritypolicy_enabled: "{{ psp_enabled }}"
+    kube_encrypt_secret_data: true
+    apiserver_extra_volumes:
+      - name: admission-control-config
+        hostPath: /etc/kubernetes/admission-control/
+        mountPath: /etc/kubernetes/admission-control/
+        readOnly: true
+    preinstall_selinux_state: "{{ selinux_mode | default('disabled') }}"
+    tls_cipher_suites:
+      - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+      - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+      - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+    etcd_extra_vars:
+      ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
+
+- hosts: k8s-cluster
+  tasks:
+    - name: restart docker daemon to recreate iptables rules
+      systemd: name=docker state=restarted
+      become: yes
+    - name: restart kubelet to trigger static pods recreation
+      systemd: name=kubelet state=restarted
+      become: yes
+    # note: fix for the issue mentioned here:
+    # https://github.com/kubernetes-sigs/kubespray/blob/58f48500b1adac3f18466fa1c5cf8aa9d9838150/docs/flannel.md#flannel
+    - name: check if flannel.1 interface exists
+      stat:
+        path: /sys/class/net/flannel.1
+      when: kube_network_plugin == "flannel"
+      register: flannel_endpoint
+    - name: disable offloading features on flannel.1
+      command: ethtool --offload flannel.1 rx off tx off
+      become: yes
+      when:
+        - kube_network_plugin == "flannel"
+        - flannel_endpoint.stat.exists
+
+- hosts: etcd
+  tasks:
+    - name: change /var/lib/etcd owner
+      file:
+        path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
+        owner: etcd
+        group: etcd
+        recurse: true
+        state: directory
+        mode: 0700
+    - name: change /var/lib/etcd permissions
+      file:
+        path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
+        owner: etcd
+        group: etcd
+        mode: '0700'
+        state: directory
+
+- hosts: k8s-cluster
+  roles:
+    - role: cluster_defaults
+      tags: defaults
+    - role: docker_registry
+      tags: registry
+    - role: dockerhub_credentials
+      when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file"
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- name: run certificate generation for mTLS in kubelet
+  import_playbook: kubelet-certificates.yml