sw_config/bmra/patched_k8s.yml

   1 # SPDX-FileCopyrightText: 2021 Intel Corporation.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 ---
   6 - hosts: 127.0.0.1
   7   connection: local
   8   tasks: []
   9   roles:
  10     - {role: kubespray_install}
  11   environment: "{{ proxy_env | d({}) }}"
  12   any_errors_fatal: true
  13
  14 - hosts: k8s-cluster
  15   tasks: []
  16   roles:
  17     - role: cluster_defaults
  18     - role: kubespray_target_setup
  19   environment: "{{ proxy_env | d({}) }}"
  20   any_errors_fatal: true
  21
  22 - hosts: all
  23   gather_facts: false
  24   tasks:
  25     - name: prepare additional kubespray facts
  26       set_fact:
  27         kubelet_node_custom_flags_prepare: >-
  28           {%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%}
  29             --reserved-cpus={{ native_cpu_manager_reserved_cpus }}
  30           {%- endif -%}
  31         enable_admission_plugins_prepare: >-
  32           [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}]
  33         bmra_docker_version: >-
  34           {% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %}
  35         flannel_backend_type: >-
  36           {% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %}
  37         kube_config_dir: /etc/kubernetes
  38     - name: set kube_cert_dir
  39       set_fact:
  40         kube_cert_dir: "{{ kube_config_dir }}/ssl"
  41         kube_csr_dir: "{{ kube_config_dir }}/csr"
  42   environment: "{{ proxy_env | d({}) }}"
  43   any_errors_fatal: true
  44
  45 - hosts: all
  46   tasks:
  47     - name: add docker runtime vars
  48       set_fact:
  49         container_manager: docker
  50         docker_iptables_enabled: true
  51         docker_dns_servers_strict: false
  52         docker_version: "{{ bmra_docker_version }}"
  53       when: container_runtime == "docker"
  54     - name: add containerd runtime vars
  55       set_fact:
  56         container_manager: containerd
  57         etcd_deployment_type: host
  58         containerd_extra_args: |2
  59                   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry_local_address }}"]
  60                     endpoint = ["https://{{ registry_local_address }}"]
  61                   [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry_local_address }}".tls]
  62                     ca_file   = "/etc/containers/certs.d/{{ registry_local_address }}/ca.crt"
  63       when: container_runtime == "containerd"
  64 - name: run kubespray
  65   import_playbook: kubespray/cluster.yml
  66   vars:
  67     kubeadm_enabled: true
  68     multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf
  69     nginx_image_tag: 1.21.1
  70     override_system_hostname: false
  71     kube_proxy_mode: iptables
  72     enable_nodelocaldns: false
  73     system_reserved: true
  74     dashboard_enabled: true
  75     system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}"
  76     kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}"
  77     kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}"
  78     kube_api_anonymous_auth: true
  79     kube_feature_gates:
  80       - CPUManager=true    # feature gate can be enabled by default, default policy is none in Kubernetes
  81       - TopologyManager={{ topology_manager_enabled | default(true) }}
  82       - RotateKubeletServerCertificate=true
  83     # Kubernetes cluster hardening
  84     kubernetes_audit: true
  85     audit_log_maxbackups: 10
  86     kube_controller_manager_bind_address: 127.0.0.1
  87     kube_scheduler_bind_address: 127.0.0.1
  88     kube_proxy_healthz_bind_address: 127.0.0.1
  89     kube_proxy_metrics_bind_address: 127.0.0.1
  90     kube_read_only_port: 0
  91     kube_override_hostname: ""
  92     kube_kubeadm_apiserver_extra_args:
  93       service-account-lookup: true
  94       service-account-key-file: "{{ kube_cert_dir }}/sa.key"
  95       admission-control-config-file: "{{ kube_config_dir }}/admission-control/config.yaml"
  96     kube_kubeadm_scheduler_extra_args:
  97       address: 127.0.0.1
  98       profiling: false
  99     kube_kubeadm_controller_extra_args:
 100       address: 127.0.0.1
 101       service-account-private-key-file: "{{ kube_cert_dir }}/sa.key"
 102     kubelet_config_extra_args:
 103       protectKernelDefaults: true
 104       cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}"
 105       topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}"
 106       eventRecordQPS: 0
 107     kube_apiserver_request_timeout: 60s
 108     kube_apiserver_enable_admission_plugins: "{{ enable_admission_plugins_prepare | from_yaml }}"
 109     podsecuritypolicy_enabled: "{{ psp_enabled }}"
 110     kube_encrypt_secret_data: true
 111     apiserver_extra_volumes:
 112       - name: admission-control-config
 113         hostPath: /etc/kubernetes/admission-control/
 114         mountPath: /etc/kubernetes/admission-control/
 115         readOnly: true
 116     preinstall_selinux_state: "{{ selinux_mode | default('disabled') }}"
 117     tls_cipher_suites:
 118       - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
 119       - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
 120       - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
 121     etcd_extra_vars:
 122       ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
 123
 124 - hosts: k8s-cluster
 125   tasks:
 126     - name: restart docker daemon to recreate iptables rules
 127       systemd: name=docker state=restarted
 128       become: yes
 129       when: container_runtime == "docker"
 130     - name: restart kubelet to trigger static pods recreation
 131       systemd: name=kubelet state=restarted
 132       become: yes
 133     # note: fix for the issue mentioned here:
 134     # https://github.com/kubernetes-sigs/kubespray/blob/58f48500b1adac3f18466fa1c5cf8aa9d9838150/docs/flannel.md#flannel
 135     - name: check if flannel.1 interface exists
 136       stat:
 137         path: /sys/class/net/flannel.1
 138       when: kube_network_plugin == "flannel"
 139       register: flannel_endpoint
 140     - name: disable offloading features on flannel.1
 141       command: ethtool --offload flannel.1 rx off tx off
 142       become: yes
 143       when:
 144         - kube_network_plugin == "flannel"
 145         - flannel_endpoint.stat.exists
 146
 147 - hosts: etcd
 148   tasks:
 149     - name: change /var/lib/etcd owner
 150       file:
 151         path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
 152         owner: etcd
 153         group: etcd
 154         recurse: true
 155         state: directory
 156         mode: 0700
 157     - name: change /var/lib/etcd permissions
 158       file:
 159         path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
 160         owner: etcd
 161         group: etcd
 162         mode: '0700'
 163         state: directory
 164
 165 - hosts: k8s-cluster
 166   roles:
 167     - role: cluster_defaults
 168       tags: defaults
 169     - role: container_registry
 170       tags: registry
 171     - role: dockerhub_credentials
 172       when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file"
 173   environment: "{{ proxy_env | d({}) }}"
 174   any_errors_fatal: true
 175
 176 - name: run certificate generation for mTLS in kubelet
 177   import_playbook: kubelet-certificates.yml