diff --git a/playbooks/sample-sap-hypervisor-redhat-ocp-virt-preconfigure.yml b/playbooks/sample-sap-hypervisor-redhat-ocp-virt-preconfigure.yml index 5f1d0bb5..c960fbca 100644 --- a/playbooks/sample-sap-hypervisor-redhat-ocp-virt-preconfigure.yml +++ b/playbooks/sample-sap-hypervisor-redhat-ocp-virt-preconfigure.yml @@ -6,13 +6,21 @@ sap_hypervisor_node_preconfigure_platform: redhat_ocp_virt tasks: - - name: Use kubeconfig file specified in environment variable K8S_AUTH_KUBECONFIG if sap_hypervisor_node_preconfigure_ocp_kubeconfig_path is not defined + - name: Use kubeconfig file specified in environment variable K8S_AUTH_KUBECONFIG | KUBECONFIG if sap_hypervisor_node_preconfigure_ocp_kubeconfig_path is not defined. when: > - sap_hypervisor_node_preconfigure_ocp_kubeconfig_path is not defined or - sap_hypervisor_node_preconfigure_ocp_kubeconfig_path == None or - sap_hypervisor_node_preconfigure_ocp_kubeconfig_path == '' + sap_hypervisor_node_preconfigure_kubeconfig is not defined or + sap_hypervisor_node_preconfigure_kubeconfig == None or + sap_hypervisor_node_preconfigure_kubeconfig == '' ansible.builtin.set_fact: - sap_hypervisor_node_preconfigure_ocp_kubeconfig_path: "{{ lookup('env', 'K8S_AUTH_KUBECONFIG') | default(None) }}" + sap_hypervisor_node_preconfigure_kubeconfig: "{{ lookup('env', 'K8S_AUTH_KUBECONFIG') | default(lookup('env', 'KUBECONFIG'), true) }}" + + - name: Ensure that kubeconfig is set + assert: + that: + - sap_hypervisor_node_preconfigure_kubeconfig is defined + - sap_hypervisor_node_preconfigure_kubeconfig is not none + - sap_hypervisor_node_preconfigure_kubeconfig | length > 0 + fail_msg: "sap_hypervisor_node_preconfigure_kubeconfig is required." - name: Create Tempdir on jumphost ansible.builtin.tempfile: @@ -20,77 +28,12 @@ suffix: "_sap_hypervisor_node_preconfigure" register: __sap_hypervisor_node_preconfigure_register_tmpdir_jumphost - - name: Set kubeconfig file variable - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_kubeconfig: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_jumphost.path }}/kubeconfig" - - - name: Read content of kubeconfig file - ansible.builtin.set_fact: - sap_hypervisor_node_preconfigure_ocp_kubeconfig_data: - "{{ lookup('file', sap_hypervisor_node_preconfigure_ocp_kubeconfig_path) | from_yaml }}" - - - name: Read cluster endpoint and CA certificate from kubeconfig if either is not defined - when: sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig - block: - - - name: Set sap_hypervisor_node_preconfigure_ocp_endpoint from kubeconfig - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_ocp_endpoint: - "{{ sap_hypervisor_node_preconfigure_ocp_kubeconfig_data['clusters'][0]['cluster']['server'] }}" - - - name: Write the certificate-authority-data to temp dir on jumphost - ansible.builtin.copy: - content: "{{ sap_hypervisor_node_preconfigure_ocp_kubeconfig_data['clusters'][0]['cluster']['certificate-authority-data'] | b64decode }}" - dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_jumphost.path }}/cluster-ca-cert.pem" - mode: "0666" - - - name: Set CA file variable - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_ca_cert: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_jumphost.path }}/cluster-ca-cert.pem" - - - - name: Use predefined CA cert and API endpoint - when: not sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig - - block: - - name: Set predefined OCP API Endpoint - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_ocp_endpoint: "{{ sap_hypervisor_node_preconfigure_ocp_endpoint }}" - - - name: Set predefined CA file - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_ca_cert: "{{ sap_hypervisor_node_preconfigure_ocp_ca_cert }}" - - - - name: Log into Red Hat OpenShift cluster (obtain access token) - community.okd.openshift_auth: - host: "{{ __sap_hypervisor_node_preconfigure_register_ocp_endpoint }}" - username: "{{ sap_hypervisor_node_preconfigure_ocp_admin_username }}" - password: "{{ sap_hypervisor_node_preconfigure_ocp_admin_password }}" - ca_cert: "{{ __sap_hypervisor_node_preconfigure_register_ca_cert }}" - register: __sap_vm_provision_register_ocp_auth_results - - - name: Set token in kubeconfig - ansible.builtin.set_fact: - sap_hypervisor_node_preconfigure_ocp_kubeconfig_data: >- - {{ - sap_hypervisor_node_preconfigure_ocp_kubeconfig_data | combine({ - 'users': sap_hypervisor_node_preconfigure_ocp_kubeconfig_data.users | map('combine', [{'user': {'token': __sap_vm_provision_register_ocp_auth_results.openshift_auth.api_key }}] ) - }, recursive=True) - }} - - - name: Write the updated kubeconfig - ansible.builtin.copy: - content: "{{ sap_hypervisor_node_preconfigure_ocp_kubeconfig_data | to_nice_yaml }}" - dest: "{{ __sap_hypervisor_node_preconfigure_register_kubeconfig }}" - mode: "0600" - - name: Invoke role with credentials set as environment variables delegate_to: "{{ inventory_hostname }}" delegate_facts: true environment: - KUBECONFIG: "{{ __sap_hypervisor_node_preconfigure_register_kubeconfig }}" - K8S_AUTH_KUBECONFIG: "{{ __sap_hypervisor_node_preconfigure_register_kubeconfig }}" + KUBECONFIG: "{{ sap_hypervisor_node_preconfigure_kubeconfig }}" + K8S_AUTH_KUBECONFIG: "{{ sap_hypervisor_node_preconfigure_kubeconfig }}" block: - name: Include sap_hypervisor_node_preconfigure Ansible Role diff --git a/playbooks/vars/sample-variables-sap-hypervisor-redhat-ocp-virt-preconfigure.yml b/playbooks/vars/sample-variables-sap-hypervisor-redhat-ocp-virt-preconfigure.yml index 57995756..73fc7cbb 100644 --- a/playbooks/vars/sample-variables-sap-hypervisor-redhat-ocp-virt-preconfigure.yml +++ b/playbooks/vars/sample-variables-sap-hypervisor-redhat-ocp-virt-preconfigure.yml @@ -6,28 +6,11 @@ # Red Hat OpenShift cluster connection details ########################################################### -# Admin username for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_username: - -# Admin password for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_password: - -# Path to kubeconfig file Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_kubeconfig_path: - -# If this is set to true, the API endpoint and the -# CA Certificate are extracted from the kubeconfig file. -# If set to false, sap_hypervisor_node_preconfigure_ocp_endpoint and -# sap_hypervisor_node_preconfigure_ocp_ca_cert have to be specified. -sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig: true - -# URL to the API endpoint of Red Hat OpenShift cluster -#sap_hypervisor_node_preconfigure_ocp_endpoint: - -# CA Certificate for Red Hat OpenShift cluster connection -# To extract the CA Cert from the kubeconfig, you can use -# grep certificate-authority-data ${KUBECONFIG} | awk '{ print $2 }' | base64 --decode > client-cert.pem -#sap_hypervisor_node_preconfigure_ocp_ca_cert: +# kubeconfig for Red Hat OpenShift cluster connection. +# Needs to contain a valid API token for trident storage operator to work. +# If not provided, the kubeconfig will be read from the environment variables +# KUBECONFIG or K8S_AUTH_KUBECONFIG +# sap_hypervisor_node_preconfigure_kubeconfig: ########################################################### @@ -47,12 +30,27 @@ sap_hypervisor_node_preconfigure_install_operators: true sap_hypervisor_node_preconfigure_setup_worker_nodes: true +################################################################################### +# Individual worker nodes config details, typically no need to change anything here. +# Requires sap_hypervisor_node_preconfigure_setup_worker_nodes to be set to true. +################################################################################### + +# Should the network of the nodes be configured? +sap_hypervisor_node_preconfigure_configure_worker_node_network: true + +# Should the cpumanager be enabled? This has to be set to true. +sap_hypervisor_node_preconfigure_enable_cpumanager: true + +# Should the kargs be applied to the worker nodes? +sap_hypervisor_node_preconfigure_enable_kargs: true + + ########################################################### # Configuration details ########################################################### # URL of the trident installer package to use -sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v24.06.0/trident-installer-24.06.0.tar.gz +sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v25.02.1/trident-installer-25.02.1.tar.gz # Allow unsupported NICs to be used for SR-IOV? sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: false @@ -72,6 +70,12 @@ sap_hypervisor_node_preconfigure_ocpv_namespace: openshift-cnv # Channel to be used for the Red Hat Openshift Virtualization Operator sap_hypervisor_node_preconfigure_ocpv_subscription_channel: stable +# Namespace to be used for the Red Hat Openshift Nmstate Operator +sap_hypervisor_node_preconfigure_nmstate_namespace: openshift-nmstate + +# Mountpoint for hostpath provisioner +sap_hypervisor_node_preconfigure_hpp_mountpoint: /var/localstorage + ########################################################### # Red Hat OpenShift cluster configuration details @@ -96,6 +100,10 @@ sap_hypervisor_node_preconfigure_cluster_config: password: xxxxx storage_driver: ontap-nas storage_prefix: ocpv_sap_ + nfs_mount_options: nfsvers=3 + export_policy: default + storageclass_name: sapstorage # Name of the storage class (careful, there can only be one SC with this name) + default_storageclass: true # Should this storage class be default? # CPU cores which will be reserved for kubernetes worker_kubernetes_reserved_cpus: "0,1" @@ -103,6 +111,12 @@ sap_hypervisor_node_preconfigure_cluster_config: # Storage device used for host path provisioner as local storage. worker_localstorage_device: /dev/vdb + # Name of the storage class for locastorage + worker_localstorage_storageclass_name: sapstorage + + # Should the local storage class be default? + worker_localstorage_default_storageclass: true + # detailed configuration for every worker that should be configured workers: @@ -124,39 +138,41 @@ sap_hypervisor_node_preconfigure_cluster_config: port: - name: ens1f0 # network IF name - - name: storage # an SR-IOV device - interface: ens2f0 # network IF name - type: sriov - - - name: ens2f0 # Set elevated MTU of 9000 - type: ethernet # on parent interface of - state: up # storagebridge - ipv4: - dhcp: false - enabled: false - mtu: 9000 - - - name: storagebridge # using a bridge - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.10 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - state: up - type: linux-bridge - - - name: multi # another SR-IOV device - interface: ens2f1 # network IF name - type: sriov +# Additional network interfaces, just shown here for reference +# - name: storage # an SR-IOV device +# interface: ens2f0 # network IF name +# type: sriov + +# - name: ens2f0 # Set elevated MTU of 9000 +# type: ethernet # on parent interface of +# state: up # storagebridge +# ipv4: +# dhcp: false +# enabled: false +# mtu: 9000 + +# - name: storagebridge # using a bridge +# bridge: # another bridge +# options: +# stp: +# enabled: false +# port: +# - name: ens2f0 # network IF name +# description: storage +# mtu: 9000 +# ipv4: +# address: +# - ip: 192.168.1.10 # IP config +# prefix-length: 24 +# auto-dns: false +# auto-gateway: false +# enabled: true +# state: up +# type: linux-bridge +# +# - name: multi # another SR-IOV device +# interface: ens2f1 # network IF name +# type: sriov - name: worker-1 # second worker configuration @@ -177,36 +193,38 @@ sap_hypervisor_node_preconfigure_cluster_config: port: - name: ens1f0 # network IF name - - name: storage # an SR-IOV device - interface: ens2f0 # network IF name - type: sriov - - - name: ens2f0 # Set elevated MTU of 9000 - type: ethernet # on parent interface of - state: up # storagebridge - ipv4: - dhcp: false - enabled: false - mtu: 9000 - - - name: storagebridge # create storage bridge - bridge: - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.11 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - state: up - type: linux-bridge - - - name: multi # another SR-IOV device - interface: ens2f1 # network IF name - type: sriov +# Additional network interfaces, just shown here for reference +# - name: storage # an SR-IOV device +# interface: ens2f0 # network IF name +# type: sriov +# +# - name: ens2f0 # Set elevated MTU of 9000 +# type: ethernet # on parent interface of +# state: up # storagebridge +# ipv4: +# dhcp: false +# enabled: false +# mtu: 9000 +# +# - name: storagebridge # create storage bridge +# bridge: +# options: +# stp: +# enabled: false +# port: +# - name: ens2f0 # network IF name +# description: storage +# mtu: 9000 +# ipv4: +# address: +# - ip: 192.168.1.11 # IP config +# prefix-length: 24 +# auto-dns: false +# auto-gateway: false +# enabled: true +# state: up +# type: linux-bridge +# +# - name: multi # another SR-IOV device +# interface: ens2f1 # network IF name +# type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 4091eb4d..3d63e048 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -103,23 +103,20 @@ Let's have a look at the most important variables you need to set. # Red Hat OpenShift cluster connection details ########################################################### -# Admin username for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_username: +# kubeconfig file Red Hat OpenShift cluster connection. +# Needs to contain a valid API token for trident storage operator to work. +# If not provided, the kubeconfig will be read from the environment variables +# KUBECONFIG or K8S_AUTH_KUBECONFIG +sap_hypervisor_node_preconfigure_kubeconfig: -# Admin password for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_password: - -# Path to kubeconfig file Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_kubeconfig_path: +``` +The `kubeconfig` configuration file has to be provided by either: -# If this is set to true, the API endpoint and the -# CA Certificate are extracted from the kubeconfig file. -# If set to false, sap_hypervisor_node_preconfigure_ocp_endpoint and -# sap_hypervisor_node_preconfigure_ocp_ca_cert have to be specified. -sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig: true +1. The Ansible variable `sap_hypervisor_node_kubeconfig`. +2. The environment variable `K8S_AUTH_KUBECONFIG`. +3. The environment variable `KUBECONFIG`. -``` -You need to provide username and password for the Red Hat OpenShift Cluster. The `kubeconfig` file can be specified in `sap_hypervisor_node_preconfigure_ocp_kubeconfig_path` or if omitted, the environment variable `K8S_AUTH_KUBECONFIG` has to point to it. Default is, to use the CA certificate and Red Hat OpenShift cluster API endpoint as specified in the `kubeconfig` file (controlled by variable `sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig`). Make sure to specify the username and password for the cluster: `sap_hypervisor_node_preconfigure_ocp_admin_username` and `sap_hypervisor_node_preconfigure_ocp_admin_password`. +If using the trident storage operator, the `kubeconfig` has also to contain a valid API token. Next are variables that define what storage configuration should be configured, if the operators should be installed and the configuration of the workers should be done. diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/main.yml new file mode 100644 index 00000000..1d5f4d3d --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/main.yml @@ -0,0 +1,5 @@ +--- +- name: Check if all trident config values are defined + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/assert/trident.yml" + when: sap_hypervisor_node_preconfigure_install_trident diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/trident.yml new file mode 100644 index 00000000..a5398028 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/assert/trident.yml @@ -0,0 +1,37 @@ +--- +- name: Define required trident keys + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_required_trident_keys: + - management + - data + - svm + - backend + - aggregate + - username + - password + - storage_driver + - storage_prefix + - nfs_mount_options + - export_policy + - storageclass_name + - default_storageclass + +- name: Initialize list of missing or empty trident keys + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_trident_missing_or_empty_keys: [] + +- name: Check each trident key for existence and non-empty value + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_trident_missing_or_empty_keys: "{{ __sap_hypervisor_node_preconfigure_trident_missing_or_empty_keys + [item] }}" + when: > + sap_hypervisor_node_preconfigure_cluster_config.trident[item] is not defined or + (sap_hypervisor_node_preconfigure_cluster_config.trident[item] | string | trim | length == 0) + loop: "{{ __sap_hypervisor_node_preconfigure_required_trident_keys }}" + +- name: Assert all trident keys are present and non-empty + ansible.builtin.assert: + that: + - __sap_hypervisor_node_preconfigure_trident_missing_or_empty_keys | length == 0 + fail_msg: >- + The following trident keys are missing or empty: + {{ __sap_hypervisor_node_preconfigure_trident_missing_or_empty_keys | join(', ') }} diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/enable-featuregate-downwardmetrics.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/enable-featuregate-downwardmetrics.yml new file mode 100644 index 00000000..c5fbc088 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/enable-featuregate-downwardmetrics.yml @@ -0,0 +1,13 @@ +--- +- name: Enable DownwardMetrics Feature Gate on kubevirt-hyperconverged + kubernetes.core.k8s: + state: present + definition: + apiVersion: hco.kubevirt.io/v1beta1 + kind: HyperConverged + metadata: + name: kubevirt-hyperconverged + namespace: "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" + spec: + featureGates: + downwardMetrics: true diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/wait-mcp-finished-updating.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/wait-mcp-finished-updating.yml new file mode 100644 index 00000000..55de648f --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/cluster/wait-mcp-finished-updating.yml @@ -0,0 +1,21 @@ +--- +- name: Wait for worker MachineConfigPool to start updating + kubernetes.core.k8s_info: + api_version: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + name: worker + ignore_errors: true + retries: 5 + delay: 60 + register: __sap_hypervisor_node_preconfigure_register_mcp_info + until: (__sap_hypervisor_node_preconfigure_register_mcp_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Updating'))[0].status == 'True' + +- name: Wait for worker MachineConfigPool to be updated + kubernetes.core.k8s_info: + api_version: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + name: worker + retries: 60 + delay: 60 + register: __sap_hypervisor_node_preconfigure_register_mcp_info + until: (__sap_hypervisor_node_preconfigure_register_mcp_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Updated'))[0].status == 'True' diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml deleted file mode 100644 index bbdbdfff..00000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Create SAP bridge NodeNetworkConfigurationPolicy - kubernetes.core.k8s: - state: present - definition: - apiVersion: nmstate.io/v1 - kind: NodeNetworkConfigurationPolicy - metadata: - name: "sap-bridge-policy-{{ worker.name }}" - spec: - nodeSelector: - kubernetes.io/hostname: "{{ worker.name }}" - desiredState: - interfaces: - - name: sapbridge - description: "Linux bridge with {{ worker.sap_bridge_interface }} as physical port to access SAP network" - type: linux-bridge - state: up - ipv4: - enabled: false - bridge: - options: - stp: - enabled: false - port: - - name: "{{ worker.sap_bridge_interface }}" - - -- name: Create SAP bridge NetworkAttachmentDefinition - kubernetes.core.k8s: - state: present - definition: - apiVersion: "k8s.cni.cncf.io/v1" - kind: NetworkAttachmentDefinition - metadata: - kubernetes.io/hostname: "{{ worker.name }}" - machineconfiguration.openshift.io/role: "{{ worker.name }}" - namespace: "{{ vm_namespace }}" - name: sap-bridge-network-definition - annotations: - k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/sapbridge - spec: - config: '{ - "cniVersion": "0.3.1", - "name": "sap-bridge-network-definition", - "type": "cnv-bridge", - "bridge": "sapbridge", - "macspoofchk": true - }' diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml deleted file mode 100644 index 57a52da2..00000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Label worker with invtsc flag - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: Namespace - metadata: - name: default - labels: - 'feature.node.kubernetes.io/cpu-feature-invtsc': enabled diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index cb982cd6..5c2501d1 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -1,4 +1,8 @@ --- +- name: Check configuration + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/assert/main.yml" + - name: Get a list of all nodes from any namespace kubernetes.core.k8s_info: kind: Node @@ -9,7 +13,7 @@ __sap_hypervisor_node_preconfigure_register_worker_node_name_list: "{{ __sap_hypervisor_node_preconfigure_register_worker_node_name_list | \ d([]) + [__sap_hypervisor_node_preconfigure_register_worker_node.name] }}" - with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" loop_control: loop_var: __sap_hypervisor_node_preconfigure_register_worker_node @@ -18,7 +22,7 @@ __sap_hypervisor_node_preconfigure_register_nodes: "{{ __sap_hypervisor_node_preconfigure_register_nodes | \ d([]) + [__sap_hypervisor_node_preconfigure_register_host] }}" - with_items: "{{ __sap_hypervisor_node_preconfigure_register_node_list['resources'] }}" + loop: "{{ __sap_hypervisor_node_preconfigure_register_node_list['resources'] }}" loop_control: loop_var: __sap_hypervisor_node_preconfigure_register_host when: __sap_hypervisor_node_preconfigure_register_host.metadata.name in __sap_hypervisor_node_preconfigure_register_worker_node_name_list @@ -52,57 +56,52 @@ # if system < 512GiB memory use 32GiB as upper boundary, 64GB otherwise as upper boundary - name: Calculate amount of hugepages to reserve (host memory < 512 GiB) ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int \ + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int \ - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int < 512 - name: Calculate amount of hugepages to reserve (host memory >= 512 GiB) ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int \ + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int \ - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int >= 512 - name: Include prepare - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/prepare.yml" + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/prepare.yml" - name: Include patch cluster masters unschedulable - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/patch-cluster-masters-unschedulable.yml" + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/master/patch-cluster-masters-unschedulable.yml" - name: Include tuned virtual host - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/tuned-virtual-host.yml" - -- name: Include install CNV operator - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/install-cnv-operator.yml" - when: sap_hypervisor_node_preconfigure_install_operators + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/worker/tuned-virtual-host.yml" -- name: Include install sriov operator - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/install-sriov-operator.yml" - when: sap_hypervisor_node_preconfigure_install_operators - -- name: Include install nmstate operator - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/install-nmstate-operator.yml" +- name: Include install operators + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/operators/main.yml" when: sap_hypervisor_node_preconfigure_install_operators - name: Include setup worker nodes - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/setup-worker-nodes.yml" + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/worker/setup-worker-nodes.yml" when: sap_hypervisor_node_preconfigure_setup_worker_nodes -- name: Wait for all k8s nodes to be ready - kubernetes.core.k8s_info: - kind: Node - wait: yes - wait_condition: - status: True - type: Ready - wait_sleep: 20 - wait_timeout: 600 +- name: Include task enable featuregate downwardmetrics + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/cluster/enable-featuregate-downwardmetrics.yml" - name: Include Trident installation - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/install-trident.yml" + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/storage/install-trident.yml" when: sap_hypervisor_node_preconfigure_install_trident - name: Include local storage creation (HPP) - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/install-hpp.yml" + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/storage/install-hpp.yml" when: sap_hypervisor_node_preconfigure_install_hpp - name: Remove temporary directory diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/patch-cluster-masters-unschedulable.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/master/patch-cluster-masters-unschedulable.yml similarity index 100% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/patch-cluster-masters-unschedulable.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/master/patch-cluster-masters-unschedulable.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-cnv-operator.yml similarity index 87% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-cnv-operator.yml index 83c21603..9f3e9d05 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-cnv-operator.yml @@ -46,7 +46,7 @@ namespace: "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" register: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name until: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.resources[0].status.installPlanRef.name is defined - changed_when: True + changed_when: true - name: Set Install Plan Name ansible.builtin.set_fact: @@ -84,6 +84,17 @@ delay: 60 until: hco_webhook_pod.resources | selectattr('status.phase', 'equalto', 'Running') | list | length == hco_webhook_pod.resources | length +- name: Wait and check for pod with name hco-webhook under "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" namespace + kubernetes.core.k8s_info: + kind: Pod + namespace: "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" + label_selectors: + - name=hco-webhook + register: __sap_hypervisor_node_preconfigure_register_hco_webhook_pod + retries: 5 + delay: 60 + until: __sap_hypervisor_node_preconfigure_register_hco_webhook_pod.resources | selectattr('status.phase', 'equalto', 'Running') | list | length == __sap_hypervisor_node_preconfigure_register_hco_webhook_pod.resources | length + - name: Wait and check if hco-webhook-service exists in "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" namespace kubernetes.core.k8s_info: kind: Service @@ -113,7 +124,7 @@ namespace: "{{ sap_hypervisor_node_preconfigure_ocpv_namespace }}" wait_condition: type: Available - status: "True" + status: true wait_timeout: 600 register: hyperconverged_status diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-nmstate-operator.yml similarity index 82% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-nmstate-operator.yml index 12f493a7..db9c171f 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-nmstate-operator.yml @@ -99,3 +99,14 @@ retries: 30 delay: 10 until: nmstate_status.resources | length > 0 + +- name: Wait and check for pod with label name nmstate-webhook under "{{ sap_hypervisor_node_preconfigure_nmstate_namespace }}" namespace + kubernetes.core.k8s_info: + kind: Pod + namespace: "{{ sap_hypervisor_node_preconfigure_nmstate_namespace }}" + label_selectors: + - name=nmstate-webhook + register: sap_hypervisor_node_preconfigure_register_webhook_pod + retries: 5 + delay: 60 + until: sap_hypervisor_node_preconfigure_register_webhook_pod.resources | selectattr('status.phase', 'equalto', 'Running') | list | length == sap_hypervisor_node_preconfigure_register_webhook_pod.resources | length diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-sriov-operator.yml similarity index 100% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/install-sriov-operator.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/main.yml new file mode 100644 index 00000000..eed42e0e --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/operators/main.yml @@ -0,0 +1,12 @@ +--- +- name: Include install CNV operator + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/operators/install-cnv-operator.yml" + +- name: Include install sriov operator + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/operators/install-sriov-operator.yml" + +- name: Include install nmstate operator + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/operators/install-nmstate-operator.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh deleted file mode 100644 index 6cec1a67..00000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# in order to allow unsupported SRIOV nics such as Mellanox -oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-hpp.yml similarity index 54% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-hpp.yml index d2f3e9bb..67b09697 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-hpp.yml @@ -1,4 +1,12 @@ --- +- name: Include worker node mkfs loop + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/storage/worker-mkfs-loop.yml" + loop: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + - name: Create systemd files for local storage handling kubernetes.core.k8s: state: present @@ -17,42 +25,22 @@ systemd: units: - contents: | - [Unit] - Description=Create mountpoint /var/localstorage and initialize filesystem - Before=var-localstorage.mount - [Service] - Type=oneshot - ExecStart=/bin/bash -c "if [[ $(lsblk -o FSTYPE {{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} --noheadings) != 'xfs' ]]; then mkfs.xfs -f {{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }}; fi" - ExecStart=/bin/mkdir -p /var/localstorage - enabled: true - name: create-mountpoint-var-localstorage.service - - contents: | - [Unit] - After=create-mountpoint-var-localstorage.service - Requires=create-mountpoint-var-localstorage.service - [Mount] - What={{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} - Where=/var/localstorage - Type=xfs [Install] WantedBy=local-fs.target - enabled: true - name: var-localstorage.mount - - contents: | [Unit] - Description=Set SELinux chcon for hostpath provisioner + Description=Create mountpoint {{ sap_hypervisor_node_preconfigure_hpp_mountpoint }}, mount and set selinux flag Before=kubelet.service - After=var-localstorage.mount [Service] - ExecStart=/usr/bin/chcon -Rt container_file_t /var/localstorage - [Install] - WantedBy=multi-user.target + Type=oneshot + ExecStart=/bin/mkdir -p {{ sap_hypervisor_node_preconfigure_hpp_mountpoint }} + ExecStart=mount {{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} {{ sap_hypervisor_node_preconfigure_hpp_mountpoint }} + ExecStart=/usr/bin/chcon -Rt container_file_t {{ sap_hypervisor_node_preconfigure_hpp_mountpoint }} enabled: true - name: hostpath-provisioner.service + name: create-mount-localstorage.service -- name: Wait for mountpoint to be ready - ansible.builtin.pause: - minutes: 3 +- name: Include wait mcp finished updating + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/cluster/wait-mcp-finished-updating.yml" - name: Create hostpath provisioner (HPP) kubernetes.core.k8s: @@ -66,7 +54,7 @@ imagePullPolicy: IfNotPresent storagePools: - name: localstorage - path: /var/localstorage + path: "{{ sap_hypervisor_node_preconfigure_hpp_mountpoint }}" workload: nodeSelector: kubernetes.io/os: linux @@ -79,7 +67,7 @@ wait: true wait_condition: type: Available - status: "True" + status: true reason: Complete wait_sleep: 30 wait_timeout: 300 @@ -91,9 +79,9 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: local + name: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_storageclass_name }}" annotations: - storageclass.kubernetes.io/is-default-class: "true" + storageclass.kubernetes.io/is-default-class: "{{ 'true' if sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_default_storageclass is true else 'false' }}" provisioner: kubevirt.io.hostpath-provisioner reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-trident.yml similarity index 86% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-trident.yml index 6cf235a6..416c8af4 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/install-trident.yml @@ -1,5 +1,5 @@ --- -- name: Download trident +- name: Download and extract trident ansible.builtin.unarchive: remote_src: true src: "{{ sap_hypervisor_node_preconfigure_install_trident_url }}" @@ -35,9 +35,9 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: nas + name: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storageclass_name }}" annotations: - storageclass.kubernetes.io/is-default-class: "true" + storageclass.kubernetes.io/is-default-class: "{{ 'true' if sap_hypervisor_node_preconfigure_cluster_config.trident.default_storageclass is true else 'false' }}" provisioner: csi.trident.netapp.io parameters: backendType: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/trident-backend.json.j2 similarity index 79% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/trident-backend.json.j2 index e422aab1..b92969bb 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/trident-backend.json.j2 @@ -1,8 +1,8 @@ { - "nfsMountOptions": "nfsvers=3", - "defaults": { - "exportPolicy": "default" - }, + "nfsMountOptions": "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.nfs_mount_options }}", + "defaults": { + "exportPolicy": "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.export_policy }}" + }, "debug":false, "managementLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.management }}", "dataLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.data }}", @@ -15,4 +15,3 @@ "storagePrefix":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_prefix }}", "version":1 } - diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/worker-mkfs-loop.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/worker-mkfs-loop.yml new file mode 100644 index 00000000..a5a87102 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/storage/worker-mkfs-loop.yml @@ -0,0 +1,32 @@ +--- +- name: Create pod that initializes filesystem + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Pod + metadata: + name: "make-filesystem-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + namespace: default + spec: + nodeName: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + containers: + - name: "make-filesystem-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + image: quay.io/fedora/fedora:latest + command: + - /bin/bash + - -c + - | + dnf install -y xfsprogs && \ + mkfs.xfs -f "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }}" + securityContext: + privileged: true + volumeMounts: + - mountPath: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }}" + name: rawdisk + volumes: + - name: rawdisk + hostPath: + path: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }}" + type: BlockDevice + restartPolicy: Never diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/99-kargs-worker.yml.j2 similarity index 100% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/99-kargs-worker.yml.j2 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-cpumanager.yml similarity index 73% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-cpumanager.yml index 50acdc0e..8b310eb3 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-cpumanager.yml @@ -1,11 +1,4 @@ --- -- name: Include configure worker - ansible.builtin.include_tasks: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/configure-worker-node.yml" - with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" - loop_control: - loop_var: __sap_hypervisor_node_preconfigure_register_worker - index_var: __sap_hypervisor_node_preconfigure_register_worker_nr - - name: Enable CPU Manager by patching MCP worker kubernetes.core.k8s: state: patched @@ -69,13 +62,6 @@ cpuManagerPolicy: static cpuManagerReconcilePeriod: 5s -- name: Render template - ansible.builtin.template: - src: 99-kargs-worker.yml.j2 - dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_role.path }}/99-kargs-worker.yml" - mode: "0644" - -- name: Enable hugepages - kubernetes.core.k8s: - state: present - src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_role.path }}/99-kargs-worker.yml" +- name: Include wait mcp finished updating + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/cluster/wait-mcp-finished-updating.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-kargs.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-kargs.yml new file mode 100644 index 00000000..6ee89292 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/enable-kargs.yml @@ -0,0 +1,15 @@ +--- +- name: Render template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_role.path }}/99-kargs-worker.yml" + mode: "0644" + +- name: Enable hugepages + kubernetes.core.k8s: + state: present + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir_role.path }}/99-kargs-worker.yml" + +- name: Include wait mcp finished updating + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/cluster/wait-mcp-finished-updating.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/node-network.yml similarity index 91% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/node-network.yml index b8fc1e41..9a1212bc 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/node-network.yml @@ -21,6 +21,7 @@ - "{{ __sap_hypervisor_node_preconfigure_register_worker_network }}" when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' or __sap_hypervisor_node_preconfigure_register_worker_network.type == 'ethernet' + or __sap_hypervisor_node_preconfigure_register_worker_network.type == 'vlan' - name: "Create NetworkAttachmentDefinition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" kubernetes.core.k8s: @@ -34,13 +35,13 @@ annotations: k8s.v1.cni.cncf.io/resourceName: "bridge.network.kubevirt.io/{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" spec: - config: '{ - "cniVersion": "0.3.1", - "name": "sapbridge-network-definition", - "type": "cnv-bridge", - "bridge": "sapbridge", - "macspoofchk": true - }' + config: "{{ { + 'cniVersion': '0.3.1', + 'name': __sap_hypervisor_node_preconfigure_register_worker_network.name, + 'type': 'cnv-bridge', + 'bridge': __sap_hypervisor_node_preconfigure_register_worker_network.name, + 'macspoofchk': true + } | to_json }}" when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' - name: Label the node with feature.node.kubernetes.io/network-sriov.capable=true diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/setup-worker-nodes.yml new file mode 100644 index 00000000..fdb14244 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/setup-worker-nodes.yml @@ -0,0 +1,19 @@ +--- +- name: Include worker node network loop + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/worker/worker-node-network-loop.yml" + loop: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + when: sap_hypervisor_node_preconfigure_configure_worker_node_network + +- name: Include enable cpumanager + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/worker/enable-cpumanager.yml" + when: sap_hypervisor_node_preconfigure_enable_cpumanager + +- name: Include enable kargs + ansible.builtin.include_tasks: + file: "platform/{{ sap_hypervisor_node_preconfigure_platform }}/worker/enable-kargs.yml" + when: sap_hypervisor_node_preconfigure_enable_kargs diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/tuned-virtual-host.yml similarity index 89% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/tuned-virtual-host.yml index e2dd4f48..642c05e7 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/tuned-virtual-host.yml @@ -11,8 +11,8 @@ spec: profile: - data: | - [main] - include=virtual-host + [main] + include=virtual-host name: virtual-host recommend: - match: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/worker-node-network-loop.yml similarity index 53% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/worker-node-network-loop.yml index 9680189e..3da13649 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/worker/worker-node-network-loop.yml @@ -1,18 +1,8 @@ --- - name: Include node network ansible.builtin.include_tasks: node-network.yml - with_items: "{{ __sap_hypervisor_node_preconfigure_register_worker.networks }}" + loop: "{{ __sap_hypervisor_node_preconfigure_register_worker.networks }}" loop_control: loop_var: __sap_hypervisor_node_preconfigure_register_worker_network index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined - -- name: Wait for all k8s nodes to be ready - kubernetes.core.k8s_info: - kind: Node - wait: true - wait_condition: - status: true - type: Ready - wait_sleep: 20 - wait_timeout: 600 diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index cc85822f..e7f08b94 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -6,28 +6,11 @@ # Red Hat OpenShift cluster connection details ########################################################### -# Admin username for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_username: - -# Admin password for Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_admin_password: - -# Path to kubeconfig file Red Hat OpenShift cluster connection -sap_hypervisor_node_preconfigure_ocp_kubeconfig_path: - -# If this is set to true, the API endpoint and the -# CA Certificate are extracted from the kubeconfig file. -# If set to false, sap_hypervisor_node_preconfigure_ocp_endpoint and -# sap_hypervisor_node_preconfigure_ocp_ca_cert have to be specified. -sap_hypervisor_node_preconfigure_ocp_extract_kubeconfig: true - -# URL to the API endpoint of Red Hat OpenShift cluster -#sap_hypervisor_node_preconfigure_ocp_endpoint: - -# CA Certificate for Red Hat OpenShift cluster connection -# To extract the CA Cert from the kubeconfig, you can use -# grep certificate-authority-data ${KUBECONFIG} | awk '{ print $2 }' | base64 --decode > client-cert.pem -#sap_hypervisor_node_preconfigure_ocp_ca_cert: +# kubeconfig for Red Hat OpenShift cluster connection. +# Needs to contain a valid API token for trident storage operator to work. +# If not provided, the kubeconfig will be read from the environment variables +# KUBECONFIG or K8S_AUTH_KUBECONFIG +# sap_hypervisor_node_preconfigure_kubeconfig: ########################################################### @@ -47,6 +30,21 @@ sap_hypervisor_node_preconfigure_install_operators: true sap_hypervisor_node_preconfigure_setup_worker_nodes: true +################################################################################### +# Individual worker nodes config details, typically no need to change anything here. +# Requires sap_hypervisor_node_preconfigure_setup_worker_nodes to be set to true. +################################################################################### + +# Should the network of the nodes be configured? +sap_hypervisor_node_preconfigure_configure_worker_node_network: true + +# Should the cpumanager be enabled? This has to be set to true. +sap_hypervisor_node_preconfigure_enable_cpumanager: true + +# Should the kargs be applied to the worker nodes? +sap_hypervisor_node_preconfigure_enable_kargs: true + + ########################################################### # Configuration details ########################################################### @@ -69,6 +67,12 @@ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: false # Namespace to be used for the Red Hat Openshift Virtualization Operator sap_hypervisor_node_preconfigure_ocpv_namespace: openshift-cnv +# Namespace to be used for the Red Hat Openshift Nmstate Operator +sap_hypervisor_node_preconfigure_nmstate_namespace: openshift-nmstate + +# Mountpoint for hostpath provisioner +sap_hypervisor_node_preconfigure_hpp_mountpoint: /var/localstorage + ########################################################### # Red Hat OpenShift cluster configuration details @@ -93,6 +97,10 @@ sap_hypervisor_node_preconfigure_cluster_config: password: xxxxx storage_driver: ontap-nas storage_prefix: ocpv_sap_ + nfs_mount_options: nfsvers=3 + export_policy: default + storageclass_name: sapstorage # Name of the storage class (careful, there can only be one SC with this name) + default_storageclass: true # Should this storage class be default? # CPU cores which will be reserved for kubernetes worker_kubernetes_reserved_cpus: "0,1" @@ -100,6 +108,12 @@ sap_hypervisor_node_preconfigure_cluster_config: # Storage device used for host path provisioner as local storage. worker_localstorage_device: /dev/vdb + # Name of the storage class for locastorage + worker_localstorage_storageclass_name: sapstorage + + # Should the local storage class be default? + worker_localstorage_default_storageclass: true + # detailed configuration for every worker that should be configured workers: