From 97e315879bf4fc435e0c5e3cce79afb133f9d456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Duarte?= Date: Sun, 7 Apr 2024 18:40:40 +0100 Subject: [PATCH 1/4] First cilium installation working --- ansible.cfg | 2 + deploy-playbook.yaml | 2 - k8s/templates/config-rke-additional.j2 | 1 + k8s/templates/config-rke-first.yaml | 1 + networking/templates/router-bird.conf.j2 | 2 +- services/cilium/bgp-peering-policy.yaml | 12 +++ services/cilium/deploy.sh | 12 +++ services/cilium/values.yaml | 102 +++++++++++++++++++++++ 8 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 ansible.cfg create mode 100644 services/cilium/bgp-peering-policy.yaml create mode 100755 services/cilium/deploy.sh create mode 100644 services/cilium/values.yaml diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..14c8065 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +host_key_checking = False diff --git a/deploy-playbook.yaml b/deploy-playbook.yaml index 810bc43..de69202 100644 --- a/deploy-playbook.yaml +++ b/deploy-playbook.yaml @@ -1,8 +1,6 @@ --- - name: Create a set new SSH key for clusters and routers ansible.builtin.import_playbook: networking/add-ssh-key-to-nodes-playbook.yaml -- name: Accept ssh keys for the first time - ansible.builtin.import_playbook: networking/accept-ssh-keys-playbook.yaml - name: Pre-setup - get correct interfaces ansible.builtin.import_playbook: networking/get-interface-playbook.yaml - name: Networking - Router BGP diff --git a/k8s/templates/config-rke-additional.j2 b/k8s/templates/config-rke-additional.j2 index ddfce3e..19f0782 100644 --- a/k8s/templates/config-rke-additional.j2 +++ b/k8s/templates/config-rke-additional.j2 @@ -3,3 +3,4 @@ server: https://10.11.11.1:9345 selinux: true tls-san: - 10.11.11.1 +cni: none diff --git a/k8s/templates/config-rke-first.yaml b/k8s/templates/config-rke-first.yaml index 49ab397..18ecfd2 100644 --- a/k8s/templates/config-rke-first.yaml +++ b/k8s/templates/config-rke-first.yaml @@ -1,3 +1,4 @@ selinux: true tls-san: - 10.11.11.1 +cni: none diff --git a/networking/templates/router-bird.conf.j2 b/networking/templates/router-bird.conf.j2 index 37f21d2..c25b49e 100644 --- a/networking/templates/router-bird.conf.j2 +++ b/networking/templates/router-bird.conf.j2 @@ -1,5 +1,5 @@ define myas = 65512; -router id 10.11.11.1; +router id 10.10.0.254; protocol device { scan time 10; diff --git a/services/cilium/bgp-peering-policy.yaml b/services/cilium/bgp-peering-policy.yaml new file mode 100644 index 0000000..83c358c --- /dev/null +++ b/services/cilium/bgp-peering-policy.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: niployments-bgp +spec: + virtualRouters: + - localASN: 65512 + exportPodCIDR: true + neighbors: + - peerAddress: "10.10.0.254/32" + peerASN: 65512 \ No newline at end of file diff --git a/services/cilium/deploy.sh b/services/cilium/deploy.sh new file mode 100755 index 0000000..9d8aad1 --- /dev/null +++ b/services/cilium/deploy.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +helm repo add cilium https://helm.cilium.io +helm repo update +helm upgrade --install cilium cilium/cilium \ + --version 1.15.3\ + --namespace kube-system\ + --values $(dirname $0)/values.yaml + +cilium-cli status --wait + +kubectl apply -f $(dirname $0)/bgp-peering-policy.yaml \ No newline at end of file diff --git a/services/cilium/values.yaml b/services/cilium/values.yaml new file mode 100644 index 0000000..b2fb64b --- /dev/null +++ b/services/cilium/values.yaml @@ -0,0 +1,102 @@ + +# @schema +# type: [null, string] +# @schema +# -- upgradeCompatibility helps users upgrading to ensure that the configMap for +# Cilium will not change critical values to ensure continued operation +# This flag is not required for new installations. +# For example: '1.7', '1.8', '1.9' +upgradeCompatibility: '1.15' +rbac: + # -- Enable creation of Resource-Based Access Control configuration. + create: true +k8sServicePort: "" +terminationGracePeriodSeconds: 1 +# -- Install the cilium agent resources. +agent: true +# -- Agent container name. +name: cilium +# -- Roll out cilium agent pods automatically when configmap is updated. +rollOutCiliumPods: true +nodeSelector: + kubernetes.io/os: linux +# -- Node tolerations for agent scheduling to nodes with taints +# ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ +tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" +# -- The priority class to use for cilium-agent. +priorityClassName: "" +# -- DNS policy for Cilium agent pods. +# Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy +dnsPolicy: "" +# -- Additional containers added to the cilium DaemonSet. +extraContainers: [] +# -- Additional agent container arguments. +extraArgs: [] +# -- Additional agent container environment variables. +extraEnv: [] +# -- Additional agent hostPath mounts. +extraHostPathMounts: [] +# - name: host-mnt-data +# mountPath: /host/mnt/data +# hostPath: /mnt/data +# hostPathType: Directory +# readOnly: true +# mountPropagation: HostToContainer + +# -- Additional agent volumes. +extraVolumes: [] +# -- Additional agent volumeMounts. +extraVolumeMounts: [] +# -- extraConfig allows you to specify additional configuration parameters to be +# included in the cilium-config configmap. +extraConfig: {} +# my-config-a: "1234" +# my-config-b: |- +# test 1 +# test 2 +# test 3 + +# -- Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) +annotations: {} +# -- Security Context for cilium-agent pods. +podSecurityContext: {} +# -- Annotations to be added to agent pods +podAnnotations: {} +# -- Labels to be added to agent pods +podLabels: {} +# -- Agent resource limits & requests +# ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +resources: {} +# limits: +# cpu: 4000m +# memory: 4Gi +# requests: +# cpu: 100m +# memory: 512Mi + +# -- resources & limits for the agent init containers +initResources: {} +# -- Cilium agent update strategy +updateStrategy: + type: RollingUpdate + rollingUpdate: + # @schema + # type: [integer, string] + # @schema + maxUnavailable: 2 +# Configuration Values for cilium-agent +# -- Annotate k8s node upon initialization with Cilium's metadata. +annotateK8sNode: false +# -- Configure BGP +# -- This feature set enables virtual BGP routers to be created via +# CiliumBGPPeeringPolicy CRDs. +bgpControlPlane: + # -- Enables the BGP control plane. + enabled: true + # -- Enable the BGPv2 APIs. + v2Enabled: false \ No newline at end of file From 3d02b481617210641150665cd317104a2cb25ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Duarte?= Date: Sun, 7 Apr 2024 20:53:14 +0100 Subject: [PATCH 2/4] Disable native kube-proxy in favor of cilium kube-proxy in eBPF --- k8s/templates/config-rke-additional.j2 | 1 + k8s/templates/config-rke-first.yaml | 1 + services/cilium/values.yaml | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/k8s/templates/config-rke-additional.j2 b/k8s/templates/config-rke-additional.j2 index 19f0782..d80988a 100644 --- a/k8s/templates/config-rke-additional.j2 +++ b/k8s/templates/config-rke-additional.j2 @@ -4,3 +4,4 @@ selinux: true tls-san: - 10.11.11.1 cni: none +disable-kube-proxy: "true" \ No newline at end of file diff --git a/k8s/templates/config-rke-first.yaml b/k8s/templates/config-rke-first.yaml index 18ecfd2..654c428 100644 --- a/k8s/templates/config-rke-first.yaml +++ b/k8s/templates/config-rke-first.yaml @@ -2,3 +2,4 @@ selinux: true tls-san: - 10.11.11.1 cni: none +disable-kube-proxy: "true" diff --git a/services/cilium/values.yaml b/services/cilium/values.yaml index b2fb64b..dd8e692 100644 --- a/services/cilium/values.yaml +++ b/services/cilium/values.yaml @@ -10,7 +10,6 @@ upgradeCompatibility: '1.15' rbac: # -- Enable creation of Resource-Based Access Control configuration. create: true -k8sServicePort: "" terminationGracePeriodSeconds: 1 # -- Install the cilium agent resources. agent: true @@ -99,4 +98,7 @@ bgpControlPlane: # -- Enables the BGP control plane. enabled: true # -- Enable the BGPv2 APIs. - v2Enabled: false \ No newline at end of file + v2Enabled: false +kubeProxyReplacement: true +k8sServiceHost: 10.11.11.1 +k8sServicePort: 6443 \ No newline at end of file From 3aad4fe94d348be07fcf6bc22747015a473b0009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Duarte?= Date: Tue, 9 Apr 2024 16:09:00 +0100 Subject: [PATCH 3/4] Swtich to cilium on kind, add LB Pool --- dev/setup-kind-cluster.sh | 67 ++++++++++++--------- dev/test-cluster.kind.yaml | 3 + services/cilium/load-balancer-pool-dev.yaml | 8 +++ services/cilium/load-balancer-pool.yaml | 8 +++ services/cilium/values.yaml | 8 ++- 5 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 services/cilium/load-balancer-pool-dev.yaml create mode 100644 services/cilium/load-balancer-pool.yaml diff --git a/dev/setup-kind-cluster.sh b/dev/setup-kind-cluster.sh index 7cce011..4bca0a7 100755 --- a/dev/setup-kind-cluster.sh +++ b/dev/setup-kind-cluster.sh @@ -2,11 +2,15 @@ KIND_EXECUTABLE=kind KUBECTL_EXECUTABLE=kubectl +CILIUM_EXECUTABLE=cilium-cli +HELM_EXECUTABLE=helm # first check if the kind executable exists command -v $KIND_EXECUTABLE >/dev/null 2>&1 || { echo >&2 "I require '$KIND_EXECUTABLE' but it's not installed. Aborting."; exit 1; } command -v $KUBECTL_EXECUTABLE >/dev/null 2>&1 || { echo >&2 "I require '$KUBECTL_EXECUTABLE' but it's not installed. Aborting."; exit 1; } +command -v $CILIUM_EXECUTABLE >/dev/null 2>&1 || { echo >&2 "I require '$CILIUM_EXECUTABLE' but it's not installed. Aborting."; exit 1; } +command -v $HELM_EXECUTABLE >/dev/null 2>&1 || { echo >&2 "I require '$CILIUM_EXECUTABLE' but it's not installed. Aborting."; exit 1; } # Create "kind" network, deleting any old ones if they exist @@ -29,33 +33,40 @@ docker network create "$KIND_NETWORK_NAME" \ $KIND_EXECUTABLE create cluster --config "$(dirname "$0")"/test-cluster.kind.yaml -# install MetalLB so services are assigned an IP address on creation. - -$KUBECTL_EXECUTABLE apply -f https://github.com/metallb/metallb/raw/main/config/manifests/metallb-native.yaml - -$KUBECTL_EXECUTABLE wait --namespace metallb-system \ - --for=condition=ready pod \ - --selector=app=metallb \ - --timeout=120s - -echo "\ -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool +# deploy cilium + +$HELM_EXECUTABLE repo add cilium https://helm.cilium.io +$HELM_EXECUTABLE repo update +$HELM_EXECUTABLE upgrade --install cilium cilium/cilium \ + --version 1.15.3\ + --namespace kube-system\ + --values $(dirname $0)/../services/cilium/values.yaml\ + --set k8sServiceHost=niployments-test-cluster-external-load-balancer\ + --set k8sServicePort=6443\ + --set bgpControlPlane.enabled=false\ + --set l2announcements.enabled=true\ + --set ipam.mode=kubernetes\ + --set ipv4NativeRoutingCIDR=172.28.0.0/16\ + --set enableIPv4Masquerade=true\ + --set autoDirectNodeRoutes=true\ + --set routingMode=native + +$CILIUM_EXECUTABLE status --wait + +$KUBECTL_EXECUTABLE apply -f $(dirname $0)/../services/cilium/load-balancer-pool-dev.yaml + +cat < Date: Thu, 11 Apr 2024 15:40:51 +0100 Subject: [PATCH 4/4] fix: remove router default management route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Luís Duarte --- networking/router-setup-playbook.yaml | 15 +++++++++++++++ networking/templates/01-vagrant.network.j2 | 8 ++++++++ 2 files changed, 23 insertions(+) create mode 100644 networking/templates/01-vagrant.network.j2 diff --git a/networking/router-setup-playbook.yaml b/networking/router-setup-playbook.yaml index 5eac0ac..390b11d 100644 --- a/networking/router-setup-playbook.yaml +++ b/networking/router-setup-playbook.yaml @@ -3,6 +3,21 @@ - name: Networking - Router basic setup hosts: routers tasks: + - name: Disable vagrant default route on dev cluster + when: dev_cluster == 'true' + block: + - name: Copy template for vagrant systemd config + become: true + ansible.builtin.template: + src: templates/01-vagrant.network.j2 + dest: /etc/systemd/network/01-vagrant.network + mode: "644" + - name: Restart network service on VM + become: true + ansible.builtin.systemd: + name: systemd-networkd + state: restarted + - name: Enable Layer 3 forwarding become: true ansible.posix.sysctl: diff --git a/networking/templates/01-vagrant.network.j2 b/networking/templates/01-vagrant.network.j2 new file mode 100644 index 0000000..4ff9311 --- /dev/null +++ b/networking/templates/01-vagrant.network.j2 @@ -0,0 +1,8 @@ +[Match] +Name=eth0 + +[Network] +DHCP=yes + +[DHCP] +UseRoutes=false \ No newline at end of file