Router VM setup on physical cluster

Vagrantfile

@@ -30,9 +30,10 @@ end
 
 def configure_router(i, config)
     config.vm.define "router#{i}" do |router|
-        router.vm.box = "generic/debian11"
+        router.vm.box = "generic/debian12"
+        router.vm.provision "shell", reboot: true, inline: "sudo systemctl enable systemd-networkd.service"
         lip = $ip.clone
-        router.vm.provision "shell", reboot: true, path:"dev/router-networking.sh", args: [lip]
+        router.vm.provision "shell", reboot: true, path:"dev/router-networking.sh", args: [lip, $host_only.to_s]
         if $host_only == false then
             if $bridge_interface != nil then
                     router.vm.network "public_network",
@@ -59,7 +60,7 @@ def configure_router(i, config)
         end
 
         configure_ram(router, $router_ram)
-        configure_private_network(router, true)
+        configure_private_network(router, false)
         router.vm.provision "shell" do |s|
             s.inline = "hostnamectl set-hostname $1"
             s.args = ["router"+i.to_s]
@@ -70,8 +71,21 @@ end
 def configure_cluster_node(i, config)
     config.vm.define "cluster#{i}" do |clustervm|
         clustervm.vm.box = "NIAEFEUP/rocky-NInux"
-        clustervm.vm.box_version = "0.4.1"
+        clustervm.vm.box_version = "0.5.1"
         lip = $ip.clone
+
+        # We enable nested virtualization for vm build tests in vagrant
+        clustervm.vm.provider "virtualbox" do |vb|
+            vb.customize ['modifyvm', :id, '--nested-hw-virt', 'on']
+        end
+
+        clustervm.vm.provider :libvirt do |libvirt|
+            # Enable KVM nested virtualization
+            libvirt.nested = true
+            libvirt.cpu_mode = "host-model"
+        end
+
+
         clustervm.vm.provision "shell" do |s|
             s.path = "dev/node-networking.sh"
             s.args = [lip]

ansible-inventory.example.ini

@@ -4,10 +4,11 @@
 
 [routers]
 #while unusual, you can define multiple routers.
-#You MUST always have only 1 router that is master, for VRRP 
+#You MUST always have only 1 router that is master, for VRRP.
+# while bootstrapping you MUST leave this group out, but you will be asked to fill it in.
 
-#10.0.0.1 master=true
-#10.0.0.2 master=false
+#router1 ansible_ssh_host=10.0.0.1 master=true
+#router2 ansible_ssh_host=10.0.0.2 master=false
 
 [controlplane]
 # These are the kind of nodes that are responsible for managing
@@ -17,10 +18,13 @@
 # if you wish, you can specify an alias for a node, or you can just specify
 # the ip address as shown below:
 
-#node1 ansible_ssh_host=10.0.0.2
+# you need to always define the ansible_ssh_host and ansible_ssh_private_key because they will be changed automatically
+# you maybe to need to define the external interface (that will be given to the router, alongside the whole PCI device). 
+
+
+#node1 ansible_ssh_host=10.0.0.2 ansible_ssh_private_key=/path/to/private_key
+#node1 ansible_ssh_host=10.0.0.2 ansible_ssh_private_key=/path/to/private_key external_interface=enp1f0
 
-#10.0.0.3
-#10.0.0.4
 
 [workers]
 # These kinds of nodes are also connected to Kubernetes cluster,

deploy-playbook.yaml

@@ -1,13 +1,44 @@
 ---
+- name: Install Router VMs
+  ansible.builtin.import_playbook: router/build-router-vm-playbook.yaml
 - name: Create a set new SSH key for clusters and routers
   ansible.builtin.import_playbook: networking/add-ssh-key-to-nodes-playbook.yaml
-- name: Accept ssh keys for the first time
-  ansible.builtin.import_playbook: networking/accept-ssh-keys-playbook.yaml
+- name: Networking - Setup static internal IPs
+  ansible.builtin.import_playbook: networking/dhcp-server-config-playbook.yaml
+- name: Wait for connection
+  hosts: all
+  connection: local
+  gather_facts: false
+  tasks:
+    - name: Wait for nodes to change dhcp address
+      ansible.builtin.wait_for:
+        port: 22
+        host: '{{ (ansible_ssh_host | default(ansible_host)) | default(inventory_hostname) }}'
+        search_regex: OpenSSH
+        delay: 10
+        timeout: 120
+- name: Pre-setup - enable NTP syncronization
+  hosts: all
+  tasks:
+    - name: "Enable NTP client"
+      become: true
+      ansible.builtin.command: /usr/bin/timedatectl set-ntp on
+      changed_when: true
+    - name: "Switch to UTC timezone"
+      become: true
+      community.general.timezone:
+        name: "UTC"
 - name: Pre-setup - get correct interfaces
   ansible.builtin.import_playbook: networking/get-interface-playbook.yaml
+- name: Pre-setup - Node re-enable NetworkManager
+  ansible.builtin.import_playbook: node/reenable-networkmanager-playbook.yaml
+- name: Networking - Router initial config
+  ansible.builtin.import_playbook: networking/router-setup-playbook.yaml
 - name: Networking - Router BGP
   ansible.builtin.import_playbook: networking/router-bgp-playbook.yaml
 - name: Networking - VRRP
   ansible.builtin.import_playbook: networking/router-vrrp-playbook.yaml
 - name: Networking - Router Controlplane HA
   ansible.builtin.import_playbook: networking/controlplane-ha-playbook.yaml
+- name: Nodes - Enable EPEL repositories
+  ansible.builtin.import_playbook: node/add-epel-repos-playbook.yaml

dev/node-networking.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
-sudo ip r del 0.0.0.0
-sudo nmcli device modify ens5 ipv4.never-default yes
-sudo nmcli con add type ethernet con-name main-network ifname ens6 ip4 10.10.0.$1/24 \
-    gw4 10.10.0.254
-sudo nmcli con up main-network ifname ens6
+
+#TODO(luis): remove static ip configuration when dhcp server can be configured, to better replicate the physical node configuration
+# sudo systemctl disable dhcpcd
+# sudo systemctl enable --now NetworkManager
+sudo ip r del default || true
+# sudo nmcli device modify ens5 ipv4.never-default yes
+# sudo nmcli con add type ethernet con-name main-network ifname ens6 ip4 10.10.0.$1/24 \
+#     gw4 10.10.0.254
+# sudo nmcli con up main-network ifname ens6

dev/router-networking.sh

@@ -1,21 +1,42 @@
 #!/bin/sh
+apt-get purge -y ifupdown 
 echo "net.ipv4.ip_forward = 1" > /etc/sysctl.d/10-router.conf
 
+echo "[Match]
+Name=eth0
+
+[Network]
+DHCP=yes
+DefaultRouteOnDevice=false
+" > /etc/systemd/network/01-vagrant.network
+
+if ["$2" -eq "true"]; then
+echo "Configuring host-only"
+echo "[Match]
+Name=eth1
+
+[Network]
+Address=10.69.0.2/24
+Gateway=10.69.0.1
+DefaultRouteOnDevice=true
+" > /etc/systemd/network/00-external.network
+else 
+echo "Public network... fallback to dhcp"
+fi
+echo "[Match]
+Name=*
+
+[Network]
+DHCP=yes
+
+[Network]
+LinkLocalAddressing=yes
+IPv4LLRoute=true" > /etc/systemd/network/99-default-ipv4ll.network
+
 echo "
-allow-hotplug eth0
-auto lo
-iface lo inet loopback
-iface eth0 inet dhcp
-    post-up ip route del default dev eth0 || true
-
-auto eth2
-iface eth2 inet static
-    address 10.10.0.$1
-    netmask 255.255.255.0
-" >> /etc/network/interfaces
-
-nft add table nat
-nft add chain nat postrouting { type nat hook postrouting priority 100 \; }
-nft add rule nat postrouting ip saddr 10.10.0.0/24 oif eth1 masquerade
-nft list ruleset > /etc/nftables.conf
-systemctl enable nftables
+nameserver 1.1.1.1
+" >> /etc/resolvconf/resolv.conf.d/tail
+
+apt-get install -y avahi-daemon avahi-utils avahi-autoipd
+
+sed -i 's/publish-workstation=no/publish-workstation=yes/g' /etc/avahi/avahi-daemon.conf 

networking/add-ssh-key-to-nodes-playbook.yaml

@@ -39,7 +39,7 @@
       when: inventory_hostname | regex_replace('\d', '') == 'cluster'
       throttle: 1
       block:
-        - name: Propagate the publicg key
+        - name: Propagate the public key
           become: true
           ansible.posix.authorized_key:
             user: ni
@@ -50,16 +50,16 @@
           connection: local
           ansible.builtin.lineinfile:
             path: '{{ inventory_dir }}/ansible-inventory-dev.ini'
-            regexp: '^{{ inventory_hostname }}(.*)ansible_ssh_private_key_file=(.*)( .*)?'
-            line: '{{ inventory_hostname }}\1ansible_ssh_private_key_file={{ playbook_dir | dirname }}/.ssh/new_key\3'
+            regexp: '^{{ inventory_hostname }}(.*)ansible_ssh_private_key_file=(\S*) *(.*)'
+            line: '{{ inventory_hostname }}\1ansible_ssh_private_key_file={{ playbook_dir | dirname }}/.ssh/new_key \3'
             backrefs: true
 
     - name: Add the key and mofidy inventory if router
       when: inventory_hostname | regex_replace('\d', '') == 'router'
       block:
         - name: Propagate public key to this node
           ansible.posix.authorized_key:
-            user: vagrant
+            user: "{{ 'vagrant' if dev_cluster == 'true' else 'ni' }}"
             state: present
             key: "{{ new_ssh_key.stdout }}"
             exclusive: "{{ not dev_cluster }}"
@@ -69,7 +69,7 @@
           connection: local
           ansible.builtin.lineinfile:
             path: '{{ inventory_dir }}/ansible-inventory-dev.ini'
-            regexp: '^{{ inventory_hostname }}(.*)ansible_ssh_private_key_file=(.*) +(.*)'
+            regexp: '^{{ inventory_hostname }}(.*)ansible_ssh_private_key_file=(\S*) *(.*)'
             line: '{{ inventory_hostname }}\1ansible_ssh_private_key_file={{ playbook_dir | dirname }}/.ssh/new_key \3'
             backrefs: true