Refactor vagrantfile and make vrrp work properly

This also refactors a bit of the ansible logic, handles multiple router setup and tries to accept all ssh fingerprints
NIAEFEUP · Jan 3, 2024 · 245f1fe · 245f1fe
1 parent 1bbebc8
commit 245f1fe
Show file tree

Hide file tree

Showing 14 changed files with 125 additions and 45 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -1,11 +1,12 @@
 require 'yaml'
 
 config = File.exists?('local-dev-cluster.yaml') ? YAML.load_file('local-dev-cluster.yaml') : YAML.load_file('dev-cluster.yaml')
-cluster_vm_ram = config["cluster"]["node"]["ram"]
+$cluster_vm_ram = config["cluster"]["node"]["ram"]
 num_of_nodes = config["cluster"]["nodeCount"]
-router_ram = config["router"]["ram"]
-bridge_interface = config["networking"] == nil ? nil : config["networking"]["bridgeInterface"]
-host_only = config["networking"] == nil ? false : (config["networking"]["hostOnly"] || false)
+$router_ram = config["router"]["ram"]
+router_count = config["router"]["count"] || 1 
+$bridge_interface = config["networking"] == nil ? nil : config["networking"]["bridgeInterface"]
+$host_only = config["networking"] == nil ? false : (config["networking"]["hostOnly"] || false)
 $ip = 2 # start with 2 because virtualbox adapter makes 10.10.0.1 reserved for the host 
 
 def configure_ram(vm, ram)
@@ -27,47 +28,70 @@ def configure_private_network(vm, auto_config)
     $ip = $ip + 1
 end
 
-Vagrant.configure("2") do |config|
-    config.vm.synced_folder '.', '/vagrant', disabled: true
-    config.vm.define "router" do |router|
+def configure_router(i, config)
+    config.vm.define "router#{i}" do |router|
         router.vm.box = "generic/debian11"
-        if host_only == false then
-            if bridge_interface != nil then
+        lip = $ip.clone
+        router.vm.provision "shell", reboot: true, path:"dev/router-networking.sh", args: [lip]
+        if $host_only == false then
+            if $bridge_interface != nil then
                     router.vm.network "public_network",
-                    :dev => bridge_interface
+                    :dev => $bridge_interface
             else
                 router.vm.network "public_network"
             end
         else
             router.vm.network "private_network", 
+            virtualbox__intnet: "outgoing_network",
             :libvirt__forward_mode => "nat",
             :libvirt__network_name => "outgoing",
             :libvirt__host_ip => "10.69.0.1",
-            :ip => "10.69.0.2",
+            :ip => "10.69.0."+ (i+1).to_s,
             :libvirt__dhcp_enable => false
 
+            router.vm.provision "shell",
+            run: "always",
+            inline: "ip r add default via 10.69.0.1" 
+
             #NOTE (luisd): i think virtualbox doesnt have this problem
             # it mostly applies to wireless configs or  you don't want to
             # expose the router to your network
         end
 
-        configure_ram(router, router_ram)
+        configure_ram(router, $router_ram)
         configure_private_network(router, true)
-
-        router.vm.provision "shell", reboot: true, path:"dev/router-networking.sh"
-
+        router.vm.provision "shell" do |s|
+            s.inline = "hostnamectl set-hostname $1"
+            s.args = ["router"+i.to_s]
+        end
     end
-    for i in 1..num_of_nodes do 
-        config.vm.define "cluster#{i}" do |clustervm|
-            clustervm.vm.box = "generic/ubuntu2204" 
-            clustervm.vm.provision "shell" do |s|
-                s.path = "dev/node-networking.sh"
-                s.args = [$ip]
-            end
-            configure_ram(clustervm, cluster_vm_ram)
-            configure_private_network(clustervm, false)
-
+end
+
+def configure_cluster_node(i, config)
+    config.vm.define "cluster#{i}" do |clustervm|
+        clustervm.vm.box = "generic/ubuntu2204"
+        lip = $ip.clone
+        clustervm.vm.provision "shell" do |s|
+            s.path = "dev/node-networking.sh"
+            s.args = [lip]
         end
+
+        clustervm.vm.provision "shell" do |s|
+            s.inline = "hostnamectl set-hostname $1"
+            s.args = ["cluster"+i.to_s]
+        end
+        configure_ram(clustervm, $cluster_vm_ram)
+        configure_private_network(clustervm, false)
+
     end
+end
 
-end
+Vagrant.configure("2") do |config|
+    config.vm.synced_folder '.', '/vagrant', disabled: true
+    for i in 1..router_count do
+        configure_router(i, config)
+    end
+    for i in 1..num_of_nodes do 
+        configure_cluster_node(i, config)
+    end
+end
diff --git a/ansible-inventory.example.ini b/ansible-inventory.example.ini
@@ -34,4 +34,5 @@ controlplane
 workers
 
 [all:vars]
-dev_cluster=false
+dev_cluster=false
+ansible_python_interpreter="/usr/bin/env python3"
diff --git a/deploy-playbook.yaml b/deploy-playbook.yaml
@@ -1,4 +1,6 @@
 ---
+- name: Accept ssh keys for the first time
+  ansible.builtin.import_playbook: networking/accept-ssh-keys-playbook.yaml
 - name: Pre-setup - get correct interfaces
   ansible.builtin.import_playbook: networking/get-interface-playbook.yaml
 - name: Networking - Router BGP

diff --git a/dev/generate-ansible-inventory.sh b/dev/generate-ansible-inventory.sh
@@ -85,6 +85,6 @@ then
 INVENTORY+="\n[workers]\n"
 fi
 INVENTORY+="\n[nodes:children]\ncontrolplane\nworkers\n
-[all:vars]\ndev_cluster=true"
+[all:vars]\ndev_cluster=true\nansible_python_interpreter='/usr/bin/env python3'"
 
 echo -e $INVENTORY > "ansible-inventory-dev.ini"
diff --git a/dev/node-networking.sh b/dev/node-networking.sh
@@ -17,7 +17,7 @@ echo "
                 addresses: [10.10.0.$1/24]
                 routes:
                 -   to: default
-                    via: 10.10.0.2 
+                    via: 10.10.0.254 
                     metric: 0
                 nameservers:
                     addresses: [1.1.1.1, 1.0.0.1]

diff --git a/dev/router-networking.sh b/dev/router-networking.sh
@@ -1,22 +1,18 @@
 #!/bin/sh
 echo "net.ipv4.ip_forward = 1" > /etc/sysctl.d/10-router.conf
 
-echo '
+echo "
 allow-hotplug eth0
 auto lo
 iface lo inet loopback
-auto eth1
 iface eth0 inet dhcp
-    post-up ip route del default dev $IFACE || true
-
-auto eth1
-iface eth1 inet dhcp
+    post-up ip route del default dev eth0 || true
 
 auto eth2
 iface eth2 inet static
-    address 10.10.0.2
+    address 10.10.0.$1
     netmask 255.255.255.0
-' > /etc/network/interfaces
+" >> /etc/network/interfaces
 
 nft add table nat
 nft add chain nat postrouting { type nat hook postrouting priority 100 \; }

diff --git a/networking/accept-ssh-keys-playbook.yaml b/networking/accept-ssh-keys-playbook.yaml
@@ -0,0 +1,10 @@
+- name: Trust all ssh hosts if they don't exist yet
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Accept SSH key for each host
+      connection: local
+      ansible.builtin.known_hosts:
+        state: present
+        name: "{{ hostvars[inventory_hostname]['ansible_ssh_host'] }}"
+        key: "{{ lookup('pipe', 'ssh-keyscan -T 10 -H -t ssh-ed25519 ' + hostvars[inventory_hostname]['ansible_ssh_host']) }}"
diff --git a/networking/controlplane-ha-playbook.yaml b/networking/controlplane-ha-playbook.yaml
@@ -11,8 +11,17 @@
         src: templates/router-haproxy.cfg.j2
         dest: /etc/haproxy/haproxy.cfg
         mode: "644"
+    - name: Enable HAProxy
+      become: true
+      when: hostvars[inventory_hostname]['master'] == 'true'
+      ansible.builtin.systemd:
+        service: haproxy
+        enabled: true
     - name: Restart HAProxy
       become: true
+      when: hostvars[inventory_hostname]['master'] == 'true'
       ansible.builtin.systemd:
         service: haproxy
         state: restarted
+      retries: 3
+      delay: 5
diff --git a/networking/router-bgp-playbook.yaml b/networking/router-bgp-playbook.yaml
@@ -21,7 +21,14 @@
         src: templates/router-bird.conf.j2
         dest: /etc/bird/bird.conf
         mode: "644"
+    - name: Enable bird2
+      when: hostvars[inventory_hostname]['master'] == 'true'
+      become: true
+      ansible.builtin.systemd:
+        name: bird
+        enabled: true
     - name: Restart bird2
+      when: hostvars[inventory_hostname]['master'] == 'true'
       become: true
       ansible.builtin.systemd:
         name: bird

diff --git a/networking/router-vrrp-playbook.yaml b/networking/router-vrrp-playbook.yaml
@@ -11,6 +11,12 @@
       become: true
       ansible.builtin.apt:
         name: keepalived
+    - name: Copy notify script
+      become: true
+      ansible.builtin.copy:
+        src: templates/vrrp-notify-script.sh
+        dest: /etc/keepalived/vrrp-notify-script.sh
+        mode: "777"
     - name: Configure Keepalived
       become: true
       ansible.builtin.template:

diff --git a/networking/templates/keepalived.conf.j2 b/networking/templates/keepalived.conf.j2
@@ -2,14 +2,15 @@ vrrp_instance VI_1 {
         state {{ "MASTER" if hostvars[inventory_hostname]["master"] == "true" else "BACKUP"}}
         interface {{ hostvars[inventory_hostname]["target_interface"] }}
         virtual_router_id 51
-        priority 254
+        priority {{250 if hostvars[inventory_hostname]["master"] == "true" else 240}}
         advert_int 1
         authentication {
-              auth_type PASS
-              auth_pass {{ hostvars[inventory_hostname]["vrrp_secret"] }}
+            auth_type PASS
+            auth_pass {{ hostvars[inventory_hostname]["vrrp_secret"] }}
         }
         virtual_ipaddress {
-              10.11.11.1/24
-              10.11.11.2/24
+            10.11.11.1/24 brd 10.11.11.255
+            10.10.0.254/24 brd 10.10.0.255
         }
+        notify "/etc/keepalived/vrrp-notify-script.sh"
 }
diff --git a/networking/templates/router-bird.conf.j2 b/networking/templates/router-bird.conf.j2
@@ -18,15 +18,16 @@ protocol static {
   ipv4;
 }
 
-{% for nodename in groups["controlplane"]%}
+{% for nodename in groups["nodes"]%}
 protocol bgp {{ nodename }}_bgp{
-  local 10.11.11.1 as myas;
+  local 10.10.0.254 as myas;
   neighbor {{ hostvars[nodename]["ansible_"~
     hostvars[nodename]["ansible_facts"]["target_interface"]]['ipv4']['address']
   }} as myas;
   direct;
   ipv4 {
     import all;
+    export all;
   };
 }
 

diff --git a/networking/templates/router-haproxy.cfg.j2 b/networking/templates/router-haproxy.cfg.j2
@@ -31,7 +31,7 @@ defaults
 # apiserver frontend which proxys to the control plane nodes
 #---------------------------------------------------------------------
 frontend apiserver
-    bind 10.11.11.2:6443
+    bind 10.11.11.1:6443
     mode tcp
     option tcplog
     default_backend apiserver

diff --git a/networking/templates/vrrp-notify-script.sh b/networking/templates/vrrp-notify-script.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+TYPE=$1
+NAME=$2
+STATE=$3
+
+case $STATE in
+        "MASTER") systemctl start bird
+                  systemctl start haproxy
+                  exit 0
+                  ;;
+        "BACKUP") systemctl stop bird
+                  systemctl stop haproxy
+                  exit 0
+                  ;;
+        "FAULT")  systemctl stop bird
+                  systemctl stop haproxy
+                  exit 0
+                  ;;
+        *)        echo "unknown state"
+                  exit 1
+                  ;;
+esac