fix apt locks

k8s/kubeadm_config.yaml

@@ -39,3 +39,4 @@ networking:
   dnsDomain: cluster.local
   podSubnet: ${pod_cidr}
 scheduler: {}
+

k8s/main.tf

@@ -24,6 +24,21 @@ resource "null_resource" "k8s" {
     private_key = file(format("/home/%s/.ssh/id_rsa", var.ssh_user))
   }
 
+  provisioner "local-exec" {
+    # Loop through the list of IP addresses and ping each one.
+    command = <<-EOT
+          for ip in "${join("\" \"", var.node_list)}"; do
+            until ping -c1 -W 1 "$ip"; do sleep 5; done
+          done
+        EOT
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "cloud-init status --wait"
+    ]
+  }
+
   provisioner "file" {
 
     content = templatefile("${path.module}/kubeadm_config.yaml", {

k8s/master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -xe
 
 # Initialize the Kubernetes cluster with the provided configuration file
 sudo kubeadm init --config /tmp/kubeadm_config.yaml

k8s/node.sh

@@ -39,9 +39,9 @@ function wait_for_api_server() {
 
 if ! grep -qa container=lxc /proc/1/environ; then
     enable_ip_forwarding
-    set_huge_pages "${nr_hugepages}"
-    sudo apt-get -y install linux-modules-extra-$(uname -r)
-    add_kernel_modules nvme-tcp nvmet
+    #set_huge_pages "${nr_hugepages}"
+    #sudo apt-get -y install linux-modules-extra-$(uname -r)
+    #add_kernel_modules nvme-tcp nvmet
 fi
 
 wait_for_api_server "${master_ip}" "${token}"

k8s/repo.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 set -x
 
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmour -o /etc/apt/trusted.gpg.d/docker.gpg --yes
@@ -38,6 +39,16 @@ sudo sysctl --system
 sudo mkdir -p /etc/containerd
 sudo containerd config default | sudo tee /etc/containerd/config.toml
 
+
+# add registry
+#
+cat << EOF | sudo tee -a /etc/containerd/config.toml
+[plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.1.4:5000"]
+endpoint = ["http://192.168.1.4:5000"]
+[plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.1.4:5000"]
+insecure = true
+EOF
+
 # ensure reboot does not take forever
 sudo mkdir -p /etc/systemd/system/containerd.service.d
 sudo tee /etc/systemd/system/containerd.service.d/override.conf >/dev/null <<EOF

libvirt/cloud_init.cfg

@@ -28,16 +28,16 @@ autoinstall:
       - arches: [amd64, i386]
         uri: http://nl.archive.ubuntu.com/ubuntu
 
-#package_update: true
-#package_upgrade: true
+package_update: true
+package_upgrade: true
 packages:
     - qemu-guest-agent
-#    - curl
-#    - gnupg2
-#    - software-properties-common
-#    - apt-transport-https
-#    - ca-certificates
-#
+    - curl
+    - gnupg2
+    - software-properties-common
+    - apt-transport-https
+    - ca-certificates
+
 # cgroups v2 does not work and causes the controller and scheduler to crash.
 
 runcmd:

libvirt/main.tf

@@ -65,8 +65,10 @@ resource "libvirt_cloudinit_disk" "commoninit" {
     }
   )
 
-  network_config = templatefile("${path.module}/network_config.cfg", {})
-  pool           = libvirt_pool.tpool.name
+  network_config = templatefile("${path.module}/network_config.cfg", {
+    "addresses" : cidrhost("192.168.1.0/24", 200 + count.index)
+  })
+  pool = libvirt_pool.tpool.name
 }
 
 
@@ -80,9 +82,10 @@ resource "libvirt_domain" "domain-ubuntu" {
   cloudinit = libvirt_cloudinit_disk.commoninit[count.index].id
 
   network_interface {
-    hostname       = format("${var.host_name}-%d", count.index + 1)
-    bridge         = "br0"
-    wait_for_lease = true
+    hostname  = format("${var.host_name}-%d", count.index + 1)
+    addresses = [cidrhost("192.168.1.0/24", 200 + count.index)]
+    bridge    = "br0"
+    #wait_for_lease = true
   }
 
   console {
@@ -113,8 +116,21 @@ resource "libvirt_domain" "domain-ubuntu" {
     listen_type = "address"
     autoport    = true
   }
+
+  provisioner "remote-exec" {
+    inline = ["cloud-init status --wait"]
+    on_failure = continue
+    connection {
+      type = "ssh"
+      user = var.ssh_user
+      private_key = file(format("/home/%s/.ssh/id_rsa", var.ssh_user))
+      host = cidrhost("192.168.1.0/24", 200 + count.index)
+    }
+  }
 }
 
+
+
 output "nodes" {
   value = libvirt_domain.domain-ubuntu.*.network_interface.0.addresses.0
 }

libvirt/network_config.cfg

@@ -0,0 +1,9 @@
+version: 2
+ethernets:
+  ens3:
+    addresses:
+      - '${addresses}/24'
+    gateway4: 192.168.1.1
+    nameservers:
+      addresses:
+        - 8.8.8.8

main.tf

@@ -1,9 +1,3 @@
-module "k8s" {
-  source = "./k8s"
-  ssh_user     = var.ssh_user
-  node_list    = module.provider.nodes
-  overlay_cidr = var.overlay_cidr
-}
 
 module "provider" {
   source      = "./libvirt"
@@ -13,6 +7,16 @@ module "provider" {
   qcow2_image = var.qcow2_image
 }
 
+module "k8s" {
+  source = "./k8s"
+  ssh_user     = var.ssh_user
+  node_list    = module.provider.nodes
+  overlay_cidr = var.overlay_cidr
+  depends_on = [
+    module.provider.result
+  ]
+}
+
 output "data" {
   value = module.provider.result
 }

variables.tf

@@ -7,7 +7,7 @@ variable "num_nodes" {
 variable "qcow2_image" {
   type        = string
   description = "Base install image. Ubuntu cloud images are assumed."
-  default     = "./kinetic-server-cloudimg-amd64.img"
+  default     = "./jammy-server-cloudimg-amd64.img"
 }
 
 variable "ssh_user" {