apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: name: ${CLUSTER_NAME}-worker-a spec: template: spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} preKubeadmCommands: - | sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab swapoff -a mount -a cat < /etc/modules-load.d/containerd.conf overlay br_netfilter EOF modprobe overlay modprobe br_netfilter cat < /etc/sysctl.d/99-kubernetes-cri.conf net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF sysctl --system export DEBIAN_FRONTEND=noninteractive apt-get update -y apt-get remove -y docker docker-engine containerd runc apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg chmod a+r /etc/apt/keyrings/docker.gpg chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list apt-get update -y TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} cat < /etc/crictl.yaml runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock EOF containerd config default > /etc/containerd/config.toml sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} spec: clusterNetwork: pods: cidrBlocks: - ${POD_CIDR:=192.168.0.0/16} services: cidrBlocks: - ${SERVICE_CIDR:=172.26.0.0/16} controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketCluster name: ${CLUSTER_NAME} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} pool: worker-a name: ${CLUSTER_NAME}-worker-a spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} pool: worker-a template: metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} pool: worker-a spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate name: ${CLUSTER_NAME}-worker-a clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate name: ${CLUSTER_NAME}-worker-a version: ${KUBERNETES_VERSION} --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane metadata: name: ${CLUSTER_NAME}-control-plane spec: kubeadmConfigSpec: clusterConfiguration: apiServer: extraArgs: cloud-provider: external controllerManager: extraArgs: cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} joinConfiguration: nodeRegistration: ignorePreflightErrors: - DirAvailable--etc-kubernetes-manifests kubeletExtraArgs: cloud-provider: external provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} postKubeadmCommands: - | cat <> /etc/network/interfaces auto lo:0 iface lo:0 inet static address {{ .controlPlaneEndpoint }} netmask 255.255.255.255 EOF systemctl restart networking mkdir -p $HOME/.kube cp /etc/kubernetes/admin.conf $HOME/.kube/config echo "source <(kubectl completion bash)" >> $HOME/.bashrc echo "alias k=kubectl" >> $HOME/.bashrc echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc if [ -f "/run/kubeadm/kubeadm.yaml" ]; then export KUBECONFIG=/etc/kubernetes/admin.conf export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.7.0}/deployment.yaml export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}''' kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) fi preKubeadmCommands: - | sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab swapoff -a mount -a cat < /etc/modules-load.d/containerd.conf overlay br_netfilter EOF modprobe overlay modprobe br_netfilter cat < /etc/sysctl.d/99-kubernetes-cri.conf net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF sysctl --system export DEBIAN_FRONTEND=noninteractive apt-get update -y apt-get remove -y docker docker-engine containerd runc apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1) if [ "$major_vers" -ge 20 ]; then apt-get install -y kubetail fi install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg chmod a+r /etc/apt/keyrings/docker.gpg chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list apt-get update -y TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} containerd config default > /etc/containerd/config.toml cat < /etc/crictl.yaml runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock EOF sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate name: ${CLUSTER_NAME}-control-plane replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketCluster metadata: name: ${CLUSTER_NAME} spec: metro: ${METRO} projectID: ${PROJECT_ID} vipManager: CPEM --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane spec: template: spec: billingCycle: hourly machineType: ${CONTROLPLANE_NODE_TYPE} os: ${NODE_OS:=ubuntu_20_04} sshKeys: - ${SSH_KEY} tags: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate metadata: name: ${CLUSTER_NAME}-worker-a spec: template: spec: billingCycle: hourly machineType: ${WORKER_NODE_TYPE} os: ${NODE_OS:=ubuntu_20_04} sshKeys: - ${SSH_KEY} tags: []