1
0

Starting point

This commit is contained in:
2025-07-01 21:39:08 +02:00
commit 11061780bf
7 changed files with 661 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*.exe

8
README.org Normal file
View File

@@ -0,0 +1,8 @@
#+TITLE=README
#+DATE=2025-07-01
* Testing kubernetes
Note
Manually download virtctl and clusterctl

356
capi-quickstart.yaml Normal file
View File

@@ -0,0 +1,356 @@
apiVersion: cluster.x-k8s.io/v1beta1
kind: ClusterClass
metadata:
name: quick-start
namespace: default
spec:
controlPlane:
machineHealthCheck:
unhealthyConditions:
- status: Unknown
timeout: 300s
type: Ready
- status: "False"
timeout: 300s
type: Ready
machineInfrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
name: quick-start-control-plane
ref:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
name: quick-start-control-plane
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerClusterTemplate
name: quick-start-cluster
patches:
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository
valueFrom:
variable: imageRepository
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
description: Sets the imageRepository used for the KubeadmControlPlane.
enabledIf: '{{ ne .imageRepository "" }}'
name: imageRepository
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd
valueFrom:
template: |
local:
imageTag: {{ .etcdImageTag }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
description: Sets tag to use for the etcd image in the KubeadmControlPlane.
name: etcdImageTag
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns
valueFrom:
template: |
imageTag: {{ .coreDNSImageTag }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
description: Sets tag to use for the etcd image in the KubeadmControlPlane.
name: coreDNSImageTag
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/customImage
valueFrom:
template: |
kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }}
selector:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
matchResources:
machineDeploymentClass:
names:
- default-worker
- jsonPatches:
- op: add
path: /spec/template/spec/template/customImage
valueFrom:
template: |
kindest/node:{{ .builtin.machinePool.version | replace "+" "_" }}
selector:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
matchResources:
machinePoolClass:
names:
- default-worker
- jsonPatches:
- op: add
path: /spec/template/spec/customImage
valueFrom:
template: |
kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
selector:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
matchResources:
controlPlane: true
description: Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments.
name: customImage
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs
value:
admission-control-config-file: /etc/kubernetes/kube-apiserver-admission-pss.yaml
- op: add
path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes
value:
- hostPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml
mountPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml
name: admission-pss
pathType: File
readOnly: true
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files
valueFrom:
template: |
- content: |
apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1{{ if semverCompare "< v1.25" .builtin.controlPlane.version }}beta1{{ end }}
kind: PodSecurityConfiguration
defaults:
enforce: "{{ .podSecurityStandard.enforce }}"
enforce-version: "latest"
audit: "{{ .podSecurityStandard.audit }}"
audit-version: "latest"
warn: "{{ .podSecurityStandard.warn }}"
warn-version: "latest"
exemptions:
usernames: []
runtimeClasses: []
namespaces: [kube-system]
path: /etc/kubernetes/kube-apiserver-admission-pss.yaml
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
description: Adds an admission configuration for PodSecurity to the kube-apiserver.
enabledIf: '{{ .podSecurityStandard.enabled }}'
name: podSecurityStandard
variables:
- name: imageRepository
required: true
schema:
openAPIV3Schema:
default: ""
description: imageRepository sets the container registry to pull images from.
If empty, nothing will be set and the from of kubeadm will be used.
example: registry.k8s.io
type: string
- name: etcdImageTag
required: true
schema:
openAPIV3Schema:
default: ""
description: etcdImageTag sets the tag for the etcd image.
example: 3.5.3-0
type: string
- name: coreDNSImageTag
required: true
schema:
openAPIV3Schema:
default: ""
description: coreDNSImageTag sets the tag for the coreDNS image.
example: v1.8.5
type: string
- name: podSecurityStandard
required: false
schema:
openAPIV3Schema:
properties:
audit:
default: restricted
description: audit sets the level for the audit PodSecurityConfiguration mode. One of privileged, baseline, restricted.
type: string
enabled:
default: true
description: enabled enables the patches to enable Pod Security Standard via AdmissionConfiguration.
type: boolean
enforce:
default: baseline
description: enforce sets the level for the enforce PodSecurityConfiguration mode. One of privileged, baseline, restricted.
type: string
warn:
default: restricted
description: warn sets the level for the warn PodSecurityConfiguration mode. One of privileged, baseline, restricted.
type: string
type: object
workers:
machineDeployments:
- class: default-worker
machineHealthCheck:
unhealthyConditions:
- status: Unknown
timeout: 300s
type: Ready
- status: "False"
timeout: 300s
type: Ready
template:
bootstrap:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: quick-start-default-worker-bootstraptemplate
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
name: quick-start-default-worker-machinetemplate
machinePools:
- class: default-worker
template:
bootstrap:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: quick-start-default-worker-bootstraptemplate
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
name: quick-start-default-worker-machinepooltemplate
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerClusterTemplate
metadata:
name: quick-start-cluster
namespace: default
spec:
template:
spec: {}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
metadata:
name: quick-start-control-plane
namespace: default
spec:
template:
spec:
kubeadmConfigSpec:
clusterConfiguration:
apiServer:
certSANs:
- localhost
- 127.0.0.1
- 0.0.0.0
- host.docker.internal
initConfiguration:
nodeRegistration: {}
joinConfiguration:
nodeRegistration: {}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: quick-start-control-plane
namespace: default
spec:
template:
spec:
extraMounts:
- containerPath: /var/run/docker.sock
hostPath: /var/run/docker.sock
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: quick-start-default-worker-machinetemplate
namespace: default
spec:
template:
spec:
extraMounts:
- containerPath: /var/run/docker.sock
hostPath: /var/run/docker.sock
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
metadata:
name: quick-start-default-worker-machinepooltemplate
namespace: default
spec:
template:
spec:
template: {}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: quick-start-default-worker-bootstraptemplate
namespace: default
spec:
template:
spec:
joinConfiguration:
nodeRegistration: {}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: capi-quickstart
namespace: default
spec:
clusterNetwork:
pods:
cidrBlocks:
- 192.168.0.0/16
serviceDomain: cluster.local
services:
cidrBlocks:
- 10.128.0.0/12
topology:
class: quick-start
controlPlane:
metadata: {}
replicas: 1
variables:
- name: imageRepository
value: ""
- name: etcdImageTag
value: ""
- name: coreDNSImageTag
value: ""
- name: podSecurityStandard
value:
audit: restricted
enabled: true
enforce: baseline
warn: restricted
version: v1.33.0
workers:
machineDeployments:
- class: default-worker
name: md-0
replicas: 3

243
cluster-template.yaml Normal file
View File

@@ -0,0 +1,243 @@
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: ${CLUSTER_NAME}-worker-a
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
kubeletExtraArgs:
cloud-provider: external
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
preKubeadmCommands:
- |
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
swapoff -a
mount -a
cat <<EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
export DEBIAN_FRONTEND=noninteractive
apt-get update -y
apt-get remove -y docker docker-engine containerd runc
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
chmod a+r /etc/apt/keyrings/docker.gpg
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
apt-get update -y
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
cat <<EOF > /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
EOF
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
systemctl restart containerd
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: ${CLUSTER_NAME}
spec:
clusterNetwork:
pods:
cidrBlocks:
- ${POD_CIDR:=192.168.0.0/16}
services:
cidrBlocks:
- ${SERVICE_CIDR:=172.26.0.0/16}
controlPlaneRef:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
name: ${CLUSTER_NAME}-control-plane
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketCluster
name: ${CLUSTER_NAME}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
labels:
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
pool: worker-a
name: ${CLUSTER_NAME}-worker-a
spec:
clusterName: ${CLUSTER_NAME}
replicas: ${WORKER_MACHINE_COUNT}
selector:
matchLabels:
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
pool: worker-a
template:
metadata:
labels:
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
pool: worker-a
spec:
bootstrap:
configRef:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: ${CLUSTER_NAME}-worker-a
clusterName: ${CLUSTER_NAME}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketMachineTemplate
name: ${CLUSTER_NAME}-worker-a
version: ${KUBERNETES_VERSION}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
name: ${CLUSTER_NAME}-control-plane
spec:
kubeadmConfigSpec:
clusterConfiguration:
apiServer:
extraArgs:
cloud-provider: external
controllerManager:
extraArgs:
cloud-provider: external
initConfiguration:
nodeRegistration:
kubeletExtraArgs:
cloud-provider: external
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
joinConfiguration:
nodeRegistration:
ignorePreflightErrors:
- DirAvailable--etc-kubernetes-manifests
kubeletExtraArgs:
cloud-provider: external
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
postKubeadmCommands:
- |
cat <<EOF >> /etc/network/interfaces
auto lo:0
iface lo:0 inet static
address {{ .controlPlaneEndpoint }}
netmask 255.255.255.255
EOF
systemctl restart networking
mkdir -p $HOME/.kube
cp /etc/kubernetes/admin.conf $HOME/.kube/config
echo "source <(kubectl completion bash)" >> $HOME/.bashrc
echo "alias k=kubectl" >> $HOME/.bashrc
echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc
if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
export KUBECONFIG=/etc/kubernetes/admin.conf
export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.7.0}/deployment.yaml
export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
fi
preKubeadmCommands:
- |
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
swapoff -a
mount -a
cat <<EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
export DEBIAN_FRONTEND=noninteractive
apt-get update -y
apt-get remove -y docker docker-engine containerd runc
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
if [ "$major_vers" -ge 20 ]; then
apt-get install -y kubetail
fi
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
chmod a+r /etc/apt/keyrings/docker.gpg
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
apt-get update -y
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
containerd config default > /etc/containerd/config.toml
cat <<EOF > /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
EOF
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
systemctl restart containerd
ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketMachineTemplate
name: ${CLUSTER_NAME}-control-plane
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketCluster
metadata:
name: ${CLUSTER_NAME}
spec:
metro: ${METRO}
projectID: ${PROJECT_ID}
vipManager: CPEM
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketMachineTemplate
metadata:
name: ${CLUSTER_NAME}-control-plane
spec:
template:
spec:
billingCycle: hourly
machineType: ${CONTROLPLANE_NODE_TYPE}
os: ${NODE_OS:=ubuntu_20_04}
sshKeys:
- ${SSH_KEY}
tags: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: PacketMachineTemplate
metadata:
name: ${CLUSTER_NAME}-worker-a
spec:
template:
spec:
billingCycle: hourly
machineType: ${WORKER_NODE_TYPE}
os: ${NODE_OS:=ubuntu_20_04}
sshKeys:
- ${SSH_KEY}
tags: []

7
clusterconfig.yaml Normal file
View File

@@ -0,0 +1,7 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker

8
create-cluster.ps1 Normal file
View File

@@ -0,0 +1,8 @@
kind create cluster --config clusterconfig.yaml --name kubevirt-test
kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/v1.5.2/kubevirt-operator.yaml"
kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/v1.5.2/kubevirt-cr.yaml"
write-host "Waiting until kubevirt has had a change to get deployed"
while(kubectl get kubervirt.kubevirt.io/kubevirt -n kubevirt -o=jsonpath="{.status.phase}" !== "Deployed"){}
start-sleep -Seconds 300
kubectl apply -f vm.yaml

37
vm.yaml Normal file
View File

@@ -0,0 +1,37 @@
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
name: testvm
spec:
runStrategy: Halted
template:
metadata:
labels:
kubevirt.io/size: small
kubevirt.io/domain: testvm
spec:
domain:
devices:
disks:
- name: containerdisk
disk:
bus: virtio
- name: cloudinitdisk
disk:
bus: virtio
interfaces:
- name: default
masquerade: {}
resources:
requests:
memory: 64M
networks:
- name: default
pod: {}
volumes:
- name: containerdisk
containerDisk:
image: quay.io/kubevirt/cirros-container-disk-demo
- name: cloudinitdisk
cloudInitNoCloud:
userDataBase64: SGkuXG4=