apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.20.11
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    - name: node-ip
      value: 10.10.20.11
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
controlPlaneEndpoint: k8s-api.lab.local:6443
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12

sudo hostnamectl set-hostname <NODE_NAME>

sudo timedatectl set-ntp true
sudo systemctl enable --now systemd-timesyncd || true

ip addr
ip route
getent hosts k8s-api.lab.local || true

hostname -f || hostname
ip link
sudo cat /sys/class/dmi/id/product_uuid || true

sudo swapoff -a
sudo sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
free -h

cat <<'EOF' | sudo tee /etc/modules-load.d/kubernetes.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<'EOF' | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

lsmod | grep -E 'overlay|br_netfilter'
sysctl net.bridge.bridge-nf-call-iptables
sysctl net.ipv4.ip_forward

sudo apt-get update
sudo apt-get install -y \
  ca-certificates \
  curl \
  gnupg \
  lsb-release \
  apt-transport-https \
  jq \
  yq \
  git \
  vim \
  bash-completion \
  conntrack \
  socat \
  ebtables \
  ethtool \
  iproute2 \
  iptables \
  nftables \
  nfs-common \
  open-iscsi

sudo dnf install -y \
  ca-certificates \
  curl \
  gnupg2 \
  jq \
  git \
  vim \
  bash-completion \
  conntrack-tools \
  socat \
  ebtables \
  ethtool \
  iproute \
  iptables \
  nftables \
  nfs-utils \
  iscsi-initiator-utils

sudo zypper refresh
sudo zypper install -y \
  ca-certificates \
  curl \
  gpg2 \
  jq \
  git \
  vim \
  bash-completion \
  conntrack-tools \
  socat \
  ebtables \
  ethtool \
  iproute2 \
  iptables \
  nftables \
  nfs-client \
  open-iscsi

getenforce || true
sudo setenforce 0 || true
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config || true

sudo aa-status || true

sudo ufw status verbose
# Nur im Lab, wenn du die Wirkung verstehst:
# sudo ufw disable

sudo firewall-cmd --state || true
sudo firewall-cmd --list-all || true

sudo firewall-cmd --permanent --add-port=6443/tcp
sudo firewall-cmd --permanent --add-port=2379-2380/tcp
sudo firewall-cmd --permanent --add-port=10250/tcp
sudo firewall-cmd --permanent --add-port=10257/tcp
sudo firewall-cmd --permanent --add-port=10259/tcp
sudo firewall-cmd --reload

sudo apt-get update
sudo apt-get install -y containerd

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

sudo systemctl enable --now containerd
sudo systemctl restart containerd
sudo systemctl status containerd --no-pager

sudo ctr version
sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock info || true

sudo dnf install -y containerd

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

sudo systemctl enable --now containerd
sudo systemctl restart containerd

export KUBERNETES_MINOR="v1.36"
export CRIO_VERSION="v1.36"

sudo systemctl enable --now crio
sudo systemctl status crio --no-pager
sudo crictl --runtime-endpoint unix:///var/run/crio/crio.sock info

cat <<'EOF' | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

cat <<'EOF' | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///var/run/crio/crio.sock
image-endpoint: unix:///var/run/crio/crio.sock
timeout: 10
debug: false
EOF

sudo crictl info
sudo crictl images

# /etc/containerd/config.toml, Ausschnitt
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
  endpoint = ["https://registry.lab.local:5000"]

[plugins."io.containerd.grpc.v1.cri".registry.configs."registry.lab.local:5000".tls]
  ca_file = "/etc/containerd/certs.d/registry.lab.local/ca.crt"

sudo systemctl restart containerd
sudo crictl pull registry.k8s.io/pause:3.10 || true

export KUBERNETES_MINOR="v1.36"

export KUBERNETES_MINOR="v1.36"

sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
sudo mkdir -p -m 755 /etc/apt/keyrings

curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/deb/Release.key" \
  | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/deb/ /" \
  | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl
sudo systemctl enable --now kubelet

export KUBERNETES_MINOR="v1.36"

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubelet

RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
ARCH="amd64"
DOWNLOAD_DIR="/usr/local/bin"

sudo mkdir -p "${DOWNLOAD_DIR}"
cd "${DOWNLOAD_DIR}"
sudo curl -L --remote-name-all "https://dl.k8s.io/release/${RELEASE}/bin/linux/${ARCH}/{kubeadm,kubelet,kubectl}"
sudo chmod +x kubeadm kubelet kubectl

kubeadm version
kubectl version --client
kubelet --version
sudo systemctl status kubelet --no-pager

mkdir -p ~/kubeadm
cat <<'EOF' > ~/kubeadm/kubeadm-single-node.yaml
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.20.11
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    - name: node-ip
      value: 10.10.20.11
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: stable
clusterName: bm-single
controlPlaneEndpoint: 10.10.20.11:6443
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
  dnsDomain: cluster.local
apiServer:
  certSANs:
    - 10.10.20.11
    - bm-single.lab.local
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
EOF

sudo kubeadm init --config ~/kubeadm/kubeadm-single-node.yaml

mkdir -p "$HOME/.kube"
sudo cp -i /etc/kubernetes/admin.conf "$HOME/.kube/config"
sudo chown "$(id -u):$(id -g)" "$HOME/.kube/config"

kubectl cluster-info
kubectl get nodes -o wide

kubectl taint nodes --all node-role.kubernetes.io/control-plane- || true
kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints

kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

cilium install --version <CILIUM_VERSION>
cilium status --wait

kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/<CALICO_VERSION>/manifests/calico.yaml

kubectl get nodes -o wide
kubectl get pods -A -o wide
kubectl -n kube-system get pods
kubectl run smoke --image=registry.k8s.io/e2e-test-images/agnhost:2.53 --restart=Never -- sleep 3600
kubectl wait --for=condition=Ready pod/smoke --timeout=120s
kubectl delete pod smoke

apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.20.11
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    - name: node-ip
      value: 10.10.20.11
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: stable
clusterName: bm-multi
controlPlaneEndpoint: k8s-api.lab.local:6443
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
apiServer:
  certSANs:
    - k8s-api.lab.local
    - 10.10.20.10
    - 10.10.20.11
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

sudo kubeadm init --config kubeadm-multi.yaml

kubeadm token create --print-join-command

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH>

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH>

kubectl get nodes -o wide
kubectl get pods -A -o wide

kubectl label node k8s-w-01 node-role.kubernetes.io/worker=worker
kubectl label node k8s-w-02 node-role.kubernetes.io/worker=worker
kubectl label node k8s-w-03 node-role.kubernetes.io/worker=worker

kubectl label node k8s-w-01 topology.kubernetes.io/zone=rack-a
kubectl label node k8s-w-02 topology.kubernetes.io/zone=rack-b
kubectl label node k8s-w-03 topology.kubernetes.io/zone=rack-c

kubectl create namespace smoke
kubectl create deployment web -n smoke --image=nginx:stable --replicas=3
kubectl expose deployment web -n smoke --port=80 --target-port=80
kubectl get pod -n smoke -o wide
kubectl run curl -n smoke --rm -it --image=curlimages/curl --restart=Never -- curl -sS http://web.smoke.svc.cluster.local
kubectl delete namespace smoke

global
    log /dev/log local0
    log /dev/log local1 notice
    daemon
    maxconn 4096

defaults
    log global
    mode tcp
    option tcplog
    timeout connect 10s
    timeout client  1m
    timeout server  1m

frontend kubernetes_api
    bind *:6443
    default_backend kubernetes_control_plane

backend kubernetes_control_plane
    balance roundrobin
    option tcp-check
    server cp1 10.10.20.11:6443 check
    server cp2 10.10.20.12:6443 check
    server cp3 10.10.20.13:6443 check

sudo systemctl enable --now haproxy
sudo systemctl status haproxy --no-pager
nc -vz k8s-api.lab.local 6443

vrrp_instance K8S_API {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 110
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass CHANGE_ME
    }
    virtual_ipaddress {
        10.10.20.10/24
    }
}

sudo systemctl enable --now keepalived
ip addr show eth0

apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.20.11
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    - name: node-ip
      value: 10.10.20.11
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: stable
clusterName: bm-ha
controlPlaneEndpoint: k8s-api.lab.local:6443
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
apiServer:
  certSANs:
    - k8s-api.lab.local
    - 10.10.20.10
    - 10.10.20.11
    - 10.10.20.12
    - 10.10.20.13
controllerManager: {}
scheduler: {}
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

sudo kubeadm init --config kubeadm-ha.yaml --upload-certs

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH> \
  --control-plane \
  --certificate-key <CERTIFICATE_KEY>

kubectl get nodes -o wide
kubectl -n kube-system get pods -o wide
kubectl get --raw='/readyz?verbose'

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH>

kubectl -n kube-system rollout restart deployment coredns
kubectl -n kube-system rollout status deployment coredns
kubectl -n kube-system get pods -l k8s-app=kube-dns -o wide

apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: k8s-api.lab.local:6443
etcd:
  external:
    endpoints:
      - https://10.10.40.11:2379
      - https://10.10.40.12:2379
      - https://10.10.40.13:2379
    caFile: /etc/kubernetes/pki/etcd/ca.crt
    certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
    keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12

kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
kubectl -n kube-flannel get pods -o wide

kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/<CALICO_VERSION>/manifests/calico.yaml
kubectl -n calico-system get pods -o wide || kubectl -n kube-system get pods -o wide | grep calico

cilium install --version <CILIUM_VERSION>
cilium status --wait
cilium connectivity test

kubectl get nodes -o wide
kubectl get pods -A -o wide
kubectl -n kube-system get daemonsets
kubectl run net-a --image=busybox:1.36 --restart=Never -- sleep 3600
kubectl run net-b --image=busybox:1.36 --restart=Never -- sleep 3600
kubectl wait --for=condition=Ready pod/net-a --timeout=120s
kubectl wait --for=condition=Ready pod/net-b --timeout=120s
kubectl get pods -o wide
kubectl exec net-a -- ping -c 3 <POD_IP_VON_NET_B>
kubectl delete pod net-a net-b

apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: lab-pool
  namespace: metallb-system
spec:
  addresses:
    - 10.10.20.200-10.10.20.240
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: lab-l2
  namespace: metallb-system
spec:
  ipAddressPools:
    - lab-pool

kubectl get pods -n metallb-system
kubectl apply -f metallb-pool.yaml
kubectl create deployment lb-test --image=nginx:stable
kubectl expose deployment lb-test --port=80 --type=LoadBalancer
kubectl get svc lb-test -w
curl http://<EXTERNAL_IP>
kubectl delete svc lb-test
kubectl delete deployment lb-test

apiVersion: metallb.io/v1beta2
kind: BGPPeer
metadata:
  name: router-a
  namespace: metallb-system
spec:
  myASN: 64512
  peerASN: 64500
  peerAddress: 10.10.20.1
---
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: bgp-pool
  namespace: metallb-system
spec:
  addresses:
    - 10.10.50.0/24
---
apiVersion: metallb.io/v1beta1
kind: BGPAdvertisement
metadata:
  name: bgp-adv
  namespace: metallb-system
spec:
  ipAddressPools:
    - bgp-pool

helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update
helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \
  --namespace ingress-nginx \
  --create-namespace \
  --set controller.service.type=LoadBalancer

kubectl -n ingress-nginx get svc,pods
kubectl get ingressclass

kubectl get gatewayclass
kubectl get gateways -A
kubectl get httproutes -A

kubectl get storageclass
kubectl get csidrivers
kubectl get pv,pvc -A

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: storage-smoke
  namespace: default
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: <STORAGE_CLASS>
---
apiVersion: v1
kind: Pod
metadata:
  name: storage-smoke
  namespace: default
spec:
  containers:
    - name: app
      image: busybox:1.36
      command: ["sh", "-c", "echo ok > /data/test.txt && sleep 3600"]
      volumeMounts:
        - name: data
          mountPath: /data
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: storage-smoke

kubectl apply -f storage-smoke.yaml
kubectl wait --for=condition=Ready pod/storage-smoke --timeout=180s
kubectl exec storage-smoke -- cat /data/test.txt
kubectl delete -f storage-smoke.yaml

kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
kubectl -n kube-system rollout status deployment metrics-server
kubectl top nodes
kubectl top pods -A

helm repo add jetstack https://charts.jetstack.io
helm repo update
helm upgrade --install cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
  --set crds.enabled=true

kubectl -n cert-manager get pods

kubectl get pods -A | grep -E 'prometheus|grafana|loki|otel|tempo|metrics'
kubectl get servicemonitors -A 2>/dev/null || true
kubectl get podmonitors -A 2>/dev/null || true

kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
kubectl -n argocd rollout status deployment/argocd-server

flux check --pre
flux bootstrap git \
  --url=ssh://git@example.local/platform/kubernetes.git \
  --branch=main \
  --path=clusters/bm-lab-01

sudo sed -i 's/^#\?PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config
sudo sed -i 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config
sudo systemctl reload ssh || sudo systemctl reload sshd

kubectl auth can-i get pods -A
kubectl auth can-i '*' '*' --as system:serviceaccount:default:default
kubectl get clusterrolebinding -o wide

kubectl label namespace default pod-security.kubernetes.io/enforce=baseline --overwrite
kubectl label namespace default pod-security.kubernetes.io/audit=restricted --overwrite
kubectl label namespace default pod-security.kubernetes.io/warn=restricted --overwrite

apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: <BASE64_32_BYTE_KEY>
      - identity: {}

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: demo
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns
  namespace: demo
spec:
  podSelector: {}
  policyTypes:
    - Egress
  egress:
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: kube-system
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53

kubeadm config images list --kubernetes-version <VERSION>

sudo kubeadm config images pull --kubernetes-version <VERSION> --cri-socket unix:///run/containerd/containerd.sock

sudo kubeadm config images pull \
  --kubernetes-version <VERSION> \
  --image-repository registry.lab.local/kubernetes

apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: v1.36.0
imageRepository: registry.lab.local/kubernetes

curl -k https://registry.lab.local/v2/_catalog || true
sudo crictl pull registry.lab.local/kubernetes/pause:3.10

curl -sfL https://get.k3s.io | sh -

sudo k3s kubectl get nodes
sudo cat /etc/rancher/k3s/k3s.yaml

mkdir -p ~/.kube
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
sudo chown "$(id -u):$(id -g)" ~/.kube/config
kubectl get nodes

curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="\
  --cluster-cidr=10.42.0.0/16 \
  --service-cidr=10.43.0.0/16 \
  --disable traefik \
  --disable servicelb" sh -

curl -sfL https://get.k3s.io | K3S_TOKEN="<CLUSTER_SECRET>" sh -s - server \
  --cluster-init \
  --tls-san k3s-api.lab.local \
  --disable traefik \
  --disable servicelb

curl -sfL https://get.k3s.io | K3S_TOKEN="<CLUSTER_SECRET>" sh -s - server \
  --server https://k3s-api.lab.local:6443 \
  --tls-san k3s-api.lab.local \
  --disable traefik \
  --disable servicelb

curl -sfL https://get.k3s.io | K3S_TOKEN="<CLUSTER_SECRET>" sh -s - agent \
  --server https://k3s-api.lab.local:6443

kubectl get nodes -o wide
kubectl get pods -A -o wide
sudo systemctl status k3s --no-pager
sudo journalctl -u k3s -n 100 --no-pager

curl -sfL https://get.rke2.io | sudo sh -
sudo mkdir -p /etc/rancher/rke2
cat <<'EOF' | sudo tee /etc/rancher/rke2/config.yaml
token: <CLUSTER_SECRET>
tls-san:
  - rke2-api.lab.local
node-ip: 10.10.20.11
cni: cilium
EOF

sudo systemctl enable --now rke2-server

mkdir -p ~/.kube
sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config
sudo chown "$(id -u):$(id -g)" ~/.kube/config
export PATH=$PATH:/var/lib/rancher/rke2/bin
kubectl get nodes

curl -sfL https://get.rke2.io | sudo sh -
sudo mkdir -p /etc/rancher/rke2
cat <<'EOF' | sudo tee /etc/rancher/rke2/config.yaml
server: https://rke2-api.lab.local:9345
token: <CLUSTER_SECRET>
tls-san:
  - rke2-api.lab.local
node-ip: 10.10.20.12
cni: cilium
EOF

sudo systemctl enable --now rke2-server

curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_TYPE="agent" sh -
sudo mkdir -p /etc/rancher/rke2
cat <<'EOF' | sudo tee /etc/rancher/rke2/config.yaml
server: https://rke2-api.lab.local:9345
token: <CLUSTER_SECRET>
node-ip: 10.10.20.21
EOF

sudo systemctl enable --now rke2-agent

kubectl get nodes -o wide
kubectl get pods -A -o wide
sudo journalctl -u rke2-server -n 100 --no-pager
sudo journalctl -u rke2-agent -n 100 --no-pager || true

curl -sSfL https://get.k0s.sh | sudo sh
sudo k0s install controller --single
sudo k0s start
sudo k0s status
sudo k0s kubectl get nodes

mkdir -p ~/.kube
sudo k0s kubeconfig admin > ~/.kube/config
chmod 600 ~/.kube/config
kubectl get nodes

sudo k0s install controller
sudo k0s start
sudo k0s token create --role=worker > worker-token
sudo k0s token create --role=controller > controller-token

curl -sSfL https://get.k0s.sh | sudo sh
sudo k0s install worker --token-file /path/to/worker-token
sudo k0s start

sudo k0s status
sudo k0s kubectl get nodes -o wide
sudo k0s kubectl get pods -A -o wide

export CLUSTER_NAME="talos-bm-lab"
export CONTROL_PLANE_ENDPOINT="https://10.10.20.10:6443"

talosctl gen config "${CLUSTER_NAME}" "${CONTROL_PLANE_ENDPOINT}" --output-dir ./talos-out

talosctl apply-config \
  --insecure \
  --nodes 10.10.20.11 \
  --file ./talos-out/controlplane.yaml

talosctl apply-config \
  --insecure \
  --nodes 10.10.20.21 \
  --file ./talos-out/worker.yaml

talosctl bootstrap \
  --nodes 10.10.20.11 \
  --endpoints 10.10.20.11 \
  --talosconfig ./talos-out/talosconfig

talosctl kubeconfig \
  --nodes 10.10.20.11 \
  --endpoints 10.10.20.11 \
  --talosconfig ./talos-out/talosconfig \
  --force

kubectl get nodes -o wide

talosctl health --nodes 10.10.20.11 --endpoints 10.10.20.11
talosctl service kubelet --nodes 10.10.20.11
talosctl logs kubelet --nodes 10.10.20.11
talosctl dmesg --nodes 10.10.20.11
talosctl disks --nodes 10.10.20.11

[control_plane]
k8s-cp-01 ansible_host=10.10.20.11 node_ip=10.10.20.11
k8s-cp-02 ansible_host=10.10.20.12 node_ip=10.10.20.12
k8s-cp-03 ansible_host=10.10.20.13 node_ip=10.10.20.13

[workers]
k8s-w-01 ansible_host=10.10.20.21 node_ip=10.10.20.21
k8s-w-02 ansible_host=10.10.20.22 node_ip=10.10.20.22
k8s-w-03 ansible_host=10.10.20.23 node_ip=10.10.20.23

[kubernetes:children]
control_plane
workers

ansible/
  inventory.ini
  group_vars/
    all.yaml
  playbooks/
    00-preflight.yaml
    10-os-baseline.yaml
    20-container-runtime.yaml
    30-kubernetes-packages.yaml
    40-kubeadm-init.yaml
    50-kubeadm-join.yaml
    60-addons.yaml
    90-validate.yaml
  roles/
    os_baseline/
    containerd/
    kubeadm/
    cni/
    metallb/
    ingress/
    observability/

- name: Kubernetes Kernelmodule laden
  ansible.builtin.copy:
    dest: /etc/modules-load.d/kubernetes.conf
    content: |
      overlay
      br_netfilter

- name: Module sofort laden
  ansible.builtin.command: modprobe {{ item }}
  loop:
    - overlay
    - br_netfilter
  changed_when: false

- name: sysctl setzen
  ansible.builtin.copy:
    dest: /etc/sysctl.d/99-kubernetes-cri.conf
    content: |
      net.bridge.bridge-nf-call-iptables = 1
      net.bridge.bridge-nf-call-ip6tables = 1
      net.ipv4.ip_forward = 1

- name: sysctl anwenden
  ansible.builtin.command: sysctl --system
  changed_when: false

# user-data, stark gekürzt
autoinstall:
  version: 1
  identity:
    hostname: k8s-cp-01
    username: ubuntu
    password: "$6$REPLACE_WITH_HASH"
  ssh:
    install-server: true
    authorized-keys:
      - ssh-ed25519 AAAA... workshop
  packages:
    - curl
    - git
    - jq
    - vim
  late-commands:
    - curtin in-target --target=/target -- systemctl enable ssh

text
reboot
network --bootproto=dhcp --device=link --activate
rootpw --iscrypted <HASH>
user --name=admin --groups=wheel --iscrypted --password=<HASH>
%packages
@^minimal-environment
curl
git
jq
vim
%end
%post
systemctl enable sshd
%end

sudo swapoff -a
sudo sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab

cat <<'EOF' | sudo tee /etc/modules-load.d/kubernetes.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter

cat <<'EOF' | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system

sudo apt-get update
sudo apt-get install -y containerd
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo systemctl enable --now containerd
sudo systemctl restart containerd

export KUBERNETES_MINOR="v1.36"

sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/deb/Release.key" \
  | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/deb/ /" \
  | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl
sudo systemctl enable --now kubelet

getent hosts k8s-api.lab.local
nc -vz k8s-api.lab.local 6443 || true

cat <<'EOF' > kubeadm-ha.yaml
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.20.11
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    - name: node-ip
      value: 10.10.20.11
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: stable
clusterName: bm-ha
controlPlaneEndpoint: k8s-api.lab.local:6443
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
apiServer:
  certSANs:
    - k8s-api.lab.local
    - 10.10.20.10
    - 10.10.20.11
    - 10.10.20.12
    - 10.10.20.13
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
EOF

sudo kubeadm init --config kubeadm-ha.yaml --upload-certs

mkdir -p "$HOME/.kube"
sudo cp -i /etc/kubernetes/admin.conf "$HOME/.kube/config"
sudo chown "$(id -u):$(id -g)" "$HOME/.kube/config"
kubectl get nodes -o wide

kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
kubectl -n kube-flannel get pods -w

cilium install --version <CILIUM_VERSION>
cilium status --wait

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH> \
  --control-plane \
  --certificate-key <CERTIFICATE_KEY>

sudo kubeadm join k8s-api.lab.local:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH>

kubectl get nodes -o wide
kubectl get pods -A -o wide
kubectl -n kube-system rollout status deployment coredns
kubectl get --raw='/readyz?verbose'

kubectl get ns metallb-system || true
# Installation nach Projektstandard: Helm, Kustomize oder Manifeste.

apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: lab-pool
  namespace: metallb-system
spec:
  addresses:
    - 10.10.20.200-10.10.20.240
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: lab-l2
  namespace: metallb-system
spec:
  ipAddressPools:
    - lab-pool

kubectl create deployment nginx --image=nginx:stable
kubectl expose deployment nginx --port=80 --type=LoadBalancer
kubectl get svc nginx -w
curl http://<EXTERNAL_IP>
kubectl delete svc nginx
kubectl delete deployment nginx

hostnamectl
uname -a
ip addr
ip route
timedatectl
free -h
swapon --show
lsmod | grep -E 'overlay|br_netfilter'
sysctl net.ipv4.ip_forward

sudo systemctl status containerd --no-pager || true
sudo systemctl status crio --no-pager || true
sudo crictl info
sudo crictl ps -a
sudo crictl images

kubeadm version
kubelet --version
kubectl version --client
sudo systemctl status kubelet --no-pager
sudo journalctl -u kubelet -n 100 --no-pager

kubectl cluster-info
kubectl get nodes -o wide
kubectl describe node <NODE>
kubectl get componentstatuses 2>/dev/null || true
kubectl get --raw='/readyz?verbose'
kubectl get --raw='/livez?verbose'

kubectl -n kube-system get pods -o wide
kubectl -n kube-system get pods -l component=kube-apiserver -o wide
kubectl -n kube-system get pods -l component=kube-controller-manager -o wide
kubectl -n kube-system get pods -l component=kube-scheduler -o wide
kubectl -n kube-system get pods -l component=etcd -o wide

kubectl run dns-test --image=busybox:1.36 --restart=Never -- sleep 3600
kubectl wait --for=condition=Ready pod/dns-test --timeout=120s
kubectl exec dns-test -- nslookup kubernetes.default.svc.cluster.local
kubectl exec dns-test -- nslookup kube-dns.kube-system.svc.cluster.local
kubectl delete pod dns-test

kubectl create namespace svc-test
kubectl create deployment echo -n svc-test --image=registry.k8s.io/e2e-test-images/agnhost:2.53 -- /agnhost netexec --http-port=8080
kubectl expose deployment echo -n svc-test --port=80 --target-port=8080
kubectl run curl -n svc-test --rm -it --image=curlimages/curl --restart=Never -- curl -sS http://echo
kubectl delete namespace svc-test

kubectl create deployment lb-smoke --image=nginx:stable
kubectl expose deployment lb-smoke --port=80 --type=LoadBalancer
kubectl get svc lb-smoke -w
curl http://<EXTERNAL_IP>
kubectl delete svc lb-smoke
kubectl delete deployment lb-smoke

kubectl get storageclass
kubectl get pv,pvc -A
kubectl get volumeattachments -A 2>/dev/null || true

kubectl auth can-i get pods -A
kubectl get serviceaccounts -A
kubectl get roles,rolebindings -A
kubectl get clusterroles,clusterrolebindings
kubectl get ns --show-labels | grep pod-security || true

sudo journalctl -u kubelet -n 200 --no-pager
sudo crictl info
swapon --show
cat /var/lib/kubelet/config.yaml 2>/dev/null || true

kubectl describe node <NODE>
kubectl get pods -A -o wide
kubectl -n kube-system get daemonsets

kubectl -n kube-system describe deployment coredns
kubectl -n kube-system logs deployment/coredns
kubectl get nodes -o wide

kubectl describe pod <POD> -n <NAMESPACE>
kubectl get events -n <NAMESPACE> --sort-by=.lastTimestamp
kubectl describe nodes | grep -A5 -E 'Allocated resources|Taints'

kubectl describe pod <POD> -n <NAMESPACE>
sudo crictl pull <IMAGE>

kubectl get svc,endpoints,endpointslices -n <NAMESPACE>
kubectl get pods -n <NAMESPACE> --show-labels
kubectl describe svc <SERVICE> -n <NAMESPACE>

kubectl -n metallb-system get pods
kubectl -n metallb-system get ipaddresspools,l2advertisements,bgppeers,bgpadvertisements
kubectl describe svc <SERVICE> -n <NAMESPACE>

getent hosts k8s-api.lab.local
nc -vz k8s-api.lab.local 6443
curl -k https://k8s-api.lab.local:6443/readyz
sudo journalctl -u haproxy -n 100 --no-pager || true
sudo journalctl -u keepalived -n 100 --no-pager || true

kubectl -n kube-system get pods -l component=etcd -o wide
kubectl -n kube-system logs <ETCD_POD>
sudo crictl ps | grep etcd

sudo ETCDCTL_API=3 etcdctl endpoint health \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

ip link
tracepath <ZIEL_IP>
ping -M do -s 1472 <ZIEL_IP>

sudo ETCDCTL_API=3 etcdctl snapshot save /var/backups/etcd-snapshot-$(date +%F-%H%M).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

sudo ETCDCTL_API=3 etcdctl snapshot status /var/backups/etcd-snapshot-*.db -w table

kubectl cordon <NODE>
kubectl drain <NODE> --ignore-daemonsets --delete-emptydir-data
# Wartung durchführen
kubectl uncordon <NODE>

sudo apt-mark unhold kubeadm || true
sudo apt-get update
sudo apt-get install -y kubeadm=<VERSION>
sudo apt-mark hold kubeadm

sudo kubeadm upgrade plan
sudo kubeadm upgrade apply <VERSION>

sudo apt-mark unhold kubelet kubectl || true
sudo apt-get install -y kubelet=<VERSION> kubectl=<VERSION>
sudo apt-mark hold kubelet kubectl
sudo systemctl daemon-reload
sudo systemctl restart kubelet

sudo kubeadm certs check-expiration

sudo kubeadm certs renew all
sudo systemctl restart kubelet

kubectl get ns
kubectl delete namespace <NAMESPACE>

kubectl drain <NODE> --ignore-daemonsets --delete-emptydir-data
kubectl delete node <NODE>

sudo kubeadm reset
sudo systemctl stop kubelet
sudo systemctl stop containerd || true
sudo systemctl stop crio || true

sudo rm -rf /etc/cni/net.d
sudo rm -rf /var/lib/cni
sudo rm -rf /var/lib/kubelet
sudo rm -rf /etc/kubernetes
sudo rm -rf /var/lib/etcd
sudo iptables -F || true
sudo iptables -t nat -F || true
sudo iptables -t mangle -F || true
sudo ipvsadm --clear || true

sudo apt-mark unhold kubelet kubeadm kubectl || true
sudo apt-get purge -y kubelet kubeadm kubectl kubernetes-cni || true
sudo apt-get autoremove -y

sudo yum remove -y kubelet kubeadm kubectl kubernetes-cni || true

sudo /usr/local/bin/k3s-uninstall.sh || true
sudo /usr/local/bin/k3s-agent-uninstall.sh || true

sudo /usr/local/bin/rke2-uninstall.sh || true
sudo /usr/local/bin/rke2-agent-uninstall.sh || true

sudo k0s stop || true
sudo k0s reset || true

talosctl reset --nodes <NODE_IP> --endpoints <ENDPOINT_IP> --graceful=false --reboot

swapon --show
lsmod | grep br_netfilter
sysctl net.ipv4.ip_forward
sudo crictl info

kubectl get nodes
kubectl get pods -A
kubectl run test --image=busybox:1.36 --restart=Never -- echo ok

kubectl get nodes -o wide
kubectl create deployment spread --image=nginx:stable --replicas=3
kubectl get pods -o wide

kubectl get svc -A | grep LoadBalancer
curl http://<EXTERNAL_IP>

kubectl get pvc,pv
kubectl exec storage-smoke -- cat /data/test.txt

kubectl get nodes
kubectl -n kube-system get pods -l component=kube-apiserver -o wide
kubectl get --raw='/readyz?verbose'

hostnamectl
ip addr
ip route
getent hosts <NAME>
timedatectl
free -h
swapon --show
lsblk
findmnt
sudo cat /sys/class/dmi/id/product_uuid || true

sudo systemctl status containerd --no-pager
sudo systemctl restart containerd
sudo crictl info
sudo crictl ps -a
sudo crictl logs <CONTAINER_ID>
sudo ctr namespaces list

kubeadm version
sudo kubeadm config images list
sudo kubeadm config images pull
sudo kubeadm init --config kubeadm.yaml --upload-certs
kubeadm token create --print-join-command
sudo kubeadm join <ENDPOINT>:6443 --token <TOKEN> --discovery-token-ca-cert-hash sha256:<HASH>
sudo kubeadm reset
sudo kubeadm certs check-expiration

kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods -A -o wide
kubectl describe node <NODE>
kubectl get events -A --sort-by=.lastTimestamp | tail -50
kubectl get --raw='/readyz?verbose'

kubectl get svc,endpoints,endpointslices -A
kubectl get ingress -A
kubectl get gatewayclass,gateway,httproute -A 2>/dev/null || true
kubectl get networkpolicy -A
ss -tulpn
nc -vz <HOST> <PORT>
tracepath <IP>

kubectl get storageclass
kubectl get pv,pvc -A
kubectl describe pvc <PVC> -n <NAMESPACE>
kubectl get volumeattachments -A 2>/dev/null || true
lsblk
findmnt

sudo journalctl -u kubelet -f
sudo journalctl -u containerd -f
sudo journalctl -u crio -f
kubectl logs <POD> -n <NAMESPACE>
kubectl logs <POD> -n <NAMESPACE> --previous
kubectl logs deployment/<DEPLOYMENT> -n <NAMESPACE> --all-containers

kubernetes-workshop/
  docs/
    environments/
      environment-native-baremetal-kubernetes-DE.md
    architecture-kubernetes-DE.md
    concepts-kubernetes-DE.md
    commands-kubernetes-DE.md
    best-practices-kubernetes-DE.md
  ansible/
    inventory.ini
    group_vars/
    playbooks/
    roles/
  clusters/
    bm-single/
      kubeadm.yaml
      cni/
      metallb/
      ingress/
      storage/
    bm-ha/
      kubeadm-ha.yaml
      loadbalancer/
      addons/
  scripts/
    preflight-baremetal.sh
    validate-cluster.sh
    reset-node.sh
  examples/
    smoke/
    network/
    storage/
    security/

#!/usr/bin/env bash
set -euo pipefail

echo "== Host =="
hostnamectl || true
uname -a

echo "== Network =="
ip addr
ip route

echo "== Time =="
timedatectl || true

echo "== Swap =="
swapon --show || true
free -h

echo "== Kernel modules =="
lsmod | grep -E 'overlay|br_netfilter' || true

echo "== sysctl =="
sysctl net.bridge.bridge-nf-call-iptables || true
sysctl net.ipv4.ip_forward || true

echo "== Runtime =="
systemctl is-active containerd || true
systemctl is-active crio || true
crictl info || true

#!/usr/bin/env bash
set -euo pipefail

kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods -A -o wide
kubectl get --raw='/readyz?verbose'

kubectl run validation-dns --image=busybox:1.36 --restart=Never -- sleep 3600
kubectl wait --for=condition=Ready pod/validation-dns --timeout=120s
kubectl exec validation-dns -- nslookup kubernetes.default.svc.cluster.local
kubectl delete pod validation-dns

#!/usr/bin/env bash
set -euo pipefail

read -r -p "Dieser Vorgang löscht Kubernetes-Zustand auf diesem Node. Fortfahren? [yes/NO] " answer
if [[ "${answer}" != "yes" ]]; then
  echo "Abbruch."
  exit 1
fi

sudo kubeadm reset -f || true
sudo systemctl stop kubelet || true
sudo systemctl stop containerd || true
sudo systemctl stop crio || true
sudo rm -rf /etc/cni/net.d /var/lib/cni /var/lib/kubelet /etc/kubernetes
sudo rm -rf /var/lib/etcd