From 91b6263cf3859c231d755b2bc43cf70b19ce4984 Mon Sep 17 00:00:00 2001 From: Nishan Date: Thu, 10 Apr 2025 23:55:30 +0530 Subject: [PATCH] Migrate from kind to k3d for cluster fromation --- facilitator/assets/exams/labs.json | 8 +-- jumphost/Dockerfile | 14 ++++- jumphost/scripts/cleanup-exam-env.sh | 12 ++-- jumphost/scripts/prepare-exam-env.sh | 1 + kind-cluster/Dockerfile | 20 +------ kind-cluster/entrypoint.sh | 11 +++- kind-cluster/scripts/env-cleanup | 41 +++++++------ kind-cluster/scripts/env-setup | 89 +++++++++++++++------------- 8 files changed, 101 insertions(+), 95 deletions(-) diff --git a/facilitator/assets/exams/labs.json b/facilitator/assets/exams/labs.json index 0cf26da..1c49b38 100644 --- a/facilitator/assets/exams/labs.json +++ b/facilitator/assets/exams/labs.json @@ -6,7 +6,7 @@ "name": "CKAD Comprehensive Lab - 1", "category": "CKAD", "description": "Hands-on exercises covering core Kubernetes concepts, pod configuration, deployments, and services with practical tasks for application deployment in a cluster.", - "warmUpTimeInSeconds": 180, + "warmUpTimeInSeconds": 260, "difficulty": "Medium", "examDurationInMinutes": 120 }, @@ -26,7 +26,7 @@ "name": "CKS Practice Lab - Kubernetes Security Essentials", "category": "CKS", "description": "Practice essential Kubernetes security concepts including network policies, RBAC, supply chain security, and runtime protection", - "warmUpTimeInSeconds": 180, + "warmUpTimeInSeconds": 260, "difficulty": "Hard", "examDurationInMinutes": 120 }, @@ -36,7 +36,7 @@ "name": "CKA Practice Lab - Core Concepts", "category": "CKA", "description": "Practice essential Kubernetes administrator tasks covering cluster management, networking, storage, and security", - "warmUpTimeInSeconds": 160, + "warmUpTimeInSeconds": 260, "difficulty": "Easy", "examDurationInMinutes": 60 }, @@ -56,7 +56,7 @@ "name": "Helm Fundamentals Lab", "category": "Other", "description": "Comprehensive lab covering Helm basics including chart installation, repositories, customization, packaging, debugging, and advanced Helm operations for Kubernetes applications.", - "warmUpTimeInSeconds": 120, + "warmUpTimeInSeconds": 260, "difficulty": "Medium", "examDurationInMinutes": 90 } diff --git a/jumphost/Dockerfile b/jumphost/Dockerfile index 355bb9d..48cb737 100644 --- a/jumphost/Dockerfile +++ b/jumphost/Dockerfile @@ -20,11 +20,19 @@ RUN apt-get update && apt-get install -y \ docker.io\ jq +# Accept build-time architecture argument +ARG TARGETARCH -#kubectl -RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ +#install kubectl for x86_64 and ARM64 architectures +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ && chmod +x kubectl \ - && mv kubectl /usr/local/bin/kubectl + && mv kubectl /usr/local/bin/kubectl; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl; \ + fi # Install Helm RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \ diff --git a/jumphost/scripts/cleanup-exam-env.sh b/jumphost/scripts/cleanup-exam-env.sh index 954c2eb..26daf3f 100644 --- a/jumphost/scripts/cleanup-exam-env.sh +++ b/jumphost/scripts/cleanup-exam-env.sh @@ -1,4 +1,5 @@ #!/bin/bash +exec >> /proc/1/fd/1 2>&1 # cleanup-exam-env.sh # @@ -20,14 +21,9 @@ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null candidate@k8s-ap #cleanup docker env log "Cleaning up docker environment" -docker system prune -a --volumes -f -docker rm -f $(docker ps -a -q) -docker rmi -f $(docker images -q) -docker volume rm $(docker volume ls -q) -docker network rm $(docker network ls -q) -docker container rm $(docker container ls -q) -docker image rm $(docker image ls -q) -docker volume rm $(docker volume ls -q) +docker system prune -a --volumes -fa +docker network prune -fa +docker image prune -fa # Remove the exam environment directory log "Removing exam environment directory" diff --git a/jumphost/scripts/prepare-exam-env.sh b/jumphost/scripts/prepare-exam-env.sh index 6e7183a..33d01c9 100644 --- a/jumphost/scripts/prepare-exam-env.sh +++ b/jumphost/scripts/prepare-exam-env.sh @@ -1,4 +1,5 @@ #!/bin/bash +exec >> /proc/1/fd/1 2>&1 # Log function with timestamp diff --git a/kind-cluster/Dockerfile b/kind-cluster/Dockerfile index ab7918c..37cb4aa 100644 --- a/kind-cluster/Dockerfile +++ b/kind-cluster/Dockerfile @@ -3,24 +3,8 @@ FROM docker:dind # Accept build-time architecture argument ARG TARGETARCH -# install kind for x86_64 architecture -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - wget https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64 && \ - chmod +x kind-linux-amd64 && \ - mv kind-linux-amd64 /usr/local/bin/kind && \ - echo "Building for x86_64 platform"; \ - touch /amd64-ready; \ - fi - -# install kind for ARM64 architecture -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - wget https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-arm64 && \ - chmod +x kind-linux-arm64 && \ - mv kind-linux-arm64 /usr/local/bin/kind && \ - echo "Building for ARM64 platform"; \ - touch /arm64-ready; \ - fi - +#install curl +RUN apk add --no-cache curl # use our own to start kind cluster COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/kind-cluster/entrypoint.sh b/kind-cluster/entrypoint.sh index c0511d0..ba1eaa7 100644 --- a/kind-cluster/entrypoint.sh +++ b/kind-cluster/entrypoint.sh @@ -8,8 +8,12 @@ echo "$(date '+%Y-%m-%d %H:%M:%S') | ===== INITIALIZATION STARTED =====" echo "$(date '+%Y-%m-%d %H:%M:%S') | Executing container startup script..." -# Execute current entrypoint -sh /usr/local/bin/startup.sh & +# Execute current entrypoint script +if [ -f /usr/local/bin/startup.sh ]; then + sh /usr/local/bin/startup.sh & +else + echo "$(date '+%Y-%m-%d %H:%M:%S') | [INFO] Default startup script not found at /usr/local/bin/startup.sh" +fi # =============================================================================== # Docker Readiness Check @@ -36,6 +40,9 @@ adduser -S -D -H -s /sbin/nologin -G sshd sshd #start ssh service /usr/sbin/sshd -D & +#install k3d +wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.8.3 bash + sleep 10 touch /ready diff --git a/kind-cluster/scripts/env-cleanup b/kind-cluster/scripts/env-cleanup index 3e8a922..1339e05 100644 --- a/kind-cluster/scripts/env-cleanup +++ b/kind-cluster/scripts/env-cleanup @@ -1,43 +1,48 @@ -# Cleanup script to remove the Kind cluster +#!/bin/bash +exec >> /proc/1/fd/1 2>&1 + +# Cleanup script to remove the K3d cluster CLUSTER_NAME=${1:-cluster} -#check if CLUSTER_NAME is set +# Check if CLUSTER_NAME is set if [ -z "$CLUSTER_NAME" ]; then echo "CLUSTER_NAME is not set" exit 1 fi -# Check if the Kind cluster is running -if kind get clusters | grep -q "$CLUSTER_NAME"; then - # Delete the Kind cluster - kind delete cluster --name "$CLUSTER_NAME" +# Check if the K3d cluster is running +if k3d cluster list | grep -q "$CLUSTER_NAME"; then + # Delete the K3d cluster + k3d cluster delete "$CLUSTER_NAME" # Wait until the cluster is fully deleted - while kind get clusters -q | grep -q "$CLUSTER_NAME"; do + while k3d cluster list | grep -q "$CLUSTER_NAME"; do sleep 1 done - echo "Kind cluster $CLUSTER_NAME has been deleted." + echo "K3d cluster '$CLUSTER_NAME' has been deleted." else - echo "Kind cluster $CLUSTER_NAME is not running." + echo "K3d cluster '$CLUSTER_NAME' is not running." exit 0 fi -# Perform a full Docker system prune (removes all unused data) but do not remove kindest/node images -docker system prune -a -f --filter "label=org.opencontainers.image.name=kindest/node" +# Perform a full Docker system prune (removes all unused containers, networks, images, and build cache) +# This does NOT remove K3d images, unless you manually want to docker volume prune -f +docker network prune -f +docker image prune -f --filter "label!=ghcr.io/k3d-io" echo "Docker system cleaned up." -#delete kind-config.yaml if present -if [ -f "/tmp/kind-config.yaml" ]; then - rm -f /tmp/kind-config.yaml - echo "Kind-config.yaml file deleted." +# Delete K3d config file if present +if [ -f "/tmp/k3d-config.yaml" ]; then + rm -f /tmp/k3d-config.yaml + echo "K3d config file deleted." fi -#delete kubeconfig file if present in the /root/.kube/kubeconfig -if [ -f "/root/.kube/kubeconfig" ]; then - rm -f /root/.kube/kubeconfig +# Delete kubeconfig file if present in /home/candidate/.kube/kubeconfig (adjust path if needed) +if [ -f "/home/candidate/.kube/kubeconfig" ]; then + rm -f /home/candidate/.kube/kubeconfig echo "Kubeconfig file deleted." fi diff --git a/kind-cluster/scripts/env-setup b/kind-cluster/scripts/env-setup index 1604e19..64ac2ce 100755 --- a/kind-cluster/scripts/env-setup +++ b/kind-cluster/scripts/env-setup @@ -1,54 +1,58 @@ #!/bin/sh +exec >> /proc/1/fd/1 2>&1 # =============================================================================== -# KIND Cluster Creation +# K3D Cluster Creation # this script access the parametter i.e num of nodes i.e NUM_WORKERS # example command to run this script ./setup.sh 3 cluster1 # =============================================================================== NUM_WORKERS=${1:-0} # Default to 1 worker if not provided CLUSTER_NAME=${2:-cluster} -NODE_IMAGE=${3:-kindest/node:v1.32.3} -#delete kind-config.yaml if present -if [ -f "kind-config.yaml" ]; then - echo "kind-config.yaml already exists, deleting it" - rm -f kind-config.yaml +# Delete cluster config if exists +if [ -f "k3d-config.yaml" ]; then + echo "k3d-config.yaml already exists, deleting it" + rm -f k3d-config.yaml fi -cat < /tmp/kind-config.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - extraPortMappings: - - containerPort: 6443 - hostPort: 6443 - kubeadmConfigPatches: - - | - apiVersion: kubeadm.k8s.io/v1beta3 - kind: ClusterConfiguration - apiServer: - certSANs: - - "k8s-api-server" - - "127.0.0.1" - - "localhost" +# Generate K3d config +cat < /tmp/k3d-config.yaml +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: $CLUSTER_NAME +servers: 1 +agents: $NUM_WORKERS +ports: + - port: "$API_PORT:6443" + nodeFilters: + - loadbalancer +kubeAPI: + host: "127.0.0.1" + hostPort: "$API_PORT" +options: + k3s: + extraArgs: + - arg: "--tls-san=k8s-api-server" + nodeFilters: + - server:* + - arg: "--tls-san=127.0.0.1" + nodeFilters: + - server:* + - arg: "--tls-san=localhost" + nodeFilters: + - server:* EOF -for i in $(seq 1 "$NUM_WORKERS"); do - echo " - role: worker" >> /tmp/kind-config.yaml -done +echo "Cluster config with $NUM_WORKERS worker nodes generated: /tmp/k3d-config.yaml" -echo "Cluster config with $NUM_WORKERS worker nodes generated: /tmp/kind-config.yaml" - - -echo "$(date '+%Y-%m-%d %H:%M:%S') | Creating KIND cluster with configuration..." +echo "$(date '+%Y-%m-%d %H:%M:%S') | Creating K3d cluster with configuration..." echo "$(date '+%Y-%m-%d %H:%M:%S') | ├── Name: $CLUSTER_NAME" -echo "$(date '+%Y-%m-%d %H:%M:%S') | ├── Image: $NODE_IMAGE" -echo "$(date '+%Y-%m-%d %H:%M:%S') | ├── Config: /tmp/kind-config.yaml" -echo "$(date '+%Y-%m-%d %H:%M:%S') | └── Number of nodes: $NUM_WORKERS" +echo "$(date '+%Y-%m-%d %H:%M:%S') | ├── Config: /tmp/k3d-config.yaml" +echo "$(date '+%Y-%m-%d %H:%M:%S') | └── Number of nodes: $((NUM_WORKERS + 1)) (1 server + $NUM_WORKERS agents)" -# Create kind cluster -kind create cluster --name $CLUSTER_NAME --image $NODE_IMAGE --config /tmp/kind-config.yaml +# Create k3d cluster +k3d cluster create --config /tmp/k3d-config.yaml # =============================================================================== # Cluster Readiness Check @@ -57,14 +61,14 @@ kind create cluster --name $CLUSTER_NAME --image $NODE_IMAGE --config /tmp/kind- echo "$(date '+%Y-%m-%d %H:%M:%S') | Verifying cluster status..." CLUSTER_CHECK_COUNT=0 -# Wait for kind cluster to be ready -while ! kind get clusters | grep "$CLUSTER_NAME"; do +# Wait for k3d cluster to be ready +while ! k3d cluster list | grep -q "$CLUSTER_NAME"; do CLUSTER_CHECK_COUNT=$((CLUSTER_CHECK_COUNT+1)) - echo "$(date '+%Y-%m-%d %H:%M:%S') | [WAITING] KIND cluster not ready yet... (attempt $CLUSTER_CHECK_COUNT)" - sleep 10 + echo "$(date '+%Y-%m-%d %H:%M:%S') | [WAITING] K3d cluster '$CLUSTER_NAME' not ready yet... (attempt $CLUSTER_CHECK_COUNT)" + sleep 5 done -echo "$(date '+%Y-%m-%d %H:%M:%S') | [SUCCESS] KIND cluster $CLUSTER_NAME is ready and operational" +echo "$(date '+%Y-%m-%d %H:%M:%S') | ✅ K3d cluster '$CLUSTER_NAME' is up and running!" # =============================================================================== # Setup Complete @@ -74,9 +78,10 @@ echo "$(date '+%Y-%m-%d %H:%M:%S') | Docker and KIND environment is ready for us # Save kubeconfig and set API server address cp /home/candidate/.kube/config /home/candidate/.kube/kubeconfig -sed -i 's|server: https://.*|server: https://k8s-api-server:6443|' /home/candidate/.kube/kubeconfig -echo "127.0.0.1 k8s-api-server" >> /etc/hosts +sed -i 's|server: https://127\.0\.0\.1:\([0-9]*\)|server: https://k8s-api-server:\1|' /home/candidate/.kube/kubeconfig + export KUBECONFIG=/home/candidate/.kube/kubeconfig + #info on config file setup done echo "$(date '+%Y-%m-%d %H:%M:%S') | Config file setup done"