EKS Cluster Operations

Step 1: Pre-Upgrade

bash# Check current state
aws eks describe-cluster --name prod-cluster --query "cluster.version"
kubectl get nodes -o wide  # Verify kubelet alignment
eksctl get addon --cluster prod-cluster -o table

# Validate HA control plane (3+ AZs)
aws eks describe-cluster --name prod-cluster --query "cluster.resourcesVpcConfig"

Checklist:

• Review Kubernetes 1.30→1.31 release notes (API deprecations)

• 5+ free IPs per subnet (aws ec2 describe-subnets)

• Cluster Autoscaler matches control plane version

• All add-ons compatible (VPC CNI, CoreDNS, kube-proxy)

Step 2: Backup Everything

bash# etcd snapshot (EKS managed, but validate)
aws eks update-kubeconfig --name prod-cluster
velero backup create eks-pre-upgrade --include-namespaces="*"

# Git commit critical manifests
git add helm/ manifests/ && git commit -m "Pre-upgrade backup 1.31"

# Helm chart values
helm repo update && helm list -A > helm-backup-$(date +%F).txt

Step 3: Centralized Monitoring Dashboard

text# Grafana Dashboard: EKS Cluster Health
apiVersion: v1
kind: ConfigMap
metadata:
  name: eks-dashboard
  namespace: monitoring
data:
  dashboard.json: |
    {
      "nodes_ready": "sum(kube_node_status_condition{condition='Ready',status='true'})",
      "etcd_quorum": "histogram_quantile(0.99, rate(apiserver_request_duration_seconds_bucket[5m]))",
      "pending_pods": "sum(kube_pod_status_phase{phase='Pending'})"
    }

Datadog Monitors:

text- Alert: Nodes NotReady > 10% (5m)
- Critical: API server latency > 5s P99
- Warning: Pending pods > 5

Step 4: Control Plane Upgrade (Zero-Downtime)

bash# Upgrade control plane first
eksctl upgrade cluster --name prod-cluster --version 1.31 --approve

# Monitor status (takes 10-15 mins)
watch 'aws eks describe-cluster --name prod-cluster --query "cluster.status"'

Step 5: Data Plane (Node Groups) Rolling Upgrade

bash# Batch upgrade node groups (10% at a time)
eksctl upgrade nodegroup \
  --cluster prod-cluster \
  --name workers-critical \
  --version 1.31.0-2025.01.18 \
  --max-unavailable 3

# Validate each batch
kubectl get nodes -l node.kubernetes.io/role!=control-plane --sort-by='{.metadata.creationTimestamp}'

Step 6: Add-on Synchronization

bash# Critical add-ons must match control plane
eksctl upgrade addon --name vpc-cni --cluster prod-cluster --approve
eksctl upgrade addon --name coredns --cluster prod-cluster --approve
eksctl upgrade addon --name kube-proxy --cluster prod-cluster --approve

# Cluster Autoscaler (match kubelet version)
kubectl apply -f cluster-autoscaler/eks-autoscaler-rbac.yaml
kubectl set image deployment.apps/cluster-autoscaler -n kube-system \
  cluster-autoscaler=k8s.gcr.io/autoscaling/cluster-autoscaler:v1.31.0

Step 7: HA Control Plane Validation

bash# Verify etcd quorum across AZs
kubectl get endpoints kubernetes -o yaml | grep -A5 etcd

# Test AZ failure simulation
aws ec2 reboot-instances --instance-ids i-xxx # (test AZ only)

# Validate control plane endpoints
kubectl get ep kubernetes -o jsonpath='{.subsets[*].addresses[*].ip}'

Step 8: Post-Upgrade Validation (2 Weeks)

bash# Chaos validation
kubectl apply -f chaos/experiments/pod-delete.yaml
kubectl apply -f chaos/experiments/node-drain.yaml

# Long-term monitoring
kubectl top nodes --sort-by=cpu
kubectl top pods --all-namespaces --sort-by=cpu

Step 9: Cluster Outage Notifications (Datadog)

text# datadog-agent-cluster-agent.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-agent
spec:
  template:
    spec:
      containers:
      - name: cluster-agent
        env:
        - name: DD_CLUSTER_CHECKS_ENABLED
          value: "true"
        - name: DD_LOGS_CONFIG_KUBELET_ENRICHER_ENABLED
          value: "true"

Datadog Monitors:

textTrigger: cluster:status != "Active" for 2m
Recovery: All nodes Ready + API healthy
Notify: #cluster-outages @oncall-sre