☸️ Kubernetes 완전 가이드

#!/bin/bash
# 모든 노드에서 실행 (Control Plane + Worker 공통)

# 1. 호스트명 설정 (각 노드마다 다르게)
hostnamectl set-hostname k8s-master-01   # Worker: k8s-worker-01, k8s-worker-02

# 2. /etc/hosts 에 모든 노드 등록
cat >> /etc/hosts << 'EOF'
192.168.1.10  k8s-master-01
192.168.1.11  k8s-worker-01
192.168.1.12  k8s-worker-02
EOF

# 3. swap 비활성화 (필수 — kubelet 요구사항)
swapoff -a
sed -i '/swap/d' /etc/fstab

# 3-1. SELinux 비활성화 (RHEL/Rocky Linux 계열 필수)
if [ -f /etc/selinux/config ]; then
  setenforce 0
  sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
fi

# 4. 커널 모듈 로드
cat > /etc/modules-load.d/k8s.conf << 'EOF'
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter

# 5. sysctl 네트워크 파라미터
cat > /etc/sysctl.d/k8s.conf << 'EOF'
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

# 6. 방화벽 포트 개방

# [Debian/Ubuntu 계열 - ufw 사용 시]
# Control Plane:
# ufw allow 6443/tcp    # API Server
# ufw allow 2379:2380/tcp # etcd
# ufw allow 10250/tcp   # kubelet API
# ufw allow 10259/tcp   # kube-scheduler
# ufw allow 10257/tcp   # kube-controller-manager
# Worker Node:
# ufw allow 10250/tcp   # kubelet API
# ufw allow 30000:32767/tcp # NodePort range
# ufw allow from 192.168.1.0/24 # 내부망 전체 허용

# [RHEL/Rocky Linux 계열 - firewalld 사용 시]
# Control Plane:
# firewall-cmd --permanent --add-port=6443/tcp
# firewall-cmd --permanent --add-port=2379-2380/tcp
# firewall-cmd --permanent --add-port=10250/tcp
# firewall-cmd --permanent --add-port=10259/tcp
# firewall-cmd --permanent --add-port=10257/tcp
# Worker Node:
# firewall-cmd --permanent --add-port=10250/tcp
# firewall-cmd --permanent --add-port=30000-32767/tcp
# firewall-cmd --reload

#!/bin/bash
# containerd 설치 (모든 노드 - Linux 배포판별 선택 실행)

# ── 1. Debian/Ubuntu 계열 (APT) ──────────────────
if command -v apt-get &> /dev/null; then
  apt-get update
  apt-get install -y ca-certificates curl gnupg
  install -m 0755 -d /etc/apt/keyrings
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg |     gpg --dearmor -o /etc/apt/keyrings/docker.gpg
  chmod a+r /etc/apt/keyrings/docker.gpg

  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg]   https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" |     tee /etc/apt/sources.list.d/docker.list

  apt-get update
  apt-get install -y containerd.io
fi

# ── 2. RHEL/Rocky Linux 계열 (DNF) ───────────────
if command -v dnf &> /dev/null && ! command -v apt-get &> /dev/null; then
  dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
  dnf install -y containerd.io
fi

# ── 3. containerd 기본 설정 생성 및 SystemdCgroup 활성화 ──
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
# SystemdCgroup 활성화 (kubelet의 cgroup driver와 일치화)
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

systemctl restart containerd
systemctl enable containerd
systemctl status containerd

#!/bin/bash
# kubeadm, kubelet, kubectl 설치 (모든 노드 - Linux 배포판별 선택 실행)

# ── 1. Debian/Ubuntu 계열 (APT) ──────────────────
if command -v apt-get &> /dev/null; then
  apt-get update
  apt-get install -y apt-transport-https ca-certificates curl gpg

  curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.36/deb/Release.key |     gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

  echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg]   https://pkgs.k8s.io/core:/stable:/v1.36/deb/ /" |     tee /etc/apt/sources.list.d/kubernetes.list

  apt-get update
  apt-get install -y kubelet kubeadm kubectl
  apt-mark hold kubelet kubeadm kubectl   # 자동 업그레이드 방지
fi

# ── 2. RHEL/Rocky Linux 계열 (DNF) ───────────────
if command -v dnf &> /dev/null && ! command -v apt-get &> /dev/null; then
  cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

  dnf install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
fi

# ── 3. 서비스 활성화 ──────────────────────────────
systemctl enable --now kubelet

# 버전 확인
kubeadm version
kubectl version --client

#!/bin/bash
# Control Plane 노드에서만 실행

# kubeadm init — Pod CIDR은 CNI 선택에 맞춰야 함
# Calico: 192.168.0.0/16 / Flannel: 10.244.0.0/16
kubeadm init \
  --control-plane-endpoint "k8s-master-01:6443" \
  --pod-network-cidr=192.168.0.0/16 \
  --service-cidr=10.96.0.0/12 \
  --kubernetes-version=v1.36.0 \
  --cri-socket unix:///run/containerd/containerd.sock \
  2>&1 | tee /root/kubeadm-init.log

# 성공 시 출력 예:
# Your Kubernetes control-plane has initialized successfully!
# kubeadm join k8s-master-01:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>

# kubectl 설정 (일반 사용자)
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

# 클러스터 상태 확인 (CNI 설치 전이라 NotReady)
kubectl get nodes
kubectl get pods -n kube-system

#!/bin/bash
# Worker 노드에서 실행
# kubeadm init 출력에서 복사한 join 명령 실행

kubeadm join k8s-master-01:6443 \
  --token <your-token> \
  --discovery-token-ca-cert-hash sha256:<your-hash>

# ── 토큰 만료(24h) 후 재발급 방법 (Control Plane에서) ────────
# kubeadm token create --print-join-command

# Control Plane에서 클러스터 전체 상태 확인

# 노드 목록 (CNI 전: NotReady)
kubectl get nodes -o wide

# 시스템 Pod 상태
kubectl get pods -n kube-system

# etcd 헬스체크
kubectl exec -n kube-system etcd-k8s-master-01 -- \
  etcdctl endpoint health \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# API Server 응답 확인 (kubectl get componentstatuses는 deprecated — /healthz?verbose 사용 권장)
kubectl cluster-info
kubectl get --raw='/healthz?verbose'

# HA Control Plane 추가 (3-node HA 구성 시)
# Control Plane #2, #3 에서 실행

# kubeadm init 출력의 --control-plane 명령 사용
kubeadm join k8s-master-01:6443 \
  --token <token> \
  --discovery-token-ca-cert-hash sha256:<hash> \
  --control-plane \
  --certificate-key <cert-key>

# certificate-key 재발급 (만료 시)
# kubeadm init phase upload-certs --upload-certs

# HA 완료 후 로드밸런서로 API Server 분산 필요
# HAProxy / keepalived / 클라우드 LB를 VIP(192.168.1.100:6443)로 설정
# --control-plane-endpoint는 VIP 또는 DNS로 지정

# Calico 설치 (권장)
# kubeadm init 시 --pod-network-cidr=192.168.0.0/16 지정 필요

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml

cat > calico-custom.yaml << 'EOF'
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  calicoNetwork:
    ipPools:
    - blockSize: 26
      cidr: 192.168.0.0/16
      encapsulation: VXLANCrossSubnet   # 멀티 서브넷 환경
      natOutgoing: Enabled
      nodeSelector: all()
  nodeMetricsPort: 9091
---
apiVersion: operator.tigera.io/v1
kind: APIServer
metadata:
  name: default
spec: {}
EOF
kubectl create -f calico-custom.yaml

# 설치 완료 확인 (모든 Pod Running)
watch kubectl get pods -n calico-system

# 노드 Ready 확인
kubectl get nodes

# NetworkPolicy — Namespace 간 격리 예시
# default namespace의 Pod는 같은 namespace 내에서만 통신 허용
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-ingress
  namespace: production
spec:
  podSelector: {}          # 모든 Pod에 적용
  policyTypes:
  - Ingress
---
# API 서버만 DB 접근 허용
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-api-to-db
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: postgres
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          role: api-server
    ports:
    - port: 5432

# CoreDNS 및 DNS 해석 확인
kubectl get pods -n kube-system -l k8s-app=kube-dns

# DNS 동작 테스트
kubectl run dns-test --image=busybox:1.36 --rm -it --restart=Never -- \
  nslookup kubernetes.default

# Service DNS 해석 형식
# <service>.<namespace>.svc.cluster.local
# 예: postgres-svc.database.svc.cluster.local

# local-path-provisioner 설치 (단일 노드 개발 환경)
kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.28/deploy/local-path-storage.yaml

# default StorageClass로 지정
kubectl patch storageclass local-path \
  -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

kubectl get storageclass

# NFS 서버 설치 (스토리지 노드에서)
apt-get install -y nfs-kernel-server
mkdir -p /data/nfs
chmod 777 /data/nfs
echo "/data/nfs 192.168.1.0/24(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
exportfs -ra
systemctl enable --now nfs-server

# NFS 클라이언트 설치 (모든 Worker 노드)
apt-get install -y nfs-common

# NFS 서브디렉터리 Provisioner (Helm)
helm repo add nfs-subdir-external-provisioner \
  https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm repo update

helm install nfs-provisioner \
  nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
  --namespace kube-system \
  --set nfs.server=192.168.1.50 \
  --set nfs.path=/data/nfs \
  --set storageClass.name=nfs-client \
  --set storageClass.defaultClass=true \
  --set storageClass.reclaimPolicy=Retain

kubectl get storageclass

# 1) Gateway API CRD 설치 (GatewayClass/Gateway/HTTPRoute 등록) — standard channel
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml

# 2) Gateway Controller 설치 — 여기서는 Contour 예시 (Envoy Gateway, Cilium 등으로 대체 가능)
#    Contour Gateway Provisioner가 GatewayClass "contour"를 자동 생성
kubectl apply -f https://projectcontour.io/quickstart/contour-gateway-provisioner.yaml
kubectl get gatewayclass contour

# 팀별 위임(HTTPProxy)·정책 설정 등 심화 내용은 K8s 심화 가이드의
# "Contour & Gateway API" 섹션(/learn/kubernetes-advanced/#contour-gateway)에서 이어집니다.

# ingress2gateway 설치 (Krew 플러그인 또는 바이너리 다운로드)
# https://github.com/kubernetes-sigs/ingress2gateway/releases

# 현재 클러스터의 ingress-nginx 기반 Ingress를 Gateway API 리소스로 변환해 출력
ingress2gateway print --providers=ingress-nginx > gateway-resources.yaml

# 결과 검토 후 적용 (기존 Ingress는 자동으로 삭제되지 않으므로 병행 운영하며 트래픽 전환 가능)
kubectl apply -f gateway-resources.yaml

# 전환 검증 후 레거시 Ingress 및 ingress-nginx 컨트롤러 제거
# kubectl delete ingress myapp-ingress -n production
# helm uninstall ingress-nginx -n ingress-nginx

# ── Legacy 참고용 — 신규 구축에는 권장하지 않음 (ingress-nginx는 2026-03-24 EOL) ──
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update

helm install ingress-nginx ingress-nginx/ingress-nginx \
  --namespace ingress-nginx --create-namespace \
  --set controller.service.type=NodePort \
  --set controller.service.nodePorts.http=30080 \
  --set controller.service.nodePorts.https=30443

kubectl get pods -n ingress-nginx -w

# kube-prometheus-stack 설치
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

cat > monitoring-values.yaml << 'EOF'
grafana:
  adminPassword: "ChangeMe!2024"
  persistence:
    enabled: true
    size: 10Gi
  ingress:
    enabled: true
    ingressClassName: nginx
    annotations:
      cert-manager.io/cluster-issuer: letsencrypt-prod
    hosts:
    - grafana.aidevops.kr
    tls:
    - secretName: grafana-tls
      hosts:
      - grafana.aidevops.kr

prometheus:
  prometheusSpec:
    retention: 30d
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          resources:
            requests:
              storage: 50Gi
    serviceMonitorSelectorNilUsesHelmValues: false  # 모든 ServiceMonitor 수집

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          resources:
            requests:
              storage: 5Gi

nodeExporter:
  enabled: true

kubeStateMetrics:
  enabled: true
EOF

helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  -f monitoring-values.yaml

kubectl get pods -n monitoring -w

# Grafana 주요 대시보드 ID (grafana.com/dashboards)
# 웹 UI에서 Import → ID 입력

# K8s 클러스터 전체 현황
# Dashboard ID: 15661 (Kubernetes / Views / Global)

# 노드 메트릭 (CPU/메모리/디스크/네트워크)
# Dashboard ID: 1860 (Node Exporter Full)

# Pod 메트릭 (namespace별)
# Dashboard ID: 6417 (Kubernetes Pods)

# Spring Boot 애플리케이션
# Dashboard ID: 19004 (Spring Boot 3.x Statistics)

# Gateway Controller (Envoy 기반 — Contour/Envoy Gateway 등)
# Dashboard ID: 11021 (Envoy Global), 11022 (Envoy Clusters)
# Legacy ingress-nginx를 아직 쓴다면: Dashboard ID 9614 (NGINX Ingress controller)

# Alertmanager Slack 연동
kubectl create secret generic alertmanager-slack \
  --from-literal=slack_api_url=https://hooks.slack.com/services/XXX \
  -n monitoring

# ConfigMap 생성 (비민감 데이터)
kubectl create configmap myapp-config \
  --from-literal=DB_HOST=postgres \
  --from-literal=APP_ENV=production

# Secret 생성
kubectl create secret generic myapp-secret \
  --from-literal=DB_PASSWORD=s3cr3t

# 파드에서 환경변수로 사용
envFrom:
- configMapRef:
    name: myapp-config
- secretRef:
    name: myapp-secret

# Base64는 누구나 즉시 복원 가능 — "인코딩"일 뿐 "암호화"가 아님을 직접 확인
kubectl get secret myapp-secret -o jsonpath='{.data.DB_PASSWORD}' | base64 -d
# → s3cr3t (권한만 있으면 누구든 원문 확인 가능)

# etcd Encryption at Rest 활성화 여부는 kube-apiserver 매니페스트로 확인
# (kubeadm 클러스터는 /etc/kubernetes/manifests/kube-apiserver.yaml)
grep -- "--encryption-provider-config" /etc/kubernetes/manifests/kube-apiserver.yaml
# 아무 출력이 없다면 etcd에 Secret이 평문으로 저장되고 있다는 뜻

# 환경별 Namespace 생성
kubectl create namespace development
kubectl create namespace staging
kubectl create namespace production
kubectl create namespace monitoring
kubectl create namespace database
kubectl create namespace infra

# 레이블 부여 (NetworkPolicy, ResourceQuota 셀렉터)
kubectl label namespace production  env=prod  team=backend
kubectl label namespace development env=dev   team=backend
kubectl label namespace monitoring  env=infra team=ops

# Namespace 목록
kubectl get namespaces --show-labels