[k8s] Ubuntu 22.04 설치 정리 231029

# 개인 메모장이므로 소스 버전과 사설IP 등 수정이 필요합니다.

# 환경 : proxmox, VM(ubuntu 22.04.3)

# 매우 의존한 자료... : https://tech.hostway.co.kr/2022/08/30/1374/

 

---

# OS 세팅

# VM 생성 과정에 사용자를 'worker' 생성

# 사용자(worker) sudo NOPASSWD 권한추가
echo 'worker  ALL=(ALL) NOPASSWD:ALL' | sudo tee -a /etc/sudoers > /dev/null

# 초기 패키지 설치
sudo apt-get update
sudo apt-get install -y net-tools gcc vim sysstat ca-certificates curl gnupg apt-transport-https

# alias 등록
echo 'alias vi=vim' >> ~/.bashrc && source ~/.bashrc

# 시간동기화(타임존 설정)
sudo timedatectl set-timezone Asia/Seoul

---

# install Docker Engine

# 참고 : https://tech.hostway.co.kr/2022/08/30/1374/

 

# swap off

sudo swapoff /swap.img
sudo sed -i -e '/swap.img/d' /etc/fstab

 

# docker engine

curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh
sudo systemctl enable --now docker && sudo systemctl status docker --no-pager
sudo usermod -aG docker worker
sudo docker container ls

 

# cri-docker Install

# https://github.com/Mirantis/cri-dockerd#installing-manually
VER=$(curl -s https://api.github.com/repos/Mirantis/cri-dockerd/releases/latest|grep tag_name | cut -d '"' -f 4|sed 's/v//g')
echo $VER
wget https://github.com/Mirantis/cri-dockerd/releases/download/v${VER}/cri-dockerd-${VER}.amd64.tgz
tar xvf cri-dockerd-${VER}.amd64.tgz
sudo mv cri-dockerd/cri-dockerd /usr/local/bin/

cri-dockerd --version

 

# cri-docker 서비스 등록

wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service
wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket
sudo mv cri-docker.socket cri-docker.service /etc/systemd/system/
sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service

 

sudo systemctl daemon-reload
sudo systemctl enable cri-docker.service
sudo systemctl enable --now cri-docker.socket

 

# cri-docker Active Check
sudo systemctl restart docker && sudo systemctl restart cri-docker
sudo systemctl status cri-docker.socket --no-pager

 

# Docker cgroup Change Require to Systemd
sudo mkdir -p /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

sudo systemctl restart docker && sudo systemctl restart cri-docker
sudo docker info | grep Cgroup

# Kernel Forwarding 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sudo sysctl --system # read values from all system directories.

---

# All node

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#k8s-install-0

# OS, k8s 버전에 따라 명령어 변경됨.

curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl

kubectl version --short # 버전 확인

sudo apt-mark hold kubelet kubeadm kubectl # 버전 고정

 

 

---

# k8s Init – Controller Node (Node-01)

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#initializing-your-control-plane-node

 

sudo kubeadm config images pull --cri-socket unix:///run/cri-dockerd.sock
MASTERIP=$(hostname -I |awk  '{print $1}') # 컨트롤 노드의 IP이므로 워커노드 작업을 위해 기억해야함.

echo ${MASTERIP}

sudo kubeadm init --ignore-preflight-errors=all --pod-network-cidr=192.168.0.0/16 --apiserver-advertise-address=${MASTERIP} --cri-socket /var/run/cri-dockerd.sock

# Your Kubernetes control-plane has initialized successfully!

# 찬찬히 읽으면서 진행

# kubeadm join ~~ 복사.

 

# 사용자(worker)에게 kubeadm 권한 추가

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

 

# CNI network plugin

# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/ 

# https://projectcalico.docs.tigera.io/manifests

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/tigera-operator.yaml

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.3/manifests/custom-resources.yaml

watch kubectl get pods -n calico-system # Wait until each pod has the STATUS of Running

kubectl taint nodes --all node-role.kubernetes.io/control-plane-
kubectl taint nodes --all node-role.kubernetes.io/master-

kubectl get nodes -o wide

# NAME      STATUS 
#node-01   Ready # 정상

 

 

# calicoctl 명령어 등록 및 status 확인

# 버전 업데이트되므로 최신 버전 확인 필요. https://github.com/projectcalico/calico/releases

cd /usr/local/bin

sudo curl -L https://github.com/projectcalico/calico/releases/download/v3.26.3/calicoctl-linux-amd64 -o calicoctl
sudo chmod +x calicoctl

 

## CNI Type Check 
calicoctl get ippool -o wide

## Block Check
sudo calicoctl ipam show --show-blocks

## BGP Protocol Check
sudo calicoctl node status

## Node Endpoint Check
calicoctl get workloadendpoint -A

 

---

# Node Join – Worker Node (Node-02 , Node-03)

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#join-nodes

# 워커노드에서 복사해둔 join 명령어 실행.

sudo kubeadm join 10.0.0.111:6443 --ignore-preflight-errors=all --cri-socket unix:///var/run/cri-dockerd.sock --token qulp94.lrck9a3m61n24mia --discovery-token-ca-cert-hash sha256:6a09d49f212720fa43a8c997fa913733ad015cdc70e07a73708925a0a761fda5

 

# 컨트롤 노드에서 등록 확인

kubectl get nodes

완료

# pod 생성 테스트

apiVersion: v1
kind: Pod
metadata:
  name: counter
spec:
  containers:
    - name: count
      image: busybox
      args: [/bin/sh, -c, 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']

k apply -f pod.yml

k get pod -o wide

성공. node-03에 올라갔다.

 

 

# alias 등록

echo 'alias k=kubectl' >> ~/.bashrc && source ~/.bashrc

k get nodes # 사용 예시

 

# (Option)사용자(worker)에게 kubeadm 권한 추가. 컨트롤 노드의 설정 그대로 가져오기.

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#optional-controlling-your-cluster-from-machines-other-than-the-control-plane-node

cd ~

scp -r worker@10.0.0.111:/home/worker/.kube/ .

 

# Clean up

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#tear-down

 

 

# 그 외

작업 디렉토리 생성 및 링크

sudo ln -s /home/worker/k8s /k8s

 

---