KubeClipper Development Guide
July 8, 2026 · View on GitHub
1. Development Environment
1.1 Architecture
Arch
+------------------+ +-------------------+
| Ubuntu 24.04 2C4G| | Ubuntu 22.04 |
| Development | | KC-Agent(Optional)|
+--------+---------+ +----------+--------+
| |
| |
| |
+---------------------+------------------------------------+--------------------+
Development Environment：
- Docker & containerd are both in
/usr/bin/containerd - If deploy cluster, new containerd will be installed in
/usr/local/bin/containerdby kubeclipper, which could co-exist with old containerd
Config SSH (permit root login with id_rsa) & apt update -y && apt upgrade -y
# cat /etc/os-release
PRETTY_NAME="Ubuntu 24.04.2 LTS"
NAME="Ubuntu"
VERSION_ID="24.04"
VERSION="24.04.2 LTS (Noble Numbat)"
VERSION_CODENAME=noble
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=noble
LOGO=ubuntu-logo
# uname -a
Linux VM-4-12-ubuntu 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux
1.2 Install Golang in Development Environment
wget https://golang.google.cn/dl/go1.24.0.linux-amd64.tar.gz
# wget https://golang.google.cn/dl/go1.24.3.linux-arm64.tar.gz
tar zxvf go1.24.0.linux-amd64.tar.gz
mkdir -p /opt/go
vi /etc/profile
export GOROOT=/usr/local/go
export GOPATH=/opt/go
export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
export GOPROXY=https://mirrors.aliyun.com/goproxy/,direct
1.3 Install Docker for Development Environment
apt install docker.io docker-compose -y
systemctl enable docker --now
Confirm Version
root@VM-4-12-ubuntu:~# which docker
/usr/bin/docker
root@VM-4-12-ubuntu:~# which containerd
/usr/bin/containerd
root@VM-4-12-ubuntu:~# containerd -version
containerd github.com/containerd/containerd 1.7.24
root@VM-4-12-ubuntu:~# docker version
Client:
Version: 26.1.3
API version: 1.45
Go version: go1.22.2
Git commit: 26.1.3-0ubuntu1~24.04.1
Built: Mon Oct 14 14:29:26 2024
OS/Arch: linux/amd64
Context: default
Server:
Engine:
Version: 26.1.3
API version: 1.45 (minimum version 1.24)
Go version: go1.22.2
Git commit: 26.1.3-0ubuntu1~24.04.1
Built: Mon Oct 14 14:29:26 2024
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.7.24
GitCommit:
runc:
Version: 1.1.12-0ubuntu3.1
GitCommit:
docker-init:
Version: 0.19.0
GitCommit:
Config Docker Mirror in China Mainland
vi /etc/docker/daemon.json
{
"registry-mirrors": [
"https://docker.m.daocloud.io",
"https://mirror.baidubce.com",
"http://hub-mirror.c.163.com"
]
}
Restart Docker service
systemctl daemon-reload
systemctl restart docker
Test Docker
docker run hello-world
1.4 Install Prerequisite Packages
apt install build-essential git curl wget net-tools -y
apt install ntp -y
1.5 Clone & compile
git clone git@github.com:duicikeyihangaolou/kubeclipper.git
cd kubeclipper
git checkout release-1.4
# go clean -modcache
# go mod tidy
# go mod download
make build
Check assembly results:
# ls dist/
kcctl kubeclipper-agent kubeclipper-server
2. Installing & Debugging
2.1 Deploy KC in Deployment Environment
Deploy kcctl
# curl -sfLk https://oss.kubeclipper.io/get-kubeclipper.sh | KC_REGION=cn KC_VERSION=release-1.4 bash -
[INFO] The release-1.4 version will be installed
[INFO] KC_REGION is assigned cn, kc.env file will be created
[INFO] env: Creating environment file /etc/kc/kc.env}
mkdir: created directory '/etc/kc'
[INFO] Downloading package https://kubeclipper.oss-ap-southeast-1.aliyuncs.com/kc/release-1.4/kc-linux-amd64.tar.gz
kubeclipper-server
kubeclipper-agent
kcctl
[INFO] Installing kcctl to /usr/local/bin/kcctl
kcctl has been installed successfully!
Run 'kcctl version' to view the version.
Run 'kcctl -h' for more help.
__ ______________________
/ //_/ ____/ ____/_ __/ /
/ ,< / / / / / / / /
/ /| / /___/ /___ / / / /___
/_/ |_\____/\____/ /_/ /_____/
repository: github.com/kubeclipper
Deploy KC with AIO mode
# Make sure you can ssh localhost without password
# deploy MUST with passwd or pk-file FOR 'kcctl resource' cmd
#
# ssh-keygen -t rsa
# cat /root/.ssh/id_rsa.pub >> authorized_keys
# KC_VERSION=release-1.4 kcctl deploy
KC_VERSION=release-1.4 kcctl deploy --server 10.0.4.12 --agent 10.0.4.12 --pk-file ~/.ssh/id_rsa
# kcctl deploy --help
# kcctl deploy --user root --passwd {local-host-user-pwd} --pkg kc-minimal.tar.gz
# kcctl deploy --server $IPADDR_SERVER --agent $IPADDR_AGENT --passwd xxx # or --pk-file
DO NOT USE KC_VERSION=release-1.4 kcctl deploy since it will not add pkFile & privateKey
in .kc/deploy-config.yaml, which will cause kcctl resource cmd to fail with notification: "one
of pkfile or password must be specify,please config it in ".
# KC_VERSION=release-1.4 kcctl deploy --server 10.0.4.12 --agent 10.0.4.12 --pk-file ~/.ssh/id_rsa
2025-05-30T10:08:59+08:00 INFO Using auto detected IPv4 address on interface eth0: 10.0.4.12/22
[2025-05-30T10:08:59+08:00][INFO] node-ip-detect inherits from ip-detect: first-found
[2025-05-30T10:08:59+08:00][INFO] run in aio mode.
[2025-05-30T10:08:59+08:00][INFO] ============>kc-etcd PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>kc-etcd PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>kc-server PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>kc-server PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>kc-agent PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>kc-agent PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>TIME-LAG PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] BaseLine Time: 2025-05-30T10:08:59+08:00
[2025-05-30T10:08:59+08:00][INFO] [10.0.4.12] -0.949546294 seconds
[2025-05-30T10:08:59+08:00][INFO] all nodes time lag less then 5 seconds
[2025-05-30T10:08:59+08:00][INFO] ============>TIME-LAG PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>NTP PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>NTP PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>sudo PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>sudo PRECHECK OK!
[2025-05-30T10:08:59+08:00][INFO] ============>ipDetect PRECHECK ...
[2025-05-30T10:08:59+08:00][INFO] ============>ipDetect PRECHECK OK!
[2025-05-30T10:09:02+08:00][INFO] ------ Send packages ------
--2025-05-30 10:09:02-- https://kubeclipper.oss-ap-southeast-1.aliyuncs.com/release/release-1.4/kc-amd64.tar.gz
2025-05-30 10:11:29 (6.44 MB/s) - ‘kc-amd64.tar.gz’ saved [982526790/982526790]
10.0.4.12: done!
[2025-05-30T10:11:44+08:00][INFO] ------ Install kc-etcd ------
[2025-05-30T10:11:50+08:00][INFO] ------ Install kc-server ------
[2025-05-30T10:12:03+08:00][INFO] ------ Install kc-agent ------
[2025-05-30T10:12:03+08:00][INFO] ------ Install kc-console ------
[2025-05-30T10:12:04+08:00][INFO] ------ Delete intermediate files ------
[2025-05-30T10:12:04+08:00][INFO] ------ Dump configs ------
[2025-05-30T10:12:04+08:00][INFO] ------ Upload configs ------
10.0.4.12: done!
_ __ _ _____ _ _
| | / / | | / __ \ (_)
| |/ / _ _| |__ ___| / \/ |_ _ __ _ __ ___ _ __
| \| | | | '_ \ / _ \ | | | | '_ \| '_ \ / _ \ '__|
| |\ \ |_| | |_) | __/ \__/\ | | |_) | |_) | __/ |
\_| \_/\__,_|_.__/ \___|\____/_|_| .__/| .__/ \___|_|
| | | |
|_| |_|
repository: github.com/kubeclipper
2.2 Check Services
# systemctl list-unit-files | grep kc
kc-agent.service enabled enabled
kc-console.service enabled enabled
kc-etcd.service enabled enabled
kc-server.service enabled enabled
2.3 Clean
kcctl clean --help
# Uninstall the entire kubeclipper platform.
kcctl clean
kcctl clean -A
2.4 Debug kcctl using gdb
apt install gdb -y
export GOLDFLAGS=""
cd ~/kubeclipper
make build
# or
# make build-cli
# Start GDB debugger and load the kcctl binary for debugging
gdb dist/kcctl
# Set a breakpoint at line 278 of resource.go to pause execution
b resource.go:278
# Run kcctl with login command to authenticate against local server
r login --host http://127.0.0.1 --username admin --password Thinkbig1
# Run kcctl to list cluster resources (triggers breakpoint if hit)
r resource list
# Quit debug session
q
OR, debug with VSCode, see .vscode/launch.json.
3. Tarball k8s
3.1 Tarball
Take v1.32.2 as example:
# wget caas-cd-node/blob/kc/k8s/tarball-kubernetes.sh
# bash tarball-kubernetes.sh -a amd64 -v 1.32.5 -o /tmp -s https://github.com/duicikeyihangaolou/kubeclipper-packages/raw/refs/heads/master/packages
bash tarball-kubernetes.sh -a amd64 -v 1.23.17 -o /tmp -s https://github.com/duicikeyihangaolou/kubeclipper-packages/raw/refs/heads/master/packages
cd /tmp
After tarball-kubernetes.sh, you will get a k8s folder like this, Always keep one version per
time
# find k8s
k8s
k8s/v1.23.17
k8s/v1.23.17/amd64
k8s/v1.23.17/amd64/manifest.json
k8s/v1.23.17/amd64/images.tar.gz
k8s/v1.23.17/amd64/configs.tar.gz
Then, push k8s with kcctl
cd /tmp
k8s_ver='v1.23.17'
tar -zcvf k8s-${k8s_ver}-amd64.tar.gz k8s
kcctl login --host http://127.0.0.1 --username admin --password Thinkbig1
kcctl resource push --pkg k8s-${k8s_ver}-amd64.tar.gz --type k8s
# kcctl resource push --pkg k8s-v1.23.17-amd64.tar.gz --type k8s
# kcctl resource push --pkg k8s-v1.32.5-amd64.tar.gz --type k8s
After that, you can check the k8s version with kcctl resource list
# kcctl resource list
+-----------+---------------+---------------+----------+-------+
| 10.0.4.12 | TYPE | NAME | VERSION | ARCH |
+-----------+---------------+---------------+----------+-------+
| 1. | cni | calico | v3.26.1 | amd64 |
| 2. | cri | containerd | 1.6.4 | amd64 |
| 3. | k8s | k8s | v1.23.17 | amd64 |
| 4. | k8s | k8s | v1.27.4 | amd64 |
| 5. | k8s | k8s | v1.32.5 | amd64 |
| 6. | k8s-extension | k8s-extension | v1 | amd64 |
| 7. | kc-extension | kc-extension | latest | amd64 |
+-----------+---------------+---------------+----------+-------+
Also check directory /opt/kubeclipper-server/resource/
# cat /opt/kubeclipper-server/resource/metadata.json | jq | grep minor_version
"minor_version": "v1.18",
"minor_version": "v1.19",
"minor_version": "v1.20",
"minor_version": "v1.21",
"minor_version": "v1.22",
"minor_version": "v1.23",
"minor_version": "v1.24",
"minor_version": "v1.25",
"minor_version": "v1.26",
"minor_version": "v1.27",
"minor_version": "v1.28",
# ls /opt/kubeclipper-server/resource/k8s
v1.23.17 v1.27.4 v1.32.5
3.2 Add k8s info
1.23.17 K8S could be created from webpage, however CNI yaml validate error:
Normal should be:
Modify metadata.json and change the CNI version as shown below. This still fails with 1.23.17, so the behavior likely depends on additional hardcoded logic that needs follow-up validation.
And as above, new k8s resources didn't add to metadata.json, may be a bug, we need to add it manually now. Without it, in webpage: http://kc-console/cluster/create，we could select containerd version with 1.29+ k8s version.
vi /opt/kubeclipper-server/resource/metadata.json
Modify metadata.json:
@@ -414,14 +414,14 @@
{
"name": "calico",
"type": "cni",
- "recommend_version": "v3.22.4",
+ "recommend_version": "v3.26.1",
"min_version": "",
"max_version": ""
},
{
"name": "nfs",
"type": "csi",
- "recommend_version": "v4.0.2",
+ "recommend_version": "v4.1.0",
"min_version": "",
"max_version": ""
},
@@ -771,6 +771,68 @@
"max_version": ""
}
]
+ },
+ {
+ "name": "k8s",
+ "minor_version": "v1.32",
+ "version_control": [
+ {
+ "name": "containerd",
+ "type": "cri",
+ "recommend_version": "1.6.4",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "calico",
+ "type": "cni",
+ "recommend_version": "v3.26.1",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "nfs",
+ "type": "csi",
+ "recommend_version": "v4.1.0",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "cinder",
+ "type": "csi",
+ "recommend_version": "v1.23",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "ceph",
+ "type": "csi",
+ "recommend_version": "v3.4.0",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "kubesphere",
+ "type": "app",
+ "recommend_version": "v3.2.1",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "metallb",
+ "type": "lb",
+ "recommend_version": "v0.13.7",
+ "min_version": "",
+ "max_version": ""
+ },
+ {
+ "name": "mig",
+ "type": "gpu",
+ "recommend_version": "v0.8.2",
+ "min_version": "",
+ "max_version": ""
+ }
+ ]
}
],
"addon_manifests": [
3.3 Deploy k8s using kcctl, add pause tag
Known issue: in some environments the generated containerd config omits the pause image tag, so it must be corrected manually before retrying deployment.
Modify /etc/containerd/config.toml
vi /etc/containerd/config.toml
# from
# sandbox_image = "registry.k8s.io/pause:"
# to
# sandbox_image = "registry.k8s.io/pause:3.10"
Then, systemctl restart containerd
Pause installation from webpage, then resume from the breakpoint.
You also can backup /tmp/.k8s and reboot host server, however pause & resume from webpage is much eaiser.
# Just log the steps here FYI.
mkdir ~/bak -p
cp /tmp/.k8s ~/bak -a
# after reboot
cp ~/bak/.k8s /tmp -a
# continue deploy k8s with web GUI
Check version:
# kubectl get po -A
... # all running
# kubectl version
Client Version: v1.32.5
Kustomize Version: v5.5.0
Server Version: v1.32.5
4. Demo with kubeedge
4.1 kind deploy k8s
# go 1.16+ and docker, podman or nerdctl installed
# cat portmapping.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraPortMappings:
- containerPort: 10000
hostPort: 10000
- containerPort: 10001
hostPort: 10001
- containerPort: 10002
hostPort: 10002
- containerPort: 10003
hostPort: 10003
- containerPort: 10004
hostPort: 10004
# optional: set the bind address on the host
# 0.0.0.0 is the current default
# listenAddress: "127.0.0.1"
# optional: set the protocol to one of TCP, UDP, SCTP.
# TCP is the default
protocol: TCP
kind create cluster --config portmapping.yaml
# kind delete cluster
# cp to host computer
docker exec kind-control-plane tar -c -f - /usr/bin/kubectl > kubectl.tar
tar xf kubectl.tar
mv usr/bin/kubectl /usr/bin/
4.2 deploy kubeedge with keadm
# cloudnode
wget https://github.com/kubeedge/kubeedge/releases/download/v1.20.0/keadm-v1.20.0-linux-amd64.tar.gz
# scp keadm-v1.20.0-linux-amd64.tar.gz edgenode
tar -zxvf keadm-v1.20.0-linux-amd64.tar.gz
cp keadm-v1.20.0-linux-amd64/keadm/keadm /usr/local/bin/keadm
keadm version
wget https://github.com/kubeedge/kubeedge/releases/download/v1.20.0/kubeedge-v1.20.0-linux-amd64.tar.gz
wget https://github.com/kubeedge/kubeedge/releases/download/v1.20.0/edgesite-v1.20.0-linux-amd64.tar.gz
mkdir /etc/kubeedge
cp kubeedge-v1.20.0-linux-amd64.tar.gz edgesite-v1.20.0-linux-amd64.tar.gz /etc/kubeedge
docker pull kubeedge/installation-package:v1.20.0
docker pull kubeedge/iptables-manager:v1.20.0
docker pull kubeedge/cloudcore:v1.20.0
kind load docker-image kubeedge/installation-package:v1.20.0 --name kind
kind load docker-image kubeedge/iptables-manager:v1.20.0 --name kind
kind load docker-image kubeedge/cloudcore:v1.20.0 --name kind
# keadm init --advertise-address="THE-EXPOSED-IP" --kubeedge-version=v1.20.0
keadm init --advertise-address=192.168.1.208 -v 7
docker exec kind-control-plane ss -tuln
kubectl get all -n kubeedge
kubectl get nodes
# kubectl delete nodes k8s-node1
keadm gettoken
# edgenode
vi /etc/containerd/config.toml
runtime_type = "io.containerd.runc.v2"
# disabled_plugins = ["cri"]
disabled_plugins = []
SystemdCgroup = true
# edge node install cni for cri
wget https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz
mkdir -p /etc/cni/net.d/
$ cat >/etc/cni/net.d/10-containerd-net.conflist <<EOF
{
"cniVersion": "1.0.0",
"name": "containerd-net",
"plugins": [
{
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"promiscMode": true,
"ipam": {
"type": "host-local",
"ranges": [
[{
"subnet": "10.88.0.0/16"
}],
[{
"subnet": "2001:db8:4860::/64"
}]
],
"routes": [
{ "dst": "0.0.0.0/0" },
{ "dst": "::/0" }
]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true}
}
]
}
EOF
systemctl restart containerd.service
ctr -n k8s.io image ls
docker pull kubeedge/installation-package:v1.20.0
docker save kubeedge/installation-package:v1.20.0 -o keip.tar
ctr -n k8s.io image import keip.tar
docker pull pause:3.8
docker save pause:3.8 -o pause.3.8.tar
ctr -n k8s.io image import pause.3.8.tar
ctr -n k8s.io image pull docker.io/library/eclipse-mosquitto:1.6.15
# or
docker pull eclipse-mosquitto:1.6.15
docker save eclipse-mosquitto:1.6.15 -o tt.tar
ctr -n k8s.io image import tt.tar
docker pull docker.io/kindest/kindnetd:v20250214-acbabc1a
docker save docker.io/kindest/kindnetd:v20250214-acbabc1a -o kek.tar
ctr -n k8s.io image import kek.tar
docker pull registry.k8s.io/kube-proxy:v1.32.2
docker save registry.k8s.io/kube-proxy:v1.32.2 -o kep.tar
ctr -n k8s.io image import kep.tar
ctr -n k8s.io image ls
ctr -n kube-system c ls
ctr -n kube-system t ls
ctr -n kube-system i ls
keadm join --cloudcore-ipport=172.18.0.2:10000 \
--token=keadm-gettoken-cloudnode --kubeedge-version=v1.20.0 \
-p=unix:///var/run/docker.sock
systemctl status edgecore.service
# systemctl restart edgecore.service
journalctl -u edgecore.service -xe
4.3 Config kube api in kubeedge node
4.3.1 CloudCore side
kubectl edit cm cloudcore -n kubeedge
dynamicController:
enable: true
docker exec -it kind-control-plane bash
crictl pods
# restart cloudcore pod
crictl stopp cloudcore-5d9ccb9dc8-lv2qb
4.3.2 EdgeCore Side
vi /etc/kubeedge/config/edgecore.yaml
modules:
...
edgeMesh:
enable: false
...
metaManager:
metaServer:
enable: true
vi /etc/kubeedge/config/edgecore.yaml
modules:
...
edged:
...
tailoredKubeletConfig:
...
clusterDNS:
- 169.254.96.16
clusterDomain: cluster.local
systemctl restart edgecore.service
# metaServer port 10550
netstat -natp|grep 10550
curl 127.0.0.1:10550/api/v1/services
curl http://127.0.0.1:10550/api/v1/namespaces/kube-system/pods|jq '.items.[].metadata.name'
"coredns-668d6bf9bc-j8cq4"
"coredns-668d6bf9bc-kf4cl"
"etcd-kind-control-plane"
"kindnet-5bgh7"
"kindnet-qsllc"
"kube-apiserver-kind-control-plane"
"kube-controller-manager-kind-control-plane"
"kube-proxy-92w6x"
"kube-proxy-rcjfq"
"kube-scheduler-kind-control-plane"
# cloud
kubectl get pods -n kube-system -o wide
# edge memroy
ps -p <PID> -o rss
4.4 Deploy hostnetwork application
4.4.1 edge node
docker pull nginx
docker save nginx:latest -o nginx.tar
ctr -n k8s.io image import nginx.tar
ctr image import nginx.tar
ctr -n k8s.io image list
ctr image list
netstat -natp|grep nginx
4.4.2 cloud node
kubectl apply -f hostnginx.yaml
kubectl get pods -o wide
kubectl describe pod nginx-deployment-799c65967c-q8nvt
4.4.3 Application yaml file
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
hostNetwork: true
containers:
- name: nginx
image: docker.io/library/nginx:latest
imagePullPolicy: Never
ports:
- containerPort: 80
hostPort: 80