蜷缩的蜗牛
蜷缩的蜗牛

Karmada多集群管理高可用部署

kbsonlong 收录于 Karmada  多集群  云原生
 约 1400 字 预计阅读 3 分钟 - 次阅读  

Karmada 多集群管理高可用部署

实验环境:

  • kind 创建两套Kubernetes集群
  • docker 运行一套额外的 ETCD 集群

Karmada Ha

从图中可以看到整个 Karmada 集群的单点是ETCD,实验环境部署单节点ETCD集群模拟实验，实际生产环境部署跨多可用区的ETCD集群，以保证高可用。

环境准备 

1
2
3
4
5
6

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64

chmod +x cfssl_1.6.4_linux_amd64 cfssljson_1.6.4_linux_amd64
mv cfssl_1.6.4_linux_amd64 /usr/local/bin/cfssl
mv cfssljson_1.6.4_linux_amd64 /usr/local/bin/cfssljson

生成ETCD集群证书 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

set -ex
VIP=10.90.209.105
expiry_houry=87600h
clusterDomain="cluster.local"
basePath="/opt/certs"
mkdir -p ${basePath}/{etcd,karmada}

cat << EOF >${basePath}/etcd/ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}"
    },
    "profiles": {
      "etcd": {
         "expiry": "${expiry_houry}",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat << EOF >${basePath}/etcd/ca-csr.json
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Guangzhou",
            "ST": "Guangzhou"
        }
    ]
}
EOF

cfssl gencert -initca ${basePath}/etcd/ca-csr.json | cfssljson -bare ${basePath}/etcd/ca

cat << EOF >${basePath}/etcd/server-csr.json
{
    "CN": "etcd",
    "hosts": [
      "kubernetes.default.svc",
      "*.etcd.karmada-system.svc.${clusterDomain}",
      "*.karmada-system.svc.${clusterDomain}",
      "*.karmada-system.svc",
      "localhost",
      "127.0.0.1",
      "${VIP}"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Guangzhou",
            "ST": "Guangzhou"
        }
    ]
}
EOF

cfssl gencert -ca=${basePath}/etcd/ca.pem -ca-key=${basePath}/etcd/ca-key.pem -config=${basePath}/etcd/ca-config.json -profile=etcd ${basePath}/etcd/server-csr.json | cfssljson  -f - -bare ${basePath}/etcd/server

运行 ETCD 集群 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

docker run -d --name etcd1 -p 2379:2379 \
    -v ${basePath}/etcd:/opt/etcd/ssl/ \
    registry.cn-hangzhou.aliyuncs.com/seam/etcd:3.5.9-0 \
    etcd --name etcd1 --initial-advertise-peer-urls https://${VIP}:2380 \
     --listen-peer-urls https://0.0.0.0:2380 \
     --listen-client-urls https://0.0.0.0:2379 \
     --advertise-client-urls https://${VIP}:2379 \
     --initial-cluster-token etcd-cluster-1 \
     --initial-cluster-state new \
     --client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
     --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem \
     --peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
     --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem

docker exec -ti etcd1 sh

ETCDCTL_API=3 etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem  endpoint health
#  127.0.0.1:2379 is healthy: successfully committed proposal: took = 6.695865ms

ETCDCTL_API=3 etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem  get --keys-only --prefix=true "/"

exit

生成Karmada集群证书 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

openssl req -x509 -sha256 -new -nodes -days 3650 -newkey rsa:2048 -keyout "${basePath}/karmada/server-ca.key" -out "${basePath}/karmada/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
openssl req -x509 -sha256 -new -nodes -days 3650 -newkey rsa:2048 -keyout "${basePath}/karmada/front-proxy-ca.key" -out "${basePath}/karmada/front-proxy-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"

cat << EOF >${basePath}/karmada/server-ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}",
      "usages": [
        "signing",
        "key encipherment",
        "client auth",
        "server auth"
      ]
    }
  }
}
EOF

cat << EOF  | cfssl gencert -ca=${basePath}/karmada/server-ca.crt -ca-key=${basePath}/karmada/server-ca.key -config=${basePath}/karmada/server-ca-config.json - | cfssljson -bare ${basePath}/karmada/karmada
{
  "CN": "system:admin",
  "hosts": [
    "kubernetes.default.svc",
    "*.etcd.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc",
    "localhost",
    "127.0.0.1",
    "${VIP}"
  ],
  "names": [
    {
      "O": "system:masters"
    }
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  }
}
EOF

cat << EOF > ${basePath}/karmada/front-proxy-ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}",
      "usages": [
        "signing",
        "key encipherment",
        "client auth",
        "server auth"
      ]
    }
  }
}
EOF

cat << EOF | cfssl gencert -ca=${basePath}/karmada/front-proxy-ca.crt -ca-key=${basePath}/karmada/front-proxy-ca.key -config=${basePath}/karmada/front-proxy-ca-config.json - | cfssljson -bare ${basePath}/karmada/front-proxy-client
{
  "CN": "front-proxy-client",
  "hosts": [
    "kubernetes.default.svc",
    "*.etcd.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc",
    "localhost",
    "127.0.0.1",
    "${VIP}"
  ],
  "names": [
    {
      "O": "system:masters"
    }
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  }
}
EOF

karmada_ca=$(cat ${basePath}/karmada/server-ca.crt | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_ca_key=$(cat ${basePath}/karmada/server-ca.key | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_crt=$(cat ${basePath}/karmada/karmada.pem | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_key=$(cat ${basePath}/karmada/karmada-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_ca=$(cat ${basePath}/karmada/front-proxy-ca.crt | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_client_crt=$(cat ${basePath}/karmada/front-proxy-client.pem | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_client_key=$(cat ${basePath}/karmada/front-proxy-client-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")

etcdcaCrt=$(cat ${basePath}/etcd/ca.pem | sed ":tag;N;s/\n/\\\n/;b tag")
etcdcrt=$(cat ${basePath}/etcd/server.pem | sed ":tag;N;s/\n/\\\n/;b tag")
etcdkey=$(cat ${basePath}/etcd/server-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")

创建 Kubernetes 集群 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

cat << EOF > primary.yaml
kind: Cluster
apiVersion: "kind.x-k8s.io/v1alpha4"
kubeadmConfigPatches:
- |
  apiVersion: kubeadm.k8s.io/v1beta3
  kind: ClusterConfiguration
  metadata:
    name: primary
  imageRepository: registry.aliyuncs.com/google_containers
  networking:
    dnsDomain: "${clusterDomain}"
networking:
  apiServerAddress: "${VIP}"
  podSubnet: "10.8.0.0/16"
  serviceSubnet: "10.9.0.0/16"
  dnsSearch: ["${clusterDomain}"]
nodes:
  - role: control-plane
    image: registry.cn-hangzhou.aliyuncs.com/seam/node:v1.29.0
    extraPortMappings:
      - containerPort: 5443
        hostPort: 5443
        protocol: TCP
EOF


kind create cluster --name=primary --kubeconfig=primary --config=primary.yaml --retain


cat << EOF > secondary.yaml
kind: Cluster
apiVersion: "kind.x-k8s.io/v1alpha4"
kubeadmConfigPatches:
- |
  apiVersion: kubeadm.k8s.io/v1beta3
  kind: ClusterConfiguration
  metadata:
    name: secondary
  imageRepository: registry.aliyuncs.com/google_containers
  networking:
    dnsDomain: "${clusterDomain}"
networking:
  apiServerAddress: "${VIP}"
  podSubnet: "10.10.0.0/16"
  serviceSubnet: "10.11.0.0/16"
  dnsSearch: ["${clusterDomain}"]
nodes:
  - role: control-plane
    image: registry.cn-hangzhou.aliyuncs.com/seam/node:v1.29.0
    extraPortMappings:
      - containerPort: 5443
        hostPort: 7443
        protocol: TCP
EOF

kind create cluster --name=secondary --kubeconfig=secondary --config=secondary.yaml --retain

部署 Karmada 集群 

1
2
3
4

helm --namespace karmada-system upgrade --install   karmada karmada-charts/karmada -f values.json --kubeconfig primary --create-namespace


helm --namespace karmada-system upgrade --install   karmada karmada-charts/karmada -f values.json --kubeconfig secondary --create-namespace

获取 Karmada 控制面的 kubeconfig 

1
2
3

# kubectl  get secret -n karmada-system --kubeconfig primary karmada-kubeconfig -o jsonpath="{.data.kubeconfig}" | base64 -d | sed "s/karmada-apiserver.karmada-system.svc.${clusterDomain}:5443/${VIP}:5443/g" > karmada-primary

# kubectl get secret -n karmada-system --kubeconfig secondary karmada-kubeconfig -o jsonpath="{.data.kubeconfig}" | base64 -d | sed "s/karmada-apiserver.karmada-system.svc.${clusterDomain}:5443/${VIP}:7443/g" > karmada-secondary

注册成员集群 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# kubectl karmada join primary --kubeconfig=karmada-primary --cluster-kubeconfig=primary
cluster(primary) is joined successfully

# kubectl karmada join secondary --kubeconfig=karmada-primary --cluster-kubeconfig=secondary
cluster(secondary) is joined successfully

# kubectl get cluster --kubeconfig karmada-primary
NAME        VERSION   MODE   READY   AGE
primary     v1.29.0   Push   True    20s
secondary   v1.29.0   Push   True    15s
# kubectl get cluster --kubeconfig karmada-secondary
NAME        VERSION   MODE   READY   AGE
primary     v1.29.0   Push   True    12s
secondary   v1.29.0   Push   True    10s
1
2

# karmadactl join primary --kubeconfig=karmada-primary --cluster-kubeconfig=primary --cluster-context kind-primary
# karmadactl join secondary --kubeconfig=karmada-primary --cluster-kubeconfig=secondary --cluster-context kind-secondary

资源分发 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

cat << EOF >propagationpolicy.yaml
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-propagation
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: nginx
  placement:
    clusterAffinity:
      clusterNames:
        - primary
        - secondary
    replicaScheduling:
      replicaDivisionPreference: Weighted
      replicaSchedulingType: Divided
EOF

cat << EOF >deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx
        name: nginx
EOF
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#  kubectl apply -f deployment.yaml --kubeconfig karmada-primary
deployment.apps/nginx created
#  kubectl apply -f propagationpolicy.yaml --kubeconfig karmada-primary
propagationpolicy.policy.karmada.io/nginx-propagation created

#  kubectl get pod  --kubeconfig secondary
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7854ff8877-6m42g   1/1     Running   0          40m
#  kubectl get pod  --kubeconfig primary
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7854ff8877-zzj45   1/1     Running   0          40m
#

查看Karmada注册到ETCD数据 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

kubectl exec -ti --kubeconfig secondary -n karmada-system etcd-0  \
  -- sh -c 'ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/server-ca.crt \
            ETCDCTL_CERT=/etc/kubernetes/pki/etcd/karmada.crt \
            ETCDCTL_KEY=/etc/kubernetes/pki/etcd/karmada.key \
            ETCDCTL_API=3  \
            etcdctl \
              get \
              --keys-only \
              --prefix=true \
              "/registry/karmada/" '
更新于 2024-03-04 
CC BY-NC 4.0
阅读原始文档
LINE Myspace
Karmada多集群Multicluster
返回 | 主页
0%