mirror of
https://github.com/open-cluster-management-io/ocm.git
synced 2026-08-23 22:26:49 +00:00
📖 add a solution to join cluster through proxy server (#311)
Signed-off-by: Yang Le <yangle@redhat.com>
This commit is contained in:
Yang Le
@@ -0,0 +1,71 @@
|
||||
# Join hub through a forward proxy server
|
||||
|
||||
This doc shows how a cluster joins an Open Cluster Management (OCM) hub cluster through a forward proxy server.
|
||||
|
||||
## Prerequisite
|
||||
|
||||
- [kind](https://kind.sigs.k8s.io) must be installed on your local machine. The Kubernetes version must be >= 1.19, see [kind user guide](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster) for more details.
|
||||
|
||||
- Download and install [clusteradm](https://github.com/open-cluster-management-io/clusteradm/releases). For Linux OS, run the following commands:
|
||||
|
||||
```
|
||||
wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_linux_amd64.tar.gz | sudo tar -xvz -C /usr/local/bin/
|
||||
|
||||
sudo chmod +x /usr/local/bin/clusteradm
|
||||
```
|
||||
Note: In order to run the scripts provided in this doc successfully, the clusteradm version must be > 0.7.1. You can also build it from the latest [source code](https://github.com/open-cluster-management-io/clusteradm) that contains the desired feature.
|
||||
|
||||
## Setup the proxy server
|
||||
|
||||
Run `./setup-proxy.sh`. It creates a Kind cluster and runs an open source proxy server [squid](http://www.squid-cache.org/) on it. HTTP GET method is enabled on this proxy server and it is serving on ports 31280 and 31290 for HTTP and HTTPS service respectively.
|
||||
```bash
|
||||
kubectl -n squid get svc -o wide
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
|
||||
squid NodePort 10.96.223.82 <none> 3128:31280/TCP,3129:31290/TCP 3h49m app=squid
|
||||
```
|
||||
|
||||
## Setup OCM hub cluster and managed clusters
|
||||
|
||||
Run `./setup-ocm.sh` to create three Kind clusters:
|
||||
- `hub`, the cluster on which the OCM hub is running;
|
||||
- `cluster1`, it joins the hub cluster through the HTTP proxy;
|
||||
- `cluster2`, it joins the hub cluster through the HTTPS proxy;
|
||||
|
||||
Once it is completed, `cluster1` and `cluster2` successfully join the hub cluster through the forward proxy server.
|
||||
```bash
|
||||
$ kubectl get managedcluster
|
||||
NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE
|
||||
cluster1 true https://cluster1-control-plane:6443 True True 111s
|
||||
cluster2 true https://cluster2-control-plane:6443 True True 44s
|
||||
```
|
||||
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
1. Check if the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret` contains the proxy settings
|
||||
|
||||
During bootstrapping, the Klusterlet agent uses the `bootstrap-hub-kubeconfig` to connect to the hub cluster. The proxy settings should be included in this kubeconfig.
|
||||
```bash
|
||||
kubectl -n open-cluster-management-agent get secret bootstrap-hub-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d | grep proxy-url
|
||||
proxy-url: http://proxy-control-plane:31280
|
||||
```
|
||||
Once the cluster registered to the hub cluster, the generated `hub-kubeconfig-secret` should also include the same proxy settings.
|
||||
```bash
|
||||
kubectl -n open-cluster-management-agent get secret hub-kubeconfig-secret -o jsonpath='{.data.kubeconfig}' | base64 -d | grep proxy-url
|
||||
proxy-url: http://proxy-control-plane:31280
|
||||
```
|
||||
If you enabled any add-on, the generated hub kubeconfig for add-on will have this proxy settings as well.
|
||||
|
||||
2. Check if the proxy CA certificate is included in the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret`.
|
||||
|
||||
If HTTPS proxy is used, the CA certificate of the proxy server should be appended to the `certificate-authority-data` of the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret` secret.
|
||||
|
||||
```bash
|
||||
kubectl -n open-cluster-management-agent get secret bootstrap-hub-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d | grep certificate-authority-data | awk '{print $2}' | base64 -d
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIC/jCCA...O62WrVM=
|
||||
-----END CERTIFICATE-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDDTCCA...INDFwtk=
|
||||
-----END CERTIFICATE-----
|
||||
```
|
||||
@@ -0,0 +1,12 @@
|
||||
[ req_ext ]
|
||||
subjectAltName = @alt_names
|
||||
|
||||
[ alt_names ]
|
||||
DNS.1 = <host_name>
|
||||
|
||||
[ v3_ext ]
|
||||
authorityKeyIdentifier=keyid,issuer:always
|
||||
basicConstraints=CA:FALSE
|
||||
keyUsage=keyEncipherment,dataEncipherment
|
||||
extendedKeyUsage=serverAuth,clientAuth
|
||||
subjectAltName=@alt_names
|
||||
@@ -0,0 +1,40 @@
|
||||
acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN)
|
||||
acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN)
|
||||
acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN)
|
||||
acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines
|
||||
acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN)
|
||||
acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN)
|
||||
acl localnet src fc00::/7 # RFC 4193 local private network range
|
||||
acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines
|
||||
acl SSL_ports port 443
|
||||
acl SSL_ports port 6443
|
||||
acl Safe_ports port 80 # http
|
||||
acl Safe_ports port 21 # ftp
|
||||
acl Safe_ports port 443 # https
|
||||
acl Safe_ports port 70 # gopher
|
||||
acl Safe_ports port 210 # wais
|
||||
acl Safe_ports port 1025-65535 # unregistered ports
|
||||
acl Safe_ports port 280 # http-mgmt
|
||||
acl Safe_ports port 488 # gss-http
|
||||
acl Safe_ports port 591 # filemaker
|
||||
acl Safe_ports port 777 # multiling http
|
||||
acl CONNECT method CONNECT
|
||||
http_access deny !Safe_ports
|
||||
http_access deny CONNECT !SSL_ports
|
||||
http_access allow localhost manager
|
||||
http_access deny manager
|
||||
http_access allow localhost
|
||||
http_access allow localnet
|
||||
http_access deny all
|
||||
http_port 3128
|
||||
https_port 3129 tls-cert=/etc/squid/squid-cert-key.pem
|
||||
coredump_dir /var/spool/squid
|
||||
refresh_pattern ^ftp: 1440 20% 10080
|
||||
refresh_pattern ^gopher: 1440 0% 1440
|
||||
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
|
||||
refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
|
||||
refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
|
||||
refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
|
||||
refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
|
||||
refresh_pattern . 0 20% 4320
|
||||
logfile_rotate 0
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 129 KiB |
@@ -0,0 +1,50 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: squid
|
||||
namespace: squid
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: squid
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: squid
|
||||
spec:
|
||||
containers:
|
||||
- name: squid
|
||||
image: ubuntu/squid:5.2-22.04_beta
|
||||
ports:
|
||||
- containerPort: 3128
|
||||
name: http
|
||||
protocol: TCP
|
||||
- containerPort: 3129
|
||||
name: https
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: squid-config-volume
|
||||
mountPath: /etc/squid/squid.conf
|
||||
subPath: squid.conf
|
||||
- name: squid-tls-volume
|
||||
mountPath: /etc/squid/squid-cert-key.pem
|
||||
subPath: squid-cert-key.pem
|
||||
- name: squid-data-volume
|
||||
mountPath: /var/spool/squid
|
||||
volumes:
|
||||
- name: squid-config-volume
|
||||
configMap:
|
||||
name: squid-config
|
||||
items:
|
||||
- key: squid
|
||||
path: squid.conf
|
||||
- name: squid-tls-volume
|
||||
secret:
|
||||
secretName: squid-cert-key
|
||||
items:
|
||||
- key: squid-cert-key.pem
|
||||
path: squid-cert-key.pem
|
||||
- name: squid-data-volume
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: squid
|
||||
namespace: squid
|
||||
labels:
|
||||
app: squid
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- port: 3128
|
||||
nodePort: 31280
|
||||
name: http
|
||||
- port: 3129
|
||||
nodePort: 31290
|
||||
name: https
|
||||
selector:
|
||||
app: squid
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
proxy=${HUB_NAME:-proxy}
|
||||
hub=${HUB_NAME:-hub}
|
||||
cluster1=${CLUSTER_NAME:-cluster1}
|
||||
cluster2=${CLUSTER_NAME:-cluster2}
|
||||
|
||||
hubctx="kind-${hub}"
|
||||
cluster1ctx="kind-${cluster1}"
|
||||
cluster2ctx="kind-${cluster2}"
|
||||
|
||||
# setup hub
|
||||
kind create cluster --name "${hub}"
|
||||
kubectl config use ${hubctx}
|
||||
echo "Initialize the ocm hub cluster"
|
||||
joincmd=$(clusteradm init --use-bootstrap-token | grep clusteradm)
|
||||
kubectl wait --for=condition=HubRegistrationDegraded=false clustermanager cluster-manager --timeout=60s
|
||||
|
||||
# register cluster1
|
||||
kind create cluster --name "${cluster1}"
|
||||
kubectl config use ${cluster1ctx}
|
||||
echo "Join ${cluster1} to ${hub} through HTTP proxy http://${proxy}-control-plane:31280"
|
||||
$(echo ${joincmd} --singleton --force-internal-endpoint-lookup --proxy-url=http://${proxy}-control-plane:31280 --wait | sed "s/<cluster_name>/$cluster1/g")
|
||||
|
||||
kubectl config use ${hubctx}
|
||||
echo "Accept join of ${cluster1} on ${hub}"
|
||||
clusteradm accept --clusters ${cluster1} --wait
|
||||
|
||||
kubectl get managedclusters
|
||||
|
||||
# register cluster2
|
||||
kind create cluster --name "${cluster2}"
|
||||
kubectl config use ${cluster2ctx}
|
||||
echo "Join ${cluster2} to ${hub} through HTTPS proxy https://${proxy}-control-plane:31290"
|
||||
$(echo ${joincmd} --singleton --force-internal-endpoint-lookup --proxy-url=https://${proxy}-control-plane:31290 --proxy-ca-file=tls/root-ca.crt --wait | sed "s/<cluster_name>/$cluster2/g")
|
||||
|
||||
kubectl config use ${hubctx}
|
||||
echo "Accept join of ${cluster2} on ${hub}"
|
||||
clusteradm accept --clusters ${cluster2} --wait
|
||||
|
||||
kubectl get managedclusters
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
proxy=${HUB_NAME:-proxy}
|
||||
proxyctx="kind-${proxy}"
|
||||
|
||||
# create server certificate for proxy server
|
||||
rm -rf tls
|
||||
mkdir tls
|
||||
openssl genrsa -out tls/root-ca.key 2048
|
||||
openssl req -key tls/root-ca.key -subj "/CN=example.com" -new -x509 -days 3650 -out tls/root-ca.crt
|
||||
openssl genrsa -out tls/squid.key 2048
|
||||
openssl req -new -key tls/squid.key -subj "/CN=${proxy}-control-plane" -out tls/squid.csr
|
||||
cat config/server-cert.conf | sed "s/<host_name>/$proxy-control-plane/g" > tls/server-cert.conf
|
||||
openssl x509 -req -in tls/squid.csr -CA tls/root-ca.crt -CAkey tls/root-ca.key -CAcreateserial -out tls/squid.crt -days 3600 -extensions v3_ext -extfile tls/server-cert.conf
|
||||
cat tls/squid.crt tls/squid.key >> tls/squid-cert-key.pem
|
||||
|
||||
kind create cluster --name "${proxy}"
|
||||
|
||||
kubectl config use ${proxyctx}
|
||||
echo "setup proxy server - squid"
|
||||
kubectl create ns squid
|
||||
kubectl -n squid create secret generic squid-cert-key --from-file=squid-cert-key.pem=tls/squid-cert-key.pem
|
||||
kubectl -n squid create configmap squid-config --from-file=squid=config/squid.conf
|
||||
kubectl apply -f manifests/squid-deploy.yaml
|
||||
kubectl apply -f manifests/squid-service.yaml
|
||||
|
||||
kubectl -n squid get pods
|
||||
Reference in New Issue
Block a user