From 6fe1727061cf5301a9ce0bd8c3a4efe59a376d87 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 5 Apr 2019 09:13:49 -0500 Subject: [PATCH 1/6] Add Compose file to start a simple k8s control plane --- .../docker-compose.yaml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 compose/simple-k8s-control-plane/docker-compose.yaml diff --git a/compose/simple-k8s-control-plane/docker-compose.yaml b/compose/simple-k8s-control-plane/docker-compose.yaml new file mode 100644 index 00000000..baeafe41 --- /dev/null +++ b/compose/simple-k8s-control-plane/docker-compose.yaml @@ -0,0 +1,28 @@ +version: "3" + +services: + + pause: + ports: + - 8080:8080 + image: k8s.gcr.io/pause + + etcd: + network_mode: "service:pause" + image: k8s.gcr.io/etcd:3.3.10 + command: etcd + + kube-apiserver: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-apiserver --etcd-servers http://127.0.0.1:2379 --address 0.0.0.0 --disable-admission-plugins=ServiceAccount + + kube-controller-manager: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-controller-manager --master http://localhost:8080 + + kube-scheduler: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-scheduler --master http://localhost:8080 From 1c76e23525dde1e5fc166ef2cb61351058f089e4 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 6 Apr 2019 12:01:42 -0500 Subject: [PATCH 2/6] Add Compose file and Kubernetes YAML for kube-router lab --- .../docker-compose.yaml | 29 ++++ .../kuberouter.yaml | 128 ++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 compose/kube-router-k8s-control-plane/docker-compose.yaml create mode 100644 compose/kube-router-k8s-control-plane/kuberouter.yaml diff --git a/compose/kube-router-k8s-control-plane/docker-compose.yaml b/compose/kube-router-k8s-control-plane/docker-compose.yaml new file mode 100644 index 00000000..0abbd689 --- /dev/null +++ b/compose/kube-router-k8s-control-plane/docker-compose.yaml @@ -0,0 +1,29 @@ +version: "3" + +services: + + pause: + ports: + - 8080:8080 + image: k8s.gcr.io/pause + + etcd: + network_mode: "service:pause" + image: k8s.gcr.io/etcd:3.3.10 + command: etcd + + kube-apiserver: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-apiserver --etcd-servers http://127.0.0.1:2379 --address 0.0.0.0 --disable-admission-plugins=ServiceAccount --allow-privileged + + kube-controller-manager: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-controller-manager --master http://localhost:8080 --allocate-node-cidrs --cluster-cidr=10.CLUSTER.0.0/16 + "Edit the CLUSTER placeholder first. Then, remove this line.": + + kube-scheduler: + network_mode: "service:pause" + image: k8s.gcr.io/hyperkube:v1.14.0 + command: kube-scheduler --master http://localhost:8080 diff --git a/compose/kube-router-k8s-control-plane/kuberouter.yaml b/compose/kube-router-k8s-control-plane/kuberouter.yaml new file mode 100644 index 00000000..242eba8f --- /dev/null +++ b/compose/kube-router-k8s-control-plane/kuberouter.yaml @@ -0,0 +1,128 @@ +--- +apiVersion: |+ + + + Make sure you update the line with --master=http://X.X.X.X:8080 below. + Then remove this section from this YAML file and try again. + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "ipam":{ + "type":"host-local" + } + } + ] + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router + imagePullPolicy: Always + args: + - "--run-router=true" + - "--run-firewall=true" + - "--run-service-proxy=true" + - "--master=http://X.X.X.X:8080" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-conf-dir + - mountPath: /etc/kube-router + name: kube-router-cfg + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + From 41eb91681145ec97d74dbc80025eb7fbcd925ffb Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 6 Apr 2019 12:23:40 -0500 Subject: [PATCH 3/6] Add kubebins command (install Kubernetes binaries but do not setup cluster) --- prepare-vms/lib/commands.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 27b7dbdd..c416526a 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -103,6 +103,37 @@ _cmd_deploy() { info "$0 cards $TAG" } +_cmd kubebins "Install Kubernetes and CNI binaries but don't start anything" +_cmd_kubebins() { + TAG=$1 + need_tag + + pssh --timeout 300 " + set -e + cd /usr/local/bin + if ! [ -x etcd ]; then + curl -L https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz \ + | sudo tar --strip-components=1 --wildcards -zx '*/etcd' '*/etcdctl' + fi + if ! [ -x hyperkube ]; then + curl -L https://dl.k8s.io/v1.14.0/kubernetes-server-linux-amd64.tar.gz \ + | sudo tar --strip-components=3 -zx kubernetes/server/bin/hyperkube + fi + if ! [ -x kubelet ]; then + for BINARY in kubectl kube-apiserver kube-scheduler kube-controller-manager kubelet kube-proxy; + do + sudo ln -s hyperkube \$BINARY + done + fi + sudo mkdir -p /opt/cni/bin + cd /opt/cni/bin + if ! [ -x bridge ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-amd64-v0.7.5.tgz \ + | sudo tar -zx + fi + " +} + _cmd kube "Setup kubernetes clusters with kubeadm (must be run AFTER deploy)" _cmd_kube() { TAG=$1 From 46ce3d0b3dcd192dd5cea2f8eb90a81454a05203 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 6 Apr 2019 12:28:47 -0500 Subject: [PATCH 4/6] Add disableaddrchecks command (to allow network labs on AWS) --- prepare-vms/lib/commands.sh | 8 ++++++++ prepare-vms/lib/infra.sh | 4 ++++ prepare-vms/lib/infra/aws.sh | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index c416526a..8e98265c 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -314,6 +314,14 @@ _cmd_opensg() { infra_opensg } +_cmd disableaddrchecks "Disable source/destination IP address checks" +_cmd_disableaddrchecks() { + TAG=$1 + need_tag + + infra_disableaddrchecks +} + _cmd pssh "Run an arbitrary command on all nodes" _cmd_pssh() { TAG=$1 diff --git a/prepare-vms/lib/infra.sh b/prepare-vms/lib/infra.sh index 08cefbca..6735a7c3 100644 --- a/prepare-vms/lib/infra.sh +++ b/prepare-vms/lib/infra.sh @@ -24,3 +24,7 @@ infra_quotas() { infra_opensg() { warning "infra_opensg is unsupported on $INFRACLASS." } + +infra_disableaddrchecks() { + warning "infra_disableaddrchecks is unsupported on $INFRACLASS." +} \ No newline at end of file diff --git a/prepare-vms/lib/infra/aws.sh b/prepare-vms/lib/infra/aws.sh index bb4ac084..cb5f8fd0 100644 --- a/prepare-vms/lib/infra/aws.sh +++ b/prepare-vms/lib/infra/aws.sh @@ -88,6 +88,14 @@ infra_opensg() { --cidr 0.0.0.0/0 } +infra_disableaddrchecks() { + IDS=$(aws_get_instance_ids_by_tag $TAG) + for ID in $IDS; do + info "Disabling source/destination IP checks on: $ID" + aws ec2 modify-instance-attribute --source-dest-check "{\"Value\": false}" --instance-id $ID + done +} + wait_until_tag_is_running() { max_retry=50 i=0 From 3368e21831d9690fb55acc471f38c4bf9d3e346f Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 7 Apr 2019 10:26:56 -0500 Subject: [PATCH 5/6] Add FRR route reflector --- compose/frr-route-reflector/conf/bgpd.conf | 8 +++++ compose/frr-route-reflector/conf/vtysh.conf | 0 compose/frr-route-reflector/conf/zebra.conf | 2 ++ .../frr-route-reflector/docker-compose.yaml | 34 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 compose/frr-route-reflector/conf/bgpd.conf create mode 100644 compose/frr-route-reflector/conf/vtysh.conf create mode 100644 compose/frr-route-reflector/conf/zebra.conf create mode 100644 compose/frr-route-reflector/docker-compose.yaml diff --git a/compose/frr-route-reflector/conf/bgpd.conf b/compose/frr-route-reflector/conf/bgpd.conf new file mode 100644 index 00000000..495ecdee --- /dev/null +++ b/compose/frr-route-reflector/conf/bgpd.conf @@ -0,0 +1,8 @@ +hostname frr +router bgp 64512 +bgp log-neighbor-changes +neighbor kube peer-group +neighbor kube remote-as 64512 +neighbor kube route-reflector-client +bgp listen range 0.0.0.0/0 peer-group kube +log stdout diff --git a/compose/frr-route-reflector/conf/vtysh.conf b/compose/frr-route-reflector/conf/vtysh.conf new file mode 100644 index 00000000..e69de29b diff --git a/compose/frr-route-reflector/conf/zebra.conf b/compose/frr-route-reflector/conf/zebra.conf new file mode 100644 index 00000000..5926118a --- /dev/null +++ b/compose/frr-route-reflector/conf/zebra.conf @@ -0,0 +1,2 @@ +hostname frr +log stdout diff --git a/compose/frr-route-reflector/docker-compose.yaml b/compose/frr-route-reflector/docker-compose.yaml new file mode 100644 index 00000000..a5d13b92 --- /dev/null +++ b/compose/frr-route-reflector/docker-compose.yaml @@ -0,0 +1,34 @@ +version: "3" + +services: + bgpd: + image: ajones17/frr:662 + volumes: + - ./conf:/etc/frr + - ./run:/var/run/frr + network_mode: host + entrypoint: /usr/lib/frr/bgpd -f /etc/frr/bgpd.conf --log=stdout --log-level=debug --no_kernel + restart: always + + zebra: + image: ajones17/frr:662 + volumes: + - ./conf:/etc/frr + - ./run:/var/run/frr + network_mode: host + entrypoint: /usr/lib/frr/zebra -f /etc/frr/zebra.conf --log=stdout --log-level=debug + restart: always + + vtysh: + image: ajones17/frr:662 + volumes: + - ./conf:/etc/frr + - ./run:/var/run/frr + network_mode: host + entrypoint: vtysh -c "show ip bgp" + + chmod: + image: alpine + volumes: + - ./run:/var/run/frr + command: chmod 777 /var/run/frr From ad9f845184dbd681839e102f5415f2f03daf5e7d Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 7 Apr 2019 11:23:38 -0500 Subject: [PATCH 6/6] Add export of 1.0.0.2/32 route for testing --- compose/frr-route-reflector/conf/bgpd.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/frr-route-reflector/conf/bgpd.conf b/compose/frr-route-reflector/conf/bgpd.conf index 495ecdee..19ea74ac 100644 --- a/compose/frr-route-reflector/conf/bgpd.conf +++ b/compose/frr-route-reflector/conf/bgpd.conf @@ -1,5 +1,6 @@ hostname frr router bgp 64512 +network 1.0.0.2/32 bgp log-neighbor-changes neighbor kube peer-group neighbor kube remote-as 64512