In this post, I’m going to replace the network plugin from default “noops” to “cni”, and use Calico to connect each pod.
We follow official installation manual “Installing Calico for policy and networking“.
There are basically two types of installation of available. One uses kubernetes API server (and eventually backend etcd) to store data, and the other uses other etcd datastore. I use the former to utilise existing kubernetes setup.
1. Modify kubelet and controller-manager to use Calico
In order to use Calico, we need to modify two services.
- kubelet … specify “cni” for network-plugin flag
- controller-manager … add “cluster-cidr” and “allocate-node-cidrs” flags to pass network parameter when it launches pod.
[ controller-1 ]
root@controller-1:~$ cat /etc/systemd/system/kube-controller-manager.service --<snip>-- ExecStart=/usr/local/bin/kube-controller-manager --master=http://127.0.0.1:8080 --service-account-private-key-file=/var/lib/kubernetes/cert/sa-key.pem --cluster-cidr=10.220.0.0/16 --allocate-node-cidrs=true --<snip>--
[ worker ]
root@worker-1:~# cat /lib/systemd/system/kubelet.service --<snip>-- ExecStart=/usr/bin/kubelet --kubeconfig=/var/lib/kubelet/worker.kubeconfig --client-ca-file=/var/lib/kubelet/cert/ca.pem --tls-cert-file=/var/lib/kubelet/cert/worker-1-kubelet.pem --tls-private-key-file=/var/lib/kubelet/cert/worker-1-kubelet-key.pem --cluster-dns=10.32.0.10 --cluster-domain="cluster.local" --network-plugin=cni --<snip>--
kubelet and controller-manager are ready. Let’s follow the tutorial in Official site to install Calico.
2. Create service account for Calico
root@controller-1:~# kubectl apply -f \ > https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml clusterrole.rbac.authorization.k8s.io "calico-node" created clusterrolebinding.rbac.authorization.k8s.io "calico-node" created root@controller-1:~# kubectl describe clusterrole/calico-node Name: calico-node Labels: <none> Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"rbac.authorization.k8s.io/v1beta1","kind":"ClusterRole","metadata":{"annotations":{},"name":"calico-node","namespace":""},"rules":[{"api... PolicyRule: Resources Non-Resource URLs Resource Names Verbs --------- ----------------- -------------- ----- endpoints [] [] [get] namespaces [] [] [get list watch] nodes [] [] [get list update watch] pods [] [] [get list watch] pods/status [] [] [patch] serviceaccounts [] [] [get list watch] services [] [] [get] bgpconfigurations.crd.projectcalico.org [] [] [create get list update watch] bgppeers.crd.projectcalico.org [] [] [create get list update watch] clusterinformations.crd.projectcalico.org [] [] [create get list update watch] felixconfigurations.crd.projectcalico.org [] [] [create get list update watch] globalbgpconfigs.crd.projectcalico.org [] [] [create get list update watch] globalfelixconfigs.crd.projectcalico.org [] [] [create get list update watch] globalnetworkpolicies.crd.projectcalico.org [] [] [create get list update watch] globalnetworksets.crd.projectcalico.org [] [] [create get list update watch] hostendpoints.crd.projectcalico.org [] [] [create get list update watch] ippools.crd.projectcalico.org [] [] [create get list update watch] networkpolicies.crd.projectcalico.org [] [] [create get list update watch] networkpolicies.extensions [] [] [get list watch] networkpolicies.networking.k8s.io [] [] [watch list]
3. Install Calico
In this step, we need to change default yaml file so that it fits our needs.
root@controller-1:~# wget https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml root@controller-1:~# cat calico.yaml # Calico Version v3.3.0 # https://docs.projectcalico.org/v3.3/releases#v3.3.0 # This manifest includes the following component versions: # calico/node:v3.3.0 # calico/cni:v3.3.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap ~ # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", "cniVersion": "0.3.0", "plugins": [ { "type": "calico", "log_level": "info", "datastore_type": "kubernetes", "nodeName": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { "type": "host-local", "subnet": "usePodCidr" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} } ] } --- ~ --- # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: extensions/v1beta1 metadata: name: calico-node namespace: kube-system ~ template: ~ spec: ~ serviceAccountName: calico-node ~ containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: quay.io/calico/node:v3.3.0 env: ~ # ip-in-ip is on by de default. You can change this to "off" to use BGP routing. I use ip-in-ip here because of GCP network restriction # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "Always" ~ # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. # You need to change this to match with the cidr specified in controller-manager - name: CALICO_IPV4POOL_CIDR value: "10.220.0.0/16" ~ # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni image: quay.io/calico/cni:v3.3.0 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" ~ # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config ~ volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir volumes: ~ # Used to install CNI. - name: cni-bin-dir hostPath: path: /opt/cni/bin - name: cni-net-dir hostPath: path: /etc/cni/net.d --- --<snip>--
As you can see, calico create two containers in one pod. One is to initialize CNI(create cni config under /etc/cni/net.d, and place cni binaries in to /opt/cni/bin, this container makes it possible for each node to use cni), the other is calico services(composed of felix, bird, this makes all routing possible).
Let’s apply this manifest to start using calico.
root@controller-1:~# kubectl apply -f calico.yaml configmap "calico-config" created service "calico-typha" created deployment.apps "calico-typha" created poddisruptionbudget.policy "calico-typha" created daemonset.extensions "calico-node" created serviceaccount "calico-node" created customresourcedefinition.apiextensions.k8s.io "felixconfigurations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "bgppeers.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "bgpconfigurations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "ippools.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "hostendpoints.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "clusterinformations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "globalnetworkpolicies.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "globalnetworksets.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "networkpolicies.crd.projectcalico.org" created
4. Confirmation
Let’s check if it’s working!
# check if calico is working on each nodes root@controller-1:~# kubectl get pods -o wide -n kube-system NAME READY STATUS RESTARTS AGE IP NODE calico-node-8lx78 2/2 Running 0 57m 10.240.0.21 worker-1 calico-node-xpnwh 2/2 Running 0 57m 10.240.0.22 worker-2 coredns-jz689 1/1 Running 3 10d 10.220.1.2 worker-2 coredns-sl95h 1/1 Running 0 1h 10.220.0.2 worker-1 # check if relevant cni config is generated in each worker-node root@worker-1:~# ls /etc/cni/net.d/ 10-calico.conflist calico-kubeconfig root@worker-1:~# cat /etc/cni/net.d/10-calico.conflist { "name": "k8s-pod-network", "cniVersion": "0.3.0", "plugins": [ { "type": "calico", "log_level": "info", "datastore_type": "kubernetes", "nodeName": "worker-1", "mtu": 1440, "ipam": { "type": "host-local", "subnet": "usePodCidr" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "/etc/cni/net.d/calico-kubeconfig" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} } ] } root@worker-1:~# cat /etc/cni/net.d/calico-kubeconfig # Kubeconfig file for Calico CNI plugin. apiVersion: v1 kind: Config clusters: - name: local cluster: server: https://[10.32.0.1]:443 certificate-authority-data: ***= users: - name: calico user: token: *** contexts: - name: calico-context context: cluster: local user: calico current-context: calico-context root@worker-1:~# # check pod to pod communication root@controller-1:~# kubectl run busybox --image=busybox -- sleep 3600 deployment.apps "busybox" created root@controller-1:~# kubectl run nginx --image=nginx deployment.apps "nginx" created root@controller-1:~# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE busybox-5ccc978d8d-k8g67 1/1 Running 0 26s 10.220.0.6 worker-1 nginx-65899c769f-x8f2l 1/1 Running 0 8s 10.220.1.4 worker-2 root@controller-1:~# kubectl exec -it busybox-5ccc978d8d-k8g67 -- sh / # wget -S 10.220.1.4 Connecting to 10.220.1.4 (10.220.1.4:80) HTTP/1.1 200 OK Server: nginx/1.15.5 Date: Sat, 27 Oct 2018 21:35:24 GMT Content-Type: text/html Content-Length: 612 Last-Modified: Tue, 02 Oct 2018 14:49:27 GMT Connection: close ETag: "5bb38577-264" Accept-Ranges: bytes index.html 100% |************************************************************************| 612 0:00:00 ETA / # # check routing table on worker-node root@worker-1:~# ip route show default via 10.240.0.1 dev ens4 10.200.1.0/24 dev docker0 proto kernel scope link src 10.200.1.1 linkdown 10.220.0.2 dev cali4c16a3de178 scope link 10.220.0.6 dev cali11df0bf4071 scope link 10.220.1.0/24 via 10.240.0.22 dev tunl0 proto bird onlink --> tunnel to pod subnet in other worker node is registered 10.240.0.1 dev ens4 scope link