In this post, I’m going to replace the network plugin from default “noops” to “cni”, and use Calico to connect each pod.
We follow official installation manual “Installing Calico for policy and networking“.
There are basically two types of installation of available. One uses kubernetes API server (and eventually backend etcd) to store data, and the other uses other etcd datastore. I use the former to utilise existing kubernetes setup.
1. Modify kubelet and controller-manager to use Calico
In order to use Calico, we need to modify two services.
- kubelet … specify “cni” for network-plugin flag
- controller-manager … add “cluster-cidr” and “allocate-node-cidrs” flags to pass network parameter when it launches pod.
[ controller-1 ]
root@controller-1:~$ cat /etc/systemd/system/kube-controller-manager.service --<snip>-- ExecStart=/usr/local/bin/kube-controller-manager --master=http://127.0.0.1:8080 --service-account-private-key-file=/var/lib/kubernetes/cert/sa-key.pem --cluster-cidr=10.220.0.0/16 --allocate-node-cidrs=true --<snip>--
[ worker ]
root@worker-1:~# cat /lib/systemd/system/kubelet.service --<snip>-- ExecStart=/usr/bin/kubelet --kubeconfig=/var/lib/kubelet/worker.kubeconfig --client-ca-file=/var/lib/kubelet/cert/ca.pem --tls-cert-file=/var/lib/kubelet/cert/worker-1-kubelet.pem --tls-private-key-file=/var/lib/kubelet/cert/worker-1-kubelet-key.pem --cluster-dns=10.32.0.10 --cluster-domain="cluster.local" --network-plugin=cni --<snip>--
kubelet and controller-manager are ready. Let’s follow the tutorial in Official site to install Calico.
2. Create service account for Calico
root@controller-1:~# kubectl apply -f \
> https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
clusterrole.rbac.authorization.k8s.io "calico-node" created
clusterrolebinding.rbac.authorization.k8s.io "calico-node" created
root@controller-1:~# kubectl describe clusterrole/calico-node
Name: calico-node
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"rbac.authorization.k8s.io/v1beta1","kind":"ClusterRole","metadata":{"annotations":{},"name":"calico-node","namespace":""},"rules":[{"api...
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
endpoints [] [] [get]
namespaces [] [] [get list watch]
nodes [] [] [get list update watch]
pods [] [] [get list watch]
pods/status [] [] [patch]
serviceaccounts [] [] [get list watch]
services [] [] [get]
bgpconfigurations.crd.projectcalico.org [] [] [create get list update watch]
bgppeers.crd.projectcalico.org [] [] [create get list update watch]
clusterinformations.crd.projectcalico.org [] [] [create get list update watch]
felixconfigurations.crd.projectcalico.org [] [] [create get list update watch]
globalbgpconfigs.crd.projectcalico.org [] [] [create get list update watch]
globalfelixconfigs.crd.projectcalico.org [] [] [create get list update watch]
globalnetworkpolicies.crd.projectcalico.org [] [] [create get list update watch]
globalnetworksets.crd.projectcalico.org [] [] [create get list update watch]
hostendpoints.crd.projectcalico.org [] [] [create get list update watch]
ippools.crd.projectcalico.org [] [] [create get list update watch]
networkpolicies.crd.projectcalico.org [] [] [create get list update watch]
networkpolicies.extensions [] [] [get list watch]
networkpolicies.networking.k8s.io [] [] [watch list]3. Install Calico
In this step, we need to change default yaml file so that it fits our needs.
root@controller-1:~# wget https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
root@controller-1:~# cat calico.yaml
# Calico Version v3.3.0
# https://docs.projectcalico.org/v3.3/releases#v3.3.0
# This manifest includes the following component versions:
# calico/node:v3.3.0
# calico/cni:v3.3.0
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
~
# The CNI network configuration to install on each node. The special
# values in this config will be automatically populated.
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.0",
"plugins": [
{
"type": "calico",
"log_level": "info",
"datastore_type": "kubernetes",
"nodeName": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "host-local",
"subnet": "usePodCidr"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
}
]
}
---
~
---
# This manifest installs the calico/node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: calico-node
namespace: kube-system
~
template:
~
spec:
~
serviceAccountName: calico-node
~
containers:
# Runs calico/node container on each Kubernetes node. This
# container programs network policy and routes on each
# host.
- name: calico-node
image: quay.io/calico/node:v3.3.0
env:
~
# ip-in-ip is on by de default. You can change this to "off" to use BGP routing. I use ip-in-ip here because of GCP network restriction
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
~
# The default IPv4 pool to create on startup if none exists. Pod IPs will be
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr`.
# You need to change this to match with the cidr specified in controller-manager
- name: CALICO_IPV4POOL_CIDR
value: "10.220.0.0/16"
~
# This container installs the Calico CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: quay.io/calico/cni:v3.3.0
command: ["/install-cni.sh"]
env:
# Name of the CNI config file to create.
- name: CNI_CONF_NAME
value: "10-calico.conflist"
~
# The CNI network config to install on each node.
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: calico-config
key: cni_network_config
~
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
volumes:
~
# Used to install CNI.
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
---
--<snip>--As you can see, calico create two containers in one pod. One is to initialize CNI(create cni config under /etc/cni/net.d, and place cni binaries in to /opt/cni/bin, this container makes it possible for each node to use cni), the other is calico services(composed of felix, bird, this makes all routing possible).
Let’s apply this manifest to start using calico.
root@controller-1:~# kubectl apply -f calico.yaml configmap "calico-config" created service "calico-typha" created deployment.apps "calico-typha" created poddisruptionbudget.policy "calico-typha" created daemonset.extensions "calico-node" created serviceaccount "calico-node" created customresourcedefinition.apiextensions.k8s.io "felixconfigurations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "bgppeers.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "bgpconfigurations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "ippools.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "hostendpoints.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "clusterinformations.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "globalnetworkpolicies.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "globalnetworksets.crd.projectcalico.org" created customresourcedefinition.apiextensions.k8s.io "networkpolicies.crd.projectcalico.org" created
4. Confirmation
Let’s check if it’s working!
# check if calico is working on each nodes
root@controller-1:~# kubectl get pods -o wide -n kube-system
NAME READY STATUS RESTARTS AGE IP NODE
calico-node-8lx78 2/2 Running 0 57m 10.240.0.21 worker-1
calico-node-xpnwh 2/2 Running 0 57m 10.240.0.22 worker-2
coredns-jz689 1/1 Running 3 10d 10.220.1.2 worker-2
coredns-sl95h 1/1 Running 0 1h 10.220.0.2 worker-1
# check if relevant cni config is generated in each worker-node
root@worker-1:~# ls /etc/cni/net.d/
10-calico.conflist calico-kubeconfig
root@worker-1:~# cat /etc/cni/net.d/10-calico.conflist
{
"name": "k8s-pod-network",
"cniVersion": "0.3.0",
"plugins": [
{
"type": "calico",
"log_level": "info",
"datastore_type": "kubernetes",
"nodeName": "worker-1",
"mtu": 1440,
"ipam": {
"type": "host-local",
"subnet": "usePodCidr"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
}
]
}
root@worker-1:~# cat /etc/cni/net.d/calico-kubeconfig
# Kubeconfig file for Calico CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: https://[10.32.0.1]:443
certificate-authority-data: ***=
users:
- name: calico
user:
token: ***
contexts:
- name: calico-context
context:
cluster: local
user: calico
current-context: calico-context
root@worker-1:~#
# check pod to pod communication
root@controller-1:~# kubectl run busybox --image=busybox -- sleep 3600
deployment.apps "busybox" created
root@controller-1:~# kubectl run nginx --image=nginx
deployment.apps "nginx" created
root@controller-1:~# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE
busybox-5ccc978d8d-k8g67 1/1 Running 0 26s 10.220.0.6 worker-1
nginx-65899c769f-x8f2l 1/1 Running 0 8s 10.220.1.4 worker-2
root@controller-1:~# kubectl exec -it busybox-5ccc978d8d-k8g67 -- sh
/ # wget -S 10.220.1.4
Connecting to 10.220.1.4 (10.220.1.4:80)
HTTP/1.1 200 OK
Server: nginx/1.15.5
Date: Sat, 27 Oct 2018 21:35:24 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 02 Oct 2018 14:49:27 GMT
Connection: close
ETag: "5bb38577-264"
Accept-Ranges: bytes
index.html 100% |************************************************************************| 612 0:00:00 ETA
/ #
# check routing table on worker-node
root@worker-1:~# ip route show
default via 10.240.0.1 dev ens4
10.200.1.0/24 dev docker0 proto kernel scope link src 10.200.1.1 linkdown
10.220.0.2 dev cali4c16a3de178 scope link
10.220.0.6 dev cali11df0bf4071 scope link
10.220.1.0/24 via 10.240.0.22 dev tunl0 proto bird onlink --> tunnel to pod subnet in other worker node is registered
10.240.0.1 dev ens4 scope link