k8s 14: Calico IP-in-IP

In this post, I’m going to replace the network plugin from default “noops” to “cni”, and use Calico to connect each pod.

We follow official installation manual “Installing Calico for policy and networking“.

There are basically two types of installation of available. One uses kubernetes API server (and eventually backend etcd) to store data, and the other uses other etcd datastore. I use the former to utilise existing kubernetes setup.

1. Modify kubelet and controller-manager to use Calico

In order to use Calico, we need to modify two services.

  • kubelet … specify “cni” for network-plugin flag
  • controller-manager … add “cluster-cidr” and “allocate-node-cidrs” flags to pass network parameter when it launches pod.

[ controller-1 ]

root@controller-1:~$ cat /etc/systemd/system/kube-controller-manager.service 
ExecStart=/usr/local/bin/kube-controller-manager  --master= --service-account-private-key-file=/var/lib/kubernetes/cert/sa-key.pem --cluster-cidr= --allocate-node-cidrs=true

[ worker ]

root@worker-1:~# cat /lib/systemd/system/kubelet.service 
ExecStart=/usr/bin/kubelet --kubeconfig=/var/lib/kubelet/worker.kubeconfig --client-ca-file=/var/lib/kubelet/cert/ca.pem --tls-cert-file=/var/lib/kubelet/cert/worker-1-kubelet.pem --tls-private-key-file=/var/lib/kubelet/cert/worker-1-kubelet-key.pem --cluster-dns= --cluster-domain="cluster.local" --network-plugin=cni

kubelet and controller-manager are ready. Let’s follow the tutorial in Official site to install Calico.

2. Create service account for Calico 

root@controller-1:~# kubectl apply -f \
> https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
clusterrole.rbac.authorization.k8s.io "calico-node" created
clusterrolebinding.rbac.authorization.k8s.io "calico-node" created
root@controller-1:~# kubectl describe clusterrole/calico-node
Name:         calico-node
Labels:       <none>
Annotations:  kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"rbac.authorization.k8s.io/v1beta1","kind":"ClusterRole","metadata":{"annotations":{},"name":"calico-node","namespace":""},"rules":[{"api...
  Resources                                    Non-Resource URLs  Resource Names  Verbs
  ---------                                    -----------------  --------------  -----
  endpoints                                    []                 []              [get]
  namespaces                                   []                 []              [get list watch]
  nodes                                        []                 []              [get list update watch]
  pods                                         []                 []              [get list watch]
  pods/status                                  []                 []              [patch]
  serviceaccounts                              []                 []              [get list watch]
  services                                     []                 []              [get]
  bgpconfigurations.crd.projectcalico.org      []                 []              [create get list update watch]
  bgppeers.crd.projectcalico.org               []                 []              [create get list update watch]
  clusterinformations.crd.projectcalico.org    []                 []              [create get list update watch]
  felixconfigurations.crd.projectcalico.org    []                 []              [create get list update watch]
  globalbgpconfigs.crd.projectcalico.org       []                 []              [create get list update watch]
  globalfelixconfigs.crd.projectcalico.org     []                 []              [create get list update watch]
  globalnetworkpolicies.crd.projectcalico.org  []                 []              [create get list update watch]
  globalnetworksets.crd.projectcalico.org      []                 []              [create get list update watch]
  hostendpoints.crd.projectcalico.org          []                 []              [create get list update watch]
  ippools.crd.projectcalico.org                []                 []              [create get list update watch]
  networkpolicies.crd.projectcalico.org        []                 []              [create get list update watch]
  networkpolicies.extensions                   []                 []              [get list watch]
  networkpolicies.networking.k8s.io            []                 []              [watch list]

3. Install Calico

In this step, we need to change default yaml file so that it fits our needs.

root@controller-1:~# wget https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
root@controller-1:~# cat calico.yaml
# Calico Version v3.3.0
# https://docs.projectcalico.org/v3.3/releases#v3.3.0
# This manifest includes the following component versions:
#   calico/node:v3.3.0
#   calico/cni:v3.3.0

# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
  # The CNI network configuration to install on each node.  The special
  # values in this config will be automatically populated.
  cni_network_config: |-
      "name": "k8s-pod-network",
      "cniVersion": "0.3.0",
      "plugins": [
          "type": "calico",
          "log_level": "info",
          "datastore_type": "kubernetes",
          "nodeName": "__KUBERNETES_NODE_NAME__",
          "mtu": __CNI_MTU__,
          "ipam": {
            "type": "host-local",
            "subnet": "usePodCidr"
          "policy": {
              "type": "k8s"
          "kubernetes": {
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          "type": "portmap",
          "snat": true,
          "capabilities": {"portMappings": true}
# This manifest installs the calico/node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: extensions/v1beta1
  name: calico-node
  namespace: kube-system
      serviceAccountName: calico-node
        # Runs calico/node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: quay.io/calico/node:v3.3.0
# ip-in-ip is on by de default. You can change this to "off" to use BGP routing. I use ip-in-ip here because of GCP network restriction
            # Enable IPIP
            - name: CALICO_IPV4POOL_IPIP
              value: "Always"
            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
# You need to change this to match with the cidr specified in controller-manager
            - name: CALICO_IPV4POOL_CIDR
              value: ""
        # This container installs the Calico CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
          image: quay.io/calico/cni:v3.3.0
          command: ["/install-cni.sh"]
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            # The CNI network config to install on each node.
            - name: CNI_NETWORK_CONFIG
                  name: calico-config
                  key: cni_network_config
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
        # Used to install CNI.
        - name: cni-bin-dir
            path: /opt/cni/bin
        - name: cni-net-dir
            path: /etc/cni/net.d

As you can see, calico create two containers in one pod. One is to initialize CNI(create cni config under /etc/cni/net.d, and place cni binaries in to /opt/cni/bin, this container makes it possible for each node to use cni), the other is calico services(composed of felix, bird, this makes all routing possible).

Let’s apply this manifest to start using calico.

root@controller-1:~# kubectl apply -f calico.yaml 
configmap "calico-config" created
service "calico-typha" created
deployment.apps "calico-typha" created
poddisruptionbudget.policy "calico-typha" created
daemonset.extensions "calico-node" created
serviceaccount "calico-node" created
customresourcedefinition.apiextensions.k8s.io "felixconfigurations.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "bgppeers.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "bgpconfigurations.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "ippools.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "hostendpoints.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "clusterinformations.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "globalnetworkpolicies.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "globalnetworksets.crd.projectcalico.org" created
customresourcedefinition.apiextensions.k8s.io "networkpolicies.crd.projectcalico.org" created

4. Confirmation

Let’s check if it’s working!

# check if calico is working on each nodes
root@controller-1:~# kubectl get pods -o wide -n kube-system
NAME                READY     STATUS    RESTARTS   AGE       IP            NODE
calico-node-8lx78   2/2       Running   0          57m   worker-1
calico-node-xpnwh   2/2       Running   0          57m   worker-2
coredns-jz689       1/1       Running   3          10d    worker-2
coredns-sl95h       1/1       Running   0          1h    worker-1

# check if relevant cni config is generated in each worker-node
root@worker-1:~# ls /etc/cni/net.d/
10-calico.conflist  calico-kubeconfig
root@worker-1:~# cat /etc/cni/net.d/10-calico.conflist 
  "name": "k8s-pod-network",
  "cniVersion": "0.3.0",
  "plugins": [
      "type": "calico",
      "log_level": "info",
      "datastore_type": "kubernetes",
      "nodeName": "worker-1",
      "mtu": 1440,
      "ipam": {
        "type": "host-local",
        "subnet": "usePodCidr"
      "policy": {
          "type": "k8s"
      "kubernetes": {
          "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
      "type": "portmap",
      "snat": true,
      "capabilities": {"portMappings": true}
root@worker-1:~# cat /etc/cni/net.d/calico-kubeconfig 
# Kubeconfig file for Calico CNI plugin.
apiVersion: v1
kind: Config
- name: local
    server: https://[]:443
    certificate-authority-data: ***=
- name: calico
    token: ***
- name: calico-context
    cluster: local
    user: calico
current-context: calico-context

# check pod to pod communication
root@controller-1:~# kubectl run busybox --image=busybox -- sleep 3600
deployment.apps "busybox" created
root@controller-1:~# kubectl run nginx --image=nginx 
deployment.apps "nginx" created
root@controller-1:~# kubectl get pods -o wide
NAME                       READY     STATUS    RESTARTS   AGE       IP           NODE
busybox-5ccc978d8d-k8g67   1/1       Running   0          26s   worker-1
nginx-65899c769f-x8f2l     1/1       Running   0          8s   worker-2
root@controller-1:~# kubectl exec -it busybox-5ccc978d8d-k8g67 -- sh
/ # wget -S
Connecting to (
  HTTP/1.1 200 OK
  Server: nginx/1.15.5
  Date: Sat, 27 Oct 2018 21:35:24 GMT
  Content-Type: text/html
  Content-Length: 612
  Last-Modified: Tue, 02 Oct 2018 14:49:27 GMT
  Connection: close
  ETag: "5bb38577-264"
  Accept-Ranges: bytes
index.html           100% |************************************************************************|   612  0:00:00 ETA
/ # 

# check routing table on worker-node
root@worker-1:~# ip route show
default via dev ens4 dev docker0  proto kernel  scope link  src linkdown dev cali4c16a3de178  scope link dev cali11df0bf4071  scope link via dev tunl0  proto bird onlink  --> tunnel to pod subnet in other worker node is registered dev ens4  scope link