In the next few posts, we will secure the communication between each services one by one. In this first post, we will secure the communication between your local machine and API server (in my case in GCP), which goes across the internet and considered to be the most vulnerable part in our cluster at this moment. After completing this post, the cluster communication will be something like below.
Place certificate in directory
1. Move certificates/keys to relevance directory
In the previous post, I generated all the cert under /root/cert, let’s move them to appropriate directory.
[ controller-1 ]
root@controller-1:~# mkdir -p /var/lib/kubernetes root@controller-1:~# mv ./cert /var/lib/kubernetes
Modify API server with certificate
1. Modify service file
Modify system service file so that it starts with appropriate keys
[ controller-1 ]
# modify system service file
root@controller-1:~# vim /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-apiserver --etcd-servers=http://127.0.0.1:2379 --service-cluster-ip-range=10.32.0.0/24 --insecure-bind-address=0.0.0.0 --kubelet-https=false --client-ca-file=/var/lib/kubernetes/cert/ca.pem --tls-cert-file=/var/lib/kubernetes/cert/apiserver.pem --tls-private-key-file=/var/lib/kubernetes/cert/apiserver-key.pem
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
# reload service and confirm it started successfully
root@controller-1:~# systemctl daemon-reload
root@controller-1:~# systemctl restart kube-apiserver
root@controller-1:~# systemctl status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: enabled)
Active: active (running) since Sun 2018-10-14 07:33:13 UTC; 9s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 2553 (kube-apiserver)
Tasks: 7
Memory: 314.9M
CPU: 7.333s
CGroup: /system.slice/kube-apiserver.service
└─2553 /usr/local/bin/kube-apiserver --etcd-servers=http://127.0.0.1:2379 --service-cluster-ip-rang
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.159548 2553 customresource_discovery_cont
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.159855 2553 naming_controller.go:276] Sta
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.160666 2553 crdregistration_controller.go
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.160956 2553 controller_utils.go:1019] Wai
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.255408 2553 cache.go:39] Caches are synce
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.257407 2553 cache.go:39] Caches are synce
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.268264 2553 controller_utils.go:1026] Cac
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.270127 2553 autoregister_controller.go:13
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.270451 2553 cache.go:32] Waiting for cach
Oct 14 07:33:21 controller-1 kube-apiserver[2553]: I1014 07:33:21.371129 2553 cache.go:39] Caches are synce
root@controller-1:~#
Modify kubectl
1. Generate kubeconfig file
[ controller-1 ]
# create kubeconfig named kubectl.kubeconfig
# we embed certificate for portability.
# specify public ip for server-https
root@controller-1:~# kubectl config set-cluster k8s-demo \
> --certificate-authority=/var/lib/kubernetes/cert/ca.pem \
> --embed-certs=true \
> --server=https://x.x.x.x:6443 \
> --kubeconfig=kubectl.kubeconfig
Cluster "k8s-demo" set.
# create user admin with respective certificate and key. Note admin is the default account
root@controller-1:~# kubectl config set-credentials admin \
> --client-certificate=/var/lib/kubernetes/cert/kubectl.pem \
> --client-key=/var/lib/kubernetes/cert/kubectl-key.pem \
> --embed-certs=true \
> --kubeconfig=kubectl.kubeconfig
User "admin" set.
# set context, this name will be used to activate the context
# this cluster name and user name needs to match the ones we created in above
root@controller-1:~# kubectl config set-context default \
> --cluster=k8s-demo \
> --user=admin \
> --kubeconfig=kubectl.kubeconfig
Context "default" created.
root@controller-1:~# ls -l
total 8
-rw------- 1 root root 6061 Oct 14 08:09 kubectl.kubeconfig
# you can check what is the config in kubeconfig before distributing it
root@controller-1:~# kubectl config view --kubeconfig=kubectl.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: REDACTED
server: https://x.x.x.x:6443
name: k8s-demo
contexts:
- context:
cluster: k8s-demo
user: admin
name: default
current-context: ""
kind: Config
preferences: {}
users:
- name: admin
user:
client-certificate-data: REDACTED
client-key-data: REDACTEDOnce kubeconfig is generated, you need to copy it to your local machine.
2. Confirm if the config works
[ Local Machine ]
# set the kubectl to use "default" context from the kubeconfig file we just generated shogokobayashi k8s-demo $ kubectl config use-context default --kubeconfig=kubectl.kubeconfig Switched to context "default". # check a few kubectl commands to check if it still works shogokobayashi k8s-demo $ kubectl get nodes --kubeconfig=kubectl.kubeconfig NAME STATUS ROLES AGE VERSION worker-1 NotReady <none> 18d v1.11.3 worker-2 NotReady <none> 14d v1.11.3 # now we can see the communication via on HTTPS(hence it is no more readable on the wire). shogokobayashi k8s-demo $ tcpdump -i en0 src net x.x.x.x/32 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on en0, link-type EN10MB (Ethernet), capture size 262144 bytes 11:08:47.100220 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [S.], seq 2100300494, ack 3012496608, win 28160, options [mss 1420,sackOK,TS val 2080068646 ecr 770063841,nop,wscale 7], length 0 11:08:47.332850 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [.], ack 165, win 229, options [nop,nop,TS val 2080068984 ecr 770064244], length 0 11:08:47.334202 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [.], seq 1:1409, ack 165, win 229, options [nop,nop,TS val 2080068988 ecr 770064244], length 1408 11:08:47.334211 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1409:1680, ack 165, win 229, options [nop,nop,TS val 2080068988 ecr 770064244], length 271 11:08:47.578384 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1680:1731, ack 1502, win 251, options [nop,nop,TS val 2080069220 ecr 770064484], length 51 11:08:47.578397 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1731:1793, ack 1502, win 251, options [nop,nop,TS val 2080069220 ecr 770064484], length 62 11:08:47.803677 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [.], ack 1802, win 272, options [nop,nop,TS val 2080069463 ecr 770064717], length 0 11:08:47.803690 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1793:1835, ack 1802, win 272, options [nop,nop,TS val 2080069463 ecr 770064717], length 42 11:08:47.803693 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1835:1873, ack 1802, win 272, options [nop,nop,TS val 2080069463 ecr 770064717], length 38 11:08:47.804797 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 1873:1954, ack 1802, win 272, options [nop,nop,TS val 2080069465 ecr 770064717], length 81 11:08:47.805375 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [.], seq 1954:3362, ack 1802, win 272, options [nop,nop,TS val 2080069465 ecr 770064717], length 1408 11:08:47.806130 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [.], seq 3362:4770, ack 1802, win 272, options [nop,nop,TS val 2080069465 ecr 770064717], length 1408 11:08:47.806140 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 4770:5355, ack 1802, win 272, options [nop,nop,TS val 2080069465 ecr 770064717], length 585 11:08:48.126873 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [P.], seq 5355:5386, ack 1803, win 272, options [nop,nop,TS val 2080069693 ecr 770064951], length 31 11:08:48.126887 IP x.x.x.x.bc.googleusercontent.com.sun-sr-https > 192.168.1.107.51801: Flags [F.], seq 5386, ack 1803, win 272, options [nop,nop,TS val 2080069693 ecr 770064951], length 0