[k8s] 1. 네트워크 구현 방식

개요

k8s에서 network 통신이 어떻게 이루어지는 알아 보겠습니다. 

핵심원리 

k8s는 network namespace라는 기술을 사용하여 network를 분리 합니다. 

그리고 분리된 network에 가상의VNIC(Virtual Network Interface Card)를 할 당하고 이를 virtual swich에 연결하는 방식으로 구현이 됩니다. 

구성

2개의 물리 서버 

4개의 network namespace

basic

turnnel

구성 script 

####################################################################
#environment variable
####################################################################
NODE_IP="192.168.0.16"
NS1="NS1"
NS2="NS2"
BRIDGE_SUBNET="172.16.0.0/24"
BRIDGE_IP="172.16.1.1"
NS1_IP="172.16.0.2"
NS2_IP="172.16.0.3"

TO_NODE_IP="192.168.0.26"
TO_BRIDGE_SUBNET="172.16.1.0/24"
TO_BRIDGE_IP="172.16.1.0/24"
TO_NS1_IP="172.16.1..2"
TO_NS2_IP="172.16.1.3"

####################################################################
#step1, creating namespace
####################################################################
#setting
sudo ip netns add  $NS1
sudo ip netns add  $NS2

#confirming
ip netns show
#ip netns show
# NS2
# NS1

####################################################################
#step3, creating the veth pairs
####################################################################
#setting
sudo ip link add veth10 type veth peer name veth11
sudo ip link add veth20 type veth peer name veth21

#confirming
#sudo ip link show veth10
#18: veth10@vethe11: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
#    link/ether de:c1:42:19:be:60 brd ff:ff:ff:ff:ff:ff
#sudo ip link show veth20
#20: veth20@vethe21: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
#    link/ether 6e:af:3b:47:7d:f5 brd ff:ff:ff:ff:ff:ff

####################################################################
#step 4, Adding the veth pairs to the namespace
####################################################################
#setting
sudo ip link set veth11 netns $NS1
sudo ip link set veth21 netns $NS2

#confirming
sudo ip netns exec NS1 ip addr
#sudo ip netns exec NS1 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#23: veth11@if24: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
#    link/ether e2:a2:ec:4d:ac:48 brd ff:ff:ff:ff:ff:ff link-netnsid 0
sudo ip netns exec NS2 ip addr
#sudo ip netns exec NS2 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#21: veth21@if22: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
#    link/ether ae:d3:09:8e:3f:ef brd ff:ff:ff:ff:ff:ff link-netnsid 0

####################################################################
#step5, Configuring the interface in the networknamespace IP addr
####################################################################
#setting
sudo ip netns exec NS1 ip addr add $NS1_IP/24 dev veth11
sudo ip netns exec NS2 ip addr add $NS2_IP/24 dev veth21

#confirming
sudo ip netns exec NS1 ip addr
#sudo ip netns exec NS1 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#23: veth11@if24: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
#    link/ether e2:a2:ec:4d:ac:48 brd ff:ff:ff:ff:ff:ff link-netnsid 0
#    inet 1.1.0.2/24 scope global veth11
#       valid_lft forever preferred_lft forever
sudo ip netns exec NS2 ip addr
#sudo ip netns exec NS2 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#21: veth21@if22: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
#    link/ether ae:d3:09:8e:3f:ef brd ff:ff:ff:ff:ff:ff link-netnsid 0
#    inet 1.1.0.3/24 scope global veth21
#       valid_lft forever preferred_lft forever

####################################################################
#step6, Enabling the interfaces inside the network namespaces
####################################################################
#setting
sudo ip netns exec $NS1 ip link set dev veth11 up
sudo ip netns exec $NS2 ip link set dev veth21 up

#confirming
sudo ip netns exec NS1 ip addr
#sudo ip netns exec NS1 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#23: veth11@if24: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state LOWERLAYERDOWN group default qlen 1000
#    link/ether e2:a2:ec:4d:ac:48 brd ff:ff:ff:ff:ff:ff link-netnsid 0
#    inet 1.1.0.2/24 scope global veth11
#       valid_lft forever preferred_lft forever
sudo ip netns exec NS2 ip addr
#sudo ip netns exec NS2 ip addr
#1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#21: veth21@if22: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state LOWERLAYERDOWN group default qlen 1000
#    link/ether ae:d3:09:8e:3f:ef brd ff:ff:ff:ff:ff:ff link-netnsid 0
#    inet 1.1.0.3/24 scope global veth21
#       valid_lft forever preferred_lft forever

####################################################################
#step7, Creating the bridge
####################################################################
#setting
sudo ip link add jk-br0 type bridge

#confirming
sudo ip link show type bridge jk-br0
#sudo ip link show type bridge jk-br0
#25: jk-br0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
#    link/ether 3e:a6:57:e3:59:7d brd ff:ff:ff:ff:ff:ff

####################################################################
#step8, Adding the network namespace interface to the bridge
####################################################################
#setting
sudo ip link set dev veth10 master jk-br0
sudo ip link set dev veth20 master jk-br0

#confirming
ip link show master jk-br0 
#ip link show master jk-br0 
#22: veth20@if21: <BROADCAST,MULTICAST> mtu 1500 qdisc noop master jk-br0 state DOWN mode DEFAULT group default qlen 1000
#    link/ether c6:9e:49:ea:16:2d brd ff:ff:ff:ff:ff:ff link-netns NS2
#24: veth10@if23: <BROADCAST,MULTICAST> mtu 1500 qdisc noop master jk-br0 state DOWN mode DEFAULT group default qlen 1000
#    link/ether 06:40:f8:c3:95:b3 brd ff:ff:ff:ff:ff:ff link-netns NS1

####################################################################
#step9, Assigning the IP addr to the bridge
####################################################################
#setting
sudo ip addr add $BRIDGE_IP/24 dev jk-br0

#confirming
ip addr show jk-br0
#ip addr show jk-br0
#25: jk-br0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
#    link/ether 06:40:f8:c3:95:b3 brd ff:ff:ff:ff:ff:ff
#    inet 1.1.0.1/24 scope global jk-br0
#       valid_lft forever preferred_lft forever

####################################################################
#step10, Enabling the bridge / veth interface 
####################################################################
#setting
sudo ip link set dev jk-br0 up
sudo ip link set dev veth10 up
sudo ip link set dev veth20 up

#confirming
ip add show jk-br0
#ip add show jk-br0
#25: jk-br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
#    link/ether 06:40:f8:c3:95:b3 brd ff:ff:ff:ff:ff:ff
#    inet 1.1.0.1/24 scope global jk-br0
#       valid_lft forever preferred_lft forever
#    inet6 fe80::440:f8ff:fec3:95b3/64 scope link 
#       valid_lft forever preferred_lft forever
ip add show veth10
#ip add show veth10
#24: veth10@if23: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master jk-br0 state UP group default qlen 1000
#    link/ether 06:40:f8:c3:95:b3 brd ff:ff:ff:ff:ff:ff link-netns NS1
#    inet6 fe80::440:f8ff:fec3:95b3/64 scope link 
#       valid_lft forever preferred_lft forever
ip add show veth20
#ip add show veth20
#22: veth20@if21: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master jk-br0 state UP group default qlen 1000
#    link/ether c6:9e:49:ea:16:2d brd ff:ff:ff:ff:ff:ff link-netns NS2
#    inet6 fe80::c49e:49ff:feea:162d/64 scope link 
#       valid_lft forever preferred_lft forever


####################################################################
#step11, Enabling the loopback interface in the network namespace 
####################################################################
#setting
sudo ip netns exec $NS1 ip link set dev lo up
sudo ip netns exec $NS2 ip link set dev lo up

#confirming
sudo ip netns exec NS1 ip addr show lo
#sudo ip netns exec NS1 ip addr show lo
#1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#    inet 127.0.0.1/8 scope host lo
#       valid_lft forever preferred_lft forever
#    inet6 ::1/128 scope host 
#       valid_lft forever preferred_lft forever
sudo ip netns exec NS2 ip addr show lo
#sudo ip netns exec NS2 ip addr show lo
#1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#    inet 127.0.0.1/8 scope host lo
#       valid_lft forever preferred_lft forever
#    inet6 ::1/128 scope host 
#       valid_lft forever preferred_lft forever

####################################################################
#step12, setting default route in the network namesapces
####################################################################
#setting
sudo ip netns exec $NS1 ip route add default via $BRIDGE_IP dev veth11
sudo ip netns exec $NS2 ip route add default via $BRIDGE_IP dev veth21
sudo sysctl -w net.ipv4.ip_forward=1

#confirming
sudo ip netns exec NS2 ip route
#sudo ip netns exec NS2 ip route
#default via 1.1.0.1 dev veth21 
#1.1.0.0/24 dev veth21 proto kernel scope link src 1.1.0.3 
sudo ip netns exec NS1 ip route
#sudo ip netns exec NS1 ip route
#default via 1.1.0.1 dev veth11 
#1.1.0.0/24 dev veth11 proto kernel scope link src 1.1.0.2 
sudo sysctl net.ipv4.ip_forward=1
#net.ipv4.ip_forward = 1


####################################################################
#step13-1, specific setup
####################################################################
#setting
sudo ip route add $TO_BRIDGE_SUBNET via $TO_NODE_IP dev enp2s0


#confirming

ip route
#ip route
#default via 192.168.0.1 dev enp2s0 proto dhcp metric 100 
#default via 192.168.219.1 dev wlp3s0 proto dhcp metric 600 
#1.1.0.0/24 dev jk-br0 proto kernel scope link src 1.1.0.1 
#2.1.1.0/24 via 192.168.0.11 dev enp2s0 



####################################################################
#step13-2, specific setup for tunnel
####################################################################
TUNNEL_IP=1.1.0.100
TO_TUNNEL_IP=2.1.0.100
sudo socat UDP:$TO_NODE_IP:9000,bind=$NODE_IP:9000, TUN:$TUNNEL_IP/16,tun-name=tundudp,iff-no-pi,tun-type=tun &
sudo ip link set tundudp up
sudo ip route add $TO_BRIDGE_SUBNET via $TUNNEL_IP dev tundudp

TUNNEL_IP=2.1.0.100
TO_TUNNEL_IP=1.1.0.100
sudo socat UDP:$TO_NODE_IP:9000,bind=$NODE_IP:9000, TUN:$TUNNEL_IP/16,tun-name=tundudp,iff-no-pi,tun-type=tun &
sudo ip link set tundudp up
sudo ip route add $TO_BRIDGE_SUBNET via $TUNNEL_IP dev tundudp

####################################################################
#step14, Test
####################################################################
#To NS1
sudo ip netns exec $NS1 ping -W 1 -c 2 1.1.0.2

#To NS1
sudo ip netns exec $NS1 ping -W 1 -c 2 1.1.0.3

#To bridge
sudo ip netns exec $NS1 ping -W 1 -c 2 1.1.0.1

#To sever
sudo ip netns exec $NS1 ping -W 1 -c 2 $TO_NODE_IP

#To to_bridge
sudo ip netns exec $NS1 ping -W 1 -c 2 2.1.0.1

#To to_VM
sudo ip netns exec $NS1 ping -W 1 -c 2 2.1.0.2

참조 영상

https://www.youtube.com/watch?v=U35C0EPSwoY&list=PLSAko72nKb8QWsfPpBlsw-kOdMBD7sra-&index=2&ab_channel=TheLearningChannel