00 网络虚拟化预备知识

netns与veth

netns：只是隔离了network资源，对其他namespace资源不进行隔离，所以宿主机上能使用的命令，netns内部也能使用。网络资源的隔离，包括网络设备、IPv4以及IPv6地址、IP路由表、防火墙、/proc/net、/sys/class/net以及套接字等。
veth：veth设备是成对出现的，两端连接两个设备，一个设备收到的数据发送请求后，会将数据发送到另一个设备上去。相当一根网线（或网线+网卡），网线两端的网卡也可以设置IP，有些时候也不设置网卡IP而是只起到连接线的作用，比如连接到二层设备网桥（交换机）的port上。

实验：在宿主机上创建2个netns，然后使用veth联通两个netns。测试两个netns是否可ping通

h1LKDi-1625189168440

#创建名称空间netns1和netns2
[root@OS-network-1 ~]# ip netns add netns1
[root@OS-network-1 ~]# ip netns add netns2
[root@OS-network-1 ~]# ip netns list
netns2
netns1

#创建veth设备
[root@OS-network-1 ~]# ip link add veth1.1 type veth peer name veth1.2
[root@OS-network-1 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
3: veth1.2@veth1.1: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether c2:b2:02:c9:ae:3a brd ff:ff:ff:ff:ff:ff
4: veth1.1@veth1.2: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 0e:f4:eb:b8:08:ef brd ff:ff:ff:ff:ff:ff
    
#将veth设备的一端插入到netns1名称空间中 ，另一端插入到netns2名称空间中    
[root@OS-network-1 ~]# ip link set veth1.1 netns netns1
[root@OS-network-1 ~]# ip link set veth1.2 netns netns2
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 brd 10.211.55.255 scope global noprefixroute dynamic eth0
       valid_lft 1427sec preferred_lft 1427sec
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591992sec preferred_lft 604792sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
[root@OS-network-1 ~]# ip netns exec netns1 ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
4: veth1.1@if3: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 0e:f4:eb:b8:08:ef brd ff:ff:ff:ff:ff:ff link-netnsid 1
    
# 对netns1名称空间中的veth设备的一端(网卡)进行改名(可选)，配置IP(192.168.1.2/24)并启动网卡
[root@OS-network-1 ~]# ip netns exec netns1 ip link set veth1.1 name eth0  #可选
[root@OS-network-1 ~]# ip netns exec netns1 ip a add 192.168.1.2/24 dev eth0
[root@OS-network-1 ~]# ip netns exec netns1 ip link set eth0 up
[root@OS-network-1 ~]# ip netns exec netns1 ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
4: eth0@if3: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state LOWERLAYERDOWN group default qlen 1000
    link/ether 0e:f4:eb:b8:08:ef brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet 192.168.1.2/24 scope global eth0
       valid_lft forever preferred_lft forever


# 对netns2名称空间中的veth设备的一端(网卡)进行改名(可选)，配置IP(192.168.1.3/24)并启动网卡
[root@OS-network-1 ~]# ip netns exec netns2 ip link set veth1.2 name eth0  #可选
[root@OS-network-1 ~]# ip netns exec netns2 ip link set eth0 up
[root@OS-network-1 ~]# ip netns exec netns2 ip a add 192.168.1.3/24 dev eth0
[root@OS-network-1 ~]# ip netns exec netns2 ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
3: eth0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether c2:b2:02:c9:ae:3a brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.1.3/24 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::c0b2:2ff:fec9:ae3a/64 scope link
       valid_lft forever preferred_lft forever

#测试ping
[root@OS-network-1 ~]# ip netns exec netns1 ping -c1 192.168.1.3
PING 192.168.1.3 (192.168.1.3) 56(84) bytes of data.
64 bytes from 192.168.1.3: icmp_seq=1 ttl=64 time=0.032 ms

--- 192.168.1.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.032/0.032/0.032/0.000 ms


[root@OS-network-1 ~]# ip netns exec netns2 ping -c1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=0.019 ms

--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.019/0.019/0.019/0.000 ms

linux网桥

linux网桥是一个虚拟的交换机，是一个二层设备，但可以配置IP。

brctl命令在bridge-utils安装包中。

[root@OS-network-1 ~]# brctl -h
Usage: brctl [commands]
commands:
	addbr     	<bridge>		add bridge（创建网桥）
	delbr     	<bridge>		delete bridge（删除网桥）
	addif     	<bridge> <device>	add interface to bridge(向网桥中添加网卡)
	delif     	<bridge> <device>	delete interface from bridge(从网桥中删除网卡)
	hairpin   	<bridge> <port> {on|off}	turn hairpin on/off
	setageing 	<bridge> <time>		set ageing time
	setbridgeprio	<bridge> <prio>		set bridge priority
	setfd     	<bridge> <time>		set bridge forward delay
	sethello  	<bridge> <time>		set hello time
	setmaxage 	<bridge> <time>		set max message age
	setpathcost	<bridge> <port> <cost>	set path cost
	setportprio	<bridge> <port> <prio>	set port priority
	show      	[ <bridge> ]		show a list of bridges（查看网桥）
	showmacs  	<bridge>		show a list of mac addrs
	showstp   	<bridge>		show bridge stp info
	stp       	<bridge> {on|off}	turn stp on/off
	
# 创建、查看linux网桥	
[root@OS-network-1 ~]# brctl addbr br1
[root@OS-network-1 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
br1		8000.000000000000	no
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 brd 10.211.55.255 scope global noprefixroute dynamic eth0
       valid_lft 1761sec preferred_lft 1761sec
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591961sec preferred_lft 604761sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 22:ff:bc:44:ee:5c brd ff:ff:ff:ff:ff:ff
    
# 启动网桥设备    
[root@OS-network-1 ~]# ip link set br1 up
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 brd 10.211.55.255 scope global noprefixroute dynamic eth0
       valid_lft 1747sec preferred_lft 1747sec
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591947sec preferred_lft 604747sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 22:ff:bc:44:ee:5c brd ff:ff:ff:ff:ff:ff
    inet6 fe80::20ff:bcff:fe44:ee5c/64 scope link
       valid_lft forever preferred_lft forever

# 删除物理网卡的IP，将该IP配置到网桥br1上，并将eth0加入到br1桥上。
[root@OS-network-1 ~]# ip a del 10.211.55.64/24 dev eth0; ip a add 10.211.55.64/24 dev br1; brctl addif br1 eth0
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br1 state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591902sec preferred_lft 604702sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 scope global br1
       valid_lft forever preferred_lft forever
    inet6 fe80::20ff:bcff:fe44:ee5c/64 scope link
       valid_lft forever preferred_lft forever
       
# 为网桥br1添加路由    
[root@OS-network-1 ~]# ip route
10.211.55.0/24 dev br1 proto kernel scope link src 10.211.55.64
[root@OS-network-1 ~]# ip route add default via 10.211.55.1 dev br1
[root@OS-network-1 ~]# ip route
default via 10.211.55.1 dev br1
10.211.55.0/24 dev br1 proto kernel scope link src 10.211.55.64

查看linux Bridge的mac表：

root@pve01:~# brctl showmacs vmbr0
port no	mac addr		is local?	ageing timer
  1	62:e6:aa:d8:94:d6	no		   0.02
  1	c6:02:8a:b3:a7:be	yes		   0.00
  1	c6:02:8a:b3:a7:be	yes		   0.00
  2	ce:e2:90:f8:16:28	yes		   0.00
  2	ce:e2:90:f8:16:28	yes		   0.00
  2	d6:6c:ec:c0:5e:e1	no		   1.01

“is local?”为yes表示veth pair的一端在bridge，该MAC地址是veth设备的MAC；为no表示不在bridge的veth设备。
“ageing timer”：指的是老化时间。

说明：
brctl addbr br1 等同于 ip link add br1 type bridge
brctl addif br1 eth0 等同于 ip link set eth0 master br1
brctl show 等同于 ip link show type bridge + ip link show master br0

试验1：单宿主机“brctl网桥+netns”实现虚拟机访问外网

br-in：虚拟桥

br-ex：物理桥，与物理网卡绑定

中间的路由器和dhcp在一个netns中。

[root@OS-network-1 ~]# brctl addbr br-ex

[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 brd 10.211.55.255 scope global noprefixroute dynamic eth0
       valid_lft 1094sec preferred_lft 1094sec
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591840sec preferred_lft 604640sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br-ex: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether e6:cc:ca:11:9d:07 brd ff:ff:ff:ff:ff:ff

#启动网卡br-ex
[root@OS-network-1 ~]# ip link set br-ex up
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 brd 10.211.55.255 scope global noprefixroute dynamic eth0
       valid_lft 1783sec preferred_lft 1783sec
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591689sec preferred_lft 604489sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether e6:cc:ca:11:9d:07 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::e4cc:caff:fe11:9d07/64 scope link
       valid_lft forever preferred_lft forever
       

# 删除物理网卡的IP，将该IP配置到br-ex上，并将eth0加入到br-ex桥上。
[root@OS-network-1 ~]# ip a del 10.211.55.64/24 dev eth0; ip a add 10.211.55.64/24 dev br-ex; brctl addif br-ex eth0
[root@OS-network-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-ex state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet6 fdb2:2c26:f4e4:0:d3c1:2487:cb61:a77c/64 scope global noprefixroute dynamic
       valid_lft 2591513sec preferred_lft 604313sec
    inet6 fe80::91e:7192:7cdb:adf9/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
3: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
    inet 10.211.55.64/24 scope global br-ex
       valid_lft forever preferred_lft forever
    inet6 fe80::e4cc:caff:fe11:9d07/64 scope link
       valid_lft forever preferred_lft forever
 
 [root@OS-network-1 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
br-ex		8000.001c426cb4ed	no		eth0


# 重建内部桥br-in
[root@OS-network-1 ~]# brctl addbr br-in
[root@OS-network-1 ~]# ip link set br-in up
[root@OS-network-1 ~]# ip link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-ex state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
3: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
4: br-in: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether fa:9e:36:47:16:05 brd ff:ff:ff:ff:ff:ff
    
# 启用内核转发ip_forward
# netns内部无法独自开启内核转发ip_forward，需要在宿主机上开启，netns内部共享宿主机的内核转发
[root@OS-network-1 ~]# vi /etc/sysctl.conf
[root@OS-network-1 ~]# cat /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
net.ipv4.ip_forward = 1
# 使参数生效
[root@OS-network-1 ~]# sysctl -p
net.ipv4.ip_forward = 1


# 创建veth设备
[root@OS-network-1 ~]#  ip link add veth1.1 type veth peer name veth1.2
[root@OS-network-1 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-ex state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
3: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
4: br-in: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether fa:9e:36:47:16:05 brd ff:ff:ff:ff:ff:ff
5: veth1.2@veth1.1: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 3a:ed:d1:b6:a6:84 brd ff:ff:ff:ff:ff:ff
6: veth1.1@veth1.2: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 9e:62:ec:e7:e7:8b brd ff:ff:ff:ff:ff:ff


# 使用qemu-kvm创建虚拟机，并将虚拟机的网卡一端插入到虚拟机中，一端插入到br-in网桥中
# 如果宿主机无法联网，其使用"ip route add default via 10.211.55.1 dev br-ex"添加默认路由
[root@OS-network-1 ~]# yum install -y qemu-kvm
[root@OS-network-1 ~]# ln -sv /usr/libexec/qemu-kvm  /usr/bin/
'/usr/bin/qemu-kvm' -> '/usr/libexec/qemu-kvm'
[root@OS-network-1 ~]# mkdir -pv /images/cirros
[root@OS-network-1 ~]# cd /images/cirros/
[root@OS-network-1 cirros]# rz -be
rz waiting to receive.**B0100000063f694
[root@OS-network-1 cirros]# # Received /Users/lijuzhang/Downloads/cirros-0.4.0-x86_64-disk.img
[root@OS-network-1 cirros]# ls
cirros-0.4.0-x86_64-disk.img
[root@OS-network-1 cirros]# cp cirros-0.4.0-x86_64-disk.img test1.qcow2
[root@OS-network-1 cirros]# cp cirros-0.4.0-x86_64-disk.img test2.qcow2
# 准备网卡脚本
[root@OS-network-1 cirros]# vi /etc/qemu-ifup
[root@OS-network-1 cirros]# bash -n /etc/qemu-ifup
[root@OS-network-1 cirros]# cat /etc/qemu-ifup
#!/bin/bash

bridge=br-in

if [ -n "$1" ];then
  ip link set $1 up
  brctl addif $bridge $1 && exit 0 || exit 1
else
  echo "Error: no interface specified"
  exit 2
fi

[root@OS-network-1 cirros]# chmod +x /etc/qemu-ifup
#启动虚拟机,--nographic表示不启用VNCserver；如果要启用vnc，可以使用-daemonize后台运行
[root@OS-network-1 cirros]# qemu-kvm -m 128 -smp 1 -name vm1 -drive file=/images/cirros/test1.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:cc -net tap,ifname=veth1.0,script=/etc/qemu-ifup --nographic


[root@OS-network-1 cirros]# qemu-kvm -m 128 -smp 1 -name vm2 -drive file=/images/cirros/test2.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:dd -net tap,ifname=veth2.0,script=/etc/qemu-ifup --nographic


# 发现虚拟机的网卡加入到网桥了
[root@localhost ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
br-ex		8000.001c426cb4ed	no		eth0
br-in		8000.4a987e8803a1	no		veth1.0
																veth2.0
																
																
# 路由器配置
[root@localhost ~]# ip link add rinr type veth peer name rins
[root@localhost ~]# ip link set rinr up
[root@localhost ~]# ip link set rins up
[root@localhost ~]# brctl addif br-in rins
[root@localhost ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
br-ex		8000.001c426cb4ed	no		eth0
br-in		8000.4a987e8803a1	no		rins
																veth1.0
																veth2.0
[root@localhost ~]# ip link set rinr netns router
[root@localhost ~]# ip link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-ex state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
3: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 00:1c:42:6c:b4:ed brd ff:ff:ff:ff:ff:ff
4: br-in: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 4a:98:7e:88:03:a1 brd ff:ff:ff:ff:ff:ff
7: veth1.0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-in state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether a6:f0:bb:f9:b2:5f brd ff:ff:ff:ff:ff:ff
8: veth2.0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master br-in state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether 4a:98:7e:88:03:a1 brd ff:ff:ff:ff:ff:ff
13: rins@if14: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state LOWERLAYERDOWN mode DEFAULT group default qlen 1000
    link/ether 76:29:66:29:5e:6b brd ff:ff:ff:ff:ff:ff link-netnsid 0
[root@localhost ~]# ip netns exec router ip link set rinr name eth0  #改名，可选
[root@localhost ~]# ip netns exec router ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
14: eth0@if13: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 6e:2f:61:3f:6b:45 brd ff:ff:ff:ff:ff:ff link-netnsid 0  
[root@localhost ~]# ip netns exec router ip link set rinr up
Cannot find device "rinr"
# 启动网卡eth0，配置ip
[root@localhost ~]# ip netns exec router ip link set eth0 up
[root@localhost ~]# ip netns exec router ip a add 192.168.1.1/4 dev eth0
[root@localhost ~]# ip netns exec router ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
14: eth0@if13: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 6e:2f:61:3f:6b:45 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.1.1/4 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::6c2f:61ff:fe3f:6b45/64 scope link
       valid_lft forever preferred_lft forever 
       
       
       
#vm1配置IP和网关
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 52:54:00:aa:bb:cc brd ff:ff:ff:ff:ff:ff
    inet6 fe80::5054:ff:feaa:bbcc/64 scope link
       valid_lft forever preferred_lft forever
# ip a add 192.168.1.2/24 dev eth0
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 52:54:00:aa:bb:cc brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.2/24 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:feaa:bbcc/64 scope link
       valid_lft forever preferred_lft forever
# ping 192.168.1.1 -c1
PING 192.168.1.1 (192.168.1.1): 56 data bytes
64 bytes from 192.168.1.1: seq=0 ttl=64 time=1.466 ms

--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 1.466/1.466/1.466 ms
#配置网关
# ip route add default via 192.168.1.1


#veth连接router和br-ex
[root@localhost ~]# ip link add rexr type veth peer name rexs
[root@localhost ~]# ip link del rexs
[root@localhost ~]# ip link add rexr type veth peer name rexs
[root@localhost ~]# brctl addif br-ex rexs
[root@localhost ~]# ip link set rexs up
[root@localhost ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
br-ex		8000.001c426cb4ed	no		eth0
							rexs
br-in		8000.4a987e8803a1	no		rins
							veth1.0
							veth2.0
[root@localhost ~]# ip link set rexr netns router
[root@localhost ~]# ip netns exec router ip link set rexr name eth1
[root@localhost ~]# ip netns exec router ip link set eth1 up
[root@localhost ~]# ip netns exec router ip a add 10.211.55.222/24 dev eth1
[root@localhost ~]# ip netns exec router ping 10.211.55.1 -c1
PING 10.211.55.1 (10.211.55.1) 56(84) bytes of data.
64 bytes from 10.211.55.1: icmp_seq=1 ttl=128 time=0.093 ms

--- 10.211.55.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.093/0.093/0.093/0.000 ms

#虚拟机
#通路由器
# ping 10.211.55.222 -c1
PING 10.211.55.222 (10.211.55.222): 56 data bytes
64 bytes from 10.211.55.222: seq=0 ttl=64 time=0.822 ms

--- 10.211.55.222 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 0.822/0.822/0.822 ms
#不通物理网关，原因是到达物理网关的报文无法回到路由器上，因此需要在路由器上添加snat规则
# ping 10.211.55.1 -c1
PING 10.211.55.1 (10.211.55.1): 56 data bytes

--- 10.211.55.1 ping statistics ---
1 packets transmitted, 0 packets received, 100% packet loss

#在路由器上添加snat规则
[root@localhost ~]# ip netns exec router iptables -t nat -A POSTROUTING -s 192.168.1.0/24 ! -d 192.168.1.0/24 -j SNAT --to-source 10.211.55.222
[root@localhost ~]# ip netns exec router iptables -vnL -t nat
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
    0     0 SNAT       all  --  *      *       192.168.1.0/24      !192.168.1.0/24       to:10.211.55.222
    
#虚拟机
# ping 10.211.55.1 -c1
PING 10.211.55.1 (10.211.55.1): 56 data bytes
64 bytes from 10.211.55.1: seq=0 ttl=127 time=2.080 ms

--- 10.211.55.1 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 2.080/2.080/2.080 ms