03 Ovs桥与 Gre隧道

试验拓补图

说明：由于node2和node3上的网卡都是内部网卡，不能访问外网，为了使他们能够访问repo安装软件，会将node2和node3的管理网卡的网关指向node1。而node1会开启内核转发和SNAT规则，以作为网关使用。

环境准备

1、配置Host1作为网关

# 开启内核转发
# 临时生效：sysctl -w net.ipv4.ip_forward=1
[root@centos7-1 ~]# vi /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
net.ipv4.ip_forward = 1
[root@centos7-1 ~]# sysctl -p
net.ipv4.ip_forward = 1

# 添加SNAT规则
[root@centos7-1 ~]# iptables -t nat -A POSTROUTING -s 10.37.129.0/24 -j SNAT --to-source 10.211.55.68
[root@centos7-1 ~]# iptables -vnL -t nat
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
    0     0 SNAT       all  --  *      *       10.37.129.0/24       0.0.0.0/0            to:10.211.55.68

2、配置Host2和Host3的网关指向Host1

[root@centos7-2 ~]# ip r add default via 10.37.129.9 dev eth0

[root@centos7-3 ~]# ip r add default via 10.37.129.9 dev eth0

试验：GRE隧道

1、安装软件包openvswitch

node2和node3安装、。

cat > /etc/yum.repos.d/openstack-rocky.repo <<EOF
[openstack]
name=opentack
baseurl=https://mirrors.aliyun.com/centos/7/cloud/x86_64/openstack-rocky/
gpgcheck=0

[Virt]
name=CentOS-$releasever - Base
baseurl=https://mirrors.aliyun.com/centos/7/virt/x86_64/kvm-common/
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
EOF

yum install -y openvswitch
systemctl start openvswitch
systemctl enable openvswitch

2、安装qemu-kvm，准备启动虚拟机的镜像和脚本

yum install -y qemu-kvm
ln -sv /usr/libexec/qemu-kvm  /usr/bin/

#准备虚拟机磁盘镜像
mkdir -pv /images/cirros
cd /images/cirros/
rz -be
cp cirros-0.4.0-x86_64-disk.img test1.qcow2
cp cirros-0.4.0-x86_64-disk.img test2.qcow2

# 准备网卡脚本
cat >/etc/qemu-ovs-ifup<<"EOF"
#!/bin/bash

bridge=br-in

if [ -n "$1" ];then
  ip link set $1 up
  sleep 1
  ovs-vsctl add-port  $bridge $1 && exit 0 || exit 1
  brctl addif $bridge $1 && exit 0 || exit 1
else
  echo "Error: no port specified"
  exit 2
fi
EOF

chmod +x /etc/qemu-ovs-ifup

cat >/etc/qemu-ovs-ifdown<<"EOF"
#!/bin/bash

bridge=br-in

if [ -n "$1" ];then
  ip link set $1 down
  sleep 1
  ovs-vsctl del-port  $bridge $1 && exit 0 || exit 1
else
  echo "Error: no port specified"
  exit 2
fi
EOF

chmod +x /etc/qemu-ovs-ifdown

3、创建ovs桥br-in

node2和node3上执行

ovs-vsctl add-br br-in

4、创建DHCP服务（可选）

采用netns+veth+dnsmasq，veth设备的一端接入到netns中，另一端接入到ovs桥br-in中。

本步骤可选，如果不部署DHCP服务，则需要为每一台VM收到配置网卡信息。

[root@centos7-2 ~]# ip netns add router
[root@centos7-2 ~]# ip link add vethr type veth peer name vethb
[root@centos7-2 ~]# ip link set vethr netns router
[root@centos7-2 ~]# ip netns exec router ip link set vethr up
[root@centos7-2 ~]# ip netns exec router ip a add 192.168.1.254/24 dev vethr
[root@centos7-2 ~]# ovs-vsctl add-port br-in vethb
[root@centos7-2 ~]# ip link set vethb up

[root@centos7-2 ~]# yum install -y dnsmasq
[root@centos7-2 ~]# cat > /etc/dnsmasq.d/netns.conf <<EOF
#DNS服务器的地址
listen-address=192.168.1.254
#设置DHCP分配的地址范围和时间 
dhcp-range=192.168.1.10,192.168.1.200,1h 
#设定网关的地址：dnsmasq具体的option，可以通过dnsmasq --help dhcp查看
dhcp-option=3,192.168.1.254  #等同于dhcp-option=option:router,192.168.1.254
#设定DNS服务器
dhcp-option=option:dns-server,114.114.114.114,8.8.4.4
EOF
#启动dnsmasq，监听67/udp端口
[root@centos7-2 ~]# ip netns exec router dnsmasq --conf-file=/etc/dnsmasq.d/netns.conf

5、node2创建虚拟机vm1和vm2

从DHCP中自动获取IP

[root@centos7-2 ~]# qemu-kvm -m 128 -smp 1 -name vm1 -drive file=/images/cirros/test1.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:01 -net tap,ifname=veth1.0,script=/etc/qemu-ovs-ifup,downscript=/etc/qemu-ovs-ifdown -nographic

# 已经自动分配IP了
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 52:54:00:aa:bb:01 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.109/24 brd 192.168.1.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:feaa:bb01/64 scope link
       valid_lft forever preferred_lft forever
       
[root@centos7-2 ~]# qemu-kvm -m 128 -smp 1 -name vm2 -drive file=/images/cirros/test2.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:02 -net tap,ifname=veth2.0,script=/etc/qemu-ovs-ifup,downscript=/etc/qemu-ovs-ifdown -nographic

6、node3创建虚拟机vm3

手动指定IP，因为目前隧道还没有打通，无法为node3上的vm自动分配IP

[root@centos7-3 ~]# qemu-kvm -m 128 -smp 1 -name vm3 -drive file=/images/cirros/test1.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:03 -net tap,ifname=veth1.0,script=/etc/qemu-ovs-ifup,downscript=/etc/qemu-ovs-ifdown -nographic


# ip a add 192.168.1.13/24 dev eth0
#
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 52:54:00:aa:bb:03 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.13/24 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:feaa:bb03/64 scope link
       valid_lft forever preferred_lft forever
       
       
# ping 192.168.1.109
PING 192.168.1.109 (192.168.1.109): 56 data bytes
^C
--- 192.168.1.109 ping statistics ---
4 packets transmitted, 0 packets received, 100% packet loss      #不通

7、创建GRE隧道

需要在两个node的ovs桥上各自添加一个port，并设置这个port上的interface属性：type=gre options:remote_ip=对端ip

Node2：

[root@centos7-2 ~]# ovs-vsctl add-port br-in gre0   #gre0这个接口可以事先不存在
ovs-vsctl: Error detected while setting up 'gre0': could not add network device gre0 to ofproto (Invalid argument).  See ovs-vswitchd log for details.
ovs-vsctl: The default log directory is "/var/log/openvswitch".  #报错可以先忽略，因为此时还没有设置其属性。如果要避免出现此错误，可以在同一条命令中同时配置port和interface属性。


[root@centos7-2 ~]# ovs-vsctl list interface gre0
_uuid               : 2de30633-cab5-48e0-91af-b7ba1c3327fd
admin_state         : []
bfd                 : {}
bfd_status          : {}
cfm_fault           : []
cfm_fault_status    : []
cfm_flap_count      : []
cfm_health          : []
cfm_mpid            : []
cfm_remote_mpids    : []
cfm_remote_opstate  : []
duplex              : []
error               : "could not add network device gre0 to ofproto (Invalid argument)"
external_ids        : {}
ifindex             : []
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current        : []
link_resets         : []
link_speed          : []
link_state          : []
lldp                : {}
mac                 : []
mac_in_use          : []
mtu                 : []
mtu_request         : []
name                : "gre0"
ofport              : -1
ofport_request      : []
options             : {}   #为空
other_config        : {}
statistics          : {}
status              : {}
type                : ""   #为空

#设置接口的type和options
[root@centos7-2 ~]# ovs-vsctl set interface gre0 type=gre options:remote_ip=10.37.132.7
[root@centos7-2 ~]# ovs-vsctl list interface gre0
_uuid               : 2de30633-cab5-48e0-91af-b7ba1c3327fd
admin_state         : up
bfd                 : {}
bfd_status          : {}
cfm_fault           : []
cfm_fault_status    : []
cfm_flap_count      : []
cfm_health          : []
cfm_mpid            : []
cfm_remote_mpids    : []
cfm_remote_opstate  : []
duplex              : []
error               : []
external_ids        : {}
ifindex             : 12
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current        : []
link_resets         : 0
link_speed          : []
link_state          : up
lldp                : {}
mac                 : []
mac_in_use          : "ee:f9:11:51:ba:91"
mtu                 : []
mtu_request         : []
name                : "gre0"
ofport              : 4
ofport_request      : []
options             : {remote_ip="10.37.132.7"}
other_config        : {}
statistics          : {rx_bytes=0, rx_packets=0, tx_bytes=0, tx_packets=0}
status              : {tunnel_egress_iface="eth1", tunnel_egress_iface_carrier=up}
type                : gre

Node3：

[root@centos7-3 ~]# ovs-vsctl add-port br-in gre0 -- set interface gre0 type=gre options:remote_ip=10.37.132.6  # -- 分隔，可以在同一个命令中写多个子命令
ovs-vsctl: Error detected while setting up 'gre0': could not add network device gre0 to ofproto (Invalid argument).  See ovs-vswitchd log for details.
ovs-vsctl: The default log directory is "/var/log/openvswitch".

[root@centos7-3 ~]# ovs-vsctl list interface gre0
_uuid               : 2d4eb6ec-060c-4ef8-8039-f9c1ffc62b9e
admin_state         : up
bfd                 : {}
bfd_status          : {}
cfm_fault           : []
cfm_fault_status    : []
cfm_flap_count      : []
cfm_health          : []
cfm_mpid            : []
cfm_remote_mpids    : []
cfm_remote_opstate  : []
duplex              : []
error               : []
external_ids        : {}
ifindex             : 9
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current        : []
link_resets         : 0
link_speed          : []
link_state          : up
lldp                : {}
mac                 : []
mac_in_use          : "02:8b:a4:97:41:e4"
mtu                 : []
mtu_request         : []
name                : "gre0"
ofport              : 2
ofport_request      : []
options             : {remote_ip="10.37.132.6"}
other_config        : {}
statistics          : {rx_bytes=0, rx_packets=0, tx_bytes=0, tx_packets=0}
status              : {tunnel_egress_iface="eth1", tunnel_egress_iface_carrier=up}
type                : gre

GRE隧道建立之后，2个节点上的br-in处于同一个二层网络（大二层网络）。

8、vm3测试：ping vm1

# ping 192.168.1.109 -c1
PING 192.168.1.109 (192.168.1.109): 56 data bytes
64 bytes from 192.168.1.109: seq=0 ttl=64 time=3.389 ms

--- 192.168.1.109 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss   # 通，说明GRE隧道生效
round-trip min/avg/max = 3.389/3.389/3.389 ms

9、node3启动vm4（DHCP）

此时，由于GRE隧道已经打通，所以node3上启动的虚拟机也是可以通过DHCP自动获取IP

[root@centos7-3 ~]# qemu-kvm -m 128 -smp 1 -name vm4 -drive file=/images/cirros/test2.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:aa:bb:04 -net tap,ifname=veth2.0,script=/etc/qemu-ovs-ifup,downscript=/etc/qemu-ovs-ifdown -nographic

10、抓包分析：vm1 –> vm3

#VM1
# ping 192.168.1.13

[root@centos7-2 ~]# tcpdump -nn -i eth1
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes
15:23:00.687131 IP 10.37.132.6 > 10.37.132.7: GREv0, length 102: IP 192.168.1.109 > 192.168.1.13: ICMP echo request, id 44033, seq 149, length 64
15:23:00.688307 IP 10.37.132.7 > 10.37.132.6: GREv0, length 102: IP 192.168.1.13 > 192.168.1.109: ICMP echo reply, id 44033, seq 149, length 64
15:23:01.689727 IP 10.37.132.6 > 10.37.132.7: GREv0, length 102: IP 192.168.1.109 > 192.168.1.13: ICMP echo request, id 44033, seq 150, length 64
15:23:01.691515 IP 10.37.132.7 > 10.37.132.6: GREv0, length 102: IP 192.168.1.13 > 192.168.1.109: ICMP echo reply, id 44033, seq 150, length 64
15:23:02.692230 IP 10.37.132.6 > 10.37.132.7: GREv0, length 102: IP 192.168.1.109 > 192.168.1.13: ICMP echo request, id 44033, seq 151, length 64
15:23:02.694352 IP 10.37.132.7 > 10.37.132.6: GREv0, length 102: IP 192.168.1.13 > 192.168.1.109: ICMP echo reply, id 44033, seq 151, length 64

抓包发现，eth1网卡上传输的是GRE协议的报文，报文内部包含了ICMP协议的报文。

试验：基于GRE的VLAN

vm1和vm3划分到VLAN10；

vm2和vm4划分到VLAN20；

1、为br-in的端口打TAG

Node2：

[root@centos7-2 ~]# ovs-vsctl show
b3965e68-22d8-4a48-80a9-30c430a516b3
    Bridge br-in
        Port "veth2.0"
            Interface "veth2.0"
        Port vethb
            Interface vethb
        Port "veth1.0"
            Interface "veth1.0"
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="10.37.132.7"}
        Port br-in
            Interface br-in
                type: internal
    ovs_version: "2.11.0"
[root@centos7-2 ~]# ovs-vsctl set port veth1.0 tag=10
[root@centos7-2 ~]# ovs-vsctl set port veth2.0 tag=20
[root@centos7-2 ~]# ovs-vsctl show
b3965e68-22d8-4a48-80a9-30c430a516b3
    Bridge br-in
        Port "veth2.0"
            tag: 20
            Interface "veth2.0"
        Port vethb
            Interface vethb
        Port "veth1.0"
            tag: 10
            Interface "veth1.0"
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="10.37.132.7"}
        Port br-in
            Interface br-in
                type: internal
    ovs_version: "2.11.0"

Node3：

[root@centos7-3 ~]# ovs-vsctl show
5972c459-6d72-4984-9d88-6e0d7b2a00e8
    Bridge br-in
        Port "veth2.0"
            Interface "veth2.0"
        Port br-in
            Interface br-in
                type: internal
        Port "veth1.0"
            Interface "veth1.0"
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="10.37.132.6"}
    ovs_version: "2.11.0"
[root@centos7-3 ~]# ovs-vsctl set port veth1.0 tag=10
[root@centos7-3 ~]# ovs-vsctl set port veth2.0 tag=20
[root@centos7-3 ~]# ovs-vsctl show
5972c459-6d72-4984-9d88-6e0d7b2a00e8
    Bridge br-in
        Port "veth2.0"
            tag: 20
            Interface "veth2.0"
        Port br-in
            Interface br-in
                type: internal
        Port "veth1.0"
            tag: 10
            Interface "veth1.0"
        Port "gre0"
            Interface "gre0"
                type: gre
                options: {remote_ip="10.37.132.6"}
    ovs_version: "2.11.0"

GRE隧道，默认是trunc口。

2、测试VLAN通信

#VM1-->VM3 通
# ping 192.168.1.13 -c1
PING 192.168.1.13 (192.168.1.13): 56 data bytes
64 bytes from 192.168.1.13: seq=0 ttl=64 time=1.920 ms

--- 192.168.1.13 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 1.920/1.920/1.920 ms

#VM1-->VM4  不通
# ping 192.168.1.112 -c1
PING 192.168.1.112 (192.168.1.112): 56 data bytes

--- 192.168.1.112 ping statistics ---
1 packets transmitted, 0 packets received, 100% packet loss



#VM2-->VM3 不通
# ping 192.168.1.13 -c1
PING 192.168.1.13 (192.168.1.13): 56 data bytes

--- 192.168.1.13 ping statistics ---
1 packets transmitted, 0 packets received, 100% packet loss


#VM2-->VM4  通
# ping 192.168.1.112 -c1
PING 192.168.1.112 (192.168.1.112): 56 data bytes
64 bytes from 192.168.1.112: seq=0 ttl=64 time=13.150 ms

--- 192.168.1.112 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 13.150/13.150/13.150 ms