Keepalived+LVS/Nginx 实验

Untitled

VRRP

VRRP ( Virtual Router Redundancy Protocol ) 是一种容错协议，它保证当主机的下一跳路由器出现故障，由另外一台路由器来代替出现故障的路由器进行工作，从而保证网络通信的可靠性。

Keepalived

Keepalived 是Linux系统上VRRP协议的实现。keepalived配合LVS就能实现负载均衡和高可用，通过配置keepalived配置文件就可以调用ipvsadm命令对LVS进行配置。此外keepalived也可以与Nginx反向代理服务器等配合使用，实现高可用。

Alt text

实验一：Keepalived + LVS(NAT)

Alt text

第一步搭建实验环境

同步各节点的时间，在各节点上安装ntp服务，修改配置文件实现自动与指定的NTP服务器同步时间

# 安装ntp和ntpdate
[10@root ~]# cat ip.txt 
172.18.17.10
172.18.17.11
172.18.17.12
192.168.17.20
192.168.17.21
[10@root ~]# pssh -h ip.txt yum -y install ntp ntpdate

# 手动同步后再使用ntpd服务自动同步
[10@root ~]# pssh -h ip.txt ntpdate 172.18.17.10

# 修改ntp配置文件，实现自动同步
vim /etc/ntp.conf 
    server 172.18.17.10 iburst

[11@root ~]# chkconfig --add ntpd
[11@root ~]# chkconfig ntpd on
[11@root ~]# service ntpd start

修改防火墙规则允许相应报文通过
修改hosts文件实现各节点间可通过主机名通信(此步非必须)
设置使各节点之间的root用户可以基于密钥认证方式进行通信(此步非必须)
注意：RS1和RS2网关为 192.168.17.100
开启接收组播

ip link set multicast on dev eth0

第二步配置KA1和KA2实现虚拟路由

由于我们配置的虚拟路由与LVS(NAT)配合完成集群的高可用和负载均衡，对于LVS的NET模型，VIP和DIP需要同步，即KA1作为Masther同时拥有虚拟的VIP和DIP地址，或作为BACKUP同时失去这两个虚拟地址。使用keepalived同步组即可实现。

KA1和KA2上安装 keepalived

yum -y install keepalived

KA1上的配置文件

[11@root keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@'host'
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1

   vrrp_mcast_group4 224.0.17.17
}

vrrp_sync_group VG_1 {
    group {
        VI_1
        VI_2
    }
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }
    virtual_ipaddress {
        172.18.17.100/16
    }
}

vrrp_instance VI_2 {
    state MASTER
    interface eth1
    virtual_router_id 200
    priority 99 
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        192.168.17.100/24
    }
}

KA2上的配置文件

[12@root keepalived]# cat /etc/keepalived/keepalived.conf
# 全局配置
! Configuration File for keepalived

global_defs {
   notification_email {
     root@'host'
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2

   vrrp_mcast_group4 224.0.17.17
}

vrrp_sync_group VG_1 {
    group {
        VI_1
        VI_2
    }
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 100
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }
    virtual_ipaddress {
        172.18.17.100/16
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth1
    virtual_router_id 200
    priority 77
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        192.168.17.100/24
    }
}

第三步配置LVS

由于keepalived可以通过配置文件直接调用lvs，在keepalived的配置文件中就可以直接配置LVS规则。

KA1和KA2中lvs的配置

virtual_server 172.18.17.100 80 {
    delay_loop 6
    lb_algo rr
    lb_kind NAT
    persistence_timeout 50
    protocol TCP

    sorry_server 127.0.0.1 80

    real_server 192.168.17.20 80 {
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.17.21 80 {
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

第四步定义邮件通知

我们希望当KA1或KA2变换主被模式，或不能正常工作时发邮件通知。keepalived不能直接实现此功能，但keepalived可以触发脚本的执行，所以我们可以在脚本中实现发邮件通知的功能，由keepalived触发执行。

KA1和KA2上同时安装邮件服务 sendmail，并启动该服务

yum -y install sendmail
chkconfig --add sendmail
chkconfig sendmail on
chkconfig sendmail start

邮件通知脚本

#!/bin/bash

contact='root@localhost'

notify() {
    subject="$(hostname) to be $1, vip floating"
    body="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$body" | mail -s "$subject" $contact
}

case $1 in
master)
    notify master
    ;;
backup)
    notify backup
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac

在keepalived配置文件中添加触发机制

notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

KA1 和KA2 上 keepalived 最终配置

KA1上的配置

[11@root ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
#1. 全局配置段
global_defs {
   notification_email {
     root@'host'
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   #组播地址
   vrrp_mcast_group4 224.0.17.17
}

#2. 虚拟路由配置段
#2.1 虚拟路由组
vrrp_sync_group VG_1 {
    group {
        VI_1
        VI_2
    }
}
#2.2 虚拟路由VI_1
vrrp_instance VI_1 {
    # 状态是MASTER/BACKUP
    state MASTER
    interface eth0
    # 虚拟路由VI_1的编号
    virtual_router_id 100
    # 优先级，要求MASTER的比BACKUP高
    priority 100
    advert_int 1
    # 认证方法
    authentication {
        auth_type PASS
        auth_pass 100100
    }
    # 虚拟IP
    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }
    # 检查的端口
    track_interface {
        eth0
    }
    # 触发脚本执行
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

#2.3 虚拟路由VI_2
vrrp_instance VI_2 {
    state MASTER
    interface eth1
    virtual_router_id 200
    priority 99 
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        192.168.17.100/24
    }

    track_interface {
        eth0
    }

    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}


virtual_server 172.18.17.100 80 {
    delay_loop 6
    lb_algo wrr  
    lb_kind NAT 
    persistence_timeout 50
    protocol TCP 

    sorry_server 127.0.0.1 80

    real_server 192.168.17.20 80 {
        weight 1
        HTTP_GET {
            url { 
                path /
                status_code 200 
            }   
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }   
    }   

    real_server 192.168.17.21 80 {
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

KA2上的配置

[12@root keepalived]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@'host'
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2

   vrrp_mcast_group4 224.0.17.17
}

vrrp_sync_group VG_1 {
    group {
        VI_1
        VI_2
    }
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 100
    priority 88
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }

    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }

    track_interface {
        eth0
    }

    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth1
    virtual_router_id 200
    priority 77
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        192.168.17.100/24
    }

    track_interface {
        eth0
    }

    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

virtual_server 172.18.17.100 80 {
    delay_loop 6
    lb_algo wrr  
    lb_kind NAT 
    persistence_timeout 50
    protocol TCP 

    sorry_server 127.0.0.1 80

    real_server 192.168.17.20 80 {
        weight 1
        HTTP_GET {
            url { 
                path /
                status_code 200 
            }   
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }   
    }   

    real_server 192.168.17.21 80 {
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

实验二：Keepalived + Nginx

实验一演示了通过两台服务器，一个做主设备，一个做备用设备，实现了高可用。但这样的设计有点小遗憾，不能同时利用两台设备的硬件资源。下面的实验我们通过建立两个虚拟路由，第一个虚拟路由中KA1为主，KA2为备，第二个虚拟路由中KA1为备，KA2为主。KA1和KA2形成互为主备的关系，充分利用硬件资源。此外，这个实验演示keepalived如何实现nginx反向代理的高可用。

Alt text

第一步搭建实验环境

同步各节点时间
修改防火墙规则允许相应报文通过
修改hosts文件实现各节点间可通过主机名通信(此步非必须)
设置使各节点之间的root用户可以基于密钥认证方式进行通信(此步非必须)
开启接收组播
此处反向代理和后端WEB服务器均使用nginx，即KA1，KA2和RS0-RS3均安装nginx，此处需要提前挂载好光盘和配置好安装nginx的yum源，具体可参考我的另外一篇博客Nginx服务的安装和配置

[10@root ~]# cat ip.txt 
172.18.17.11
172.18.17.12
192.168.17.20
192.168.17.21
192.168.17.22
192.168.17.23

[10@root ~]# pssh -h ip.txt mount /dev/sr0 /mnt/cd

[10@root ~]# pssh -h ip.txt yum -y install nginx

为两台搭建虚拟路由的KA1和KA2安装 keepalived

[10@root ~]# cat ip.txt
172.18.17.11
172.18.17.12
[10@root ~]# pssh -h ip.txt yum -y install keepalived

第二步实现KA1和KA2互为主备

KA1上的 keepalived配置文件

[11@root ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.0.17.17
}

# 主
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }

    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }

    track_interface {
        eth0 
    }
}

# 备
vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 200
    priority 88
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        172.18.17.200/16 dev eth0 label eth0:2
    }

    track_interface {
        eth0 
    }
}

KA2上的 keepalived配置文件

[12@root ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_mcast_group4 224.0.17.17
}

# 备
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 100
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }

    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }

    track_interface {
        eth0 
    }
}

# 主
vrrp_instance VI_2 {
    state MASTER
    interface eth0
    virtual_router_id 200
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        172.18.17.200/16 dev eth0 label eth0:2
    }

    track_interface {
        eth0 
    }
}

第三步 Keepalived 实现 Nginx 的高可用

KA1和KA2上先实现 Nginx的反向代理功能，配置一样，nginx反向代理的配置文件如下

[11@root ~]# cat /etc/nginx/conf.d/default.conf
upstream http_servers {
    server 192.168.17.20:80; 
    server 192.168.17.21:80;
    server 192.168.17.22:80;
    server 192.168.17.23:80;
    least_conn;
}

server {
    listen 80 default_server;
    root   /usr/share/nginx/html;

    location / {
        proxy_pass  http://http_servers;
    }

    error_page 404 /404.html;
        location = /40x.html {
    }

    error_page 500 502 503 504 /50x.html;
        location = /50x.html {
    }
}

实现Nginx的高可用
keepalived 配置文件里不能够直接配置nginx的负载均衡等功能，也无从监控nginx是否运行正常。但 keepalived 可以触发脚本改变自身的参数。所以我们可以根据此功能实现 keepalived 根据脚本监控 nginx 程序是否运行正常，运行不正常就自动降低优先级成为备用设备，运行正常后再返回到原来的优先级，加入主设备的选举中。
- KA1中加入以下配置项*

...
vrrp_script chk_nginx {
    script "killall -0 nginx && exit 0 || exit 1"
    interval 1
    weith -20 
}

vrrp_instance VI_1 {
...
    track_script {
        chk_nginx
    }
...
}

KA2中加入以下配置项*

...
vrrp_script chk_nginx {
    script "killall -0 nginx && exit 0 || exit 1"
    interval 2
    weith -20 
}

vrrp_instance VI_2 {
...
    track_script {
        chk_nginx
    }
...
}

KA1 和 KA2 上keepalived完整的配置文件

KA1 上keepalived配置文件

[11@root ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.0.17.17
}

vrrp_script chk_nginx {
    script "killall -0 nginx && exit 0 || exit 1"
    interval 1
    weith -20 
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }

    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }

    track_interface {
        eth0 
    }

    track_script {
        chk_nginx
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 200
    priority 88
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        172.18.17.200/16 dev eth0 label eth0:2
    }

    track_interface {
        eth0 
    }
}

KA2 上keepalived配置文件

[12@root ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_mcast_group4 224.0.17.17
}

vrrp_script chk_nginx {
    script "killall -0 nginx && exit 0 || exit 1"
    interval 1
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 100
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 100100
    }

    virtual_ipaddress {
        172.18.17.100/16 dev eth0 label eth0:1
    }

    track_interface {
        eth0 
    }
}

vrrp_instance VI_2 {
    state MASTER
    interface eth0
    virtual_router_id 200
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 200200
    }

    virtual_ipaddress {
        172.18.17.200/16 dev eth0 label eth0:2
    }

    track_interface {
        eth0 
    }

    track_script
    {
        chk_nginx
    }
}

测试

Alt text

END

zhubiao

Keepalived+LVS/Nginx 实验

Table of Contents