Minggle'Blog

CentOS 环境下 nginx + keepalived 实现业务发布高可用

2021/11/18 CentOS 共 5928 字，约 17 分钟
闷骚的程序员

0x0 Nginx 编译安装部署

0x01 安装工具及依赖

yum -y install wget vim gcc gcc-c++ pcre pcre-devel zlib zlib-devel openssl openssl-devel

0x02 添加用户及用户组

groupadd nginx
useradd -g nginx nginx

0x03下载源码包并解压缩

cd /usr/local/src/
wget http://nginx.org/download/nginx-1.21.1.tar.gz
tar zxvf nginx-1.21.1.tar.gz

0x04 创建编译所需路径

mkdir -p /var/cache/nginx/client_temp 
mkdir -p /var/cache/nginx/proxy_temp
mkdir -p /var/cache/nginx/fastcgi_temp
mkdir -p /var/cache/nginx/uwsgi_temp
mkdir -p /var/cache/nginx/scgi_temp

0x05 编译

cd nginx-1.21.1/
./configure \
--prefix=/etc/nginx \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx.pid \
--lock-path=/var/run/nginx.lock \
--http-client-body-temp-path=/var/cache/nginx/client_temp \
--http-proxy-temp-path=/var/cache/nginx/proxy_temp \
--http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
--http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
--http-scgi-temp-path=/var/cache/nginx/scgi_temp \
--user=nginx \
--group=nginx \
--with-file-aio \
--with-threads \
--with-http_addition_module \
--with-http_auth_request_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_mp4_module \
--with-http_random_index_module \
--with-http_realip_module \
--with-http_secure_link_module \
--with-http_slice_module \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-http_sub_module \
--with-http_v2_module \
--with-mail \
--with-mail_ssl_module \
--with-stream \
--with-stream_realip_module \
--with-stream_ssl_module \
--with-stream_ssl_preread_module

0x06 安装

make && make install

0x07 编写nginx service 文件

vim /lib/systemd/system/nginx.service

---
[Unit]
Description=nginx - high performance web server
Documentation=http://nginx.org/en/docs/
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=/var/run/nginx.pid
ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx.conf
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID

[Install]
WantedBy=multi-user.target
---

0x08 防火墙放行监听端口

firewall-cmd --zone=public --add-service=http --permanent
firewall-cmd --zone=public --add-service=https --permanent
firewall-cmd --reload

0x09 其他常用命令

# 启动nginx
systemctl start nginx
# 停止nginx
systemctl stop nginx
# 重新启动nginx
systemctl restart nginx
# 设置nginx开机自启动
systemctl enable nginx
# 查看nginx状态
systemctl status nginx

# 显示nginx版本
nginx -v
# 检查nginx配置文件
nginx -t
# 重载nginx配置文件
nginx -s reload
# 配置文件路径
/etc/nginx/nginx.conf

1x0 KeepAlived 安装部署

1x01 安装keepalived

yum -y install keepalived

1x02 开启keepalived日志记录

  • 编辑/etc/sysconfig/keepalived：
KEEPALIVED_OPTIONS="-D -d -S 0"
  • 编辑/etc/rsyslog.conf：
# 配置文件最后面加上下面一行
local0.*                                        /var/log/keepalived.log
  • 重启rsyslog：
service rsyslog restart
  • 日志路径：
/var/log/keepalived.log

1x03 配置检测脚本

vim /usr/local/src/check_nginx_pid.sh

---
#!/bin/bash
#检测nginx是否启动了
A=`ps -C nginx --no-header |wc -l`        
if [ $A -eq 0 ];then    #如果nginx没有启动就启动nginx                        
      systemctl start nginx                #重启nginx
      if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then    #nginx重启失败，则停掉keepalived服务，进行VIP转移
              killall keepalived                    
      fi
fi
---

大佬写的另一个检测脚本

#!/bin/bash
# check nginx server status
# http://qicheng0211.blog.51cto.com
 
# nginx端口
PORTS="16915 16916"

function check_ports {
    for port in $PORTS;do
        nc -z 127.0.0.1 $port | grep -q succeeded
        [ "${PIPESTATUS[1]}" -eq 0 ] && mark=${mark}1
    done
    # 如果mark值为空说明两个端口都不通。
    # 如果mark等于1，说明有一个端口是通的。
    # 如果mark等于11，说明两个端口都是通的。
    echo $mark
}

ret1=$(check_ports)
# 如果nginx端口不通，会尝试重启一次nginx
if [ "$ret1" != 11 ];then
    /sbin/service nginx stop
    /sbin/service nginx start
    sleep 1
    ret2=$(check_ports)
    # 如果还是有端口不通，表示nginx服务不正常，则停掉keepalived，使VIP发生切换
    [ "$ret2" != 11 ] && /etc/init.d/keepalived stop
fi

1x04 添加脚本执行权限

chmod 755 /usr/local/src/check_nginx_pid.sh

1x05 配置keepalived

  • 配置文件路径
/etc/keepalived/keepalived.conf
  • 节点A
vim /etc/keepalived/keepalived.conf

---
#检测脚本
vrrp_script chk_nginx_status {
    script "/usr/local/src/check_nginx_pid.sh" #心跳执行的脚本，检测nginx是否启动
    interval 2                          #（检测脚本执行的间隔，单位是秒）
    weight 2                            #权重
}
#vrrp 实例定义部分
vrrp_instance VI_1 {
    state BACKUP            # 指定keepalived的角色，MASTER为主，BACKUP为备，本次两节点都是BACKUP，根据优先级自动选择master
    interface ens160         # 当前进行vrrp通讯的网卡`ip addr`
    virtual_router_id 77    # 虚拟路由编号，节点要一致，起始51
    priority 100            # 优先级，数值越大，获取处理请求的优先级越高
    advert_int 1            # 检查间隔，默认为1s(vrrp组播周期秒数)
    nopreempt               # 不抢占：如果集群里已存在MASTER状态的主机，即使优先级高于MASTER也不抢占为MASTER。只在优先级高的主机上设置即可。
    
    #授权访问
    authentication {
        auth_type PASS #设置验证类型和密码，各节点必须使用相同的密码才能正常通信
        auth_pass 1111
    }
    track_script {
        chk_nginx_status            #（调用检测脚本）
    }
    virtual_ipaddress {
        10.0.0.3            # 定义虚拟ip(VIP)，可多设，每行一个
    }
}
---
  • 节点B
vim /etc/keepalived/keepalived.conf

---
#检测脚本
vrrp_script chk_nginx_status {
    script "/usr/local/src/check_nginx_pid.sh" #心跳执行的脚本，检测nginx是否启动
    interval 2                          #（检测脚本执行的间隔，单位是秒）
    weight 2                            #权重
}
#vrrp 实例定义部分
vrrp_instance VI_1 {
    state BACKUP            # 指定keepalived的角色，MASTER为主，BACKUP为备，本次两节点都是BACKUP，根据优先级自动选择master
    interface ens160         # 当前进行vrrp通讯的网卡`ip addr`
    virtual_router_id 77    # 虚拟路由编号，节点要一致，起始51
    priority 80            # 优先级，数值越大，获取处理请求的优先级越高
    advert_int 1            # 检查间隔，默认为1s(vrrp组播周期秒数)
    
    #授权访问
    authentication {
        auth_type PASS #设置验证类型和密码，各节点必须使用相同的密码才能正常通信
        auth_pass 1111
    }
    track_script {
        chk_nginx_status            #（调用检测脚本）
    }
    virtual_ipaddress {
        10.0.0.3            # 定义虚拟ip(VIP)，可多设，每行一个
    }
}
---

1x06 防火墙开放vrrp组播地址

firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0 --in-interface ens160 --destination 224.0.0.18 --protocol vrrp -j ACCEPT
firewall-cmd --reload

1x07 其他命令

# 启动keepalived
systemctl start keepalived
# 停止keepalived
systemctl stop keepalived
# 重新启动keepalived
systemctl restart keepalived
# 设置keepalived开机自启动
systemctl enable keepalived
# 查看keepalived状态
systemctl status keepalived
# 配置文件路径
/etc/keepalived/keepalived.conf
# 日志文件路径
/var/log/keepalived.log

3x0 节点间配置同步

3x01 方式一，定时任务crond

  • 节点A
vim /usr/local/src/scp_nginx_conf.sh

---
#!/bin/bash
/usr/bin/scp -r /etc/nginx/nginx.conf root@nginx-backup-2:/etc/nginx/ >> /usr/local/src/nginx_scp.log
/usr/bin/scp -r /etc/nginx/cert root@nginx-backup-2:/etc/nginx/ >> /usr/local/src/nginx_scp.log
/usr/bin/scp -r /etc/nginx/conf.d root@nginx-backup-2:/etc/nginx/ >> /usr/local/src/nginx_scp.log
---


cront -e

---
40 3 * * * /usr/sbin/nginx -s reload
* 4 * * * /bin/bash /usr/local/src/scp_nginx_conf.sh &
---
  • 节点B
cront -e
---
10 4 * * * /usr/sbin/nginx -s reload
---

3x02 方式二：rsync+inotify-tools

见：

rsync+inotify-tools实现nginx配置文件实时同步

3x03 方式三，Rsync+sersync（待补充）

参考文档

https://www.cnblogs.com/SimpleWu/p/11004902.html

https://blog.51cto.com/qicheng0211/1695674

文档信息

Search

    Table of Contents