kubernetes中部署redis哨兵模式集群

在部署redis哨兵模式集群之前，先看一下需要做哪些准备，因为redis默认镜像中是没有哨兵模式的，所以需要把定义好的配置放到镜像中，这里需要两个配置文件分别是redis的配置文件和哨兵模式的配置文件，还有一个启动脚本。

 # tree .
 .
 ├── Dockerfile
 ├── redis.conf
 ├── run.sh
 └── sentinel.conf
 ​
 0 directories, 4 files

这个是需要放入镜像的文件，下面看一下每个文件的内容。

 cat Dockerfile
 FROM redis:7.0
 COPY *.conf /usr/local/etc/redis/
 COPY run.sh /usr/local/etc/redis/run.sh
 RUN apt update -y;apt-get install vim net-tools -y;apt-get clean && \
     chmod +x /usr/local/etc/redis/run.sh
 ​
 CMD /usr/local/etc/redis/run.sh
 ########################分割线##################################
 cat sentinel.conf
 port 26379
 # 哨兵sentinel的工作目录
 dir "/tmp"
 sentinel deny-scripts-reconfig yes
 sentinel monitor mymaster redis-sentinel-0.redis-sentinel 6379 2
 sentinel auth-pass mymaster wulaoer.org
 sentinel down-after-milliseconds mymaster 5000
 sentinel failover-timeout mymaster 15000
 sentinel parallel-syncs mymaster 2
 sentinel config-epoch mymaster 3
 sentinel leader-epoch mymaster 3
 sentinel resolve-hostnames yes
 ########################分割线##################################
 cat redis.conf
 bind 0.0.0.0
 protected-mode yes
 port 6379
 requirepass wulaoer.org
 masterauth wulaoer.org
 tcp-backlog 511
 timeout 0
 #requirepass hello
 tcp-keepalive 300
 daemonize yes
 supervised no
 pidfile /var/run/redis_6379.pid
 loglevel notice
 logfile /var/log/redis.log
 databases 32
 save 900 1
 save 300 10
 save 60 10000
 stop-writes-on-bgsave-error no
 rdbcompression no
 rdbchecksum no
 dbfilename dump.rdb
 slave-serve-stale-data yes
 slave-read-only yes
 repl-diskless-sync no
 repl-diskless-sync-delay 5
 repl-disable-tcp-nodelay no
 slave-priority 100
 appendonly yes
 appendfilename "appendonly.aof"
 appendfsync everysec
 no-appendfsync-on-rewrite no
 auto-aof-rewrite-percentage 100
 auto-aof-rewrite-min-size 64mb
 aof-load-truncated yes
 lua-time-limit 5000
 slowlog-log-slower-than 10000
 slowlog-max-len 128
 latency-monitor-threshold 0
 notify-keyspace-events ""
 hash-max-ziplist-entries 512
 hash-max-ziplist-value 64
 list-max-ziplist-size -2
 list-compress-depth 0
 set-max-intset-entries 512
 zset-max-ziplist-entries 128
 zset-max-ziplist-value 64
 hll-sparse-max-bytes 3000
 activerehashing yes
 client-output-buffer-limit normal 0 0 0
 client-output-buffer-limit slave 256mb 64mb 60
 client-output-buffer-limit pubsub 32mb 8mb 60
 hz 10
 aof-rewrite-incremental-fsync yes
 #客户端最大连接数
 maxclients 20000
 lazyfree-lazy-eviction yes
 lazyfree-lazy-expire yes
 lazyfree-lazy-server-del yes
 slave-lazy-flush yes

基础镜像文件准备好了，这里有一个需要注意的地方，在sentinel.conf文件中，配置的sentinel monitor mymaster redis-sentinel-0.redis-sentinel 6379 2这里不能直接使用redis-sentinel-0,如果直接使用会有脑裂的问题存在，最重要的是一定要加sentinel resolve-hostnames yes使用ip地址，这个是redis特有的，如果不加会失败失败。下面就是直接构建一下基础镜像，然后上传到私有仓库即可。

 docker build -t harbor.wulaoer.org/ops/redis-sentinel:7.0 .
 docker push harbor.wulaoer.org/ops/redis-sentinel:7.0

上传成功后，我们需要把哨兵模式的yaml文件看一下，然后创建集群。因为需要持久化，可以使用ntfs做持久化，可以参考：https://www.wulaoer.org/?p=2714

  cat redis.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: redis-sentinel
 spec:
   serviceName: redis-sentinel
   selector:
     matchLabels:
       app: redis-sentinel
   replicas: 3
   template:
     metadata:
       labels:
         app: redis-sentinel
     spec:
       restartPolicy: Always
       containers:
         - name: redis-sentinel
           image: harbor.wulaoer.org/ops/redis-sentinel:7.0
           imagePullPolicy: Always
           volumeMounts:
             - name: "redis-data"
               mountPath: "/data"
           env:
             - name: POD_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
           livenessProbe:
             tcpSocket:
               port: 6379
             initialDelaySeconds: 3
             periodSeconds: 5
           readinessProbe:
             tcpSocket:
               port: 6379
             initialDelaySeconds: 3
             periodSeconds: 5
           ports:
             - containerPort: 6379
           resources:
             requests:
               memory: 256Mi
               cpu: 50m
             limits:
               memory: 256Mi
               cpu: 200m
       imagePullSecrets:
         - name: image-auth #修改成自己的
   volumeClaimTemplates:
     - metadata:
         name: "redis-data"
       spec:
         resources:
           requests:
             storage: 10Gi
         accessModes:
         - ReadWriteOnce
         storageClassName: "nfs-storage"
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: redis-sentinel
 spec:
   type: ClusterIP
   ports:
     - name: redis-sentinel
       port: 6379
       targetPort: 6379
   selector:
     app: redis-sentinel

在这里我们定义了探针，并配置了数据持久化，可以根据自己的环境酌情修改。

 [root@k8s-master sentinel]# kubectl apply -f redis.yaml -f wulaoer
 [root@k8s-master sentinel]# kubectl get pod -n wulaoer
 NAME               READY   STATUS    RESTARTS   AGE
 redis-sentinel-0   1/1     Running   0          111s
 redis-sentinel-1   1/1     Running   0          2m27s
 redis-sentinel-2   1/1     Running   0          3m3s
 [root@k8s-master sentinel]# kubectl exec -it -n wulaoer redis-sentinel-0 /bin/bash
 kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
 root@redis-sentinel-0:/data# redis-cli -h redis-sentinel-0.redis-sentinel
 redis-sentinel-0.redis-sentinel:6379> auth wulaoer.org
 OK
 redis-sentinel-0.redis-sentinel:6379> INFO Replication
 # Replication
 role:master  #一主两从
 connected_slaves:2
 slave0:ip=10.244.122.42,port=6379,state=online,offset=96620,lag=1
 slave1:ip=10.244.172.186,port=6379,state=online,offset=96905,lag=0
 master_failover_state:no-failover
 master_replid:3822260b2a53b4556cb1a27f4a2dc0f48e14bcb1
 master_replid2:d27fe812d625f40007bb72711f016648fd3809bd
 master_repl_offset:96905
 second_repl_offset:60099
 repl_backlog_active:1
 repl_backlog_size:1048576
 repl_backlog_first_byte_offset:20790
 repl_backlog_histlen:76116

到此，redis集群哨兵模式部署完成，下面我们验证一下，看看把主节点kill掉后，是否会自动的选举其他节点。

 kubectl get pod -n wulaoer -o wide
 NAME               READY   STATUS    RESTARTS        AGE     IP               NODE         NOMINATED NODE   READINESS GATES
 redis-sentinel-0   1/1     Running   0               7m41s   10.244.123.233   k8s-node45   <none>           <none>
 redis-sentinel-1   1/1     Running   0               8m17s   10.244.172.186   k8s-node34   <none>           <none>
 redis-sentinel-2   1/1     Running   1 (5m54s ago)   8m54s   10.244.122.42    k8s-node31   <none>           <none>
 ​
 kubectl exec -it -n wulaoer redis-sentinel-0 /bin/bash
 kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
 root@redis-sentinel-0:/data# cat /var/run/redis_6379.pid
 14
 root@redis-sentinel-0:/data# kill 14
 root@redis-sentinel-0:/data# /usr/local/bin/redis-server /usr/local/etc/redis/redis.conf
 ​
 kubectl exec -it -n wulaoer redis-sentinel-0 /bin/bash
 kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
 root@redis-sentinel-0:/data# redis-cli -h redis-sentinel-0.redis-sentinel
 redis-sentinel-0.redis-sentinel:6379> auth wulaoer.org
 OK
 redis-sentinel-0.redis-sentinel:6379> INFO Replication
 # Replication
 role:slave
 master_host:10.244.122.42
 master_port:6379
 master_link_status:up
 master_last_io_seconds_ago:1
 master_sync_in_progress:0
 slave_read_repl_offset:129469
 slave_repl_offset:129469
 slave_priority:100
 slave_read_only:1
 replica_announced:1
 connected_slaves:0
 master_failover_state:no-failover
 master_replid:01c80432c85f6e44e758db3c943bda5329da53ef
 master_replid2:0000000000000000000000000000000000000000
 master_repl_offset:129469
 second_repl_offset:-1
 repl_backlog_active:1
 repl_backlog_size:1048576
 repl_backlog_first_byte_offset:119319
 repl_backlog_histlen:10151
 root@redis-sentinel-0:/data# redis-cli -h redis-sentinel-1.redis-sentinel -p 26379
 redis-sentinel-1.redis-sentinel:26379> info sentinel
 # Sentinel
 sentinel_masters:1
 sentinel_tilt:0
 sentinel_tilt_since_seconds:-1
 sentinel_running_scripts:0
 sentinel_scripts_queue_length:0
 sentinel_simulate_failure_flags:0
 master0:name=mymaster,status=ok,address=10.244.122.42:6379,slaves=2,sentinels=4

验证成功，说明探针起到了作用，集群没有问题了。这里需要注意，因为添加了探针，在kill之后启动服务时，如果启动的慢，探针会生效就会重启节点，推出容器，也可以不添加探针在测试的时候，但是在生成环境一定要添加探针