I am running a 3 node Redis Cluster. Having trouble starting the sentinel service with TLS errors. Description of the setup, configs and errors below.
Node Info:
Node 1:
Management IP: 10.10.200.241
Redis Bind IP : 10.20.200.241
Node 2:
Management IP: 10.10.200.242
Redis Bind IP : 10.20.200.242
Node 2:
Management IP: 10.10.200.243
Redis Bind IP : 10.20.200.243
The /etc/hosts file on each of the servers is:
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
10.10.200.241 Redis-241
10.10.200.242 Redis-242
10.10.200.243 Redis-243
#
10.20.200.241 Redis-01-infra
10.20.200.242 Redis-02-infra
10.20.200.243 Redis-03-infra
I have built Redis 6.2.6 with make BUILD_TLS=yes
REDIS CLUSTER CONFIGS
The redis.conf file is (TLS Configs Bolded at the bottom):
bind Redis-01-infra
protected-mode no
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /var/log/redis/redis.log
databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /var/lib/redis
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
# requirepass foobar
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes
tls-port 6379
port 0
tls-cert-file /etc/st2/certs/Redis-01-infra.crt
tls-key-file /etc/st2/certs/Redis-01-infra.key
tls-ca-cert-file /etc/st2/certs/ST2ca.crt
tls-ca-cert-dir "/etc/st2/certs"
tls-auth-clients no
tls-replication yes
tls-cluster yes
tls-protocols "TLSv1.1 TLSv1.2 TLSv1.3"
tls-ciphersuites "TLS_CHACHA20_POLY1305_SHA256"
tls-prefer-server-ciphers no
REDIS CLUSTER TEST
The redis cluster comes up fine and there are no errors in the /var/log/redis.log file.
Here is the redis-cli output of the cluster:
[root@Redis-241 redis]# redis-cli --tls --cert /etc/certs/Redis-01-infra.crt --key /etc/certs/Redis-01-infra.key --cacert /etc/certs/ST2ca.crt -h Redis-01-infra -p 6379
Redis-01-infra:6379>info
# Server
redis_version:6.2.6
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:e77605d678fec247
redis_mode:standalone
os:Linux 5.4.17-2136.307.3.1.el8uek.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:8.5.0
process_id:1327928
process_supervised:systemd
run_id:68e91b696fea133a180ed2ee8e685a6cdf08659f
tcp_port:6379
server_time_usec:1661139609246948
uptime_in_seconds:6603
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:195225
executable:/usr/bin/redis-server
config_file:/etc/redis/redis.conf
io_threads_active:0
# Clients
connected_clients:1
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:32
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
# Memory
used_memory:1964632
used_memory_human:1.87M
used_memory_rss:17436672
used_memory_rss_human:16.63M
used_memory_peak:2066080
used_memory_peak_human:1.97M
used_memory_peak_perc:95.09%
used_memory_overhead:1920296
used_memory_startup:810200
used_memory_dataset:44336
used_memory_dataset_perc:3.84%
allocator_allocated:2052424
allocator_active:2326528
allocator_resident:5140480
total_system_memory:16482394112
total_system_memory_human:15.35G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.13
allocator_frag_bytes:274104
allocator_rss_ratio:2.21
allocator_rss_bytes:2813952
rss_overhead_ratio:3.39
rss_overhead_bytes:12296192
mem_fragmentation_ratio:8.88
mem_fragmentation_bytes:15473856
mem_not_counted_for_evict:0
mem_replication_backlog:1048576
mem_clients_slaves:41024
mem_clients_normal:20496
mem_aof_buffer:0
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0
# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1661133024
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:0
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:311296
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0
# Stats
total_connections_received:19
total_commands_processed:13365
instantaneous_ops_per_sec:2
total_net_input_bytes:508061
total_net_output_bytes:133213
instantaneous_input_kbps:0.09
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:2
sync_partial_ok:0
sync_partial_err:2
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:95
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:584
total_forks:2
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:13356
total_writes_processed:1682
io_threaded_reads_processed:0
io_threaded_writes_processed:0
# Replication
role:master
connected_slaves:2
slave0:ip=10.20.200.242,port=6379,state=online,offset=17285,lag=0
slave1:ip=10.20.200.243,port=6379,state=online,offset=17285,lag=1
master_failover_state:no-failover
master_replid:629f36276e7cde445604e3ee6ae97745d4e4b547
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:17285
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:17285
# CPU
used_cpu_sys:4.434331
used_cpu_user:5.668672
used_cpu_sys_children:0.008261
used_cpu_user_children:0.000942
used_cpu_sys_main_thread:4.071347
used_cpu_user_main_thread:5.344037
# Modules
# Errorstats
# Cluster
cluster_enabled:0
# Keyspace
Redis-01-infra:6379>
Sentinel Config
I need to configure sentinel on this cluster. Here is the /etc/sentinel.conf file:
bind Redis-01-infra
sentinel monitor RedisMaster 10.20.200.241 6379 2
sentinel down-after-milliseconds RedisMaster 60000
logfile "/var/log/redis/sentinel.log"
protected-mode no
tls-port 26379
port 0
tls-cert-file "/etc/st2/certs/Redis-01-infra.crt"
tls-key-file "/etc/st2/certs/Redis-01-infra.key"
tls-ca-cert-file "/etc/st2/certs/ST2ca.pem"
tls-ca-cert-dir "/etc/st2/certs"
tls-replication yes
tls-protocols "TLSv1.1 TLSv1.2 TLSv1.3"
Certificate Verification
The CA Cert is self signed and here is the verification the Redis-01-infra.crt.
[root@Redis-241 certs]# openssl x509 -in Redis-01-infra.crt -text -noout
Certificate:
Data:
Version: 1 (0x0)
Serial Number: 0 (0x0)
Signature Algorithm: sha256WithRSAEncryption
Issuer: C = US, ST = CA, L = SJC, O = XXYY.ai, OU = "Eng. ", CN = CAROOT
Validity
Not Before: Aug 22 01:39:32 2022 GMT
Not After : Sep 21 01:39:32 2022 GMT
Subject: C = US, ST = CA, L = SJC, O = XXYY.ai, OU = Eng., CN = Redis-01-infra
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public-Key: (4096 bit)
Modulus:
00:cd:63:4f:cc:a9:1b:c5:c2:a0:92:c0:c4:72:1f:
2d:75:ed:1c:c8:cd:b4:74:d4:cf:b8:a5:7d:e4:cd:
97:3f:fa:66:0e:a2:ca:ba:e2:dd:53:2a:3f:62:42:
c8:4f:a2:4f:53:14:74:80:12:c4:4c:da:07:9b:0d:
5e:9c:5d:2d:f2:85:ec:39:72:fb:d4:0c:bb:be:be:
bf:03:5f:0f:79:8d:47:a7:ae:8b:53:16:de:81:0c:
fa:52:52:8c:46:68:8d:00:f2:50:bc:72:f9:ea:03:
78:58:13:f3:f5:ed:34:27:8f:be:af:fb:03:4f:37:
57:42:5d:ed:8a:8d:cb:87:fa:e3:63:08:9e:56:b0:
2a:06:97:d3:05:79:93:e0:f3:fe:d2:94:70:a1:6f:
31:2d:06:e2:05:46:64:63:34:c6:45:e4:3e:81:5d:
4e:84:95:ed:9b:52:c3:ca:8c:c6:94:cb:18:c6:17:
6f:ed:66:f1:46:d0:f9:bc:b1:65:af:da:b1:99:82:
fc:2b:35:40:0f:18:9c:2a:d9:9c:d3:eb:6a:6d:5d:
a4:32:8e:26:e1:69:ad:b0:b7:ba:db:8f:06:03:07:
8e:25:90:a2:eb:a2:af:65:eb:19:c8:f3:7c:c5:e1:
26:a1:03:5c:8c:82:65:02:ef:2c:37:00:65:b3:15:
c6:41:27:a8:e4:9e:2c:0a:d7:49:fc:42:10:64:75:
76:d6:99:67:f6:4a:ef:2e:27:8a:00:33:f9:d5:93:
c7:15:3a:df:90:2b:8b:3c:f6:c2:e1:50:38:87:54:
a5:5c:9f:02:92:8e:98:9c:11:1f:1d:98:68:0a:1d:
21:07:74:20:ba:5c:d4:0c:04:09:69:a8:86:24:e9:
70:7a:07:43:3e:1b:91:7e:91:95:50:80:81:4a:69:
90:51:ee:e8:58:dc:78:5f:80:6b:9b:57:8a:f2:40:
88:f2:59:d0:f2:14:30:96:c1:f6:bc:09:87:82:43:
30:b4:9d:22:9c:63:5a:96:3b:b0:5e:6f:96:52:67:
ab:33:5f:8f:81:ae:52:66:bd:29:b1:5f:ac:6b:a3:
8b:44:f0:e6:3e:4d:71:55:c5:11:e9:1a:fe:c0:3b:
b0:68:10:57:21:97:59:25:89:7e:94:82:31:be:bf:
26:be:d2:26:bb:40:f0:71:59:f5:3f:e5:93:76:2f:
06:2c:1f:a4:9a:91:21:ca:c4:cb:f1:74:39:7d:ae:
f5:20:bf:37:4d:34:c6:0b:eb:08:8c:af:94:20:20:
09:20:fd:b0:63:2d:28:98:21:f4:dd:72:79:1f:06:
55:ae:15:f4:cd:71:8c:7f:9a:61:61:12:75:e7:f0:
b3:9c:db
Exponent: 65537 (0x10001)
Signature Algorithm: sha256WithRSAEncryption
32:f7:02:a6:5d:e8:a7:66:5a:92:67:47:c4:4d:bb:09:75:bd:
ae:5d:fc:71:a0:c1:0f:73:8e:e0:fe:49:1b:57:49:0a:e3:1f:
34:e4:24:97:7c:93:87:7b:80:6e:77:f7:d4:85:08:28:38:88:
59:5a:11:80:5f:60:d0:70:da:ea:93:81:f2:7a:c9:a9:10:d9:
dd:c4:45:8c:8e:47:a6:1e:15:36:03:9b:76:c6:5d:1b:82:ef:
5d:ba:f3:42:86:bb:63:1a:18:0b:3d:d7:69:52:f6:a1:f2:e3:
58:38:ec:92:8f:00:da:cb:19:2a:34:0d:53:ed:2b:0d:8e:f9:
b5:f8:37:33:dc:f4:24:e9:e9:7a:02:14:1a:45:a2:9b:bf:c6:
98:72:44:f1:5c:93:2e:e0:01:5f:70:9e:fc:16:54:b9:5b:71:
18:a4:2c:ff:28:dd:6a:29:19:49:67:92:9b:ab:df:17:74:12:
69:ce:60:2c:33:6a:2d:5a:6b:76:52:38:17:79:d2:89:ec:5e:
d2:93:30:26:48:66:23:00:79:9e:9c:66:c0:be:91:ef:17:f8:
0f:a0:46:a2:27:41:ba:ac:86:07:98:47:9e:d9:ad:fe:95:27:
44:c1:97:9b:6c:ef:d8:04:b8:4c:25:2b:6e:e9:1a:0c:cc:a8:
55:fd:86:d6
[root@Redis-241 certs]#
ERROR
When I start sentinel, the sentinel.log has the following error:
# Error accepting a client connection: error:1408F10B:SSL routines:ssl3_get_record:wrong version number (conn: fd=12)
Any Idea what the configuration should be to go past this error. Please let me know how to resolve this issue.