Thank you Jesus for your answer.
As an observation the only node that gives me the error is the wazuh-elk01, and the wazuh-elk03 the 02 has no error log.
We are going to work on the elastic-tunning that you indicate, but here I wanted to ask for your recommendation :)
We have 3 elasticsearch nodes, each with the capacity to be data node and master node, each node with 16 GB of memory and the Kibana is in node 01
According to the recommendation, there should not be more than 50% of the available memory.
Each node would be in the following way, is it possible or is it a low memory or maybe is a lot of memory?
Thank you very much for your help Jesus!!!
Here the screens you asked me









[root@wazuh-elk01 ~]# top -n 1
top - 11:51:53 up 209 days, 19:40, 2 users, load average: 0.48, 0.61, 0.72
Tasks: 168 total, 2 running, 166 sleeping, 0 stopped, 0 zombie
%Cpu(s): 26.2 us, 1.5 sy, 0.0 ni, 72.3 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 16266756 total, 1268244 free, 2274368 used, 12724144 buff/cache
KiB Swap: 4190204 total, 4132188 free, 58016 used. 12671268 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
31487 elastic+ 20 0 13.2g 1.7g 241264 S 106.7 11.2 2057:59 java
11517 root 20 0 162028 2284 1548 R 6.7 0.0 0:00.01 top
1 root 20 0 191216 4080 2444 S 0.0 0.0 86:38.90 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:07.73 kthreadd
3 root 20 0 0 0 0 S 0.0 0.0 2:31.71 ksoftirqd/0
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H
7 root rt 0 0 0 0 S 0.0 0.0 0:25.97 migration/0
8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh
9 root 20 0 0 0 0 S 0.0 0.0 101:07.34 rcu_sched
10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-drain
11 root rt 0 0 0 0 S 0.0 0.0 1:12.01 watchdog/0
12 root rt 0 0 0 0 S 0.0 0.0 1:30.74 watchdog/1
13 root rt 0 0 0 0 S 0.0 0.0 0:32.49 migration/1
14 root 20 0 0 0 0 S 0.0 0.0 2:46.74 ksoftirqd/1
16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:0H
17 root rt 0 0 0 0 S 0.0 0.0 1:33.06 watchdog/2
18 root rt 0 0 0 0 S 0.0 0.0 0:25.98 migration/2
19 root 20 0 0 0 0 S 0.0 0.0 2:18.33 ksoftirqd/2
21 root 0 -20 0 0 0 S 0.0 0.0 0:00.01 kworker/2:0H
22 root rt 0 0 0 0 S 0.0 0.0 1:31.18 watchdog/3
23 root rt 0 0 0 0 S 0.0 0.0 0:28.22 migration/3
24 root 20 0 0 0 0 S 0.0 0.0 2:49.66 ksoftirqd/3
26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/3:0H
28 root 20 0 0 0 0 S 0.0 0.0 0:00.02 kdevtmpfs
29 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns
30 root 20 0 0 0 0 S 0.0 0.0 0:14.67 khungtaskd
31 root 0 -20 0 0 0 S 0.0 0.0 0:00.05 writeback
32 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd
33 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
34 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
35 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
36 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd
37 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md
38 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller
44 root 20 0 0 0 0 S 0.0 0.0 10:12.83 kswapd0
45 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd
46 root 39 19 0 0 0 S 0.0 0.0 1:28.47 khugepaged
47 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto
55 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld
57 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd
58 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kaluad
[root@wazuh-elk02 ~]# top -n 1
top - 11:52:50 up 209 days, 19:38, 1 user, load average: 0.02, 0.07, 0.11
Tasks: 165 total, 2 running, 163 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.5 us, 1.5 sy, 0.0 ni, 96.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 16266756 total, 773952 free, 6164540 used, 9328264 buff/cache
KiB Swap: 4190204 total, 4141052 free, 49152 used. 8733500 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
9984 root 20 0 162024 2276 1548 R 6.2 0.0 0:00.01 top
1 root 20 0 191208 4060 2432 S 0.0 0.0 82:42.38 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:06.03 kthreadd
3 root 20 0 0 0 0 S 0.0 0.0 1:47.79 ksoftirqd/0
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H
7 root rt 0 0 0 0 S 0.0 0.0 0:14.40 migration/0
8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh
9 root 20 0 0 0 0 R 0.0 0.0 82:41.20 rcu_sched
10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-drain
11 root rt 0 0 0 0 S 0.0 0.0 0:58.57 watchdog/0
12 root rt 0 0 0 0 S 0.0 0.0 1:16.11 watchdog/1
13 root rt 0 0 0 0 S 0.0 0.0 0:19.64 migration/1
14 root 20 0 0 0 0 S 0.0 0.0 1:38.08 ksoftirqd/1
16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:0H
17 root rt 0 0 0 0 S 0.0 0.0 1:15.89 watchdog/2
18 root rt 0 0 0 0 S 0.0 0.0 0:15.93 migration/2
19 root 20 0 0 0 0 S 0.0 0.0 1:21.53 ksoftirqd/2
21 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/2:0H
22 root rt 0 0 0 0 S 0.0 0.0 1:14.95 watchdog/3
23 root rt 0 0 0 0 S 0.0 0.0 0:16.09 migration/3
24 root 20 0 0 0 0 S 0.0 0.0 1:27.78 ksoftirqd/3
26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/3:0H
28 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs
29 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns
30 root 20 0 0 0 0 S 0.0 0.0 0:13.28 khungtaskd
31 root 0 -20 0 0 0 S 0.0 0.0 0:00.06 writeback
32 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd
33 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
34 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
35 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
36 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd
37 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md
38 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller
44 root 20 0 0 0 0 S 0.0 0.0 9:32.27 kswapd0
45 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd
46 root 39 19 0 0 0 S 0.0 0.0 1:07.13 khugepaged
47 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto
55 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld
57 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd
[root@wazuh-elk03 ~]# top -n 1
top - 11:53:35 up 209 days, 19:31, 1 user, load average: 0.13, 0.35, 0.33
Tasks: 166 total, 1 running, 165 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.0 us, 1.6 sy, 0.0 ni, 98.4 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 16266756 total, 727108 free, 6176808 used, 9362840 buff/cache
KiB Swap: 4190204 total, 4142680 free, 47524 used. 8666620 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
18163 elastic+ 20 0 18.8g 4.7g 152668 S 6.2 30.6 283:25.36 java
26021 root 20 0 162028 2268 1548 R 6.2 0.0 0:00.01 top
1 root 20 0 191204 3864 2228 S 0.0 0.0 79:51.08 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:07.32 kthreadd
3 root 20 0 0 0 0 S 0.0 0.0 1:42.28 ksoftirqd/0
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H
7 root rt 0 0 0 0 S 0.0 0.0 0:17.64 migration/0
8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh
9 root 20 0 0 0 0 S 0.0 0.0 100:52.91 rcu_sched
10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-drain
11 root rt 0 0 0 0 S 0.0 0.0 1:03.30 watchdog/0
12 root rt 0 0 0 0 S 0.0 0.0 1:19.37 watchdog/1
13 root rt 0 0 0 0 S 0.0 0.0 0:18.62 migration/1
14 root 20 0 0 0 0 S 0.0 0.0 1:51.95 ksoftirqd/1
16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:0H
17 root rt 0 0 0 0 S 0.0 0.0 1:19.22 watchdog/2
18 root rt 0 0 0 0 S 0.0 0.0 0:17.78 migration/2
19 root 20 0 0 0 0 S 0.0 0.0 1:46.94 ksoftirqd/2
21 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/2:0H
22 root rt 0 0 0 0 S 0.0 0.0 1:18.47 watchdog/3
23 root rt 0 0 0 0 S 0.0 0.0 0:21.10 migration/3
24 root 20 0 0 0 0 S 0.0 0.0 2:26.67 ksoftirqd/3
26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/3:0H
28 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs
29 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns
30 root 20 0 0 0 0 S 0.0 0.0 0:17.63 khungtaskd
31 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback
32 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd
33 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
34 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
35 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
36 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd
37 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md
38 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller
44 root 20 0 0 0 0 S 0.0 0.0 21:14.86 kswapd0
45 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd
46 root 39 19 0 0 0 S 0.0 0.0 1:04.12 khugepaged
47 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto
55 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld
57 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd
58 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kaluad
[root@wazuh-elk01 ~]# cat /etc/elasticsearch/jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms1g
-Xmx1g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## G1GC Configuration
# NOTE: G1GC is only supported on JDK version 10 or later.
# To use G1GC uncomment the lines below.
# 10-:-XX:-UseConcMarkSweepGC
# 10-:-XX:-UseCMSInitiatingOccupancyOnly
# 10-:-XX:+UseG1GC
# 10-:-XX:InitiatingHeapOccupancyPercent=75
## DNS cache policy
# cache ttl in seconds for positive DNS lookups noting that this overrides the
# JDK security property networkaddress.cache.ttl; set to -1 to cache forever
-Des.networkaddress.cache.ttl=60
# cache ttl in seconds for negative DNS lookups noting that this overrides the
# JDK security property networkaddress.cache.negative ttl; set to -1 to cache
# forever
-Des.networkaddress.cache.negative.ttl=10
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# explicitly set the stack size
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# turn off a JDK optimization that throws away stack traces for common
# exceptions because stack traces are important for debugging
-XX:-OmitStackTraceInFastThrow
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps; ensure the directory exists and
# has sufficient space
-XX:HeapDumpPath=/var/lib/elasticsearch
# specify an alternative path for JVM fatal error logs
-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
## JDK 8 GC logging
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:/var/log/elasticsearch/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
# JDK 9+ GC logging
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
# time/date parsing will break in an incompatible way for some date patterns and locals
9-:-Djava.locale.providers=COMPAT
# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
10-:-XX:UseAVX=2
[root@wazuh-elk02 ~]# cat /etc/elasticsearch/jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms4g
-Xmx4g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# explicitly set the stack size
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# turn off a JDK optimization that throws away stack traces for common
# exceptions because stack traces are important for debugging
-XX:-OmitStackTraceInFastThrow
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps; ensure the directory exists and
# has sufficient space
-XX:HeapDumpPath=/var/lib/elasticsearch
# specify an alternative path for JVM fatal error logs
-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
## JDK 8 GC logging
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:/var/log/elasticsearch/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
# JDK 9+ GC logging
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
# time/date parsing will break in an incompatible way for some date patterns and locals
9-:-Djava.locale.providers=COMPAT
# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
10-:-XX:UseAVX=2
[root@wazuh-elk03 ~]# cat /etc/elasticsearch/jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms4g
-Xmx4g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# explicitly set the stack size
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# turn off a JDK optimization that throws away stack traces for common
# exceptions because stack traces are important for debugging
-XX:-OmitStackTraceInFastThrow
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps; ensure the directory exists and
# has sufficient space
-XX:HeapDumpPath=/var/lib/elasticsearch
# specify an alternative path for JVM fatal error logs
-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
## JDK 8 GC logging
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:/var/log/elasticsearch/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
# JDK 9+ GC logging
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
# time/date parsing will break in an incompatible way for some date patterns and locals
9-:-Djava.locale.providers=COMPAT
# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
10-:-XX:UseAVX=2