
[2023-08-23T15:00:32,761][WARN ][stderr ] [node-1] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[2023-08-23T15:00:32,761][WARN ][stderr ] [node-1] SLF4J: Defaulting to no-operation (NOP) logger implementation
[2023-08-23T15:00:32,762][WARN ][stderr ] [node-1] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[2023-08-23T15:00:32,787][INFO ][o.o.s.s.t.SSLConfig ] [node-1] SSL dual mode is disabled
[2023-08-23T15:00:43,297][ERROR][o.o.s.a.s.SinkProvider ] [node-1] Default endpoint could not be created, auditlog will not work properly.
[2023-08-23T15:00:43,298][WARN ][o.o.s.a.r.AuditMessageRouter] [node-1] No default storage available, audit log may not work properly. Please check configuration.
[2023-08-23T15:00:43,299][INFO ][o.o.s.a.i.AuditLogImpl ] [node-1] Message routing enabled: false
[2023-08-23T15:00:45,780][WARN ][o.o.p.c.ThreadPoolMetricsCollector] [node-1] Fail to read queue capacity via reflection (35 Times)
[2023-08-23T15:00:47,035][WARN ][o.o.g.DanglingIndicesState] [node-1] gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually
[2023-08-23T15:00:50,428][ERROR][o.o.s.c.ConfigurationLoaderSecurity7] [node-1] Exception while retrieving configuration for [INTERNALUSERS, ACTIONGROUPS, CONFIG, ROLES, ROLESMAPPING, TENANTS, NODESDN, WHITELIST, ALLOWLIST, AUDIT] (index=.opendistro_security)
org.opensearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
at org.opensearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:205) ~[opensearch-2.6.0.jar:2.6.0]
at org.opensearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:191) ~[opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:81) ~[opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.get.TransportMultiGetAction.doExecute(TransportMultiGetAction.java:58) ~[opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:218) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:118) [opensearch-index-management-2.6.0.0.jar:2.6.0.0]
at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:232) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:149) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter.apply(PerformanceAnalyzerActionFilter.java:78) [opensearch-performance-analyzer-2.6.0.0.jar:2.6.0.0]
at org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:216) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.support.TransportAction.execute(TransportAction.java:188) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.support.TransportAction.execute(TransportAction.java:107) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:110) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.client.node.NodeClient.doExecute(NodeClient.java:97) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:465) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.client.support.AbstractClient.multiGet(AbstractClient.java:581) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.loadAsync(ConfigurationLoaderSecurity7.java:208) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.configuration.ConfigurationLoaderSecurity7.load(ConfigurationLoaderSecurity7.java:99) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.configuration.ConfigurationRepository.getConfigurationsFromIndex(ConfigurationRepository.java:372) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration0(ConfigurationRepository.java:318) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.configuration.ConfigurationRepository.reloadConfiguration(ConfigurationRepository.java:303) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at org.opensearch.security.configuration.ConfigurationRepository$1.run(ConfigurationRepository.java:163) [opensearch-security-2.6.0.0.jar:2.6.0.0]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2023-08-23T15:00:51,644][WARN ][o.o.o.i.ObservabilityIndex] [node-1] message: index [.opensearch-observability/-A-wQ7KyTsq2a488bLKUyA] already exists
[2023-08-23T15:00:52,326][ERROR][o.o.s.a.BackendRegistry ] [node-1] Not yet initialized (you may need to run securityadmin)
[2023-08-23T15:00:52,358][ERROR][o.o.s.a.BackendRegistry ] [node-1] Not yet initialized (you may need to run securityadmin)
[2023-08-23T15:00:52,364][ERROR][o.o.s.a.BackendRegistry ] [node-1] Not yet initialized (you may need to run securityadmin)
[2023-08-23T15:00:52,369][ERROR][o.o.s.a.BackendRegistry ] [node-1] Not yet initialized (you may need to run securityadmin)
[2023-08-23T15:01:54,483][INFO ][o.o.c.r.a.AllocationService] [node-1] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[wazuh-alerts-4.x-2022.10.14][2], [wazuh-monitoring-2022.41w][0], [wazuh-alerts-4.x-2022.10.14][0]]]).
[2023-08-23T15:01:54,532][INFO ][o.o.a.u.d.DestinationMigrationCoordinator] [node-1] Detected cluster change event for destination migration
[2023-08-23T15:02:50,375][INFO ][o.o.i.i.ManagedIndexCoordinator] [node-1] Cancel background move metadata process.
[2023-08-23T15:02:50,376][INFO ][o.o.i.i.ManagedIndexCoordinator] [node-1] Performing move cluster state metadata.
[2023-08-23T15:02:50,376][INFO ][o.o.i.i.MetadataService ] [node-1] Move metadata has finished.
[2023-08-23T15:05:49,596][INFO ][o.o.j.s.JobSweeper ] [node-1] Running full sweep
[2023-08-23T15:05:50,379][INFO ][o.o.i.i.PluginVersionSweepCoordinator] [node-1] Canceling sweep ism plugin version job
^[[B^[[A[2023-08-23T15:10:49,598][INFO ][o.o.j.s.JobSweeper ] [node-1] Running full sweep
and thats where its sitting
Whats missing from making indexer startup error free, and why did this change all of a sudden?
Aug 23 15:17:24 monitoring-prd systemd-entrypoint[5856]: WARNING: A terminally deprecated method in java.lang.System has been called
Aug 23 15:17:24 monitoring-prd systemd-entrypoint[5856]: WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.OpenSearch (file:/usr/share/wazuh-indexer/lib/>
Aug 23 15:17:24 monitoring-prd systemd-entrypoint[5856]: WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.OpenSearch
Aug 23 15:17:24 monitoring-prd systemd-entrypoint[5856]: WARNING: System::setSecurityManager will be removed in a future release
Aug 23 15:17:26 monitoring-prd systemd-entrypoint[5856]: WARNING: A terminally deprecated method in java.lang.System has been called
Aug 23 15:17:26 monitoring-prd systemd-entrypoint[5856]: WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.Security (file:/usr/share/wazuh-indexer/lib/op>
Aug 23 15:17:26 monitoring-prd systemd-entrypoint[5856]: WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.Security
Aug 23 15:17:26 monitoring-prd systemd-entrypoint[5856]: WARNING: System::setSecurityManager will be removed in a future release
and the follwoing error/warn in the indexer log:
[2023-08-23T15:18:04,026][WARN ][r.suppressed ] [node-1] path: /.kibana/_count, params: {index=.kibana}
org.opensearch.action.search.SearchPhaseExecutionException: all shards failed
at org.opensearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:663) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:372) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:698) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:471) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$0(AbstractSearchAsyncAction.java:273) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.action.search.AbstractSearchAsyncAction$2.doRun(AbstractSearchAsyncAction.java:350) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.threadpool.TaskAwareRunnable.doRun(TaskAwareRunnable.java:78) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:59) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:806) [opensearch-2.6.0.jar:2.6.0]
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) [opensearch-2.6.0.jar:2.6.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
