--- During this error, I'm seeing a reason for this since the server have plenty of memory and reindex process does not reach 30GB of used memory $ free -h total used free shared buff/cache available Mem: 94G 26G 24G 1.4G 43G 65G Swap: 0B 0B 0B [2022-06-27 12:58:54,802] [Index-Batch-30[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@76a8c4f7[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@699a8172[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@24c13c36]]]] WARN com.google.gerrit.server.patch.PatchListLoader : Error computing PatchListKey[BASE..48258fac6e150aa621305e6b634ed11b22d858f4 IGNORE_NONE] java.util.concurrent.ExecutionException: java.io.IOException: org.eclipse.jgit.errors.LargeObjectException$OutOfMemory: Out of memory loading unknown object at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:564) at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:525) at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:102) at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:237) at com.google.common.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2313) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2279) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2155) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2045) at com.google.common.cache.LocalCache.get(LocalCache.java:3951) at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4871) at com.google.gerrit.server.cache.h2.H2CacheImpl.get(H2CacheImpl.java:141) at com.google.gerrit.server.patch.PatchListCacheImpl.get(PatchListCacheImpl.java:105) at com.google.gerrit.server.patch.DiffSummaryLoader.call(DiffSummaryLoader.java:43) at com.google.gerrit.server.patch.DiffSummaryLoader.call(DiffSummaryLoader.java:25) at com.google.gerrit.server.cache.h2.H2CacheImpl.lambda$get$1(H2CacheImpl.java:152) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.lambda$get$0(CaffeinatedGuavaCache.java:69) at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2344) at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2342) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2325) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.get(CaffeinatedGuavaCache.java:67) at com.google.gerrit.server.cache.h2.H2CacheImpl.get(H2CacheImpl.java:141) at com.google.gerrit.server.patch.PatchListCacheImpl.getDiffSummary(PatchListCacheImpl.java:164) at com.google.gerrit.server.query.change.ChangeData.getDiffSummary(ChangeData.java:408) at com.google.gerrit.server.query.change.ChangeData.computeChangedLines(ChangeData.java:417) at com.google.gerrit.server.query.change.ChangeData.changedLines(ChangeData.java:429) at com.google.gerrit.server.index.change.ChangeField.lambda$static$34(ChangeField.java:715) at com.google.gerrit.index.FieldDef.get(FieldDef.java:142) at com.google.gerrit.index.Schema.fieldValues(Schema.java:187) at com.google.gerrit.index.Schema.lambda$buildFields$0(Schema.java:219) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.lambda$forEachRemaining$1(CollectSpliterators.java:67) at java.base/java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.forEachRemaining(CollectSpliterators.java:67) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) at com.google.gerrit.index.Schema.buildFields(Schema.java:221) at com.google.gerrit.lucene.AbstractLuceneIndex.toDocument(AbstractLuceneIndex.java:324) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:293) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:110) at com.google.gerrit.server.index.change.ChangeIndexer.indexImpl(ChangeIndexer.java:218) at com.google.gerrit.server.index.change.ChangeIndexer.doIndex(ChangeIndexer.java:183) at com.google.gerrit.server.index.change.ChangeIndexer.index(ChangeIndexer.java:179) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.index(AllChangesIndexer.java:240) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.lambda$call$1(AllChangesIndexer.java:229) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:229) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:193) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:612) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: java.io.IOException: org.eclipse.jgit.errors.LargeObjectException$OutOfMemory: Out of memory loading unknown object at com.google.gerrit.server.patch.PatchListLoader.toFileHeader(PatchListLoader.java:517) at com.google.gerrit.server.patch.PatchListLoader.getPatchListEntry(PatchListLoader.java:389) at com.google.gerrit.server.patch.PatchListLoader.readPatchList(PatchListLoader.java:216) at com.google.gerrit.server.patch.PatchListLoader.call(PatchListLoader.java:133) at com.google.gerrit.server.patch.PatchListLoader.call(PatchListLoader.java:83) at com.google.gerrit.server.cache.h2.H2CacheImpl.lambda$get$1(H2CacheImpl.java:152) at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4876) at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3529) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2278) ... 70 more Caused by: org.eclipse.jgit.errors.LargeObjectException$OutOfMemory: Out of memory loading unknown object at org.eclipse.jgit.internal.storage.file.PackFile.load(PackFile.java:912) at org.eclipse.jgit.internal.storage.file.PackFile.get(PackFile.java:284) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedObject(ObjectDirectory.java:459) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedFromSelfOrAlternate(ObjectDirectory.java:417) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openObject(ObjectDirectory.java:408) at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:132) at org.eclipse.jgit.diff.ContentSource$ObjectReaderSource.open(ContentSource.java:112) at org.eclipse.jgit.diff.ContentSource$Pair.open(ContentSource.java:259) at org.eclipse.jgit.diff.DiffFormatter.open(DiffFormatter.java:1074) at org.eclipse.jgit.diff.DiffFormatter.createFormatResult(DiffFormatter.java:1002) at org.eclipse.jgit.diff.DiffFormatter.toFileHeader(DiffFormatter.java:964) at com.google.gerrit.server.patch.PatchListLoader.lambda$toFileHeader$2(PatchListLoader.java:493) at com.google.gerrit.server.logging.LoggingContextAwareCallable.call(LoggingContextAwareCallable.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ... 3 more Caused by: java.lang.OutOfMemoryError: Java heap space --- --- Several similar warnings with different ammount of words: [2647.813s][warning][gc,alloc] Index-Batch-30[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@6fbd5311[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@54be22d0[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@497947ac]]]: Retried waiting for GCLocker too often allocating 154248 words --- --- Error computing PatchListKey - Previous attempt failed with LargeObjectException Reindexing changes: changes: 29% (292529/990478), project-slices: 40% (1668/4074), Slicing projects: 100% (6540/6540) (|) [2022-06-27 13:40:39,009] [Index-Batch-12[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@207f96cf[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@31e6ecb6[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@61eb33de]]]] WARN com.google.gerrit.server.patch.PatchListLoader : Error computing DiffSummaryKey[BASE..64a3fbf96a3ea7f178b2e05c5d10d987c67810ed IGNORE_NONE] java.util.concurrent.ExecutionException: com.google.gerrit.server.patch.PatchListObjectTooLargeException: Error computing PatchListKey[BASE..64a3fbf96a3ea7f178b2e05c5d10d987c67810ed IGNORE_NONE]. Previous attempt failed with LargeObjectException at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.get(CaffeinatedGuavaCache.java:88) at com.google.gerrit.server.cache.h2.H2CacheImpl.get(H2CacheImpl.java:141) at com.google.gerrit.server.patch.PatchListCacheImpl.getDiffSummary(PatchListCacheImpl.java:164) at com.google.gerrit.server.query.change.ChangeData.getDiffSummary(ChangeData.java:408) at com.google.gerrit.server.query.change.ChangeData.computeChangedLines(ChangeData.java:417) at com.google.gerrit.server.query.change.ChangeData.changedLines(ChangeData.java:429) at com.google.gerrit.server.index.change.ChangeField.lambda$static$34(ChangeField.java:715) at com.google.gerrit.index.FieldDef.get(FieldDef.java:142) at com.google.gerrit.index.Schema.fieldValues(Schema.java:187) at com.google.gerrit.index.Schema.lambda$buildFields$0(Schema.java:219) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.lambda$forEachRemaining$1(CollectSpliterators.java:67) at java.base/java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.forEachRemaining(CollectSpliterators.java:67) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) at com.google.gerrit.index.Schema.buildFields(Schema.java:221) at com.google.gerrit.lucene.AbstractLuceneIndex.toDocument(AbstractLuceneIndex.java:324) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:293) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:110) at com.google.gerrit.server.index.change.ChangeIndexer.indexImpl(ChangeIndexer.java:218) at com.google.gerrit.server.index.change.ChangeIndexer.doIndex(ChangeIndexer.java:183) at com.google.gerrit.server.index.change.ChangeIndexer.index(ChangeIndexer.java:179) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.index(AllChangesIndexer.java:240) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.lambda$call$1(AllChangesIndexer.java:229) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:229) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:193) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:612) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: com.google.gerrit.server.patch.PatchListObjectTooLargeException: Error computing PatchListKey[BASE..64a3fbf96a3ea7f178b2e05c5d10d987c67810ed IGNORE_NONE]. Previous attempt failed with LargeObjectException at com.google.gerrit.server.patch.PatchListCacheImpl.get(PatchListCacheImpl.java:107) at com.google.gerrit.server.patch.DiffSummaryLoader.call(DiffSummaryLoader.java:43) at com.google.gerrit.server.patch.DiffSummaryLoader.call(DiffSummaryLoader.java:25) at com.google.gerrit.server.cache.h2.H2CacheImpl.lambda$get$1(H2CacheImpl.java:152) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.lambda$get$0(CaffeinatedGuavaCache.java:69) at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2344) at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2342) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2325) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.get(CaffeinatedGuavaCache.java:67) ... 52 more --- --- In previous upgrades, we saw some timeouts during reindex and disabling all plugins before runing it, usually solved the problem, but did not worked with 3.3 [2022-06-27 13:42:16,959] [Index-Batch-14[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@3ac5d7bf[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@2f478cfe[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@5206fcd7]]]] WARN com.google.gerrit.server.patch.PatchListLoader : 5000 ms timeout reached for Diff loader in project REDACTED_PROJECT on commit 773997c02593dd3d0edad0474ba4ce840cf3f241 on path SOME/FILE/PATH/REDACTED comparing 0000000000000000000000000000000000000000..0fd0475a0857462956958a6e1eaed35b03e21972 --- --- [2022-06-27 13:46:01,379] [Diff-22] ERROR org.eclipse.jgit.internal.storage.file.ObjectDirectory : Exception caught while accessing pack file /REDACTED/PATH/REPO.git/objects/pack/pack-f957feadf06336e933a931db2888b84bb9b33e5f.pack, the pack file might be corrupt. Caught 1 consecutive errors while trying to read this pack. java.io.IOException: In-memory buffer limit exceeded at org.eclipse.jgit.internal.storage.file.PackFile.load(PackFile.java:884) at org.eclipse.jgit.internal.storage.file.PackFile.get(PackFile.java:284) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedObject(ObjectDirectory.java:459) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedFromSelfOrAlternate(ObjectDirectory.java:417) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openObject(ObjectDirectory.java:408) at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:132) at org.eclipse.jgit.diff.ContentSource$ObjectReaderSource.open(ContentSource.java:112) at org.eclipse.jgit.diff.ContentSource$Pair.open(ContentSource.java:257) at org.eclipse.jgit.diff.DiffFormatter.open(DiffFormatter.java:1074) at org.eclipse.jgit.diff.DiffFormatter.createFormatResult(DiffFormatter.java:1001) at org.eclipse.jgit.diff.DiffFormatter.toFileHeader(DiffFormatter.java:964) at com.google.gerrit.server.patch.PatchListLoader.lambda$toFileHeader$2(PatchListLoader.java:493) at com.google.gerrit.server.logging.LoggingContextAwareCallable.call(LoggingContextAwareCallable.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) --- --- [2022-06-27 14:00:33,467] [Index-Batch-28[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@bf3ca82[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@6b7dc58f[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@16603cc1]]]] WARN com.google.gerrit.server.index.change.AllChangesIndexer : Failed to index change 1202590 com.google.gerrit.exceptions.StorageException: Failed to reindex change 1202590 in index version 60 (current patch set = 1) at com.google.gerrit.server.index.change.ChangeIndexer.indexImpl(ChangeIndexer.java:224) at com.google.gerrit.server.index.change.ChangeIndexer.doIndex(ChangeIndexer.java:183) at com.google.gerrit.server.index.change.ChangeIndexer.index(ChangeIndexer.java:179) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.index(AllChangesIndexer.java:240) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.lambda$call$1(AllChangesIndexer.java:229) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:229) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:193) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:612) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: com.google.gerrit.exceptions.StorageException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:301) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:110) at com.google.gerrit.server.index.change.ChangeIndexer.indexImpl(ChangeIndexer.java:218) ... 29 more Caused by: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:564) at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:545) at com.google.common.util.concurrent.FluentFuture$TrustedFuture.get(FluentFuture.java:88) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:298) ... 31 more Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:749) at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:763) at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1567) at com.google.gerrit.lucene.AutoCommitWriter.updateDocument(AutoCommitWriter.java:98) at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1316) at com.google.gerrit.lucene.AutoCommitWriter.addDocument(AutoCommitWriter.java:51) at com.google.gerrit.lucene.AbstractLuceneIndex.lambda$insert$1(AbstractLuceneIndex.java:277) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) ... 3 more --- --- Several similar errors but with different field names [2022-06-27 14:05:26,878] [Index-Batch-25[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@65b35c2d[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@745a6c8b[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@755732a7]]]] ERROR com.google.gerrit.index.Schema : error getting field submit_record of ChangeData{Change{12527 (I9703f3c0c5c1c49423d9764caa542092f3bea009), dest=REDACTED_PROJECT,refs/heads/master, status=M}} com.google.common.util.concurrent.UncheckedExecutionException: com.google.common.util.concurrent.UncheckedExecutionException: java.util.concurrent.RejectedExecutionException: Task com.google.gerrit.server.logging.LoggingContextAwareRunnable@290ba5c0 rejected from java.util.concurrent.ThreadPoolExecutor@3f569715[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 32376] at com.github.benmanes.caffeine.guava.CaffeinatedGuavaLoadingCache.get(CaffeinatedGuavaLoadingCache.java:65) at com.google.gerrit.server.project.ProjectCacheImpl.get(ProjectCacheImpl.java:200) at com.google.gerrit.server.project.SubmitRuleEvaluator.evaluate(SubmitRuleEvaluator.java:119) at com.google.gerrit.server.query.change.ChangeData.submitRecords(ChangeData.java:841) at com.google.gerrit.server.index.change.ChangeField.formatSubmitRecordValues(ChangeField.java:943) at com.google.gerrit.index.FieldDef.get(FieldDef.java:142) at com.google.gerrit.index.Schema.fieldValues(Schema.java:187) at com.google.gerrit.index.Schema.lambda$buildFields$0(Schema.java:219) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.lambda$forEachRemaining$1(CollectSpliterators.java:67) at java.base/java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104) at com.google.common.collect.CollectSpliterators$1WithCharacteristics.forEachRemaining(CollectSpliterators.java:67) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) at com.google.gerrit.index.Schema.buildFields(Schema.java:221) at com.google.gerrit.lucene.AbstractLuceneIndex.toDocument(AbstractLuceneIndex.java:324) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:293) at com.google.gerrit.lucene.LuceneChangeIndex.insert(LuceneChangeIndex.java:110) at com.google.gerrit.server.index.change.ChangeIndexer.indexImpl(ChangeIndexer.java:218) at com.google.gerrit.server.index.change.ChangeIndexer.doIndex(ChangeIndexer.java:183) at com.google.gerrit.server.index.change.ChangeIndexer.index(ChangeIndexer.java:179) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.index(AllChangesIndexer.java:240) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.lambda$call$1(AllChangesIndexer.java:229) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:229) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:193) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:612) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: com.google.common.util.concurrent.UncheckedExecutionException: java.util.concurrent.RejectedExecutionException: Task com.google.gerrit.server.logging.LoggingContextAwareRunnable@290ba5c0 rejected from java.util.concurrent.ThreadPoolExecutor@3f569715[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 32376] at com.github.benmanes.caffeine.guava.CaffeinatedGuavaLoadingCache.get(CaffeinatedGuavaLoadingCache.java:65) at com.google.gerrit.server.cache.h2.H2CacheImpl.get(H2CacheImpl.java:130) at com.google.gerrit.server.project.ProjectCacheImpl$InMemoryLoader.load(ProjectCacheImpl.java:377) at com.google.gerrit.server.project.ProjectCacheImpl$InMemoryLoader.load(ProjectCacheImpl.java:329) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaLoadingCache$SingleLoader.load(CaffeinatedGuavaLoadingCache.java:136) at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:141) at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2344) at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2342) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2325) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:54) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaLoadingCache.get(CaffeinatedGuavaLoadingCache.java:59) ... 50 more --- --- [2022-06-27 17:54:40,743] [Index-Batch-28[java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@5a6daa41[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@a6960ea[Wrapped task = com.google.gerrit.server.logging.LoggingContextAwareRunnable@3d02a896]]]] ERROR com.google.gerrit.index.SiteIndexer : Failed to index project REDACTED_PROJECT (0/1) java.util.concurrent.ExecutionException: com.google.common.util.concurrent.ExecutionError: java.lang.OutOfMemoryError: Java heap space at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:564) at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:525) at com.google.common.util.concurrent.FluentFuture$TrustedFuture.get(FluentFuture.java:88) at com.google.gerrit.index.SiteIndexer$ErrorListener.run(SiteIndexer.java:106) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:30) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1174) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:969) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:760) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.afterRanInterruptibly(TrustedListenableFutureTask.java:133) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:133) at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) at com.google.gerrit.server.logging.LoggingContextAwareRunnable.run(LoggingContextAwareRunnable.java:103) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:612) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: com.google.common.util.concurrent.ExecutionError: java.lang.OutOfMemoryError: Java heap space at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.lambda$get$0(CaffeinatedGuavaCache.java:84) at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2344) at java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2342) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2325) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62) at com.github.benmanes.caffeine.guava.CaffeinatedGuavaCache.get(CaffeinatedGuavaCache.java:67) at com.google.gerrit.server.notedb.ChangeNotesCache.get(ChangeNotesCache.java:380) at com.google.gerrit.server.notedb.ChangeNotes.onLoad(ChangeNotes.java:610) at com.google.gerrit.server.notedb.AbstractChangeNotes.load(AbstractChangeNotes.java:158) at com.google.gerrit.server.notedb.AbstractChangeNotes.load(AbstractChangeNotes.java:138) at com.google.gerrit.server.notedb.ChangeNotes$Factory.toResult(ChangeNotes.java:328) at com.google.gerrit.server.notedb.ChangeNotes$Factory.scanOneChange(ChangeNotes.java:316) at com.google.gerrit.server.notedb.ChangeNotes$Factory.lambda$scan$2(ChangeNotes.java:301) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) at java.base/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:229) at com.google.gerrit.server.index.change.AllChangesIndexer$ProjectIndexer.call(AllChangesIndexer.java:193) at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:69) ... 9 more Caused by: java.lang.OutOfMemoryError: Java heap space ---