gerrit Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed -

RaviChandren Bose

unread,

Oct 26, 2018, 10:12:01 PM10/26/18

to Repo and Gerrit Discussion

Hi All,

Last 2 to 3 days, we are seeing the below issues on the gerrit server.

Can u Please help me to troubleshoot this.

Log:-

com.google.gerrit.server.git.UpdateException: java.util.concurrent.ExecutionException: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:470)

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:649)

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:644)

at com.google.gerrit.server.git.ReceiveCommits.insertChangesAndPatchSets(ReceiveCommits.java:859)

... 16 more

Caused by: java.util.concurrent.ExecutionException: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:476)

at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:455)

at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:79)

at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63)

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:434)

... 19 more

Caused by: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.gerrit.lucene.LuceneChangeIndex.replace(LuceneChangeIndex.java:226)

at com.google.gerrit.lucene.LuceneChangeIndex.replace(LuceneChangeIndex.java:108)

at com.google.gerrit.server.index.change.ChangeIndexer.index(ChangeIndexer.java:181)

at com.google.gerrit.server.index.change.ChangeIndexer$IndexTask.call(ChangeIndexer.java:294)

at com.google.gerrit.server.index.change.ChangeIndexer$IndexTask.call(ChangeIndexer.java:253)

at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:108)

at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:41)

at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:77)

... 8 more

Caused by: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:476)

at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:455)

at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:79)

at com.google.gerrit.lucene.LuceneChangeIndex.replace(LuceneChangeIndex.java:219)

... 15 more

Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720)

at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734)

at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1473)

at com.google.gerrit.lucene.AutoCommitWriter.updateDocument(AutoCommitWriter.java:100)

at org.apache.lucene.index.TrackingIndexWriter.updateDocument(TrackingIndexWriter.java:55)

at com.google.gerrit.lucene.AbstractLuceneIndex$4.call(AbstractLuceneIndex.java:261)

at com.google.gerrit.lucene.AbstractLuceneIndex$4.call(AbstractLuceneIndex.java:258)

at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:108)

at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:41)

at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:77)

... 3 more

Caused by: java.io.IOException: No space left on device

at sun.nio.ch.FileDispatcherImpl.write0(Native Method)

at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)

at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)

at sun.nio.ch.IOUtil.write(IOUtil.java:65)

at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)

at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)

at java.nio.channels.Channels.writeFully(Channels.java:101)

at java.nio.channels.Channels.access$000(Channels.java:61)

at java.nio.channels.Channels$1.write(Channels.java:174)

at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278)

at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)

at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)

at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)

at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80)

at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473)

at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309)

at org.apache.lucene.codecs.lucene53.Lucene53NormsConsumer.close(Lucene53NormsConsumer.java:138)

at org.apache.lucene.util.IOUtils.close(IOUtils.java:97)

at org.apache.lucene.util.IOUtils.close(IOUtils.java:84)

at org.apache.lucene.index.DefaultIndexingChain.writeNorms(DefaultIndexingChain.java:232)

at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)

at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422)

at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)

at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)

at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424)

at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286)

at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261)

at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251)

at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137)

at com.google.gerrit.lucene.WrappableSearcherManager.refreshIfNeeded(WrappableSearcherManager.java:148)

at com.google.gerrit.lucene.WrappableSearcherManager.refreshIfNeeded(WrappableSearcherManager.java:68)

at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)

at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)

at org.apache.lucene.search.ControlledRealTimeReopenThread.run(ControlledRealTimeReopenThread.java:245)

regards

Ravi

David Pursehouse

unread,

Oct 26, 2018, 10:13:40 PM10/26/18

to RaviChandren Bose, Repo and Gerrit Discussion

On Sat, Oct 27, 2018 at 11:12 AM RaviChandren Bose <rav...@gmail.com> wrote:

Hi All,

Last 2 to 3 days, we are seeing the below issues on the gerrit server.
Can u Please help me to troubleshoot this.

Caused by: java.io.IOException: No space left on device

should be enough to find the root cause...

--
--
To unsubscribe, email repo-discuss...@googlegroups.com
More info at http://groups.google.com/group/repo-discuss?hl=en

---
You received this message because you are subscribed to the Google Groups "Repo and Gerrit Discussion" group.
To unsubscribe from this group and stop receiving emails from it, send an email to repo-discuss...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

RaviChandren Bose

unread,

Nov 2, 2018, 12:53:47 PM11/2/18

to Repo and Gerrit Discussion

David,

but lot of space on my system.

gerrit2@dops-gerrit-17:/usr/local/gerrit2/logs$ df -h

Filesystem Size Used Avail Use% Mounted on

udev 32G 0 32G 0% /dev

tmpfs 6.3G 1.6M 6.3G 1% /run

/dev/mapper/dops--gerrit--17--vg-root 230G 130G 89G 60% /

tmpfs 32G 0 32G 0% /dev/shm

tmpfs 5.0M 0 5.0M 0% /run/lock

tmpfs 32G 0 32G 0% /sys/fs/cgroup

/dev/sdb1 246G 102G 132G 44% /git_repo

/dev/sda1 236M 126M 99M 57% /boot

tmpfs 6.3G 0 6.3G 0% /run/user/1000

tmpfs 6.3G 0 6.3G 0% /run/user/1001

gerrit2@dops-gerrit-17:/usr/local/gerrit2/logs$ df -hi

Filesystem Inodes IUsed IFree IUse% Mounted on

udev 7.9M 433 7.9M 1% /dev

tmpfs 7.9M 761 7.9M 1% /run

/dev/mapper/dops--gerrit--17--vg-root 15M 782K 14M 6% /

tmpfs 7.9M 1 7.9M 1% /dev/shm

tmpfs 7.9M 5 7.9M 1% /run/lock

tmpfs 7.9M 18 7.9M 1% /sys/fs/cgroup

/dev/sdb1 16M 102K 16M 1% /git_repo

/dev/sda1 61K 312 61K 1% /boot

tmpfs 7.9M 10 7.9M 1% /run/user/1000

My System Details

OS: Ubuntu 18.04.1 LTS – Server (64-Bit)

Kernel: Linux dops-gerrit-17 4.15.0-38-generic #41-Ubuntu SMP Wed Oct 10 10:59:38 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

Git: git version 2.17.1

Java Version: java version "1.8.0_101", Java(TM) SE Runtime Environment (build 1.8.0_101-b13),Java HotSpot(TM) 64-Bit Server VM (build 25.101-b13, mixed mode)

Postgresql Version: PostgreSQL 9.5.7

Gerrit Version: 2.13.9

Please help me to resolve this.

Log:

Caused by: com.google.gerrit.server.git.UpdateException: java.util.concurrent.ExecutionException: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:470)

at com.google.gerrit.server.git.MergeOp.integrateIntoHistory(MergeOp.java:470)

... 43 more

Caused by: java.util.concurrent.ExecutionException: java.io.IOException: java.util.concurrent.ExecutionException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed

at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:476)

at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:455)

at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:79)

at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63)

at com.google.gerrit.server.git.BatchUpdate.execute(BatchUpdate.java:434)

... 44 more