read streams leak memory?

[alessioalex]

Hello there,

I'm monitoring my app and I've seen my memory usage increase really oddly (it did not decrease after X hours), so I suspected a leak. I've found out that the memory seems to remain uncollected when using fs.readStream. I made a small example with an http server that serving a 2mb file and did some ab (apache benchmark) load testing on it (ab -n 5000 -c 100 a couple of times). X minutes after the load testing is done, the memory idles at 480 Mb and doesn't drop.

Am I missing something or is there really a problem with this?

index.js

var http   = require('http'),

     bytes = require('bytes'), // npm install bytes

     fs       = require('fs');                                                                                                       

                                                                                                                  

var fpath = '/path/to/a/2mb/file/in/my/case';

                                                                                                                  

http.createServer(function(req, res) {                                                                            

  fs.createReadStream(fpath).pipe(res);                                                                           

}).listen(7777);                                                                                                  

                                                                                                                  

var lastMem;                                                                                                      

                                                                                                                  

function getMem(msg) {                                                                                            

  var memUsg;                                                                                                     

                                                                                                                  

  msg = msg || '';                                                                                                

                                                                                                                  

  memUsg = bytes(process.memoryUsage().rss);                                                                      

  if (lastMem !== memUsg) {                                                                                       

    lastMem = memUsg;                                                                                             

    console.log(msg + ' ' + memUsg);                                                                              

  }                                                                                                               

}                                                                                                                 

                                                                                                                  

setInterval(function() {                                                                                          

  getMem('rss:');                                                                                                    

}, 5000);

Thanks!

[mscdex]

On Tuesday, June 25, 2013 9:39:56 AM UTC-4, alessioalex wrote:

I'm monitoring my app and I've seen my memory usage increase really oddly (it did not decrease after X hours), so I suspected a leak. I've found out that the memory seems to remain uncollected when using fs.readStream. I made a small example with an http server that serving a 2mb file and did some ab (apache benchmark) load testing on it (ab -n 5000 -c 100 a couple of times). X minutes after the load testing is done, the memory idles at 480 Mb and doesn't drop.

What version of node? 

[Eldar]

This is not an answer to your question, but just in case, doing

fs.createReadStream(fpath).pipe(res);

is not safe. Any error on response during streaming will lead to leak as a file will remain opened.

So in production you should destroy stream explicitly.

The full pipe operation is something like:

var stream = fs.createReadStream(path)

stream.on('error', function() {

  res.destroy()

})

res.on('close', function() {

  stream.destroy()

})

stream.pipe(res)

[mscdex]

On Tuesday, June 25, 2013 9:39:56 AM UTC-4, alessioalex wrote:

http.createServer(function(req, res) {                                                                            

  fs.createReadStream(fpath).pipe(res);                                                                           

What if you do `res.writeHead(200);` first? 

[alessioalex]

@Eldar: I know, but for the sake of brevity I omitted that.

@mscdex: it has little to do with http, I made a simpler example with a loop here:

https://gist.github.com/alessioalex/5861041 ( you can clone this )

And then run node --expose-gc loop.js ( the garbage collector will run each 5 seconds )

Note: I've included a sample.txt 2 Mb file.

Also, I've tested on Node 0.8.x and 0.10.x and it behaves the same, after the gc runs the memory idles at > 100 Mb, which is really odd.

This is the code from the gist:

loop.js

// run with: node --expose-gc loop.js
var fs  = require('fs');
 
// 2 Mb file
var fpath = __dirname + '/sample.txt';
 
function readStream() {
  fs.createReadStream(fpath).on('open', function() {
    process.stdout.write('o');
  }).on('data', function() {
    process.stdout.write('*');
  }).on('end', function() {
    process.stdout.write('$');
  });
}
 
for (var i = 0; i < 180; i++) {
  readStream();
}
 
function getMem(msg) {
  memUsg = (process.memoryUsage().rss / (1024 * 1024)).toFixed(2);
  console.log('[' + new Date() + '] rss: ' + memUsg + ' Mb');
}
 
var garbageCollect = gc || function() {};
 
setInterval(function() {
  garbageCollect();
  getMem();
}, 5000);

[liuyanghejerry]

--
--
Job Board: http://jobs.nodejs.org/
Posting guidelines: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
You received this message because you are subscribed to the Google
Groups "nodejs" group.
To post to this group, send email to nod...@googlegroups.com
To unsubscribe from this group, send email to
nodejs+un...@googlegroups.com
For more options, visit this group at
http://groups.google.com/group/nodejs?hl=en?hl=en
 
---
You received this message because you are subscribed to the Google Groups "nodejs" group.
To unsubscribe from this group and stop receiving emails from it, send an email to nodejs+un...@googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.
 
 

Interesting. I have about 50MB memory reported at last and 150MB reported when loop 500 times. When tried 5000, error poped out saying EMFILE.

However, I made heapdump before clone streams and after. It shows almost the same (still little different, but my chrome cannot make comparison) no matter 180 times or 500 times. Don't know why.

[blemx]

I tried your code with some modification

setInterval(function() {

  for (var i = 0; i < 180; i++) {
    readStream();
  }

}, 300);

setInterval(function() {
  garbageCollect();
  getMem();
}, 500);

and while code is looping, Its memory usage increased to about 280Mb and became stable.

I guess node reserves that space and not gc them.

2013년 6월 26일 수요일 오전 3시 42분 50초 UTC+9, alessioalex 님의 말:

[Oleg Slobodskoi]

I have similar problems right now. We switched from node 0.8.25 to 0.10.12 and now servers consuming lots of memory. I could track this down db calls using mongoose. Every time I call some url, memory usage increases 0.5-1 Mb and remains on this level. The same happens on node 0.8.25, but the memory usage goes very quickly down, within seconds. After I pumped the memory on node 0.10.12 full, I was waiting about 30-60 minutes -  and then it was near the start position again.

It looks for me like the reason is not the js code itself leaking variables, because it should never go down in this case, but it looks like gc is not doing his work as usual.

Its an urgent problem for me right now, any suggestions what to try would be helpfull.

Best,

Oleg

[Alexandru Vladutu]

Oleg I think this can solve your problems, it solved mine:

node --always-compact server.js

So run node with the always compact flag for v8 and the gc seems to be doing its thing. Let me know if it worked.

Best,

Alex

--

--
Job Board: http://jobs.nodejs.org/
Posting guidelines: https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
You received this message because you are subscribed to the Google
Groups "nodejs" group.
To post to this group, send email to nod...@googlegroups.com
To unsubscribe from this group, send email to
nodejs+un...@googlegroups.com
For more options, visit this group at
http://groups.google.com/group/nodejs?hl=en?hl=en
 
---

You received this message because you are subscribed to a topic in the Google Groups "nodejs" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/nodejs/uETidz1BUHk/unsubscribe.
To unsubscribe from this group and all its topics, send an email to nodejs+un...@googlegroups.com.

[Oleg Slobodskoi]

Thanks for the try, but it didn't helped. We have found a temporary hotfix, however the real issue seems to be a bug in the incremental gc itself.

Here is a full documentation of the issue 

https://github.com/LearnBoost/mongoose/issues/1565

Oleg Slobodskoi

twitter.com/oleg008
github.com/kof

You received this message because you are subscribed to the Google Groups "nodejs" group.

To unsubscribe from this group and stop receiving emails from it, send an email to nodejs+un...@googlegroups.com.