On Sun, Oct 28, 2012 at 4:02 AM, P. Douglas Reeder <reeder
> One of the things my app needs to do is write a large JSON file to disk.
> Currently, it's implemented naively:
> writeStream = fs.createWriteStream(process.cwd() + "/staticRoot/album.json",
> {"encoding": "utf8"});
> writeStream.addListener("error", function (error) {
> console.error("album.json:", error);
> if (! writeError) // preserve 1st error
> writeError = error;
> callback(writeError); // on error, abort writing next file
> });
> writeStream.end(JSON.stringify(album));
> This appears to work ok when the JSON file is 150k (the largest size I can
> readily test). It needs to work when the JSON file is up to 500k. i'm not
> sure how big the write buffer is, but I lack confidence that this is the
> right way to go. The bulk of the JSON file is an array of objects, so I
> could create the JSON file one array item at a time, checking the return
> value from writeStream.write() in a manner analogous to the following HTML
> file writing:
> writeStream.addListener("drain", writeUntilBufferFull);
> writeStream.write("<h1>" + title + "</h1>\n");
> writeStream.write("<table>\n");
> function writeUntilBufferFull() {
> var longDate, urlFileName, htmlFileName, row;
> console.log("writeUntilBufferFull() p=", p, " metadata:",
> JSON.stringify(pictureMetadata[p]));
> while (p < pictureMetadata.length) {
> longDate = pictureMetadata[p].date ?
> pictureMetadata[p].date.toLocaleDateString() : "";
> urlFileName = encodeURIComponent(pictureMetadata[p].fileName).replace(/'/g,
> '%27');
> htmlFileName = pictureMetadata[p].fileName.replace(/&/g,
> "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g,
> """);
> row = " <tr><td align='right'>" + (p+1) + "</td><td align='right'>" +
> longDate +
> "</td><td><a class='gallery' href='pictures/" + urlAlbumName + "/" +
> urlFileName + "' title='" + longDate + "'>" + htmlFileName + "</a></td>" +
> "<td><a download=\"" + htmlFileName + "\" href='full-size/" + urlAlbumName +
> "/" + urlFileName + "'>full-size</a></td></tr>\n";
> if (! writeStream.write(row))
> break;
> ++p;
> }
> if (p < pictureMetadata.length) {
> ++p;
> } else {
> writeStream.removeListener("drain", writeUntilBufferFull);
> writeStream.end();
> callback(writeError);
> }
> }
> What's the best strategy for writing large JSON files?
that data. If it's just a one-off or relatively rare thing, use
