Don't store resources content in memory - stream to storage (fixes #386)

.qlty/qlty.toml

@@ -0,0 +1,10 @@
+config_version = "0"
+
+exclude_patterns = [
+  "test/**",
+]
+
+[coverage]
+ignores = [
+  "benchmark/**",
+]

MIGRATION.md

@@ -1,5 +1,60 @@
 # Migration Guide
 
+## From version 6 to 7
+
+Version 7 stops storing resource content in memory ([#386](https://github.com/website-scraper/node-website-scraper/issues/386)). Resources which need no modification (images, fonts, media, scripts - everything except html and css) are streamed directly from the network to storage, and html/css content is freed as soon as the resource is saved. Memory usage no longer grows with the size of the scraped website.
+
+#### saveResource action
+
+Resource content is now exposed as a Readable stream via `resource.getContentStream()` instead of `resource.getText()`. For streamed resources the stream can be consumed only once - consume or destroy it within the action.
+
+```javascript
+// before
+registerAction('saveResource', async ({resource}) => {
+  await saveItSomewhere(resource.getFilename(), resource.getText());
+});
+
+// after
+import { buffer } from 'stream/consumers';
+registerAction('saveResource', async ({resource}) => {
+  await saveItSomewhere(resource.getFilename(), await buffer(resource.getContentStream()));
+});
+// or pipe resource.getContentStream() to your storage to avoid buffering
+```
+
+#### afterResponse action
+
+The action is called when response headers arrive, before the body is downloaded. It receives `{response}` where response is `{url, statusCode, headers, getBody()}` instead of the full got response. Returning a string is not supported anymore - return an object or null. Return an object without `body` to keep the resource streaming (no buffering).
+
+```javascript
+// before
+registerAction('afterResponse', ({response}) => {
+  if (response.statusCode === 404) return null;
+  return { body: response.body, metadata: { headers: response.headers } };
+});
+
+// after
+registerAction('afterResponse', ({response}) => {
+  if (response.statusCode === 404) return null;
+  return { metadata: { headers: response.headers } };
+});
+// call `await response.getBody()` only if you need to inspect or replace the body
+```
+
+#### scrape() result
+
+Resource content is not retained after save: `resource.getText()` returns `null` in the result and in `onResourceSaved`. Read saved files from your storage instead.
+
+#### generateFilename action
+
+`responseData` no longer contains `body` - filenames are generated from headers (`url`, `statusCode`, `mimeType`, `encoding`, `metadata`) before the body is consumed.
+
+#### Other behavior changes
+
+* Binary files are written byte-for-byte from the network (previously the content was round-tripped through a latin1 string). Output for correctly-served files is unchanged; edge-case encodings may differ.
+* Transient request errors are retried as before, but only until response headers are received. A connection which dies mid-body is not retried.
+* `responseType` in the `request` option is ignored (responses are streamed).
+
 ## From version 4 to 5
 
 #### ESM module

README.md

@@ -119,7 +119,7 @@ The difference between [maxRecursiveDepth](#maxRecursiveDepth) and [maxDepth](#m
   only html resources with depth 2 will be filtered out, last image will be downloaded
 
 #### request
-Object, custom options for http module [got](https://github.com/sindresorhus/got#options) which is used inside website-scraper. Allows to set retries, cookies, userAgent, encoding, etc.
+Object, custom options for http module [got](https://github.com/sindresorhus/got#options) which is used inside website-scraper. Allows to set retries, cookies, userAgent, encoding, etc. Note that `responseType` is controlled by the scraper (responses are streamed) and is ignored if passed here.
 ```javascript
 // use same request options for all resources
 scrape({
@@ -244,7 +244,7 @@ class MyPlugin {
 		registerAction('afterFinish', async () => {});
 		registerAction('error', async ({error}) => {console.error(error)});
 		registerAction('beforeRequest', async ({resource, requestOptions}) => ({requestOptions}));
-		registerAction('afterResponse', async ({response}) => response.body);
+		registerAction('afterResponse', async ({response}) => ({metadata: {headers: response.headers}}));
 		registerAction('onResourceSaved', ({resource}) => {});
 		registerAction('onResourceError', ({resource, error}) => {});
 		registerAction('saveResource', async ({resource}) => {});
@@ -315,15 +315,19 @@ registerAction('beforeRequest', async ({resource, requestOptions}) => {
 ```
 
 ##### afterResponse
-Action afterResponse is called after each response, it allows to customize resource or reject its saving.
+Action afterResponse is called after response headers were received, before the body is consumed. It allows to customize resource or reject its saving.
 
 Parameters - object which includes:
-* response - response object from http module [got](https://github.com/sindresorhus/got#response)
+* response - object which includes:
+  * `url` (string) - final url of the response (after redirects)
+  * `statusCode` (number)
+  * `headers` (object)
+  * `getBody()` - async function which buffers the response body and resolves with a `Buffer`. Note that calling it loads the whole body into memory - don't call it if you only need headers or status code, so the resource keeps streaming to storage
 
 Return resolved `Promise` with:
-  * object if the resource should be saved, object should contain next properties:
-    * `body` (string, required)
-    * `encoding` (`binary` or `utf8`) is used to save the file, binary is used by default.
+  * object if the resource should be saved, object may contain next properties:
+    * `body` (string or Buffer, optional) - replaces resource content; if omitted, the original body is used without buffering it in memory
+    * `encoding` (`binary` or `utf8`) is used to decode a string `body`, binary is used by default.
     * `metadata` (object) - everything you want to save for this resource (like headers, original text, timestamps, etc.), scraper will not use this field at all, it is only for the result
   * or null if the resource should be skipped
 
@@ -335,15 +339,22 @@ registerAction('afterResponse', ({response}) => {
 		return null;
 	} else {
 		return {
-			body: response.body,
-			encoding: 'utf8',
 			metadata: {
 				headers: response.headers,
 				someOtherData: [ 1, 2, 3 ]
 			}
 		}
 	}
 });
+
+// Modify html body before it is processed
+registerAction('afterResponse', async ({response}) => {
+	if (response.headers['content-type']?.includes('text/html')) {
+		const body = await response.getBody();
+		return { body: transformHtml(body.toString()), encoding: 'utf8' };
+	}
+	return {}; // everything else keeps streaming to storage
+});
 ```
 
 ##### onResourceSaved
@@ -374,7 +385,7 @@ Action generateFilename is called to determine path in file system where the res
 
 Parameters - object which includes:
 * resource - [Resource](https://github.com/website-scraper/node-website-scraper/blob/master/lib/resource.js) object
-* responseData - object returned from afterResponse action, contains `url`, `mimeType`, `body`, `metadata` properties
+* responseData - object which contains `url`, `statusCode`, `mimeType`, `encoding`, `metadata` properties. Note that it does not contain the response body - filenames are generated before the body is consumed
 
 Should return object which includes:
 * filename - String, relative to `directory` path for specified resource
@@ -419,12 +430,23 @@ Action saveResource is called to save file to some storage. Use it to save files
 Parameters - object which includes:
 * resource - [Resource](https://github.com/website-scraper/node-website-scraper/blob/master/lib/resource.js) object
 
-If multiple actions `saveResource` added - resource will be saved to multiple storages.
+Resource content is exposed as a Readable stream returned by `resource.getContentStream()`. For resources which are streamed directly from the network (all types except html and css) the stream can be consumed only once - consume it (or destroy it) within the action.
+
+If multiple actions `saveResource` added - resource will be saved to multiple storages (the scraper buffers streamed content in that case so every action can read it).
 ```javascript
+import { pipeline } from 'stream/promises';
+
 registerAction('saveResource', async ({resource}) => {
   const filename = resource.getFilename();
-  const text = resource.getText();
-  await saveItSomewhere(filename, text);
+  await pipeline(resource.getContentStream(), createMyStorageWriteStream(filename));
+});
+
+// or, if your storage needs the whole content at once
+import { buffer } from 'stream/consumers';
+
+registerAction('saveResource', async ({resource}) => {
+  const content = await buffer(resource.getContentStream());
+  await saveItSomewhere(resource.getFilename(), content);
 });
 ```
 
@@ -434,6 +456,8 @@ Array of [Resource](https://github.com/website-scraper/node-website-scraper/blob
 - `filename`: filename where page was saved (relative to `directory`)
 - `children`: array of children Resources
 
+Note that resource content is not retained in memory after the resource was saved - read the saved files from your storage if you need the content.
+
 ## Log and debug
 This module uses [debug](https://github.com/visionmedia/debug) to log events. To enable logs you should use environment variable `DEBUG`.
 The next command will log everything from website-scraper

benchmark/memory.mjs

@@ -0,0 +1,91 @@
+/*
+ * Memory benchmark for issue #386.
+ *
+ * Starts a local http server serving an html page referencing FILES_COUNT
+ * binary files of FILE_SIZE_MB each (generated chunk-wise on the fly, so the
+ * server itself stays flat), scrapes it and reports peak memory usage.
+ *
+ * Usage:
+ *   npm run benchmark:memory
+ *   FILES_COUNT=50 FILE_SIZE_MB=10 REQUEST_CONCURRENCY=8 npm run benchmark:memory
+ */
+
+import http from 'http';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import scrape from 'website-scraper';
+
+const FILES_COUNT = parseInt(process.env.FILES_COUNT || '100', 10);
+const FILE_SIZE_MB = parseInt(process.env.FILE_SIZE_MB || '20', 10);
+const REQUEST_CONCURRENCY = parseInt(process.env.REQUEST_CONCURRENCY || '8', 10);
+
+const CHUNK = Buffer.alloc(64 * 1024, 0xab);
+const CHUNKS_PER_FILE = Math.ceil(FILE_SIZE_MB * 1024 * 1024 / CHUNK.length);
+
+function startServer () {
+	const server = http.createServer((req, res) => {
+		if (req.url === '/') {
+			const imgs = Array.from({length: FILES_COUNT}, (_, i) => `<img src="blob/${i}.bin">`).join('\n');
+			res.writeHead(200, {'content-type': 'text/html'});
+			res.end(`<html><head></head><body>${imgs}</body></html>`);
+			return;
+		}
+
+		res.writeHead(200, {'content-type': 'application/octet-stream'});
+		let sent = 0;
+		const writeChunk = () => {
+			while (sent < CHUNKS_PER_FILE) {
+				sent++;
+				if (!res.write(CHUNK)) {
+					res.once('drain', writeChunk);
+					return;
+				}
+			}
+			res.end();
+		};
+		writeChunk();
+	});
+
+	return new Promise((resolve) => {
+		server.listen(0, '127.0.0.1', () => resolve(server));
+	});
+}
+
+function formatMB (bytes) {
+	return `${Math.round(bytes / 1024 / 1024)} MB`;
+}
+
+const server = await startServer();
+const baseUrl = `http://127.0.0.1:${server.address().port}/`;
+const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'website-scraper-benchmark-'));
+fs.rmSync(directory, {recursive: true, force: true}); // scraper requires the directory to not exist
+
+const peak = {rss: 0, heapUsed: 0, external: 0, arrayBuffers: 0};
+const sampler = setInterval(() => {
+	const usage = process.memoryUsage();
+	for (const key of Object.keys(peak)) {
+		peak[key] = Math.max(peak[key], usage[key]);
+	}
+}, 200);
+
+console.log(`scraping ${FILES_COUNT} files x ${FILE_SIZE_MB} MB (${FILES_COUNT * FILE_SIZE_MB} MB total), concurrency ${REQUEST_CONCURRENCY}`);
+const startedAt = Date.now();
+
+try {
+	await scrape({
+		urls: [baseUrl],
+		directory,
+		requestConcurrency: REQUEST_CONCURRENCY
+	});
+} finally {
+	clearInterval(sampler);
+	server.close();
+	fs.rmSync(directory, {recursive: true, force: true});
+}
+
+console.log(`done in ${((Date.now() - startedAt) / 1000).toFixed(1)}s`);
+console.log(`peak rss:          ${formatMB(peak.rss)}`);
+console.log(`peak heapUsed:     ${formatMB(peak.heapUsed)}`);
+console.log(`peak external:     ${formatMB(peak.external)}`);
+console.log(`peak arrayBuffers: ${formatMB(peak.arrayBuffers)}`);

lib/config/defaults.js

@@ -46,7 +46,6 @@ const config = {
 	],
 	request: {
 		throwHttpErrors: false,
-		responseType: 'buffer',
 		//cookieJar: true,
 		decompress: true,
 		headers: {

lib/plugins/save-resource-to-fs-plugin.js

@@ -1,6 +1,6 @@
 import path from 'path';
 import fs from 'fs';
-import { outputFile } from '../utils/fs.js';
+import { outputFileStream } from '../utils/fs.js';
 
 class SaveResourceToFileSystemPlugin {
 	apply (registerAction) {
@@ -20,8 +20,12 @@ class SaveResourceToFileSystemPlugin {
 
 		registerAction('saveResource', async ({resource}) => {
 			const filename = path.join(absoluteDirectoryPath, resource.getFilename());
-			const text = resource.getText();
-			await outputFile(filename, text, resource.getEncoding());
+			try {
+				await outputFileStream(filename, resource.getContentStream());
+			} catch (err) {
+				await fs.promises.rm(filename, {force: true}); // don't leave partially written files
+				throw err;
+			}
 			loadedResources.push(resource);
 		});