From 383d13719a1a41f6730b932afd25127f8e9a1591 Mon Sep 17 00:00:00 2001 From: yujiosaka <yuji.i@doboken.net> Date: Fri, 19 Jan 2018 12:38:07 +0900 Subject: [PATCH] Reduce response object in crawler rather than hccrawler --- lib/crawler.js | 21 ++++++++++++++++++++- lib/hccrawler.js | 11 ----------- test/hccrawler.test.js | 8 ++++---- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/crawler.js b/lib/crawler.js index dc2d40d..e911cd1 100644 --- a/lib/crawler.js +++ b/lib/crawler.js @@ -1,4 +1,5 @@ const { + reduce, pick, isEmpty, uniq, @@ -16,6 +17,12 @@ const GOTO_OPTIONS = [ 'timeout', 'waitUntil', ]; +const RESPONSE_FIELDS = [ + 'ok', + 'url', + 'status', + 'headers', +]; const jQueryPath = require.resolve('jquery'); @@ -42,7 +49,7 @@ class Crawler { this._collectLinks(response.url), ]) .then(([result, screenshot, links]) => ({ - response, + response: this._reduceResponse(response), result, screenshot, links, @@ -213,6 +220,18 @@ class Crawler { )) .then(() => uniq(links)); } + + /** + * @param {!Response} response + * @return {!Object} + * @private + */ + _reduceResponse(response) { + return reduce(RESPONSE_FIELDS, (memo, field) => { + memo[field] = response[field](); + return memo; + }, {}); + } } tracePublicAPI(Crawler); diff --git a/lib/hccrawler.js b/lib/hccrawler.js index 7bfc6fd..97bdb2a 100644 --- a/lib/hccrawler.js +++ b/lib/hccrawler.js @@ -6,7 +6,6 @@ const { extend, map, each, - reduce, includes, some, endsWith, @@ -58,12 +57,6 @@ const CONSTRUCTOR_OPTIONS = CONNECT_OPTIONS.concat(LAUNCH_OPTIONS).concat([ 'onSuccess', 'onError', ]); -const RESPONSE_FIELDS = [ - 'ok', - 'url', - 'status', - 'headers', -]; const EMPTY_TXT = ''; const deviceNames = Object.keys(devices); @@ -332,10 +325,6 @@ class HCCrawler extends EventEmitter { crawler.crawl() .then(res => { res = extend({}, res); - res.response = reduce(RESPONSE_FIELDS, (memo, field) => { - memo[field] = res.response[field](); - return memo; - }, {}); res.options = options; res.depth = depth; this.emit(HCCrawler.Events.RequestFinished, res); diff --git a/test/hccrawler.test.js b/test/hccrawler.test.js index 4714635..76ad13a 100644 --- a/test/hccrawler.test.js +++ b/test/hccrawler.test.js @@ -46,10 +46,10 @@ describe('HCCrawler', () => { sinon.stub(Crawler.prototype, 'crawl').returns(Promise.resolve({ options: {}, response: { - ok: (() => true), - url: (() => 'https://example.com/'), - status: (() => 200), - headers: (() => {}), + ok: true, + url: 'https://example.com/', + status: 200, + headers: {}, }, result: { title: 'Example Domain' }, links: ['http://www.iana.org/domains/example'], -- GitLab