feat(hccrawler): pass previous url to skip request

Created by: slonka

Hi,

I've found it useful to pass previousUrl to _skipRequest. Would love this to be included.

Best regards

lib/crawler.js

@@ -277,7 +277,7 @@ class Crawler {
    * @private
    */
   async _collectLinks(baseUrl) {
-    const links = [];
+    let links = [];
     await this._page.exposeFunction('pushToLinks', link => {
       const _link = resolveUrl(link, baseUrl);
       if (_link) links.push(_link);
@@ -302,6 +302,11 @@ class Crawler {
       }
       findLinks(window.document);
     });
+
+    if (links.length === 0) {
+      links = (await this._page.$$eval('a[href]', el => [...el].map(a => a.href))).map(l => resolveUrl(l, baseUrl)).filter(l => l);
+    }
+
     return uniq(links);
   }
 

lib/hccrawler.js

@@ -289,7 +289,7 @@ class HCCrawler extends EventEmitter {
    * @private
    */
   async _startRequest(options, depth, previousUrl) {
-    const skip = await this._skipRequest(options);
+    const skip = await this._skipRequest({ ...options, previousUrl });
     if (skip) {
       this.emit(HCCrawler.Events.RequestSkipped, options);
       await this._markRequested(options);