first commit

api/grabber/browserconfig.js

@@ -0,0 +1,13 @@
+module.exports = ($, done) => {
+  const icons = [];
+
+  const tileImage = $('meta[name="msapplication-TileImage"]', 'head').attr('content');
+
+  if (tileImage) {
+    icons.push({
+      src: tileImage,
+    });
+  }
+
+  return done(null, icons);
+};

api/grabber/favicon.js

@@ -0,0 +1,22 @@
+const request = require('./request.conf');
+const { URL } = require('url');
+
+module.exports = ($, done) => {
+  const url = new URL('/favicon.ico', $.baseUrl).href;
+  const reqOptions = {
+    method: 'HEAD',
+  };
+
+  request(url, reqOptions, (err, res) => {
+    // ignore errors
+    if (err) return done(null, []);
+    if (res.statusCode !== 200) return done(null, []);
+    // check image size
+    if (!(parseInt(res.headers['content-length'], 10) > 0)) return done(null, []);
+
+    return done(null, [{
+      src: url,
+      type: 'image/x-icon',
+    }]);
+  });
+};

api/grabber/index.js

@@ -0,0 +1,56 @@
+const page = require('./page');
+const URL = require('url').URL;
+const normalizeUrl = require('normalize-url');
+
+module.exports = {
+  default: (domain, options, done) => {
+    const url = `http://${domain}/`;
+
+    page(url, (err, icons, baseUrl) => {
+      if (err) return done(err);
+
+      icons.forEach((icon) => {
+        const url = new URL(icon.src, baseUrl);
+        if (options.normalizeUrl) {
+          icon.src = normalizeUrl(url.href, {
+            removeQueryParameters: [/.+/],
+            stripWWW: false,
+          });
+        } else {
+          icon.src = url.href;
+        }
+      });
+
+      const data = {
+        domain,
+        icons,
+      };
+
+      return done(null, data);
+    });
+  },
+  allurl: (url, options, done) => {
+    page(url, (err, icons, baseUrl) => {
+      if (err) return done(err);
+
+      icons.forEach((icon) => {
+        const url = new URL(icon.src, baseUrl);
+        if (options.normalizeUrl) {
+          icon.src = normalizeUrl(url.href, {
+            removeQueryParameters: [/.+/],
+            stripWWW: false,
+          });
+        } else {
+          icon.src = url.href;
+        }
+      });
+
+      const data = {
+        url,
+        icons,
+      };
+
+      return done(null, data);
+    });
+  }
+}

api/grabber/links.js

@@ -0,0 +1,29 @@
+const selectors = [
+  "link[rel='icon']",
+  "link[rel='shortcut icon']",
+  "link[rel='apple-touch-icon']",
+  "link[rel='apple-touch-icon-precomposed']",
+  "link[rel='apple-touch-startup-image']",
+  "link[rel='mask-icon']",
+  "link[rel='fluid-icon']",
+];
+
+module.exports = ($) => {
+  const icons = [];
+
+  selectors.forEach((selector) => {
+    $(selector).each((i, elem) => {
+      const { href, sizes, type } = elem.attribs;
+      if (href && href !== '#') {
+        const icon = {
+          sizes,
+          src: href,
+          type,
+        };
+        icons.push(icon);
+      }
+    });
+  });
+
+  return icons;
+};

api/grabber/manifest.js

@@ -0,0 +1,28 @@
+const request = require('./request.conf');
+const URL = require('url').URL;
+
+module.exports = ($, done) => {
+  const href = $('link[rel="manifest"]', 'head').attr('href');
+  if (!href) return done(null, []);
+
+  const url = new URL(href, $.baseUrl).href;
+
+  request(url, (err, res, manifest) => {
+    // ignore errors
+    if (err) return done(null, []);
+    if (res.statusCode !== 200) return done(null, []);
+
+    let icons = [];
+    try {
+      const parsed = JSON.parse(manifest);
+      if (Array.isArray(parsed.icons)) {
+        icons = parsed.icons.map(({ src, sizes, type }) => ({ src, sizes, type })) || [];
+      }
+    } catch (err) {
+      // ignore errors
+      if (err) return done(null, []);
+    }
+
+    return done(null, icons);
+  });
+};

api/grabber/ogimage.js

@@ -0,0 +1,21 @@
+const selectors = [
+  "meta[property='og:image']",
+];
+
+module.exports = ($) => {
+  const icons = [];
+
+  selectors.forEach((selector) => {
+    $(selector).each((i, elem) => {
+      const { content } = elem.attribs;
+      if (content && content !== '#') {
+        const icon = {
+          src: content
+        };
+        icons.push(icon);
+      }
+    });
+  });
+
+  return icons;
+};

api/grabber/page.js

@@ -0,0 +1,53 @@
+const request = require('./request.conf');
+const cheerio = require('cheerio');
+const parallel = require('async/parallel');
+
+const manifest = require('./manifest');
+const favicon = require('./favicon');
+const browserconfig = require('./browserconfig');
+const links = require('./links');
+const ogimage = require('./ogimage')
+
+const grab = (fn, $) => {
+  return (cb) => {
+    fn($, (err, icons) => {
+      cb(err, icons);
+    });
+  }
+};
+
+module.exports = (url, done) => {
+  if (/\?/.test(url)) {
+    url += `&${Date.now()}`
+  } else {
+    url += `?${Date.now()}`
+  }
+  request(url, (err, res, page) => {
+    if (err) return done(err);
+
+    // if (res.statusCode !== 200) return done(null, []);
+    if (page && typeof page !== 'string' && !(page instanceof String)) return done(null, []);
+
+    const $ = cheerio.load(page, {
+      // ignore case for tags and attribute names
+      lowerCaseTags: true,
+      lowerCaseAttributeNames: true,
+    });
+
+    $.baseUrl = `${res.request.uri.protocol}//${res.request.uri.hostname}`;
+
+    let icons = links($);
+    let ogi = ogimage($)
+    icons = [...icons, ...ogi]
+
+    parallel([
+      grab(manifest, $),
+      grab(favicon, $),
+      grab(browserconfig, $),
+    ], (err, results) => {
+      // ignore errors
+      results.forEach(arr => icons = [...icons, ...arr]);
+      return done(null, icons, $.baseUrl);
+    });
+  });
+};

api/grabber/request.conf.js

@@ -0,0 +1,13 @@
+const request = require('request');
+
+module.exports = request.defaults({
+  // follow HTTP 3xx responses as redirects
+  followRedirect: true,
+  headers: {
+    'Accept': '*/*',
+    // prevent to redirect to the mobile version of a website
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36',
+  },
+  jar: true,
+  timeout: 5000,
+});