first commit

.gitignore

@@ -0,0 +1 @@
+node_modules

Dockerfile

@@ -0,0 +1,9 @@
+FROM node:8-alpine
+ENV NODE_PORT 5000
+ENV NODE_ENV production
+RUN mkdir /data
+WORKDIR /data
+COPY . .
+RUN npm install
+EXPOSE 5000
+CMD ["npm", "start"]

api/grabber/browserconfig.js

@@ -0,0 +1,13 @@
+module.exports = ($, done) => {
+  const icons = [];
+
+  const tileImage = $('meta[name="msapplication-TileImage"]', 'head').attr('content');
+
+  if (tileImage) {
+    icons.push({
+      src: tileImage,
+    });
+  }
+
+  return done(null, icons);
+};

api/grabber/favicon.js

@@ -0,0 +1,22 @@
+const request = require('./request.conf');
+const { URL } = require('url');
+
+module.exports = ($, done) => {
+  const url = new URL('/favicon.ico', $.baseUrl).href;
+  const reqOptions = {
+    method: 'HEAD',
+  };
+
+  request(url, reqOptions, (err, res) => {
+    // ignore errors
+    if (err) return done(null, []);
+    if (res.statusCode !== 200) return done(null, []);
+    // check image size
+    if (!(parseInt(res.headers['content-length'], 10) > 0)) return done(null, []);
+
+    return done(null, [{
+      src: url,
+      type: 'image/x-icon',
+    }]);
+  });
+};

api/grabber/index.js

@@ -0,0 +1,56 @@
+const page = require('./page');
+const URL = require('url').URL;
+const normalizeUrl = require('normalize-url');
+
+module.exports = {
+  default: (domain, options, done) => {
+    const url = `http://${domain}/`;
+
+    page(url, (err, icons, baseUrl) => {
+      if (err) return done(err);
+
+      icons.forEach((icon) => {
+        const url = new URL(icon.src, baseUrl);
+        if (options.normalizeUrl) {
+          icon.src = normalizeUrl(url.href, {
+            removeQueryParameters: [/.+/],
+            stripWWW: false,
+          });
+        } else {
+          icon.src = url.href;
+        }
+      });
+
+      const data = {
+        domain,
+        icons,
+      };
+
+      return done(null, data);
+    });
+  },
+  allurl: (url, options, done) => {
+    page(url, (err, icons, baseUrl) => {
+      if (err) return done(err);
+
+      icons.forEach((icon) => {
+        const url = new URL(icon.src, baseUrl);
+        if (options.normalizeUrl) {
+          icon.src = normalizeUrl(url.href, {
+            removeQueryParameters: [/.+/],
+            stripWWW: false,
+          });
+        } else {
+          icon.src = url.href;
+        }
+      });
+
+      const data = {
+        url,
+        icons,
+      };
+
+      return done(null, data);
+    });
+  }
+}

api/grabber/links.js

@@ -0,0 +1,29 @@
+const selectors = [
+  "link[rel='icon']",
+  "link[rel='shortcut icon']",
+  "link[rel='apple-touch-icon']",
+  "link[rel='apple-touch-icon-precomposed']",
+  "link[rel='apple-touch-startup-image']",
+  "link[rel='mask-icon']",
+  "link[rel='fluid-icon']",
+];
+
+module.exports = ($) => {
+  const icons = [];
+
+  selectors.forEach((selector) => {
+    $(selector).each((i, elem) => {
+      const { href, sizes, type } = elem.attribs;
+      if (href && href !== '#') {
+        const icon = {
+          sizes,
+          src: href,
+          type,
+        };
+        icons.push(icon);
+      }
+    });
+  });
+
+  return icons;
+};

api/grabber/manifest.js

@@ -0,0 +1,28 @@
+const request = require('./request.conf');
+const URL = require('url').URL;
+
+module.exports = ($, done) => {
+  const href = $('link[rel="manifest"]', 'head').attr('href');
+  if (!href) return done(null, []);
+
+  const url = new URL(href, $.baseUrl).href;
+
+  request(url, (err, res, manifest) => {
+    // ignore errors
+    if (err) return done(null, []);
+    if (res.statusCode !== 200) return done(null, []);
+
+    let icons = [];
+    try {
+      const parsed = JSON.parse(manifest);
+      if (Array.isArray(parsed.icons)) {
+        icons = parsed.icons.map(({ src, sizes, type }) => ({ src, sizes, type })) || [];
+      }
+    } catch (err) {
+      // ignore errors
+      if (err) return done(null, []);
+    }
+
+    return done(null, icons);
+  });
+};

api/grabber/ogimage.js

@@ -0,0 +1,21 @@
+const selectors = [
+  "meta[property='og:image']",
+];
+
+module.exports = ($) => {
+  const icons = [];
+
+  selectors.forEach((selector) => {
+    $(selector).each((i, elem) => {
+      const { content } = elem.attribs;
+      if (content && content !== '#') {
+        const icon = {
+          src: content
+        };
+        icons.push(icon);
+      }
+    });
+  });
+
+  return icons;
+};

api/grabber/page.js

@@ -0,0 +1,53 @@
+const request = require('./request.conf');
+const cheerio = require('cheerio');
+const parallel = require('async/parallel');
+
+const manifest = require('./manifest');
+const favicon = require('./favicon');
+const browserconfig = require('./browserconfig');
+const links = require('./links');
+const ogimage = require('./ogimage')
+
+const grab = (fn, $) => {
+  return (cb) => {
+    fn($, (err, icons) => {
+      cb(err, icons);
+    });
+  }
+};
+
+module.exports = (url, done) => {
+  if (/\?/.test(url)) {
+    url += `&${Date.now()}`
+  } else {
+    url += `?${Date.now()}`
+  }
+  request(url, (err, res, page) => {
+    if (err) return done(err);
+
+    // if (res.statusCode !== 200) return done(null, []);
+    if (page && typeof page !== 'string' && !(page instanceof String)) return done(null, []);
+
+    const $ = cheerio.load(page, {
+      // ignore case for tags and attribute names
+      lowerCaseTags: true,
+      lowerCaseAttributeNames: true,
+    });
+
+    $.baseUrl = `${res.request.uri.protocol}//${res.request.uri.hostname}`;
+
+    let icons = links($);
+    let ogi = ogimage($)
+    icons = [...icons, ...ogi]
+
+    parallel([
+      grab(manifest, $),
+      grab(favicon, $),
+      grab(browserconfig, $),
+    ], (err, results) => {
+      // ignore errors
+      results.forEach(arr => icons = [...icons, ...arr]);
+      return done(null, icons, $.baseUrl);
+    });
+  });
+};

api/grabber/request.conf.js

@@ -0,0 +1,13 @@
+const request = require('request');
+
+module.exports = request.defaults({
+  // follow HTTP 3xx responses as redirects
+  followRedirect: true,
+  headers: {
+    'Accept': '*/*',
+    // prevent to redirect to the mobile version of a website
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36',
+  },
+  jar: true,
+  timeout: 5000,
+});

api/router.js

@@ -0,0 +1,116 @@
+const router = require('express').Router();
+const grabber = require('./grabber');
+const isDomainName = require('is-domain-name');
+const normalizeUrl = require('normalize-url');
+/**
+ * JSON in a pretty way
+ */
+
+router.all('/*', (req, res, next) => {
+  const indent = req.query.pretty === 'true' ? 2 : 0;
+  req.app.set('json spaces', indent);
+  return next();
+});
+
+router.param('domain', (req, res, next, domain) => {
+  const isCyrillic = /[а-я]+/i.test(domain);
+
+  if (isCyrillic || isDomainName(domain)) {
+    req.domain = domain;
+    return next();
+  } else {
+    return res.status(422).jsonp({
+      error: 'Invalid domain name.',
+    });
+  }
+});
+
+router.get('/grab/geticon', (req, res, next) => {
+  const options = {
+    normalizeUrl: req.query['normalize-url'] === void 0 ? true : req.query['normalize-url'] === 'true',
+  };
+
+  let url = req.query['url'] || ''
+  if (url.length === 0) {
+      return res.status(400).jsonp({
+      error: 'Unresolved domain name.',
+    });
+  }
+  url = decodeURIComponent(url)
+
+  url = normalizeUrl(url, {stripWWW: false})
+
+  grabber.allurl(url, options, (err, data) => {
+    if (err) {
+      switch (err.code) {
+        case 'ENOTFOUND':
+          return res.status(400).jsonp({
+            error: 'Unresolved domain name.',
+          });
+        case 'ETIMEDOUT':
+        case 'ESOCKETTIMEDOUT':
+          return res.status(400).jsonp({
+            error: 'The connection to a server of the domain timed out.',
+          });
+        case 'EINVAL':
+          return res.status(422).jsonp({
+            error: 'Invalid domain name.',
+          });
+        default:
+          return next(err);
+      }
+    }
+    res.status(200).jsonp(data);
+  });
+});
+
+router.get('/grab/:domain', (req, res, next) => {
+  const options = {
+    normalizeUrl: req.query['normalize-url'] === void 0 ? true : req.query['normalize-url'] === 'true',
+  };
+
+  grabber.default(req.domain, options, (err, data) => {
+    if (err) {
+      switch (err.code) {
+        case 'ENOTFOUND':
+          return res.status(400).jsonp({
+            error: 'Unresolved domain name.',
+          });
+        case 'ETIMEDOUT':
+        case 'ESOCKETTIMEDOUT':
+          return res.status(400).jsonp({
+            error: 'The connection to a server of the domain timed out.',
+          });
+        case 'EINVAL':
+          return res.status(422).jsonp({
+            error: 'Invalid domain name.',
+          });
+        default:
+          return next(err);
+      }
+    }
+    res.status(200).jsonp(data);
+  });
+});
+
+router.get('/status', (req, res) => {
+  res.status(200).jsonp({
+    status: 'OK',
+  });
+});
+
+router.use((req, res) => {
+  res.status(404).jsonp({
+    error: `Unknown API endpoint "${req.method} ${req.baseUrl}${req.url}".`,
+  });
+});
+
+router.use((err, req, res, next) => {
+  /* eslint no-unused-vars: off */
+  console.error(err);
+  res.status(500).jsonp({
+    error: 'General API error.',
+  });
+});
+
+module.exports = router;

index.js

@@ -0,0 +1,28 @@
+/**
+ * Favicon Grabber
+ */
+
+const express = require('express');
+const cors = require('cors')
+
+const routers = {
+  api: require('./api/router'),
+};
+
+const server = express();
+const NODE_PORT = parseInt(process.env.NODE_PORT, 10);
+
+server.get('/', (req, res) => {
+  res.send('ok')
+})
+
+server.use(cors({
+  origin: (origin, cb) => {
+    cb(null, !!origin)
+  }
+}))
+server.use('/api', routers.api);
+
+server.listen(NODE_PORT, () => {
+  console.log(`Server listening on port ${NODE_PORT} in ${process.env.NODE_ENV} mode...`);
+});

lib/pug-filter-markdown-it-prism.js

@@ -0,0 +1,8 @@
+const MarkdownIt = require('markdown-it');
+const prism = require('markdown-it-prism');
+
+module.exports = (markdown, options) => {
+  const md = new MarkdownIt(options);
+  md.use(prism, options.prism);
+  return md.render(markdown);
+};

package.json

@@ -0,0 +1,28 @@
+{
+  "name": "favicon-server",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "directories": {
+    "lib": "lib"
+  },
+  "scripts": {
+    "start": "node index.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "async": "^2.5.0",
+    "cheerio": "^1.0.0-rc.2",
+    "cors": "^2.8.4",
+    "express": "^4.16.3",
+    "include-media": "^1.4.9",
+    "is-domain-name": "^1.0.1",
+    "markdown-it": "^8.3.1",
+    "markdown-it-prism": "^1.1.1",
+    "normalize-url": "^2.0.1",
+    "request": "^2.81.0"
+  }
+}