This commit is contained in:
Jay 2018-09-24 00:07:40 +08:00
commit e9d4d30f57
8 changed files with 2571 additions and 0 deletions

9
.drone.yml Normal file
View File

@ -0,0 +1,9 @@
pipeline:
docker:
image: plugins/docker
registry: docker.mtfos.xyz
repo: docker.mtfos.xyz/mtfos/fblook
dockerfile: Dockerfile
tags: [latest, "${DRONE_COMMIT}"]
when:
branch: release

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
node_modules

6
Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM node:8-alpine
RUN apk add --no-cache curl ca-certificates
WORKDIR /data
COPY . .
RUN npm install
CMD ["npm", "start"]

4
config.js Normal file
View File

@ -0,0 +1,4 @@
module.exports = {
api_url: process.env.API_URL || '',
api_key: process.env.API_KEY || ''
}

105
facebook-parser.js Normal file
View File

@ -0,0 +1,105 @@
const request = require('request')
const cheerio = require('cheerio')
/**
* @typedef lastPost
* @prop {string} txt post body
* @prop {string} id post id
* @prop {string} link post link
* @prop {string} time timestamp
*/
/**
* get facebook fan page last post
* @param {string} pageid facebook fan page id
* @return {Promise<lastPost>}
*/
const getLastPost = async (pageid = '') => {
if (typeof pageid !== 'string' || pageid.trim().length === 0) return null
pageid = pageid.trim()
// console.log('access facebook fan page :::: ' + pageid)
let page = await new Promise((resolve) => {
request({
baseUrl: 'https://www.facebook.com',
url: `/${encodeURIComponent(pageid)}/posts`,
method: 'get',
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0'
}
}, (err, res, body) => {
if (err) {
resolve(null)
return
}
if (body && typeof body !== 'string' && !(body instanceof String)) {
resolve(null)
return
}
resolve(body)
})
})
if (page === null) return null
console.log(`${pageid} page length :::: `, Buffer.from(page).length)
let $ = cheerio.load(page, {
lowerCaseTags: true,
lowerCaseAttributeNames: true
})
let posts = []
$('div.userContentWrapper').each((i, el) => {
let t = cheerio.load(el)
let timeEl = t('abbr')
let time = timeEl.attr('data-utime')
let link = timeEl.parent().attr('href')
let p = t('div.userContent')
let txt = p.text() || ''
let id = p.first().attr('id')
if (!id) {
if (/[\?|&]id\=(\d+)/.test(link)) { // eslint-disable-line
let m = link.match(/[\?|&]story_fbid\=(\d+)/) // eslint-disable-line
if (m !== null && m.length > 1) {
id = m[1]
}
} else if (/\/posts\/(\d+)/.test(link)) {
let m = link.match(/\/posts\/(\d+)/)
if (m !== null && m.length > 1) {
id = m[1]
}
} else if (/\/photos\/.+?\/(\d+)/.test(link)) {
let m = link.match(/\/photos\/.+?\/(\d+)/)
if (m !== null && m.length > 1) {
id = m[1]
}
} else if (/\/videos\/(\d+)/.test(link)) {
let m = link.match(/\/videos\/(\d+)/)
if (m !== null && m.length > 1) {
id = m[1]
}
}
}
// console.log(time, link, txt, id)
if (!time || !link || !id) return null
let tmp = {
txt,
id,
link,
time
}
posts.push(tmp)
el = null
t = null
})
$ = null
if (posts.length === 0) return null
posts.sort((a, b) => {
return b.time - a.time
})
let post = posts[0]
post.link = `https://www.facebook.com/${post.link.replace(/^\//, '')}`
return post
}
module.exports = {
getLastPost
}

83
index.js Normal file
View File

@ -0,0 +1,83 @@
const cron = require('cron')
const axios = require('axios')
const config = require('./config')
const fbparser = require('./facebook-parser')
const getIDsUrl = '/api/private/pages'
const updatePostsUrl = '/api/private/pageposts'
const getIDsFromAPI = async () => {
try {
let apiResult = await axios({
baseURL: config.api_url,
url: getIDsUrl,
headers: {
'X-Mtfos-Key': config.api_key
},
method: 'get'
})
if (!('data' in apiResult) || !('list' in apiResult.data) || !Array.isArray(apiResult.data.list)) return
apiResult.data.list.forEach(t => {
getLastPost(t)
})
} catch (err) {
console.log('get ids fail :::: ', err)
}
}
const getLastPost = async (id = '') => {
console.log(`Start Get ${id} Page Post`)
if (typeof id !== 'string' || id.trim().length === 0) return
id = id.trim()
try {
let lastPost = await fbparser.getLastPost(id)
if (lastPost === null) {
console.log('get post empty')
return
}
console.log('get post ::: ', `PageID: ${id} / PostID: ${lastPost.id} / Time: ${lastPost.time} / Text: ${lastPost.txt}`)
if (!('id' in lastPost) || !('txt' in lastPost) || !('time' in lastPost) || !('link' in lastPost)) return
let minTime = Math.floor(Date.now() / 1000) - 1800
if (minTime > lastPost.time) return
let data = {
id,
post_id: lastPost.id,
text: lastPost.txt,
link: lastPost.link
}
try {
await axios({
baseURL: config.api_url,
url: updatePostsUrl,
method: 'post',
headers: {
'X-Mtfos-Key': config.api_key
},
data: {
pages: [data]
}
})
} catch (err) {
console.log('update post data fail', err)
}
} catch (err) {
console.log('get last post error', err)
}
}
// set fblook
new cron.CronJob({ //eslint-disable-line
cronTime: '00 */2 * * * *',
onTick: async () => {
console.log('Start Tick')
try {
await getIDsFromAPI()
} catch (err) {
console.log('run tick fail', err)
}
},
runOnInit: true,
start: true,
timeZone: 'Asia/Taipei'
})

2341
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

22
package.json Normal file
View File

@ -0,0 +1,22 @@
{
"name": "node-fblook",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node index.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^0.18.0",
"cheerio": "^1.0.0-rc.2",
"cron": "^1.4.1",
"request": "^2.88.0"
},
"devDependencies": {
"standard": "^12.0.1"
}
}