Skip to content

Instantly share code, notes, and snippets.

@anticlergygang
Created September 26, 2018 02:26
Show Gist options
  • Select an option

  • Save anticlergygang/d1eb2505de8834449c5c569b216ae790 to your computer and use it in GitHub Desktop.

Select an option

Save anticlergygang/d1eb2505de8834449c5c569b216ae790 to your computer and use it in GitHub Desktop.
Change 'currentBoard' for whatever board you want to parse, you can run this script concurrently in other child processes, just make sure the intervals don't line up exactly and that you are not making more than 1 request a second.
const https = require('https'),
colors = require('colors'),
moment = require('moment')
const catalogPromise = (board) => {
return new Promise((resolve, reject) => {
let path = `/${board}/catalog.json`
let url = `https://a.4cdn.org${path}`
let req = https.request({
host: 'a.4cdn.org',
path: path,
port: 443,
method: 'GET',
}, (res) => {
let data = ''
res.on('data', (chunk) => {
data = data.concat(chunk)
})
res.on('end', () => {
try {
let jsonRes = JSON.parse(data)
resolve(jsonRes)
} catch (err) {
reject(`failed to parse data:\n${util.inspect(data)}\n`)
}
})
})
req.on('error', (err) => {
reject(err)
})
req.end()
})
}
let currentBoard = 'pol'
let lastThreads = []
let currentThreads = undefined
setInterval(() => {
console.log('\n\n\n')
catalogPromise(currentBoard).then(pages => {
if (currentThreads !== undefined) {
lastThreads = currentThreads
}
currentThreads = []
pages.forEach((page, pageIndex) => {
page.threads.forEach((thread, threadIndex) => {
currentThreads.push([`https://boards.4chan.org/${currentBoard}/thread/${thread.no}`, (thread.replies / (((new Date().getTime() - thread.tim) / 1000) / 60)).toFixed(2)])
})
})
currentThreads.sort((a, b) => {
return a[1] - b[1]
})
currentThreads.forEach((thread, threadIndex) => {
lastThreads.forEach((lastThread, lastThreadIndex) => {
if(thread[0] === lastThread[0]){
if((threadIndex - lastThreadIndex) > 0){
console.log(`${thread[0]} ${thread[1]} ${threadIndex - lastThreadIndex}`.green)
}else if((threadIndex - lastThreadIndex) === 0){
console.log(`${thread[0]} ${thread[1]}`)
}else{
console.log(`${thread[0]} ${thread[1]} ${threadIndex - lastThreadIndex}`.red)
}
}
})
})
}).catch(err => {
console.log(err)
})
}, 10000)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment