Created
August 23, 2020 13:05
-
-
Save Caltrop256/4ea3bf11c8253cbec98206d400c8d3e2 to your computer and use it in GitHub Desktop.
Goes through subreddit and collects flair data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function I(username, ups, isSubmission) { | |
const obj = { | |
submissions: { | |
count: +!!isSubmission, | |
karma: isSubmission ? ups : 0 | |
}, | |
comments: { | |
count: +!isSubmission, | |
karma: !isSubmission ? ups : 0 | |
}, | |
users: new Map() | |
} | |
obj.users.set(username, 0); | |
return obj; | |
} | |
const snoowrap = require('snoowrap'), | |
fs = require('fs'), | |
r = new snoowrap({ | |
userAgent: 'node.js:Flair Inspection:v1.0.0 (by /u/caltrop_)', | |
clientId: 'client id here', | |
clientSecret: 'client secret here', | |
username: 'your reddit username', | |
password: 'your reddit passsword' | |
}), | |
sanitise = s => String(s || 'Unflaired').replace(/:[a-zA-Z0-9_-]+: ?/g, ''); | |
r.config({ | |
requestDelay: 1000, | |
continueAfterRatelimitError: true, | |
retryErrorCodes: [500, 502, 503, 504, 522], | |
maxRetryAttempts: Infinity, | |
warnings: true, | |
}); | |
let ideologies = new Map(), | |
uniqueOfType = I('_setup', 0, false); | |
uniqueOfType.comments.count = 0; | |
uniqueOfType.users.delete('_setup'); | |
const saves = fs.readdirSync('./save'); | |
for (const s of saves) { | |
fs.unlinkSync('./save/' + s); | |
} | |
time = ms => { | |
const o = { | |
d: ~~(ms / 86400000), | |
h: ~~(ms / 3600000) % 24, | |
m: ~~(ms / 60000) % 60, | |
s: ~~(ms / 1000) % 60, | |
} | |
let str = ''; | |
for (const t in o) | |
if (o[t]) str += o[t] + t + ' '; | |
return str; | |
} | |
(async function main() { | |
const subreddit = await (await r.getSubreddit('polcompball')).fetch(), | |
submissions = [], | |
absLimit = 200, | |
options = { | |
limit: 100 | |
} | |
while (true) { | |
console.log(`Fetching submissions... ${(submissions.length)} fetched.`) | |
const _subs = await subreddit.getNew(options); | |
submissions.push(..._subs); | |
if (!_subs.length || submissions.length >= absLimit) { | |
break; | |
} | |
options.after = _subs[_subs.length - 1].name; | |
} | |
const names = new Set(submissions.map(s => s.name)); | |
console.log(names.size); | |
const start = Date.now(); | |
console.log(`Done! Fetched ${submissions.length} submissions!`); | |
for (let i = 0; i < submissions.length; ++i) { | |
try { | |
process.stdout.write(`${'[' + '='.repeat(i / submissions.length * 20).padEnd(20, '.') + ']'} ${(i / submissions.length * 100).toFixed(2)}%. ${time(Date.now() - start)}`); | |
handleInfo(submissions[i], true); | |
const cs = await submissions[i].expandReplies().comments, | |
Acs = Array.from(cs); | |
comments = [...Acs]; | |
function flattenCommentTree(c) { | |
const arr = Array.from(c.replies); | |
if (arr.length) { | |
for (let i = 0; i < arr.length; ++i) { | |
comments.push(arr[i]); | |
flattenCommentTree(arr[i]); | |
} | |
} | |
} | |
for (let i = 0; i < Acs.length; ++i) { | |
flattenCommentTree(Acs[i]); | |
} | |
for (let j = 0; j < comments.length; ++j) { | |
handleInfo(comments[j], false); | |
} | |
//exportData(i); | |
process.stdout.write(` Handled ${comments.length} comments!\n`); | |
} catch (e) { | |
console.log(`Encountered Error at submission ${submissions[i].name}!`); | |
console.error(e); | |
} | |
}; | |
console.log('Finished!!!! Exporting data!'); | |
exportData(submissions.length); | |
})(); | |
function handleInfo(sub, isSubmission) { | |
const ideology = sanitise(sub.author_flair_text), | |
name = sub.author.name, | |
type = isSubmission ? 'submissions' : 'comments'; | |
uniqueOfType[type].count++; | |
uniqueOfType[type].karma += sub.ups; | |
const totalUser = uniqueOfType.users.get(name) || { | |
submissions: { | |
count: 0, | |
karma: 0 | |
}, | |
comments: { | |
count: 0, | |
karma: 0 | |
} | |
} | |
totalUser[type].count++; | |
totalUser[type].karma += sub.ups; | |
uniqueOfType.users.set(name, totalUser); | |
if (!ideologies.has(ideology)) ideologies.set(ideology, I(name, sub.ups, isSubmission)); | |
const info = ideologies.get(ideology); | |
info.users.set(name, totalUser); | |
info[type].count++; | |
info[type].karma += sub.ups; | |
} | |
function exportData(n) { | |
const flairData = { | |
total: { | |
submissions: uniqueOfType.submissions, | |
comments: uniqueOfType.comments, | |
users: uniqueOfType.users.size | |
}, | |
ideologies: {} | |
}, | |
userData = { | |
users: [] | |
}; | |
const ideologyArray = Array.from(ideologies).sort((a, b) => b[1].users.size - a[1].users.size); | |
for (let i = 0; i < ideologyArray.length; ++i) { | |
const ideo = ideologyArray[i][1]; | |
flairData.ideologies[ideologyArray[i][0]] = { | |
submissions: ideo.submissions, | |
comments: ideo.comments, | |
users: ideo.users.size | |
}; | |
const users = Array.from(ideo.users); | |
for (let j = 0; j < users.length; ++j) { | |
userData.users.push({ | |
name: users[j][0], | |
submissions: users[j][1].submissions, | |
comments: users[j][1].comments, | |
flair: ideologyArray[i][0] | |
}) | |
} | |
}; | |
userData.users = userData.users.sort((a, b) => (b.submissions.karma + b.comments.karma) - (a.submissions.karma + a.comments.karma)); | |
fs.writeFileSync('./save/flairs_' + n + '.json', JSON.stringify(flairData)); | |
fs.writeFileSync('./save/users_' + n + '.json', JSON.stringify(userData)); | |
delete uniqueOfType; | |
delete ideologies; | |
uniqueOfType = I('_setup', 0, false); | |
uniqueOfType.comments.count = 0; | |
uniqueOfType.users.delete('_setup'); | |
ideologies = new Map(); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "polcomballflairs", | |
"version": "1.0.0", | |
"description": "", | |
"main": "app.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"keywords": [], | |
"author": "", | |
"license": "ISC", | |
"dependencies": { | |
"snoowrap": "^1.21.0" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment