Last active
November 29, 2019 10:51
-
-
Save krhoyt/9b57a34d14a3de5d97f2eaed41399f69 to your computer and use it in GitHub Desktop.
Get complete list of repos and their respective details for a given GitHub user. Sorted by last updated, ascending. Output to CSV file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Holds pertinent authentiation | |
module.exports = { | |
github: { | |
token: '__your__token__here__' | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require( 'fs' ); | |
const rp = require( 'request-promise-native' ); | |
const config = require( '../configuration.js' ); | |
aggregate(); | |
async function aggregate() { | |
let ibm = []; | |
let last = 0; | |
let page = 1; | |
do { | |
console.log( 'Page: ' + page ); | |
// Get the repository list | |
// Will be paged | |
let repos = await rp( { | |
url: 'https://api.github.com/users/IBM/repos', | |
method: 'GET', | |
headers: { | |
'User-Agent': 'IBM Developer' | |
}, | |
qs: { | |
access_token: config.github.token, | |
page: page | |
}, | |
resolveWithFullResponse: true, | |
json: true | |
} ) | |
.then( ( response ) => { | |
// Page index in header | |
// Parse total pages from header | |
if( last === 0 ) { | |
const link = response.headers['link'].trim(); | |
const page = link.lastIndexOf( 'page' ) + 5; | |
const close = link.lastIndexOf( '>' ); | |
last = parseInt( link.substring( page, close ) ); | |
console.log( 'Last: ' + last ); | |
} | |
// Retrun repositories | |
return response.body; | |
} ) | |
.catch( ( error ) => { | |
console.log( 'Problem accessing API.' ); | |
console.log( error ); | |
} ); | |
// Debug | |
// console.log( repos ); | |
// Repository list results missing some data | |
// Need to get repository details specifically | |
for( let r = 0; r < repos.length; r++ ) { | |
let details = await rp( { | |
url: `https://api.github.com/repos/${repos[r].full_name}`, | |
method: 'GET', | |
headers: { | |
'User-Agent': 'IBM Developer' | |
}, | |
qs: { | |
access_token: config.github.token | |
}, | |
json: true | |
} ); | |
ibm.push( details ); | |
} | |
// Next page | |
page = page + 1; | |
// } while( page <= 3 ); | |
} while( page <= last ); | |
console.log( 'Total: ' + ibm.length ); | |
// Sort by date updated | |
ibm.sort( ( a, b ) => { | |
let a_at = new Date( a.updated_at ); | |
let b_at = new Date( b.updated_at ); | |
if( a_at.getTime() > b_at.getTime() ) return 1; | |
if( a_at.getTime() < b_at.getTime() ) return -1; | |
return 0; | |
} ); | |
// Output file | |
const file = fs.createWriteStream( 'ibmcode.csv', { | |
flags: 'a' | |
} ); | |
// Line by line | |
// Picking a few pertinent details | |
for( let i = 0; i < ibm.length; i++ ) { | |
const created_at = new Date( ibm[i].created_at ); | |
const created = `${created_at.getMonth() + 1}/${created_at.getDate()}/${created_at.getFullYear()}`; | |
const updated_at = new Date( ibm[i].updated_at ); | |
const updated = `${updated_at.getMonth() + 1}/${updated_at.getDate()}/${updated_at.getFullYear()}`; | |
const pushed_at = new Date( ibm[i].pushed_at ); | |
const pushed = `${pushed_at.getMonth() + 1}/${pushed_at.getDate()}/${pushed_at.getFullYear()}`; | |
file.write( `${ibm[i].name},${created},${updated},${pushed},${ibm[i].subscribers_count},${ibm[i].stargazers_count},${ibm[i].forks_count},${ibm[i].open_issues}\n` ); | |
} | |
// Close the file | |
file.end(); | |
console.log( 'Done.' ); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Excel = require( 'exceljs' ); | |
const fs = require( 'fs' ); | |
const rp = require( 'request-promise-native' ); | |
const config = require( './configuration.js' ); | |
report(); | |
async function report() { | |
// Read list of repos | |
// Split into array | |
const matching = fs.readFileSync( __dirname + '/patterns.txt', 'utf8' ).trim(); | |
const patterns = matching.split( '\n'); | |
// Repository details | |
let ibm = []; | |
for( let p = 0; p < patterns.length; p++ ) { | |
// Not on GitHub | |
// This is for GitHub content only | |
if( patterns[p].indexOf( 'github.com' ) === -1 ) { | |
continue; | |
} | |
// No trailing slash, thank you | |
if( patterns[p].charAt( patterns[p].length - 1 ) == '/' ) { | |
patterns[p] = patterns[p].substring( 0, patterns[p].length - 1 ); | |
} | |
// Repository full name | |
// Full name includes user account | |
const start = patterns[p].lastIndexOf( '.com/' ) + 5; | |
const name = patterns[p].substring( start ); | |
console.log( name ); | |
// Get details from GitHub API | |
let details = await rp( { | |
url: `https://api.github.com/repos/${name}`, | |
method: 'GET', | |
headers: { | |
'User-Agent': 'IBM Developer' | |
}, | |
qs: { | |
access_token: config.github.token | |
}, | |
json: true | |
} ) | |
.catch( ( error ) => { | |
if( error.statusCode == 404 ) { | |
return null; | |
} | |
} ); | |
if( details !== null ) { | |
ibm.push( details ); | |
} | |
} | |
console.log( 'Patterns: ' + ibm.length ); | |
// Sort by date updated | |
ibm.sort( ( a, b ) => { | |
let a_at = new Date( a.updated_at ); | |
let b_at = new Date( b.updated_at ); | |
if( a_at.getTime() > b_at.getTime() ) return 1; | |
if( a_at.getTime() < b_at.getTime() ) return -1; | |
return 0; | |
} ); | |
// Generate statistics | |
// Calculate manually | |
// Must prepopulate values with formulas | |
let statistics = { | |
subscribers: {sum: 0, values: [], average: 0, median: 0}, | |
stargazers: {sum: 0, values: [], average: 0, median: 0}, | |
forks: {sum: 0, values: [], average: 0, median: 0}, | |
issues: {sum: 0, values: [], average: 0, median: 0}, | |
remove: 0, | |
promote: 0 | |
}; | |
for( let i = 0; i < ibm.length; i++ ) { | |
statistics.subscribers.sum = statistics.subscribers.sum + ibm[i].subscribers_count; | |
statistics.subscribers.values.push( ibm[i].subscribers_count ); | |
statistics.stargazers.sum = statistics.stargazers.sum + ibm[i].stargazers_count; | |
statistics.stargazers.values.push( ibm[i].stargazers_count ); | |
statistics.forks.sum = statistics.forks.sum + ibm[i].forks_count; | |
statistics.forks.values.push( ibm[i].forks_count ); | |
statistics.issues.sum = statistics.issues.sum + ibm[i].open_issues; | |
statistics.issues.values.push( ibm[i].open_issues ); | |
} | |
// Average | |
statistics.subscribers.average = Math.round( statistics.subscribers.sum / ibm.length ); | |
statistics.stargazers.average = Math.round( statistics.stargazers.sum / ibm.length ); | |
statistics.forks.average = Math.round( statistics.forks.sum / ibm.length ); | |
statistics.issues.average = Math.round( statistics.issues.sum / ibm.length ); | |
// Median | |
statistics.subscribers.values.sort( numeric ); | |
statistics.subscribers.median = median( statistics.subscribers.values ); | |
statistics.stargazers.values.sort( numeric ); | |
statistics.stargazers.median = median( statistics.stargazers.values ); | |
statistics.forks.values.sort( numeric ); | |
statistics.forks.median = median( statistics.forks.values ); | |
statistics.issues.values.sort( numeric ); | |
statistics.issues.median = median( statistics.issues.values ); | |
// Setup reporting | |
let book = new Excel.Workbook(); | |
let sheet = book.addWorksheet( 'IBM Code' ); | |
sheet.columns = [ | |
{header: 'Name', width: 55, bold: true}, | |
{header: 'Created', width: 20}, | |
{header: 'Updated', width: 20}, | |
{header: 'Pushed', width: 20}, | |
{header: 'Watchers', width: 10}, | |
{header: 'Stars', width: 10}, | |
{header: 'Forks', width: 10}, | |
{header: 'Issues', width: 10}, | |
{header: 'Candidate', width: 10}, | |
]; | |
// Line by line | |
// Picking a few pertinent details | |
for( let i = 0; i < ibm.length; i++ ) { | |
// Name | |
sheet.getCell( `A${i + 2}` ).value = ibm[i].name; | |
// Created | |
const created_at = new Date( ibm[i].created_at ); | |
const created = `${created_at.getMonth() + 1}/${created_at.getDate()}/${created_at.getFullYear()}`; | |
sheet.getCell( `B${i + 2}` ).value = created; | |
// Updated | |
const updated_at = new Date( ibm[i].updated_at ); | |
const updated = `${updated_at.getMonth() + 1}/${updated_at.getDate()}/${updated_at.getFullYear()}`; | |
sheet.getCell( `C${i + 2}` ).value = updated; | |
// Pushed | |
const pushed_at = new Date( ibm[i].pushed_at ); | |
const pushed = `${pushed_at.getMonth() + 1}/${pushed_at.getDate()}/${pushed_at.getFullYear()}`; | |
sheet.getCell( `D${i + 2}` ).value = pushed; | |
// Watchers | |
// Also called subscribers in the API | |
sheet.getCell( `E${i + 2}` ).value = ibm[i].subscribers_count; | |
if( ibm[i].subscribers_count < statistics.subscribers.average ) { | |
sheet.getCell( `E${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}}; | |
} | |
if( ibm[i].subscribers_count < statistics.subscribers.median ) { | |
sheet.getCell( `E${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}}; | |
} | |
// Stargazers | |
sheet.getCell( `F${i + 2}` ).value = ibm[i].stargazers_count; | |
if( ibm[i].stargazers_count < statistics.stargazers.average ) { | |
sheet.getCell( `F${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}}; | |
} | |
if( ibm[i].stargazers_count < statistics.stargazers.median ) { | |
sheet.getCell( `F${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}}; | |
} | |
// Forks | |
sheet.getCell( `G${i + 2}` ).value = ibm[i].forks_count; | |
if( ibm[i].forks_count < statistics.forks.average ) { | |
sheet.getCell( `G${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}}; | |
} | |
if( ibm[i].forks_count < statistics.forks.median ) { | |
sheet.getCell( `G${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}}; | |
} | |
// Issues | |
sheet.getCell( `H${i + 2}` ).value = ibm[i].open_issues; | |
// Candidate for removal | |
// All measures fall below | |
if( ibm[i].subscribers_count < statistics.subscribers.median && | |
ibm[i].stargazers_count < statistics.stargazers.median && | |
ibm[i].forks_count < statistics.forks.median ) { | |
sheet.getCell( `I${i + 2}` ).value = 'X'; | |
statistics.remove = statistics.remove + 1; | |
} | |
// Outperforming | |
// All measures fall above | |
if( ibm[i].subscribers_count >= Math.round( statistics.subscribers.average ) && | |
ibm[i].stargazers_count >= Math.round( statistics.stargazers.average ) && | |
ibm[i].forks_count >= Math.round( statistics.forks.average ) ) { | |
sheet.getCell( `I${i + 2}` ).value = 'Y'; | |
statistics.promote = statistics.promote + 1; | |
} | |
} | |
// Watchers summary | |
sheet.getCell( `E${ibm.length + 2}` ).value = { | |
formula: `=MEDIAN(E2:E${ibm.length + 1})`, | |
result: statistics.subscribers.median | |
}; | |
sheet.getCell( `E${ibm.length + 3}` ).value = { | |
formula: `=ROUND(SUM(E2:E${ibm.length + 1})/${ibm.length},0)`, | |
result: statistics.subscribers.average | |
}; | |
// Stars summary | |
sheet.getCell( `F${ibm.length + 2}` ).value = { | |
formula: `=MEDIAN(F2:F${ibm.length + 1})`, | |
result: statistics.stargazers.median | |
}; | |
sheet.getCell( `F${ibm.length + 3}` ).value = { | |
formula: `=ROUND(SUM(F2:F${ibm.length + 1})/${ibm.length},0)`, | |
result: statistics.stargazers.average | |
}; | |
// Forks summary | |
sheet.getCell( `G${ibm.length + 2}` ).value = { | |
formula: `=MEDIAN(G2:G${ibm.length + 1})`, | |
result: statistics.forks.median | |
}; | |
sheet.getCell( `G${ibm.length + 3}` ).value = { | |
formula: `=ROUND(SUM(G2:G${ibm.length + 1})/${ibm.length},0)`, | |
result: statistics.forks.average | |
}; | |
// Issues summary | |
sheet.getCell( `H${ibm.length + 2}` ).value = { | |
formula: `=MEDIAN(H2:H${ibm.length + 1})`, | |
result: statistics.issues.median | |
}; | |
sheet.getCell( `H${ibm.length + 3}` ).value = { | |
formula: `=ROUND(SUM(H2:H${ibm.length + 1})/${ibm.length},0)`, | |
result: statistics.issues.average | |
}; | |
// Candidate summary | |
sheet.getCell( `I${ibm.length + 2}` ).value = { | |
formula: `=COUNTIF(I2:I${ibm.length + 1},"X")`, | |
result: statistics.remove | |
}; | |
sheet.getCell( `I${ibm.length + 3}` ).value = { | |
formula: `=COUNTIF(I2:I${ibm.length + 1},"Y")`, | |
result: statistics.promote | |
}; | |
// Summary labels | |
sheet.getCell( `D${ibm.length + 2}` ).value = 'Median'; | |
sheet.getCell( `D${ibm.length + 3}` ).value = 'Average'; | |
sheet.getCell( `J${ibm.length + 2}` ).value = 'Underperform'; | |
sheet.getCell( `J${ibm.length + 3}` ).value = 'Outperform'; | |
sheet.getCell( `E${ibm.length + 4}` ).value = 'Watchers'; | |
sheet.getCell( `F${ibm.length + 4}` ).value = 'Stars'; | |
sheet.getCell( `G${ibm.length + 4}` ).value = 'Forks'; | |
sheet.getCell( `H${ibm.length + 4}` ).value = 'Issues'; | |
// Write the file | |
await book.xlsx.writeFile( 'patterns.xlsx' ); | |
console.log( 'Done.' ); | |
} | |
function median( values ) { | |
// Odd length: Just the middle number | |
// Equal number of digits on both sides | |
let result = values[Math.floor( values.length / 2 )]; | |
// Even length: Average of two middle numbers | |
if( ( values.length % 2 ) === 0 ) { | |
const low = values.length / 2; | |
const high = low + 1; | |
result = ( values[low] + values[high] ) / 2; | |
} | |
return result; | |
} | |
// Ascending | |
function numeric( a, b ) { | |
if( a < b ) return -1; | |
if( a > b ) return 1; | |
return 0; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# List of links | |
# One per line | |
# That point to a repository hosted on GitHub | |
https://github.com/IBM/powerai-market-sentiment | |
https://github.com/IBM/connector-for-storediq/ | |
https://github.com/IBM/query-knowledge-base-with-domain-specific-documents/ | |
https://github.com/IBM/image-analysis-iot-alert | |
https://github.com/IBM/Analyze-Investment-Portfolio | |
https://github.com/IBM/xgboost-financial-predictions |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment