Skip to content

Instantly share code, notes, and snippets.

@pbrumblay

pbrumblay/merge_webpagetest_csv.mjs

Created February 14, 2022 18:54
Show Gist options
  • Save pbrumblay/40540f12e3f302e4857cc2da91602a48 to your computer and use it in GitHub Desktop.
Save pbrumblay/40540f12e3f302e4857cc2da91602a48 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
merge_webpagetest_csv.mjs
import fs from 'fs';
import path from 'path';
import csv from '@fast-csv/parse';
import { Parser } from 'json2csv';
const myArgs = process.argv.slice(2);
const directoryPath = myArgs[0];
const rows = [];
const files = await fs.promises.readdir(directoryPath);
const doWork = [];
async function readFile(file) {
const fullPath = path.join(directoryPath, file);
return new Promise((resolve, reject) => {
const s = fs.createReadStream(fullPath).pipe(csv.parse({ headers: true }));
s.on('data', row => {
if(row['full_url'].startsWith('https://web-qa') || row['full_url'].startsWith('https://web-lume-main')) {
rows.push(row);
}
});
s.on('end', resolve);
s.on('error', reject);
});
}
for(const f of files) {
if(f.endsWith('_details.csv')) {
doWork.push(readFile(f));
}
}
await Promise.all(doWork);
const fields = Object.keys(rows[0]);
const json2csvParser = new Parser({fields});
const mergedcsv = json2csvParser.parse(rows);
console.log(mergedcsv);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment