Skip to content

Instantly share code, notes, and snippets.

@sergiandreplace
Created January 19, 2020 18:07
Show Gist options
  • Save sergiandreplace/ec02521a341f29578eda8b17b5d9a550 to your computer and use it in GitHub Desktop.
Save sergiandreplace/ec02521a341f29578eda8b17b5d9a550 to your computer and use it in GitHub Desktop.
var fs = require('fs');
class LineMatcher {
constructor(regex) {
this.regex = regex
}
matches(line) {
return this.regex.test(line)
}
extract(line) {
return this.regex.exec(line)[1]
}
}
const sectionMatcher = new LineMatcher(/<h2 *id="(.*)">(.*)<\/h2>/)
const titleMatcher = new LineMatcher(/<a.*?href=".*?">(.*)<\/a>/)
const urlMatcher = new LineMatcher(/<a.*?href="(.*?)">/)
const descriptionMatcher = new LineMatcher(/<p>(.*)<\/p>/)
let parseLines = function (lines) {
let section = null
let item = {}
let items = []
for (let line of lines) {
if (sectionMatcher.matches(line)) {
section = sectionMatcher.extract(line)
item = {}
} else {
if (titleMatcher.matches(line)) {
item["title"] = titleMatcher.extract(line)
}
if (urlMatcher.matches(line)) {
item["url"] = urlMatcher.extract(line)
}
if (descriptionMatcher.matches(line)) {
item["description"] = descriptionMatcher.extract(line)
}
if (item["title"] && item["url"] && item["description"]) {
item["section"] = new String(section).toLowerCase()
items.push(item)
item = {}
}
}
}
return items
}
let prepareAnswer = function (items, issue) {
console.log(`preparing ${items.length} for issue #${issue}`)
items
.forEach(element => {
element["issue"] = parseInt(issue)
});
return items;
}
let parseIssue = function (issueContent, issueNumber) {
lines = issueContent.split("\n")
let items = parseLines(lines)
console.log(`parsed lines ${items.length}`)
let articlesList = {}
articlesList.articles = prepareAnswer(items, issueNumber)
return articlesList
}
let readIssue = function (source, issue) {
return new Promise(function (resolve, reject) {
fs.readFile(source, 'utf8', function (err, data) {
if (err) {
reject(err)
} else {
resolve(parseIssue(data, issue))
}
})
})
}
let writeJson = function (data, target) {
var json = JSON.stringify(data, null, 2);
fs.writeFile(target, json, (err) => {
if (err){
console.log(`${err}`)
console.log(`Error writing ${target}`)
};
});
}
let processFile = function (source, target) {
console.log(`Transforming ${source} into ${target}`)
let issue = new String(source).split("-").slice(-1)[0]
readIssue(source, issue)
.then(data => {
writeJson(data, target)
});
return issue;
}
let writeIssuesJson = function(target, issues) {
fs.writeFile(target + "/issues.json", JSON.stringify({"issues":issues}, null, 2));
}
let processDirectory = function (source, target) {
fs.readdir(source, function (err, files) {
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
let issues = []
files.forEach(function (file) {
processFile(source + "/" + file, target + "/" + file.split(".")[0] + ".json");
console.log(file)
issues.push(parseInt(file.split("-").slice(-1)[0].split(".")[0]))
// writeIssuesJson(target, issues);
})
})
}
let transformDirectory = function (source, target) {
if (fs.exists(target, exists => {
if (exists) {
processDirectory(source, target);
} else {
fs.mkdir(target, err => processDirectory(source, target))
}
}
)
);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment