Created
December 15, 2021 21:53
-
-
Save kami4ka/eaf55475d4b3c8e49501ead9525be12f to your computer and use it in GitHub Desktop.
Scrape Amazon products by a keyword from a file using ScrapingAnt web scraping API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Amazon Batch Scraper - create file with a list of keywords and all products would be scraped in one CSV file | |
* | |
* Installation instructions: | |
* npm install "@scrapingant/amazon-proxy-scraper" | |
* npm install json2csv | |
* | |
*/ | |
const ProductsScraper = require("@scrapingant/amazon-proxy-scraper"); | |
const readline = require('readline'); | |
const fs = require('fs'); | |
const fsPromise = require('fs').promises; | |
const Json2csvParser = require('json2csv').Parser; | |
const KEYWORDS_FILENAME = '12_14_21.csv'; | |
const MAX_PRODUCTS_FOR_KEYWORD = 10; | |
const SCRAPINGANT_API_KEY = '<YOUR_SCRAPINGANT_API_KEY>'; | |
scrapeFromCSV().then(console.log, console.error) | |
async function scrapeFromCSV() { | |
const fileStream = fs.createReadStream(KEYWORDS_FILENAME); | |
const rl = readline.createInterface({ | |
input: fileStream, | |
crlfDelay: Infinity | |
}); | |
const products = []; | |
for await (const keyword of rl) { | |
console.log(`Scraping keyword: ${keyword}`); | |
const scraper = new ProductsScraper({ | |
"apiKey": SCRAPINGANT_API_KEY, | |
"keyword": keyword, | |
"number": MAX_PRODUCTS_FOR_KEYWORD | |
}); | |
products.push(...(await scraper.startScraping())); | |
} | |
await writeDataToCSV(`results_${KEYWORDS_FILENAME}}`, products); | |
} | |
async function writeDataToCSV(filename, productsList) { | |
const productsParser = new Json2csvParser({ | |
fields: ['title', 'price', 'savings', 'rating', 'reviews-count', 'score', 'url', 'is-sponsored', 'is-amazon-choice', 'is-discounted', 'before-discount', 'amazon-id', 'thumbnail', 'high-res-image', 'short-description', 'full-description'], | |
}); | |
return fsPromise.writeFile(filename, productsParser.parse(productsList)); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment