Skip to content

Instantly share code, notes, and snippets.

@Kumawatlalit912

Kumawatlalit912/insertLargeCsv.js

Created February 24, 2023 06:15
Show Gist options
  • Save Kumawatlalit912/a6dfa315bf8da3c0d604d1358cfd1475 to your computer and use it in GitHub Desktop.
Save Kumawatlalit912/a6dfa315bf8da3c0d604d1358cfd1475 to your computer and use it in GitHub Desktop.
Download ZIP
how to insert large csv dataset in elastic using nodejs
Raw
insertLargeCsv.js
const { Client } = require('@elastic/elasticsearch');
const fs = require('fs');
const readline = require('readline');
const csv = require('csv-parser');
const client = new Client({ node: 'http://localhost:9200' });
const filePath = '/path/to/large/csv/file'; // Replace with the path to your CSV file
const chunkSize = 100000; // Replace with the desired chunk size
const rl = readline.createInterface({
input: fs.createReadStream(filePath),
crlfDelay: Infinity
});
let chunkIndex = 0;
let currentChunk = [];
rl.on('line', (line) => {
currentChunk.push(line);
if (currentChunk.length >= chunkSize) {
insertChunk(currentChunk, chunkIndex);
currentChunk = [];
chunkIndex++;
}
});
rl.on('close', () => {
if (currentChunk.length > 0) {
insertChunk(currentChunk, chunkIndex);
}
});
function insertChunk(chunk, index) {
const body = [];
chunk.forEach((line) => {
const fields = line.split(',');
body.push({
index: { _index: 'myindex', _id: index },
});
body.push({
title: fields[0],
// Add other fields here as needed
});
index++;
});
client.bulk({ body })
.then(response => console.log(`Chunk ${index} inserted successfully`))
.catch(error => console.log(`Error inserting chunk ${index}:`, error));
}
@Kumawatlalit912
Copy link
Author

Kumawatlalit912 commented Feb 24, 2023 •
edited
Loading

### for smaller files

const fs = require("fs");
const csv = require("csv-parser");
const { Client } = require("@elastic/elasticsearch");

const client = new Client({ node: "http://localhost:9200" });

fs.createReadStream("./hotflix.csv")
  .pipe(csv())
  .on("data", function (row) {
    client.index({
      index: "holamovies",
      body: row,
    });
  })
  .on("end", function () {
    console.log("csv inserted successfully");
  });

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment