import { Promise } from "bluebird";
// import fs from "fs";
import { ParameterizedContext } from "koa";
import Router from "koa-router";
import { performance, PerformanceObserver } from "perf_hooks";
const perQuery = 100;
const MongoClient = Promise.promisifyAll(require("mongodb").MongoClient);
const url = process.env.MONGODB_URI || "mongodb://127.0.0.1/OrderStats";
let db: any;
const obs = new PerformanceObserver(items => {
console.log(
items.getEntries()[0]?.name,
" _:_ ",
items.getEntries()[0].duration,
);
// performance.clearMarks();
});
obs.observe({ entryTypes: ["measure"] });
const connectToDB = async () => {
db = await MongoClient.connect(url);
db = db.db(url.split("/").splice(-1)[0]);
};
export const initScript = async (
ctx: ParameterizedContext<any, Router.IRouterParamContext<any, {}>>,
) => {
await connectToDB();
performance.mark("FetchQueryStarts");
console.log(db.collection);
const result: any = await db
.collection("oldorders")
.find({})
.sort({ _id: -1 })
.skip(0 * perQuery)
.limit(1)
.toArray();
performance.mark("FetchQueryEnds");
performance.measure("FetchFromDB", "FetchQueryStarts", "FetchQueryEnds");
delete result[0]._id;
console.log("result", result[0]._id);
await createHalfMillionRecords({ ...result[0] } as any);
performance.measure("TotalTimeTaken", "FetchQueryStarts", "InsertArrayEnd");
ctx.body = result;
};
export const createHalfMillionRecords = async (toDuplicateOrder: any) => {
performance.mark("CreateArrayStarts");
const orderChunk = new Array(5000).fill(
JSON.parse(JSON.stringify(toDuplicateOrder)),
);
const chunkCollection = new Array(100).fill([...orderChunk]);
performance.mark("CreateArrayEnds");
performance.measure("CreateArray", "CreateArrayStarts", "CreateArrayEnds");
performance.mark("InsertArrayStarts");
const result = await Promise.map(
chunkCollection,
async (i, index) => {
try {
const bulk = db
.collection("oldorders")
.initializeUnorderedBulkOp();
performance.mark(`InsertArrayStarts${index}`);
i.map((order: any) => {
bulk.insert({ ...order });
});
await bulk.execute(); // insertMany is quite faster compared to create already compared
performance.mark(`InsertArrayEnds${index}`);
performance.measure(
`InsertArray in DB ${index}`,
`InsertArrayStarts${index}`,
`InsertArrayEnds${index}`,
);
return `${index}`;
} catch (e) {
console.log(e);
return `failed ${index}`;
}
},
{ concurrency: 1 },
);
performance.mark("InsertArrayEnd");
performance.measure(
"Inserting in db",
"InsertArrayStarts",
"InsertArrayEnd",
);
console.log("result", result);
};
This has been super fast compared to the above mongoose version.