Last active
July 6, 2024 05:02
-
-
Save nestarz/1fa7ae93fb83f1eafb1b88c3a84f2e02 to your computer and use it in GitHub Desktop.
NDJSON File Streaming using browser Stream API + Loading Progress
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const parseJSON = () => | |
new TransformStream({ | |
transform(chunk, controller) { | |
controller.enqueue(JSON.parse(chunk)); | |
} | |
}); | |
const splitStream = splitOn => { | |
let buffer = ""; | |
return new TransformStream({ | |
transform(chunk, controller) { | |
buffer += chunk; | |
const parts = buffer.split(splitOn); | |
parts.slice(0, -1).forEach(part => controller.enqueue(part)); | |
buffer = parts[parts.length - 1]; | |
}, | |
flush(controller) { | |
if (buffer) controller.enqueue(buffer); | |
} | |
}); | |
}; | |
const fetchJSONLD = url => | |
fetch(url).then(response => ({ | |
response, | |
reader: response.body | |
.pipeThrough(new TextDecoderStream()) | |
// Needed to stream by line and then JSON parse the line | |
.pipeThrough(splitStream("\n")) | |
.pipeThrough(parseJSON()) | |
.getReader() | |
})); | |
export { fetchJSONLD }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(async () => { | |
const { fetchNDJSON } = await import("/fetchNDJSON.js"); | |
const results = []; | |
let progress = 0; | |
fetchJSONLD(file).then(({ response, reader }) => { | |
const length = response.headers.get("Content-Length"); | |
let received = 0; | |
const onReadChunk = async chunk => { | |
if (chunk.done) { | |
return; | |
} | |
// await timeout(1000); | |
received += chunk.value.length; | |
results.push(chunk.value); | |
progress = received / length; | |
reader.read().then(onReadChunk); | |
}; | |
reader.read().then(onReadChunk); | |
}); | |
})(); |
MDN provides a polyfill if it suits you for TextDecoderStream
https://developer.mozilla.org/en-US/docs/Web/API/TransformStream#polyfilling_textencoderstream_and_textdecoderstream
Here is a version using TextDecoder only :
const splitOn = "\n";
const newNdJsonStream = () => {
let buffer = "";
const decoder = new TextDecoder();
return new TransformStream<Uint8Array, object>({
async transform(chunk, controller) {
chunk = await chunk;
// we use `stream: true` in case the chunk ends at the middle of a character
const string = decoder.decode(chunk, { stream: true });
buffer += string;
const parts = buffer.split(splitOn);
parts
.slice(0, -1)
.forEach((part) => controller.enqueue(JSON.parse(part) as object));
// Saving the last line because it may not be ended yet
buffer = parts[parts.length - 1];
},
flush(controller) {
if (buffer) controller.enqueue(JSON.parse(buffer) as object);
},
});
};
export const fetchNdJson = (url: string) =>
fetch(url).then((response) => ({
response,
reader: response.body.pipeThrough(newNdJsonStream()).getReader(),
}));
Overall thank you very much for this @nestarz , it is very useful !
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Note: This does not work in Firefox because
TextDecoderStream
is not supported by firefox https://developer.mozilla.org/en-US/docs/Web/API/TextDecoderStream#browser_compatibility