Run this example in a full screen, or click "Open".
references:
- Efficiently loading massive D3 datasets using Apache Arrow (Chris Price, scottlogic.com)
- regl (GitHub)
Run this example in a full screen, or click "Open".
references:
<!DOCTYPE html> | |
<html> | |
<head> | |
<meta charset=utf-8> | |
<style> | |
html,body{ | |
height: 100%; | |
margin: 0; | |
} | |
.axis path { | |
display: none; | |
} | |
.axis line { | |
stroke-opacity: 0.1; | |
shape-rendering: crispEdges; | |
} | |
svg, | |
#canvas { | |
position: absolute; | |
width: 100%; | |
height: 100%; | |
} | |
#debug { | |
z-index: 10; | |
} | |
</style> | |
</head> | |
<body> | |
<div id="canvas"></div> | |
<div id='debug' style='padding:4px;background-color:#fffc;position:absolute;right:0;top:0;color:#0af;font-family:courier;font-size:12px;user-select:none'></div> | |
</body> | |
<script src="https://npmcdn.com/regl/dist/regl.min.js"></script> | |
<script src="https://d3js.org/d3.v7.min.js"></script> | |
<script src="https://cdn.jsdelivr.net/npm/apache-arrow/Arrow.es2015.min.js"></script> | |
<script> | |
const pointSize = 6, container = "#canvas"; | |
const scaleInitial = 4, nTick = 10, tickPadding = {x: 12, y: 6}; | |
const BYTESIZE = 1 // GL_UNSIGNED_BYTE | |
const FLOATSIZE = 4 // GL_FLOAT | |
const THOUSAND = 1000; | |
const maxPoints = 1 * THOUSAND * THOUSAND | |
const canvas = document.querySelector(container); | |
const svg = d3.select(container).append('svg'); | |
const gX = svg.append("g").attr("class", "axis axis--x"); | |
const gY = svg.append("g").attr("class", "axis axis--y"); | |
const regl = createREGL({container: canvas}); | |
const RGBASIZE = BYTESIZE * 4; // vec4 (r, g, b, a); uint8 is normalized and converted to float in regl(). | |
const reglData = { | |
count: 0, // the number of points to draw | |
offset: 0, | |
transform: {}, // set by reglTransform() | |
x: regl.buffer({ | |
usage: 'dynamic', | |
type: 'float', | |
length: FLOATSIZE * maxPoints | |
}), | |
y: regl.buffer({ | |
usage: 'dynamic', | |
type: 'float', | |
length: FLOATSIZE * maxPoints | |
}), | |
color: regl.buffer({ | |
usage: 'dynamic', | |
type: 'uint8', | |
length: RGBASIZE * maxPoints | |
}), | |
} | |
const drawNewPoints = (newPoints) => { | |
const max = (a, b) => a>b?a:b; | |
const numNew = newPoints.n; | |
if (numNew > maxPoints) return; | |
if (numNew + reglData.offset > maxPoints) { | |
reglData.count = reglData.offset; | |
reglData.offset = 0; | |
} | |
reglData.x.subdata(newPoints.x, reglData.offset * FLOATSIZE); | |
reglData.y.subdata(newPoints.y, reglData.offset * FLOATSIZE); | |
reglData.color.subdata(newPoints.color, reglData.offset * RGBASIZE); | |
reglData.offset += numNew | |
reglData.count = max(reglData.count, reglData.offset); | |
displayDebug(`total = ${reglData.count.toLocaleString()}`); | |
redrawRequested = true; | |
} | |
const drawPoints = () => { | |
const drawReglData = regl({ | |
profile: true, | |
depth: {enable: false}, | |
stencil: {enable: false}, | |
primitive: 'points', | |
count: regl.prop('count'), | |
frag: ` | |
precision mediump float; | |
varying vec4 fill; | |
void main() { | |
gl_FragColor = fill; | |
} | |
`, | |
vert: ` | |
precision mediump float; | |
attribute float x; | |
attribute float y; | |
attribute vec4 color; | |
uniform vec2 scale; | |
uniform vec2 offset; | |
uniform float pointSize; | |
varying vec4 fill; | |
void main() { | |
gl_PointSize = pointSize; | |
gl_Position = vec4(vec2(x, y)*scale+offset, 0, 1); | |
fill = color; | |
} | |
`, | |
uniforms: { | |
scale: regl.prop('transform.scale'), | |
offset: regl.prop('transform.offset'), | |
pointSize: pointSize, | |
}, | |
attributes: { | |
x: { // float | |
buffer: regl.prop('x'), | |
}, | |
y: { // float | |
buffer: regl.prop('y'), | |
}, | |
color: { // vec4 (r, g, b, a) | |
buffer: regl.prop('color'), | |
normalized: true, // uint8 is normalized and converted to float | |
} | |
}, | |
}); | |
regl.clear({depth: 1}); | |
drawReglData(reglData); | |
} | |
let d3Transform = transformInitial(); | |
let [widthLatest, heightLatest] = widthHeight(canvas); | |
let redrawRequested = false; | |
render(); | |
window.addEventListener('resize', resizeRender); | |
regl.frame(()=>{ | |
if (redrawRequested) {drawPoints(); redrawRequested = false;} | |
}); | |
function render() { | |
const [width, height] = widthHeight(canvas); | |
const xScale = d3.scaleLinear() | |
.domain([-width / 2, width / 2]) | |
.range([0, width]); | |
const yScale = d3.scaleLinear() | |
.domain([-height / 2, height / 2]) | |
.range([height, 0]); | |
const xAxis = d3.axisBottom(xScale) | |
.ticks(width / height * nTick) | |
.tickSize(height) | |
.tickPadding(-tickPadding.x); | |
const yAxis = d3.axisRight(yScale) | |
.ticks(nTick) | |
.tickSize(width) | |
.tickPadding(-width + tickPadding.y); | |
const zoomed = (event, _) => { | |
d3Transform = event.transform; | |
gX.call(xAxis.scale(d3Transform.rescaleX(xScale))); | |
gY.call(yAxis.scale(d3Transform.rescaleY(yScale))); | |
reglData.transform = reglTransform(); | |
redrawRequested = true; | |
} | |
const zoom = d3.zoom().on("zoom", zoomed); | |
svg.call(zoom).call(zoom.transform, d3Transform); | |
}; | |
function resizeRender() { | |
const [width, height] = widthHeight(canvas); | |
const v = (1/d3Transform.k-1)/2; | |
d3Transform = d3Transform.translate((width-widthLatest)*v, (height-heightLatest)*v); | |
[widthLatest, heightLatest] = [width, height]; | |
render(); | |
} | |
function reglTransform() { | |
const [width, height] = widthHeight(canvas); | |
const scale = [d3Transform.k/width*2, d3Transform.k/height*2]; | |
const offset = [d3Transform.x/width*2 + (d3Transform.k-1), -d3Transform.y/height*2 - (d3Transform.k-1)]; | |
return {scale: scale, offset: offset}; | |
} | |
function transformInitial(k=scaleInitial) { | |
const [width, height] = widthHeight(canvas); | |
return new d3.ZoomTransform(k, -width/2*(k-1), -height/2*(k-1)); | |
} | |
function widthHeight(canvas) { | |
const r = canvas.getBoundingClientRect(); | |
return [r.width, r.height]; | |
} | |
function displayDebug(str) { | |
document.getElementById("debug").innerText = str; | |
} | |
const url = "https://raw.githubusercontent.com/chrisprice/d3fc-webgl-hathi-explorer/master/data.arrows"; | |
const loadData = async (url) => { | |
const response = await fetch(url); | |
const reader = await Arrow.RecordBatchReader.from(response); | |
await reader.open(); | |
for await (const batch of reader) { | |
drawNewPoints(createPoints(batch)); | |
} | |
}; | |
loadData(url); | |
function getValues(arrowBatch, columnName) { | |
const i = arrowBatch.schema.fields.map(f=>f.name).indexOf(columnName); | |
if (i < 0) return; | |
return arrowBatch.data.children[i].values; | |
} | |
function createPoints(arrowBatch) { | |
const rngRGB = () => d3.hsl(Math.random() * 300 - 60, 1, 0.3 + Math.random()*0.3).rgb(); | |
const n = arrowBatch.numRows; | |
const data = { | |
n: n, | |
x: getValues(arrowBatch, 'x'), | |
y: getValues(arrowBatch, 'y'), | |
color: new Uint8Array(n*4), // vec4 (r, g, b, a); uint8 is normalized and converted to float in regl(). | |
} | |
for (let i = 0; i < n; i++) { | |
const {r, g, b} = rngRGB(); | |
data.color[i*4] = r; | |
data.color[i*4+1] = g; | |
data.color[i*4+2] = b; | |
data.color[i*4+3] = 255; | |
} | |
return data; | |
} | |
</script> | |
</html> |