Skip to content

Instantly share code, notes, and snippets.

@mattdesl

mattdesl/README.md

Last active August 9, 2023 03:44
Show Gist options
  • Save mattdesl/47a2595f5ce4c60dabad2e03c4971916 to your computer and use it in GitHub Desktop.
Save mattdesl/47a2595f5ce4c60dabad2e03c4971916 to your computer and use it in GitHub Desktop.
Download ZIP
draw HTML5 canvas with voice commands and GPT-3.5
Raw
README.md

Voice Coding with iOS Shortcuts + Node.js

Instructions:

  1. Copy and paste the server.js into a new folder called genart or whatever
  2. Run npm init -y to generate a package.json
  3. Install the dependencies:
npm install body-parser openai express canvas dotenv
  1. Add .env file with your OpenAI API key:
OPENAI_API_KEY="your_key_here"
  1. Now you should be able to run the following, it will print a URL with your local IP address:
node server.js
  1. Install the Shortcut and Add to Homescreen: https://www.icloud.com/shortcuts/f7bc8aecd68b4e53b13d68208349c589
  2. Edit the Shortcut in iOS or macOS, setup dictation if needed to be able to dictate your voice into the shortcut.
  3. Still editing the shortcut, change "your_server_ip" to the IP address printed by the server
  4. Start voice coding by running the shortcut and telling it to draw things!
Raw
draw.js
import "dotenv/config.js";
import { createCanvas } from "canvas";
import { writeFile } from "fs/promises";
import express from "express";
import { Configuration, OpenAIApi } from "openai";
import bodyParser from "body-parser";
import http from "http";
import * as path from "path";
import os from "os";
// for testing
// import axios from "axios";
export const getLocalHosts = () => {
var interfaces = os.networkInterfaces();
var addresses = new Set();
for (var k in interfaces) {
for (var k2 in interfaces[k]) {
var address = interfaces[k][k2];
if (address.family === "IPv4" && !address.internal) {
addresses.add(address.address);
}
}
}
return [...addresses];
};
const configuration = new Configuration({
apiKey: process.env.OPENAI_API_KEY,
});
const openai = new OpenAIApi(configuration);
await serve();
// optionally, send an immediate test
// await test();
async function drawCode(code) {
const width = 512;
const height = 512;
const canvas = createCanvas(width, height);
const context = canvas.getContext("2d");
const src = `(() => {
return ({ context, width, height }) => {
${code
.trim()
.split("\n")
.map((n) => ` ${n}`)
.join("\n")}
};
})();`;
try {
console.log("Eval:\n" + src);
const result = eval(src);
if (typeof result === "function") {
result({ context, width, height });
} else {
throw new Error(`expected render() to be a function`);
}
} catch (err) {
console.error(err);
}
const buf = canvas.toBuffer("image/png");
await writeFile(path.resolve(process.cwd(), "canvas.js"), src);
await writeFile(path.resolve(process.cwd(), "canvas.png"), buf);
}
async function test() {
return axios.post("http://192.168.0.32:8080/draw", {
text: "fill a blue rect",
});
}
async function serve() {
const app = express();
const messages = [];
const signature = `function render ({ context, width, height }) {`;
const defaultSrc = `/* inner body, HTML5 Canvas drawing code goes here */`;
let scriptSrc = defaultSrc;
await drawCode(scriptSrc);
app.use(bodyParser.json());
app.use("/reset", async (req, res) => {
messages.length = 0;
res.end(null);
});
app.post("/draw", async (req, res) => {
console.log("Prompt:", req.body.text);
if (messages.length == 0) {
messages.push({
role: "system",
content: `Please modify the inner body of the render() function based on the user prompt, using only HTML5 Canvas API. For each new request, only return the modified inner body of the render() function:
${signature}
${scriptSrc}
}`,
});
}
messages.push({
role: "user",
content: req.body.text,
});
const schema = {
type: "object",
properties: {
code: {
type: "string",
description: "the JavaScript drawing code",
},
},
required: ["code"],
};
try {
const completion = await openai.createChatCompletion({
model: "gpt-3.5-turbo-0613",
// model: "gpt-4-0613",
messages,
max_tokens: 256,
functions: [{ name: `set_js_code`, parameters: schema }],
function_call: { name: `set_js_code` },
temperature: 0.6,
});
const message = completion.data.choices[0].message;
if (
message.function_call &&
message.function_call.name === "set_js_code"
) {
const args = message.function_call.arguments;
try {
console.log("Trying to parse JSON:", args);
let data;
let newCode;
try {
data = eval(`(() => { return ${args}; })()`);
newCode = data.code;
} catch (err) {
console.error("Could not eval() data:", err.message);
console.log("args:", JSON.stringify(args));
}
if (!newCode) {
newCode = JSON.parse(args).code;
}
scriptSrc = newCode;
await drawCode(newCode);
messages.push({
role: "assistant",
content: `Ok, I've modified the inner body of the code, and it now looks like this:\n${scriptSrc}`,
});
} catch (err) {
console.error(err);
scriptSrc = defaultSrc;
}
console.log("New script src:", scriptSrc);
}
console.log(`Conversation:`, messages);
res.status(200).json("hello world");
} catch (err) {
console.error(err.message);
res.status(400).json({ error: err.message });
}
});
app.use(express.static("src/"));
const port = 8080;
var internalIp = getLocalHosts()[0];
var altNames = ["localhost", "127.0.0.1"];
if (internalIp && altNames.indexOf(internalIp) === -1) {
altNames.unshift(internalIp);
}
http
.createServer(app)
.listen(8080, () =>
console.log(`HTTP:\nhttp://localhost:8080/\nhttp://${internalIp}:8080/\n`)
);
}
@jpescada
Copy link

Thanks for this @mattdesl!

To avoid the syntax error "Cannot use import statement outside a module", add the module type parameter to the package.json:
{"type": "module"}

And the server.js file mentioned is the draw.js in this gist.

@jpescada
Copy link

Also worth mentioning, this setup only works if you have an active subscription for the OpenAI API. (eg, Pay as you go or free trial).
Otherwise, the chat completion call returns a 429 error.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment