Created
April 22, 2022 03:44
-
-
Save joshtwist/c977f10867c1090a18904ee9078abd22 to your computer and use it in GitHub Desktop.
Writing to BigQuery from Zuplo
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import { getTokenFromGCPServiceAccount } from "@sagi.io/workers-jwt"; | |
| import { Logger, ZuploContext } from "@zuplo/runtime"; | |
| const aud = "https://bigquery.googleapis.com/"; | |
| export class BQWriter<T> { | |
| constructor(serviceAccountJson: string, projectId: string, datasetId: string, tableId: string) { | |
| this.#serviceAccountJson = JSON.parse(serviceAccountJson); | |
| this.#insertUrl = `${aud}bigquery/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`; | |
| } | |
| #serviceAccountJson: string; | |
| #insertUrl: string; | |
| insertRows = async (entries: T[], context: ZuploContext) => { | |
| // TODO - this could likely be reused for some time | |
| const token = await getTokenFromGCPServiceAccount({ | |
| serviceAccountJSON: this.#serviceAccountJson, | |
| aud, | |
| }); | |
| context.log.info('token', token); | |
| const items = [...entries]; | |
| const rows = items.map((i) => { | |
| return { | |
| json: { ...i }, | |
| }; | |
| }); | |
| const payload = { | |
| kind: "bigquery#tableDataInsertAllResponse", | |
| rows, | |
| }; | |
| const result = await fetch(this.#insertUrl, { | |
| method: "POST", | |
| body: JSON.stringify(payload), | |
| headers: { | |
| "content-type": "application/json;charset=UTF-8", | |
| Authorization: "Bearer " + token, | |
| }, | |
| }); | |
| if (!result.ok) { | |
| const body = await result.text(); | |
| const err = new Error(`${result.status} - ${body}`); | |
| throw err; | |
| } | |
| return result; | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment