Last active
January 17, 2023 23:46
-
-
Save marcolink/9713e6d4f80c09938fc66f5d6285f785 to your computer and use it in GitHub Desktop.
Create contentful export chunks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
For very big spaces, we can create chunks of the export, and import them separately. | |
To make it work, we have to keep a specific order. | |
*/ | |
const fs = require('fs'); | |
const path = require('path'); | |
/* the order of types must be kept */ | |
const types = [ | |
'locales', | |
'contentTypes', | |
'tags', | |
'webhooks', | |
'editorInterfaces', | |
'assets', | |
'entries' | |
] | |
const chunkable = ['entries', 'assets'] | |
let sourcePath = process.argv[2]; | |
if (!sourcePath) { | |
console.error('no source file provided'); | |
process.exit(1) | |
} | |
if (!fs.existsSync(sourcePath)) { | |
console.error('file ' + sourcePath + ' not found'); | |
process.exit(1) | |
} | |
const inputData = JSON.parse(fs.readFileSync(sourcePath, 'utf8')) | |
const projectDir = path.join(process.cwd(), 'chunks', path.basename(sourcePath, '.json')) | |
if (fs.existsSync(projectDir)) { | |
fs.rmdirSync(projectDir, {recursive: true}) | |
} | |
fs.mkdirSync(projectDir, {recursive: true}) | |
const chunkSize = 50000; | |
function writeFile(fileName, content) { | |
const targetPath = path.join(projectDir, `${fileName}.json`); | |
fs.writeFileSync(targetPath, JSON.stringify(content, null, 2)) | |
} | |
let priority = 1; | |
for (const type of types) { | |
if (inputData[type]) { | |
if (chunkable.includes(type)) { | |
const chunks = Math.ceil(inputData[type].length / chunkSize); | |
for (let i = 0; i < chunks; i++) { | |
writeFile(`${priority}_${type}_${i}`, {[type]: inputData[type].splice(0, Math.min(chunkSize, inputData[type].length))}) | |
} | |
} else { | |
writeFile(`${priority}_${type}`, {[type]: inputData[type]}) | |
} | |
priority++; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Call it with the export file as single argument:
This creates a folder
/path/to/exportcontentful-export-space-id-date
that contains all the chunks.Now you can run dedicated
contentful import
commands for every chunk. It's important to keep the right order!