Last active
August 10, 2020 13:58
-
-
Save clowestab/e1b9010deef2875cb4e33005f1a01c75 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const axios = require('axios').default; | |
const mysql = require('mysql'); | |
const util = require('util'); | |
const slugify = require('slugify'); | |
const GhostAdminAPI = require('@tryghost/admin-api'); | |
const Downloader = require('nodejs-file-downloader'); | |
//Helper function to setup a database wrapper that works with promises | |
//https://codeburst.io/node-js-mysql-and-async-await-6fb25b01b628 | |
function makeDatabase() { | |
console.log(getDate() + "Running with !! PRODUCTION !! database"); | |
const host = '0.0.0.0'; | |
const connection = mysql.createConnection({ | |
host : host, | |
user : 'username', | |
password : 'password', | |
database : 'database_name', | |
charset : 'utf8mb4' | |
}); | |
return { | |
query( sql, args ) { | |
return util.promisify( connection.query ) | |
.call( connection, sql, args ); | |
}, | |
close() { | |
return util.promisify( connection.end ).call( connection ); | |
}, | |
beginTransaction() { | |
return util.promisify( connection.beginTransaction ) | |
.call( connection ); | |
}, | |
commit() { | |
return util.promisify( connection.commit ) | |
.call( connection ); | |
}, | |
rollback() { | |
return util.promisify( connection.rollback ) | |
.call( connection ); | |
} | |
}; | |
} | |
//Helper function that just returns a formatted date. Used for console output. | |
function getDate() { | |
return new Date().toISOString() + ": "; | |
} | |
//Promisify our database connection | |
const db = makeDatabase(); | |
// Ghost API Connection | |
const api = new GhostAdminAPI({ | |
url: 'https://thomasclowes.com', | |
version: "v3", | |
key: 'ghost-api-key' | |
}); | |
const instagramToken = "user-instagram-token"; | |
async function pullInstagramPosts() { | |
const url = "https://graph.instagram.com/me/media?fields=id,caption,media_type,media_url,permalink,thumbnail_url,timestamp,children{id,media_type,media_url,permalink,thumbnail_url,timestamp}&access_token=" + instagramToken; | |
var response = await axios.get(url); | |
const responseData = response.data; | |
const posts = responseData.data; | |
const paging = responseData.paging; | |
var postData = []; | |
for (post of posts) { | |
console.log(post); | |
postData.push([ | |
post.id, | |
post.id, | |
post.caption, | |
post.media_type, | |
post.media_url, | |
post.permalink, | |
post.timestamp | |
]); | |
if ('children' in post) { | |
console.log("Parent " + post.id); | |
for (child of post.children.data) { | |
const childId = child.id; | |
var caption = null; | |
postData.push([ | |
post.id, | |
child.id, | |
caption, | |
child.media_type, | |
child.media_url, | |
child.permalink, | |
child.timestamp | |
]); | |
console.log("should get " + childId); | |
} | |
} | |
} | |
const insertSql = "INSERT IGNORE INTO `ig_posts` (`parent_instagram_id`, `instagram_id`, `caption`, `media_type`, `media_url`, `permalink`, `datetime`) VALUES ?"; | |
//Insert the Strava activity row | |
await db.query(insertSql, [postData]); | |
} | |
//Helper function that returns an appropriate mobiledoc card for the post type | |
function getAppropriateCard(post, uploadedMediaUrl) { | |
if (post.media_type == "VIDEO") { | |
return ["html",{"html":"<p><video width=\"100%\" controls><source src=\"" + uploadedMediaUrl + "\" type=\"video/mp4\">Your browser does not support the video tag.</video></p>"}]; | |
} else { | |
return ["image",{"src": uploadedMediaUrl,"alt":"","title":""}]; | |
} | |
} | |
async function doWork() { | |
await pullInstagramPosts(); | |
//Get all the Instagram photos/videos that we have get to post to the blog | |
const posts = await db.query("SELECT * FROM ig_posts WHERE postId IS NULL ORDER BY parent_instagram_id DESC"); | |
const processedPosts = {}; | |
var mediaIndex = 0; | |
var lastParentId = null; | |
//Loop through each one | |
for (post of posts) { | |
//Download the media to our server | |
const parts = post.media_url.split('/'); | |
const fileName = parts[parts.length - 1].split('?')[0]; | |
const downloader = new Downloader({ | |
url: post.media_url,//If the file name already exists, a new file with the name 200MB1.zip is created. | |
directory: "./ig-images",//This folder will be created, if it doesn't exist. | |
fileName: fileName | |
}) | |
await downloader.download();//Downloader.download() returns a promise. | |
//Upload it to the Ghost blog file directory | |
const upload = await api.images.upload({ | |
file: "ig-images/" + fileName, | |
}) | |
//This is the return uploaded media url | |
const uploadedMediaUrl = upload.url; | |
//If the parent Instagram ID differs to the last one then we are working in relation to a different parent post | |
if (post.parent_instagram_id != lastParentId) { | |
//Reset our media index | |
mediaIndex = 0; | |
lastParentId = post.parent_instagram_id; | |
} | |
if (post.parent_instagram_id in processedPosts) { | |
processedPosts[post.parent_instagram_id]["media_urls"].push(post.media_url); | |
processedPosts[post.parent_instagram_id]["rowIds"].push(post.ID); | |
//Mobiledoc data | |
processedPosts[post.parent_instagram_id]["cards"].push(getAppropriateCard(post, uploadedMediaUrl)); | |
processedPosts[post.parent_instagram_id]["sections"].push([10, mediaIndex]); | |
mediaIndex++; | |
//This is the first media for this parent ID | |
} else { | |
const caption = post.caption != null ? post.caption.replace(/(?:\r\n|\r|\n)/g, '<br>') : ""; | |
const captionMobileDoc = ["html",{"html": caption}]; | |
var captionLines = post.caption.split("\n") | |
captionLines = captionLines.filter(function(e){return e}); | |
console.log(captionLines); | |
var mobileDocCards = []; | |
var mobileDocSections = []; | |
for (line of captionLines) { | |
mobileDocSections.push([1,"p",[[0,[],0,line]]]); | |
} | |
if (post.media_type != "CAROUSEL_ALBUM") { | |
mobileDocCards.push(getAppropriateCard(post, uploadedMediaUrl)); | |
mobileDocSections.push([10, mediaIndex]); | |
mediaIndex++; | |
} | |
var indexOfFullStop = caption.indexOf('.'); | |
var indexOfNewLine = caption.indexOf('\n'); | |
indexOfFullStop = indexOfFullStop !== -1 ? indexOfFullStop : 1000; | |
indexOfNewLine = indexOfNewLine !== -1 ? indexOfNewLine : 1000; | |
const indexOf = Math.min(indexOfFullStop, indexOfNewLine); | |
const postTitle = post.caption.substr(0, indexOf !== 1000 ? indexOf : caption.length); | |
const postSlug = slugify("Instagram " + postTitle); | |
var postTags = []; | |
var matchedTags = caption.match(/#[A-Za-z0-9\-]+/gi); | |
matchedTags = matchedTags != null && matchedTags.length > 0 ? matchedTags.map(tag => tag.replace("#", "")) : []; | |
postTags.push("instagram"); | |
postTags = postTags.concat(matchedTags); | |
console.log(post.datetime.toISOString()); | |
var publishedAt = post.datetime.toISOString(); | |
processedPosts[post.parent_instagram_id] = {}; | |
processedPosts[post.parent_instagram_id]["featureImage"] = uploadedMediaUrl; | |
processedPosts[post.parent_instagram_id]["publishedAt"] = publishedAt; | |
processedPosts[post.parent_instagram_id]["postTitle"] = postTitle; | |
processedPosts[post.parent_instagram_id]["postCaption"] = post.caption; | |
processedPosts[post.parent_instagram_id]["postSlug"] = postSlug; | |
processedPosts[post.parent_instagram_id]["postTags"] = postTags; | |
processedPosts[post.parent_instagram_id]["permalink"] = post.permalink; | |
processedPosts[post.parent_instagram_id]["caption"] = post.caption; | |
processedPosts[post.parent_instagram_id]["media_urls"] = [post.media_url]; | |
processedPosts[post.parent_instagram_id]["rowIds"] = [post.ID]; | |
//Mobiledoc data | |
processedPosts[post.parent_instagram_id]["cards"] = mobileDocCards; | |
processedPosts[post.parent_instagram_id]["sections"] = mobileDocSections; | |
} | |
} | |
//We will now submit the posts to the blog.. | |
//Loop through our processed posts | |
for (instagramParentId of Object.keys(processedPosts)) { | |
const processedPost = processedPosts[instagramParentId]; | |
let featureImage = processedPost["featureImage"]; | |
let publishedAt = processedPost["publishedAt"]; | |
let postTitle = processedPost["postTitle"]; | |
let postCaption = processedPost["postCaption"]; | |
let postSlug = processedPost["postSlug"]; | |
let postTags = processedPost["postTags"]; | |
let permalink = processedPost["permalink"]; | |
let rowIds = processedPost["rowIds"]; | |
let cards = processedPost["cards"]; | |
let sections = processedPost["sections"]; | |
//Add the attribution card | |
cards.push(["markdown",{"markdown":"*This post was originally posted on [my Instagram](" + permalink + ").*"}]); | |
sections.push([10, cards.length - 1]); | |
console.log(cards); | |
console.log(sections); | |
const mobiledoc = { | |
version: "0.3.1", | |
atoms: [], | |
markups: [], | |
cards: cards, | |
sections: sections, | |
} | |
const res = await api.posts | |
.add( | |
{ | |
title: postTitle, | |
slug: postSlug, | |
tags: postTags, | |
meta_description: postCaption, | |
custom_excerpt: postCaption.substring(0, 300), //Apparently there is a 300 character limit on this | |
meta_title: postTitle, | |
feature_image: featureImage, | |
status: "published", | |
published_at: publishedAt, | |
created_at: publishedAt, | |
updated_at: publishedAt, | |
mobiledoc: JSON.stringify(mobiledoc) | |
}, | |
) | |
const postId = res.id; | |
console.log("POSTED: " + postId); | |
//Mark the media rows submitted as part of this post as complete | |
const updateStravaAccessTokenSql = "UPDATE ig_posts SET postId = ? WHERE ID IN (?)"; | |
const updateResponse = await db.query(updateStravaAccessTokenSql, [postId, rowIds]); | |
console.log("MARKED AS POSTED"); | |
} | |
} | |
doWork(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment