Skip to content

Instantly share code, notes, and snippets.

@samn

samn/download-favs-from-json.rb

Created January 16, 2017 18:28
Show Gist options
  • Save samn/63cd6365b0f2d603148517ccaa3b5529 to your computer and use it in GitHub Desktop.
Save samn/63cd6365b0f2d603148517ccaa3b5529 to your computer and use it in GitHub Desktop.
Download ZIP
Archive your vine favorites
Raw
download-favs-from-json.rb
# This script uses the output from fetch-favs-json.rb
# (json files with data about posts you've liked) to download
# images and videos for these posts. It also constructs JSON objects
# formatted to be used in your archive downloaded from vine.
# This data will be written to posts.json
# Look for window.VINE_DATA in index.html from your vine archive and
# replace the value for the "posts" key with the data in posts.json
# You'll then be able to browse your favorites in a convenient web interface.
# Set this to false to only generate posts.json
# When true videos & thumnbnail images will be downloaded as wel.
require 'net/http'
require 'uri'
require 'json'
DOWNLOAD = true
def download(url, output_filename)
puts "Downloading #{url} to #{output_filename}"
uri = URI(url)
File.open(output_filename, 'w') do |file|
Net::HTTP.start(uri.hostname, :use_ssl => uri.scheme == 'https') do |http|
http.request_get(url) do |resp|
resp.read_body do |segment|
file.write(segment)
end
end
end
end
end
likes = []
Dir.glob('*.json').each do |f|
data = JSON.parse(IO.read(f))
likes += data['data']['records']
end
puts "Processing #{likes.length} likes"
puts "Ensuring images/ & videos/ exist"
Dir.mkdir('images') unless File.exists?('images')
Dir.mkdir('videos') unless File.exists?('videos')
posts = []
likes.each do |like|
postId = like['postId']
thumb_path = "images/#{postId}.jpg"
download(like['thumbnailUrl'], thumb_path) if DOWNLOAD
video_url = like['videoUrls'].select { |videoUrl| videoUrl['id'] == 'original' }.first['videoUrl']
raise "Empty video_url for #{postId}" if video_url.nil?
video_path = "videos/#{postId}.mp4"
download(video_url, video_path) if DOWNLOAD
post = {
loopsCount: like['loops']['count'],
repostsCount: like['reposts']['count'],
likesCount: like['likes']['count'],
commentsCount: like['comments']['count'],
description: like['description'],
entities: [],
created: like['created'],
username: like['username'],
userIdStr: like['userIdStr'],
videoDashUrl: like['videoDashUrl'],
shareUrl: like['shareUrl'],
postIdStr: like['postIdStr'],
thumbnailUrl: like['thumbnailUrl'],
permalinkUrl: like['permalinkUrl'],
avatarUrl: like['avatarUrl'],
videoLowURL: like['videoLowURL'],
videoUrl: like['videoUrl'],
videoPath: video_path,
thumbPath: thumb_path,
}
posts << post
end
File.open('posts.json', 'w') { |file| file.write(JSON.generate(posts)) }
puts 'Wrote post info for vine archive index.html to posts.json'
Raw
fetch-favs-json.rb
# Add values for the constants at the top of this file and then run it to
# retrieve information about posts you've liked on vine.
# They'll be saved in json files for each page of fetched results.
#
# From this data you can navigate to individual posts on the web (where they'll remain),
# or construct an index. download-favs-from-json.rb can use these json files to download
# the video & thumbnail of each post and emit json that can be spliced into your vine archive
# for easy browsing.
require 'net/http'
require 'uri'
require 'json'
# The easiest way to find these values is to log in to https://vine.co, go to your profile, and
# open web inspector. Look at the headers of an XHR request to vine.co and
# substitute the corresponding values below:
SESSION_ID = '' # From the 'vine-session-id' header
GUEST_ID = '' # From the 'vine-guest-id' header
# Your user id is in a few places. If you load your profile page on vine.co
# look for an XHR request to e.g. https://vine.co/api/users/profiles/vanity/<YOUR_USERNAME>
# the userId is in the response. There may also be a request made to
# https://vine.co/api/timelines/users/<YOUR_USER_ID>
USER_ID = ''
def fetch_likes(page, anchor)
uri = URI("https://vine.co/api/timelines/users/#{USER_ID}/likes?page=#{page}&anchor=#{anchor}&size=10")
req = Net::HTTP::Get.new(uri)
req['Host'] = 'vine.co'
req['Accept'] = 'application/json, text/javascript, */*; q=0.01'
req['x-vine-client'] = 'vinewww/2.1'
req['x-vine-client-ip'] = '0.0.0.0'
req['vine-session-id'] = SESSION_ID
req['vine-guest-id'] = GUEST_ID
req['X-Requested-With'] = 'XMLHttpRequest'
req['Referer'] = "https://vine.co/#{USER_ID}/likes"
req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0'
res = Net::HTTP.start(uri.hostname, :use_ssl => true) do |http|
puts req.uri
http.request(req)
end
case res
when Net::HTTPSuccess then
res.body
else
nil
end
end
page = 1
anchor = 0
loop do
puts "Fetching likes page #{page} anchor #{anchor}"
likes_body = fetch_likes(page, anchor)
if likes_body.nil?
puts "NIL RESPONSE at page #{page} anchor #{anchor}, ending"
break
end
likes = JSON.parse(likes_body)
if likes['data']['anchor'].nil?
puts "Empty anchor at page #{page}, ending"
break
end
File.open("#{page}.json", 'w') do |file|
file.write(likes_body)
end
page = likes['data']['nextPage']
anchor = likes['data']['anchor']
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment