Skip to content

Instantly share code, notes, and snippets.

@0x4E69676874466F78
Forked from archagon/gdc-downloader.py
Created August 5, 2013 13:53
Show Gist options
  • Save 0x4E69676874466F78/6156072 to your computer and use it in GitHub Desktop.
Save 0x4E69676874466F78/6156072 to your computer and use it in GitHub Desktop.
# GDC Vault videos can't be watched on mobile devices and this is a very sad thing indeed!
# (Note: this has changed for GDC2013, which lets you watch raw MP4 streams. Kudos!)
# This script is designed to circumvent this by downloading the lecture and slideshow
# videos which can then be re-encoded into whatever format you wish. Obviously, you
# won't be able to do this without access to the Vault. This is strictly for the
# convenience of legitimate Vault users!
# Note: this code is rather flimsy and was written as fast as possible for my own personal use.
# The code only works for the most recent GDC Vault videos, since they all use the same player
# format. If the XML format used to run the player is changed (as it has in the past), the code
# will have to be reconfigured. In the past, I was able to feed a wget-compatible cookies.txt
# file into the wget call, but I can't get it to trigger anymore. So for now, the way I download
# each video is I look at the source for the video page, find the player.html URL, and feed
# that into the script. Ugly and slow, but hey, it works.
# I generally hate reinventing the wheel and it does look like youtube-dl does some of the same
# stuff I'm doing, but I couldn't get it to work with the GDC URLs. So off to Python land we go!!!
# Usage is as follows:
# gdc-downloader.py "[GDC player.html URL]" [output dir]
#
# A GDC video URL looks like this:
# http://www.gdcvault.com/play/1015662/Creative-Panic-How-Agility-Turned
#
# A GDC player.html URL looks like this:
# http://evt.dispeak.com/ubm/gdc/sf12/player.html?xmlURL=xml/201203238_1331124629609NXXJ.xml&token=1234567890
#
# The output dir should be the name of your video. For example, suppling TestDir/GDCVid will create
# TestDir/GDCVid/GDCVid.xml, TestDir/GDCVid/GDCVid-slide.flv, etc.
# You need to have Python 2.7 and rtmpdump installed in order for this script to work. I recommend macports.
#############
# Constants #
#############
xml_prefixes = { "default":"", "sf13":"xml/" }
rtmp_suffixes = { "default":"/fcs/ident", "gdc2009":"/ondemand" }
player_regular_expression = r"^(.*[/](.*?)[/])(.*?player\.html)(.*?xml.*?=(.*?)([&].*?)?)$"
swf_name_regular_expression = r"^.*embed the Flash Content SWF when all tests are passed.*?\"src\".*?\"(.*?)\".*$"
# DEPRECATED: This URL was retrieved from the player SWF, and may change in the future.
# rtmp_url = "rtmp://fms.digitallyspeaking.com/cfx/st/ondemand/fcs/ident"
########
# Code #
########
import sys
import os
import subprocess
import re
import argparse
from urlparse import urlparse
from urllib2 import urlopen
from urllib2 import Request
from urllib2 import HTTPError
from urllib2 import URLError
from xml.dom import minidom
def text(msg):
print "[gdc_downloader] " + msg
def error(message):
print "[gdc-downloader] Error: " + message
sys.exit(1)
def message(msg):
print "[gdc-downloader] Message: " + msg
def check_dependencies():
# TODO: check rtmpdump
pass
def dump_to_file(data, dest):
dest_dir = os.path.abspath(os.path.split(dest)[0])
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
debug_file = open(dest, "w")
debug_file.write(data)
debug_file.close()
def download_url(url):
r = Request(url=url)
message("downloading url: " + url)
try:
response = urlopen(r)
except HTTPError, e:
error("http error: " + "\"" + str(e) + "\"")
except URLError, e:
error("url error (make sure you're online): " + "\"" + str(e) + "\"")
return response.read()
def retrieve_data_from_base_url(url, xml_dest):
html = url
player_regex = re.compile(player_regular_expression)
player_results = player_regex.match(html)
if not player_results:
error("player URL not found")
dump_to_file(html, xml_dest)
base_url = player_results.group(1)
event_name = player_results.group(2)
player_url = player_results.group(3)
player_arguments = player_results.group(4)
xml_url = player_results.group(5)
full_xml_url = base_url + (xml_prefixes[event_name] if event_name in xml_prefixes else xml_prefixes["default"]) + xml_url
message("player url is " + base_url + player_url)
message("event name is " + event_name)
message("player arguments are " + player_arguments)
message("xml url is " + full_xml_url)
player_html = download_url(base_url + player_url).replace('\n', '').replace('\r', '')
swf_name_regex = re.compile(swf_name_regular_expression)
swf_name_results = swf_name_regex.match(player_html)
if not swf_name_results:
error("SWF URL not found")
swf_url = base_url + swf_name_results.group(1) + ".swf"
message("swf url is " + swf_url)
data = {}
data["player_url"] = base_url + player_url + player_arguments
data["event_name"] = event_name
data["swf_url"] = swf_url
data["xml_url"] = full_xml_url
return data
def parse_xml_from_url(url, xml_dest, event_name):
xml = download_url(url)
dump_to_file(xml, xml_dest)
parsed_xml = minidom.parseString(xml)
# GDC2013 abandons the old rtmp streams in favor of raw mp4 urls
mbr_video_tags = parsed_xml.getElementsByTagName("MBRVideo")
mp4_video_tags = parsed_xml.getElementsByTagName("mp4video")
small_video_tags = parsed_xml.getElementsByTagName("smallResolutionVideo")
if (mbr_video_tags or mp4_video_tags or small_video_tags): # list the mp4 urls without downloading
text("")
text("Found raw mp4 urls, please download one of these with your favorite browser:")
mp4_urls = []
base_mp4_url = ""
# the "mp4video" tag is likely to exist, and will provide a base url for the "MBRVideo" tags
if mp4_video_tags:
if mp4_video_tags[0].firstChild is not None:
parsed_mp4_url = urlparse(mp4_video_tags[0].firstChild.nodeValue)
if not parsed_mp4_url.netloc:
message("base mp4 url not found, please do some digging or contact the gist author")
else:
base_mp4_url = "http://" + parsed_mp4_url.netloc + "/"
text(" * base mp4 url is " + base_mp4_url)
text(" * default mp4 url is " + mp4_video_tags[0].firstChild.nodeValue)
# this isn't really necessary, but included for completion
if small_video_tags:
if small_video_tags[0].firstChild is not None:
text(" * small video url is " + small_video_tags[0].firstChild.nodeValue)
for subtag in mbr_video_tags:
stream = subtag.getElementsByTagName("streamName")
if stream:
if stream[0].firstChild is not None:
# the streamName should conform to "mp4:path/to/video.mp4"
text(" * video url is " + base_mp4_url + stream[0].firstChild.nodeValue.partition(":")[2])
text("")
return {}
else: # download the rtmp streams
akamai_host_xml = parsed_xml.getElementsByTagName("akamaiHost")
speaker_video_xml = parsed_xml.getElementsByTagName("speakerVideo")
slide_video_xml = parsed_xml.getElementsByTagName("slideVideo")
if not akamai_host_xml or not speaker_video_xml or not slide_video_xml:
error("xml missing properties")
if akamai_host_xml[0].firstChild is None or speaker_video_xml[0].firstChild is None or slide_video_xml[0].firstChild is None:
error("xml missing properties")
akamai_host = "rtmp://" + akamai_host_xml[0].firstChild.nodeValue + (rtmp_suffixes[event_name] if event_name in rtmp_suffixes else rtmp_suffixes["default"])
speaker_video = speaker_video_xml[0].firstChild.nodeValue.replace(".flv", "")
slide_video = slide_video_xml[0].firstChild.nodeValue.replace(".flv", "")
message("akamai host is " + akamai_host)
message("speaker video is " + speaker_video)
message("slide video is " + slide_video)
# some of the xml files contain exta audio tracks; we want those, don't we?
audios = parsed_xml.getElementsByTagName("audios")
audio_metadata = {}
if (audios):
for audio_node in audios[0].getElementsByTagName("audio"):
audio_url = None
code = None
for (name, value) in audio_node.attributes.items():
if name == "url":
audio_url = value.replace(".flv", "")
elif name == "code":
code = value
if code:
audio_metadata[code] = audio_url
message("audio " + code + " is " + audio_url)
data = {}
data["akamai"] = akamai_host
data["speaker"] = speaker_video
data["slide"] = slide_video
data["audio"] = audio_metadata
return data
def download_video(rtmp, playpath, swf_url, page_url, filename):
args = ["rtmpdump", "--rtmp", rtmp, "--playpath", playpath, "--swfUrl", swf_url, "--pageUrl", page_url, "--flv", filename]
try:
retval = subprocess.call(args, stdin=None)
except Exception, e:
error("rtmpdump error")
return None
def download_gdc_video_at_url(url, dest=""):
dest_path = os.path.abspath(dest)
dest_name = "GDCVideo" if os.path.split(dest)[1] == "" else os.path.split(dest)[1]
# Step 0: Check dependencies.
check_dependencies()
# Step 1: Extract the following from the URL: player URL, SWF URL, XML URL.
data = retrieve_data_from_base_url(url, os.path.join(dest_path, dest_name + "-player-url.txt"))
# Step 2: Parse the XML and extract the speaker video URL, slide video URL, and metadata.
metadata = parse_xml_from_url(data["xml_url"], os.path.join(dest_path, dest_name + ".xml"), data["event_name"])
# Step 3: Download the videos.
if (metadata):
download_video(metadata["akamai"], metadata["slide"], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-slide.flv"))
download_video(metadata["akamai"], metadata["speaker"], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-speaker.flv"))
for code in metadata["audio"]:
download_video(metadata["akamai"], metadata["audio"][code], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-audio-" + code + ".flv"))
message("All done!")
def _main():
parser = argparse.ArgumentParser()
parser.add_argument("player_url", help="the full player.html URL retrieved from the video page source")
parser.add_argument("output_name", nargs='?', help="the name of the output directory")
args = parser.parse_args()
download_gdc_video_at_url(args.player_url, ("" if args.output_name is None else args.output_name));
if __name__ == "__main__":
_main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment