Skip to content

Instantly share code, notes, and snippets.

@alexmaloteaux

alexmaloteaux/get_ghidra_manuals.py

Created October 17, 2020 22:08
Show Gist options
  • Save alexmaloteaux/88a8aaa4c00c87e4532b4b2ff474d6e1 to your computer and use it in GitHub Desktop.
Save alexmaloteaux/88a8aaa4c00c87e4532b4b2ff474d6e1 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
get_ghidra_manuals.py
#! /usr/bin/env python3
import os, sys
#import requests
from urllib import request
import traceback
from shutil import copyfile
from hashlib import md5
import binascii
supported_version = "9.1.2"
class bcolors:
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
def print_success(msg):
output = bcolors.OKGREEN + msg + bcolors.ENDC
print(output)
def print_warning(msg):
output = bcolors.WARNING + msg + bcolors.ENDC
print(output)
def print_error(msg):
output = bcolors.FAIL + msg + bcolors.ENDC
print(output)
def print_usage():
print("Usage : %s ghidra_base_folder [-f]" % sys.argv[0])
if len(sys.argv) < 2 or len(sys.argv) > 3:
print_usage()
sys.exit()
ghidra_folder = os.path.join(sys.argv[1], "Ghidra")
force = False
if len(sys.argv) == 3:
if sys.argv[2] == "-f":
force = True
else:
print_usage()
sys.exit()
if not os.path.exists(ghidra_folder):
print_error("ghidra folder not found : %s" % ghidra_folder)
sys.exit()
manuals = {
'6502': [
{
'src' : "http://archive.6502.org/books/mcs6500_family_programming_manual.pdf",
'idx' : "6502.idx",
'pdf' : "mcs6500_family_programming_manual.pdf",
'md5' : "4165237c3c3fa7247714433bacb6ea4e",
}
],
'68000': [
{
'src' : "https://www.nxp.com/files-static/archives/doc/ref_manual/M68000PRM.pdf",
'idx' : "68000.idx",
'pdf' : "M68000PRM.pdf",
'md5' : "c446e98b7f33046f4ed98fcfd4f0a96c",
}
],
'8048': [
{
'src' : "http://www.decadecounter.com/vta/pdf/mcs48.pdf",
'idx' : "8048.idx",
'pdf' : "8048.pdf",
'md5' : "51634c6554dbb072d1956f564f38aa17",
}
],
'8051': [
{
'src' : "https://www.keil.com/dd/docs/datashts/intel/8xc251sx_um.pdf",
'idx' : "8051.idx",
'pdf' : "8xc251sx_um.pdf",
'md5' : "d749e5a0b3186266959dcc2a32b5af3b",
}
],
'AARCH64': [
{
'src' : "http://kib.kiev.ua/x86docs/ARMARM/DDI0487D_b_armv8_arm.pdf",
'idx' : "AARCH64.idx",
'pdf' : "DDI0487D_b_armv8_arm.pdf",
'md5' : "e2a37efc9e942f8260bd021c013c18f4",
}
],
'ARM': [
{
'src' : "https://www.cs.utexas.edu/~simon/378/resources/ARMv7-AR_TRM.pdf",
'idx' : "ARM.idx",
'pdf' : "Armv7AR_errata.pdf",
'md5' : "922f5cc42a25a02d0f737893836e89df",
}
],
'Atmel': [
{
'src' : "https://ww1.microchip.com/downloads/en/devicedoc/doc32000.pdf",
'idx' : "AVR32.idx",
'pdf' : "doc32000.pdf",
'md5' : "886df329eaf6fc184374a7e34cd7c6c6",
},
{
'src' : "http://ww1.microchip.com/downloads/en/devicedoc/atmel-0856-avr-instruction-set-manual.pdf",
'idx' : "AVR8.idx",
'pdf' : "atmel-0856-avr-instruction-set-manual.pdf",
'md5' : "8e58b391f47a50a831d38a86655ad14d",
}
],
'CR16': [
{
'src' : "https://dump.bitcheese.net/files/zujukix/Prog_16C.pdf",
'idx' : "CR16.idx",
'pdf' : "prog16c.pdf",
'md5' : "c05f0ef53c9c296272410e170d461cbb",
}
],
'HCS08': [
{
'src' : "https://www.nxp.com/docs/en/reference-manual/M68HC05TB.pdf",
'idx' : "HC05.idx",
'pdf' : "M68HC05TB.pdf",
'md5' : "da59a147fdcc43e128f229a557abc327",
},
{
'src' : "https://www.nxp.com/files-static/microcontrollers/doc/ref_manual/CPU08RM.pdf",
'idx' : "HC08.idx",
'pdf' : "CPU08RM.pdf",
'md5' : "32ee09771ebb3b9d8689a509555ecd21",
},
{
'src' : "https://www.nxp.com/files-static/microcontrollers/doc/ref_manual/HCS08RMV1.pdf",
'idx' : "HCS08.idx",
'pdf' : "HCS08RMV1.pdf",
'md5' : "b31789f3d1d6df8027479c57e0ce53b5",
}
],
'HCS12': [
{
#need to register to obtain that link
'src' : "https://www.nxp.com/webapp/Download?colCode=CPU12/CPU12X&location=null",
'idx' : "HCS12.idx",
'pdf' : "S12XCPUV2.pdf",
'md5' : "611672bd85e5ddac66e2cf77d8f46f04",
'auth' : True
}
],
'JVM': [
{
'src' : "https://docs.oracle.com/javase/specs/jvms/se8/jvms8.pdf",
'idx' : "JVM.idx",
'pdf' : "jvms8.pdf",
'md5' : "7665578d886f6d02952137e97d38e8ae",
'align' : False,
}
],
'MCS96': [
{
'src' : "http://bitsavers.trailing-edge.com/components/intel/_dataBooks/1991_Intel_16-Bit_Embedded_Controller_Handbook.pdf",
'idx' : "MCS96.idx",
'pdf' : "1991_Intel_16-Bit_Embedded_Controller_Handbook.pdf",
'md5' : "6abaf624673b5307fed14ab99012f135",
}
],
'MIPS': [
{
'src' : "https://scc.ustc.edu.cn/zlsc/lxwycj/200910/W020100308600769158777.pdf",
'idx' : "MIPS.idx",
'pdf' : "mips64v2.pdf",
'md5' : "b3cf0d829ab26af2e38d5735ae4e307f",
'align' : False,
},
{
'src' : "https://s3-eu-west-1.amazonaws.com/downloads-mips/documents/MD00087-2B-MIPS64BIS-AFP-6.06.pdf",
'idx' : "mipsM16.idx",
'pdf' : "MD00087-2B-MIPS64BIS-AFP-6.06.pdf",
'md5' : "da2d12cddfb7e7c57bedbed385abf9b1",
},
#Exactly the same file for this idx
# {
# 'src' : "https://s3-eu-west-1.amazonaws.com/downloads-mips/documents/MD00087-2B-MIPS64BIS-AFP-6.06.pdf",
# 'idx' : "mipsMic.idx",
# 'pdf' : "MD00087-2B-MIPS64BIS-AFP-6.06.pdf",
# 'md5' : "da2d12cddfb7e7c57bedbed385abf9b1",
# },
{
'src' : "https://groups.csail.mit.edu/cag/raw/documents/R4400_Uman_book_Ed2.pdf",
'idx' : "r4000.idx",
'pdf' : "r4000.pdf",
'md5' : "5fb145391e449725c38d362780d104f5",
},
],
'PA-RISC': [
{
'src' : "http://ftp.parisc-linux.org/docs/arch/pa11_acd.pdf",
'idx' : "pa11_acd.idx",
'pdf' : "pa11_acd.pdf",
'md5' : "1b7616419648bd55389ba64b7b499203",
}
],
'PIC': [
{
'src' : "https://ww1.microchip.com/downloads/en/devicedoc/40139e.pdf",
'idx' : "PIC-12.idx",
'pdf' : "PIC12_40139e.pdf",
'md5' : "a02061138f2cea8d4796c8263f2b34e5",
},
{
'src' : "https://ww1.microchip.com/downloads/en/DeviceDoc/40001761E.pdf",
'idx' : "PIC-16F.idx",
'pdf' : "PIC16F_40001761E.pdf",
'md5' : "450a29a5968b35525d2b3e740de2b912",
},
{
'src' : "https://ww1.microchip.com/downloads/en/devicedoc/33023a.pdf",
'idx' : "PIC-16.idx",
'pdf' : "PIC16_33023a.pdf",
'md5' : "13f70dba170f82edeae808f9150cf18a",
},
{
'src' : "https://ww1.microchip.com/downloads/en/devicedoc/30289b.pdf",
'idx' : "PIC-17.idx",
'pdf' : "PIC17_30289b.pdf",
'md5' : "e39da5d29ef2e86d6f69373cde367fbc",
},
{
'src' : "https://www.farnell.com/datasheets/14702.pdf",
'idx' : "PIC-18.idx",
'pdf' : "PIC18_14702.pdf",
'md5' : "9ecb61cb746febec0dddc5ec238bbc81",
},
{
'src' : "https://ww1.microchip.com/downloads/en/DeviceDoc/70157D.pdf",
'idx' : "PIC24.idx",
'pdf' : "PIC24_70157D.pdf",
'md5' : "fa5224ed0b19503a2bef2b1b2dc9a048",
'align' : False,
},
],
'PowerPC': [
{
#Description is wrong in 9.1.2 code
#more recent here but index on first : http://kib.kiev.ua/x86docs/POWER/PowerISA_V2.06B_V2_PUBLIC.pdf
'src' : "https://wiki.raptorcs.com/w/images/1/1a/PowerISA_V2.06_PUBLIC.pdf",
'idx' : "PowerISA.idx",
'pdf' : "PowerISA_V2.06_PUBLIC.pdf",
'md5' : "13eca45bbea1025adc4ccf2384a56334",
'align' : False,
},
{
'src' : "https://wiki.alcf.anl.gov/images/f/fb/PowerPC_-_Assembly_-_IBM_Programming_Environment_2.3.pdf",
'idx' : "PowerPC.idx",
'pdf' : "powerpc.pdf",
'md5' : "da742a9f2978c01702581f15a290c577",
}
],
'Sparc': [
{
'src' : "https://cr.yp.to/2005-590/sparcv9.pdf",
'idx' : "Sparc.idx",
'pdf' : "SPARCV9.pdf",
'md5' : "8d38c4ff70204cd9056b3f040b100237",
},
],
'SuperH4': [
{
'src' : "https://www.renesas.com/us/en/doc/products/mpumcu/001/rej09b0318_sh_4sm.pdf",
'idx' : "superh4.idx",
'pdf' : "rej09b0318_sh_4sm.pdf",
'md5' : "4757198e1bb055fa1a3438ca29918b36",
},
],
'TI_MSP430': [
{
'src' : "https://e2echina.ti.com/cfs-file/__key/telligent-evolution-components-attachments/00-55-01-00-00-00-61-61/MSP430x2xx-Family-User_26002300_39_3B00_s-Guide-_2800_Rev.-E_2900_.pdf",
'idx' : "MSP430.idx",
'pdf' : "MSP430.pdf",
'md5' : "e7b8c653dfae7080a8edee3428aea8ed",
},
],
'tricore': [
{
'src' : "https://www.infineon.com/dgdl/tc_v131_instructionset_v138.pdf?fileId=db3a304412b407950112b409b6dd0352",
'idx' : "tricore.idx",
'pdf' : "tc_v131_instructionset_v138.pdf",
'md5' : "f1cb5be4e5ba4d614fe7b11313a5b618",
},
{
'src' : "https://www.infineon.com/dgdl/Infineon-TC2xx_Architecture_vol2-UM-v01_00-EN.pdf?fileId=5546d46269bda8df0169ca1bf33124a8",
'idx' : "tricore2.idx",
'pdf' : "Infineon-TC2xx_Architecture_vol2-UM-v01_00-EN.pdf",
'md5' : "6a6ceb94889dfba302d8d8cb1bdc2edf",
'align' : False,
},
],
'x86': [
{
'src' : "https://community.intel.com/legacyfs/online/drupal_files/managed/a4/60/325383-sdm-vol-2abcd.pdf",
'idx' : "x86.idx",
'pdf' : "325383-sdm-vol-2abcd.pdf",
'md5' : "063a8e1060e07bb646cccdb49814a0ed",
},
],
'Z80': [
{
'src' : "http://z80.info/zip/z80cpu_um.pdf",
'idx' : "Z80.idx",
'pdf' : "UM0080.pdf",
'md5' : "6e131738b09d5b30b4d7096bc5092ca5",
},
{
'src' : "http://www.bitsavers.org/components/zilog/z180/UM005003-0703_Z8018x_User_Manual_2003.pdf",
'idx' : "Z180.idx",
'pdf' : "um0050.pdf",
'md5' : "e8a08e055d28aacff8acb9fe56f260ce",
'align' : False,
},
],
}
def download_file(name, url, fname, md5hash, wget=False) :
try:
print("Downloading \"%s\" manual into %s" % (name, fname))
f = open(fname, "wb")
req = request.Request(
url,
data=None,
headers={
'User-Agent': 'Wget/1.20.3 (linux-gnu)',
### 'Accept' : '*.*'
},
)
infile = request.urlopen(req)
while True:
buf = infile.read(4096)
if not buf:
break
f.write(buf)
f.close()
md5_download = binascii.hexlify(md5(open(cache_file, "rb").read()).digest()).decode()
if md5_download != md5hash:
raise Exception("md5 hash of downloaded file does not match")
except:
print_error('Could not download file...\n')
traceback.print_exc()
if os.path.exists(fname):
os.unlink(fname)
return False
return True
if __name__ == "__main__":
try:
version = ""
app_prop = os.path.join(ghidra_folder, "application.properties")
if not os.path.exists(app_prop):
print_warning("application.properties file not found, ghidra version may not be compatible")
else:
for line in open(app_prop, "r"):
if line.startswith("application.version="):
version = line.split("=")[1].strip()
if version != supported_version:
print_error("Version mismatch, this script may not work : %s / %s" % (version, supported_version))
else:
print_success("Correct version of ghidra detected : %s" % version)
basepath = os.path.dirname(__file__)
cache = os.path.join(basepath, "cache")
if not os.path.exists(cache):
os.mkdir(cache)
for l in manuals.keys():
print("Processor : %s" % l )
for m in manuals[l]:
pdf_name = m['pdf']
pdf = os.path.join(ghidra_folder, "Processors", l, "data/manuals", pdf_name)
idx_path = os.path.join(ghidra_folder, "Processors", l, "data/manuals", m['idx'])
if not os.path.exists(idx_path):
print_error("Index file not found : %s, skipping" % idx_path)
continue
#@SPARCV9.pdf[The SPARC Architecture Manual, Version 9 (SAV09R1459912)]
pdf_header = open(idx_path, "r").readline().split('[')[0][1:].strip()
title_name = open(idx_path, "r").readline().split('[')[1].replace(']','').strip()
if pdf_name != pdf_header:
print_error("pdf name in index is different : %s / %s" % (pdf_name, pdf_header))
continue
if not force and os.path.exists(pdf):
print_warning("manual file already exists, skipping : %s" % pdf)
continue
elif force and os.path.exists(pdf):
os.unlink(pdf)
cache_file = os.path.join(cache, pdf_name)
if not os.path.exists(cache_file):
url = m['src']
if 'auth' in m.keys() and m['auth']:
print_error("%s requires authentification and has to be manually downloaded and copy in ./cache/%s from : %s" % (title_name, pdf_name, url))
continue
if not download_file(title_name, url, cache_file, m['md5']):
print_error("Error while downloading manual from : %s" % url)
continue
else:
print("%s file exists in cache, using it..." % pdf_name)
else:
md5_cache = binascii.hexlify(md5(open(cache_file, "rb").read()).digest()).decode()
if md5_cache != m['md5']:
print_error("md5 sum of cache file %s mismatch: %s / %s" % (cache_file, md5_cache, m['md5']))
continue
copyfile(cache_file, pdf)
print_success("Manual deployed in %s" % pdf)
if 'align' in m.keys() and not m['align']:
print_warning("REM : This version of the pdf doesnt align with the idx, please provide correct link if you find one maching")
except:
print_error("Error in main, ..")
traceback.print_exc()
sys.exit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment