Skip to content

Instantly share code, notes, and snippets.

@h8rt3rmin8r

h8rt3rmin8r/ttcUpdate.sh

Created August 1, 2024 20:04
Show Gist options
  • Save h8rt3rmin8r/f9d1fc10c1a3018d44066b3f6d7bb9e8 to your computer and use it in GitHub Desktop.
Save h8rt3rmin8r/f9d1fc10c1a3018d44066b3f6d7bb9e8 to your computer and use it in GitHub Desktop.
Download ZIP
Download the latest data for the Tamriel Trade Centre addon (for the Elder Scrolls Online game) and generate a historical archive of price data with each subsequent download.
Raw
ttcUpdate.sh
#! /usr/bin/env bash
#>------------------------------------------------------------------------------
#>
#> [ ttcUpdate.sh ]
#>
#> Download the latest data for the Tamriel Trade Centre addon and generate
#> a historical archive of price data with each subsequent download.
#>
#> No inputs are required when invoking this script.
#>
#> Created on 20210603 by h8rt3rmin8r
#> Updated on 20210619 by h8rt3rmin8r (v. 1624085579)
#> Updated on 20210809 by h8rt3rmin8r (v. 1628515982)
#> - Integrated ttcParse operations for archiving price history
#> - Added automatic logging
#>
#> USAGE:
#>
#> ttcUpdate.sh <INPUT>
#> ttcUpdate.sh <OPTION>
#> ttcUpdate.sh <INPUT> <OPTION>
#>
#> where "INPUT" is an optional directory reference for output data to be
#> stored; and where "OPTION" is an optional input parameter; and where
#> "OPTION" is one of the following:
#>
#> |
#> -f, --force | Force the script to update TTC data even if the game
#> | is currently running (NOT RECOMMENDED)
#> |
#> -h, --help | Print this help text to the terminal
#> |
#> -s, --silent | Suppress verbosity messages
#> |
#> -v, --verbose | Print verbosity messages (default)
#> |
#> -V, --version | Print the script version information
#> |
#>
#> REFERENCE:
#>
#> # Official website for TamrielTradeCentre
#> https://tamrieltradecentre.com/
#>
#> # Addon Page for TamrielTradeCentre
#> https://www.esoui.com/downloads/info1245-TamrielTradeCentre.html
#>
#>------------------------------------------------------------------------------
#> VERSION: 1628857161
#>
#_______________________________________________________________________________
# Declare functions
function ttcParse_archiver() {
local proc_success=0
local final_archive="${out_dir_parse_archive}/${d_t_string}.json"
for i in "${parse_keys[@]}"; do
local cycle_key="${i}"
local cycle_key_mod="${cycle_key,,}"
local cycle_master="${sh_dir}/prices-${cycle_key}.json"
local cycle_archive="${out_dir_parse_archive}/${d_t_string}-${cycle_key_mod}"
if [[ -f "${cycle_master}" ]]; then
cat "${cycle_master}" > "${cycle_archive}"
local proc_success=1
fi
done
if [[ "${proc_success}" -eq 1 ]]; then
## if archival files were created in the previous step, combine them into one file
local cycle_counter=0
printf '%s' "${_cba}" > "${final_archive}"
for i in "${parse_keys[@]}"; do
local cycle_key="${i}"
local cycle_key_mod="${cycle_key,,}"
local cycle_part="${out_dir_parse_archive}/${d_t_string}-${cycle_key_mod}"
if [[ -f "${cycle_part}" ]]; then
## if a file exists for the current parse key, add its contents to the final archive file
if [[ "${cycle_counter}" -gt 0 ]]; then
printf '%s' ",${_q2}${cycle_key_mod}${_q2}:" >> "${final_archive}"
cat "${cycle_part}" \
| tr -d '\n' >> "${final_archive}"
else
printf '%s' "${_q2}${cycle_key_mod}${_q2}:" >> "${final_archive}"
cat "${cycle_part}" \
| tr -d '\n' >> "${final_archive}"
fi
## delete the partial archive file used in this cycle
rm "${cycle_part}"
let cycle_counter=cycle_counter+1
fi
done
printf '%s' "${_cbb}" >> "${final_archive}"
gzip -9 "${final_archive}"
fi
return $?
}
function ttcParse_logger() {
local out_string="${_q2}${d_t_string}${_q2},${_q2}${d_t_local}${_q2},${_q2}${d_t_weekday}${_q2}"
echo "${out_string}" >> "${out_file_runhistory}"
return $?
}
function ttcParse_make() {
function ttcParse_make_dirs() {
mkdir -p "${out_dir_parse}"
mkdir -p "${out_dir_parse_archive}"
return $?
}
function ttcParse_make_files() {
if [[ ! -f "${out_file_notfound}" ]]; then
echo "${out_columns_notfound}" > "${out_file_notfound}"
fi
if [[ ! -f "${out_file_runhistory}" ]]; then
echo "${out_columns_runhistory}" > "${out_file_runhistory}"
fi
return $?
}
ttcParse_make_dirs
ttcParse_make_files
return $?
}
function ttcParse_run() {
function ttcParse_run_python() {
if [[ "x${1}" == "x" ]]; then
return 1
fi
echo -e "import re\nimport json\ntranslation = {}\nprices = {}\nfil = open(\"PriceTable.lua\")\ntext = fil.read()\nfil.close()\noutputsL = re.findall(r'\[([0-9]*?)\]={\[[0-5]]={\[[0-9]*?\]={\[-1\]={.*?\"$1\"]=(.*?),',text)\nfil = open(\"ItemLookUpTable_EN.lua\")\ntext = fil.read()\nfil.close()\ntranslationL = re.findall(r'\[\"(.*?)\"\]={\[[0-9]*?]=(.[0-9]*?),}', text)\nfor x in translationL: translation[x[1]] = x[0]\nfor x in outputsL: \n\ttry: prices[translation[x[0]]] = x[1]\n\texcept KeyError: print(f\"Item {x} not found\")\nfil = open(\"prices-$1.json\",'w')\njson.dump(prices, fil)\nfil.close()" \
| python3
}
cd "${out_dir}"
ttcParse_make
for i in "${parse_keys[@]}"; do
ttcParse_run_python "${i}" \
| sed "s/^/\"$d_t\",\"$i\",\"/" \
| sed 's/$/\"/' >> "${out_file_notfound}"
done
ttcParse_archiver
ttcParse_logger
cd "${here_now}"
return $?
}
function ttcParse_unixtime2date() {
# USAGE:
#
# ttcParse_unixtime2date <OPTION> <INPUT>
#
# where "INPUT" is a valid Unix timestamp and where "OPTION" is one of the
# following:
#
# |
# -D, --day | Output only the name of the related weekday
# |
# -H, --human | Output is formatted in human-friendly format (default)
# | (does NOT preserve nano-seconds)
# |
# -i, --iso | Output is formatted as an ISO date: YYYY-MM-DD HH:MM:SS
# | (preserves nano-seconds)
# |
if [[ "x${1}" == "x" ]]; then
return 1
fi
local e_c="0"
if [[ "${1}" =~ ^[-]+[dDhHiInNsStTzZ] ]]; then
local i_n="${1//[^a-zA-Z]}"
case "${i_n}" in
D|day|weekday)
shift 1
if (($#)); then
local in_x="$@"
local DT=${in_x:0:10}
local output=$(printf "%(%A)T\n" "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
else
while read -r line; do
local in_x="${line}"
local DT=${in_x:0:10}
local output=$(printf "%(%A)T\n" "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
done
fi
if [[ "${e_c}" =~ 1 ]]; then
return 1
else
return 0
fi
;;
H|human)
shift 1
if (($#)); then
local in_x="$@"
local DT=${in_x:0:10}
local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
else
while read -r line; do
local DT=${line:0:10}
local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
done
fi
if [[ "${e_c}" =~ 1 ]]; then
return 1
else
return 0
fi
;;
i|iso)
shift 1
if (($#)); then
local in_x="$@"
local DT=${in_x:0:10}
local DT_X=${in_x#$DT}
local out_pre=$(printf "%(%Y-%m-%d %H:%M:%S)T\n" "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
if [[ "x${DT_X}" == "x" ]]; then
local output="${out_pre}"
else
local output="${out_pre}.${DT_X}"
fi
echo "${output}"
else
while read -r line; do
local in_x="${line}"
local DT=${in_x:0:10}
local DT_X=${in_x#$DT}
local out_pre=$(printf "%(%Y-%m-%d %H:%M:%S)T\n" "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
if [[ "x${DT_X}" == "x" ]]; then
local output="${out_pre}"
else
local output="${out_pre}.${DT_X}"
fi
echo "${output}"
done
fi
if [[ "${e_c}" =~ 1 ]]; then
return 1
else
return 0
fi
;;
esac
fi
if (($#)); then
local in_x="$@"
local DT=${in_x:0:10}
local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
else
while read -r line; do
local DT=${line:0:10}
local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null)
local e_c="${e_c}$?"
echo "${output}"
done
fi
if [[ "${e_c}" =~ 1 ]]; then
return 1
else
return 0
fi
}
function ttcUpdate_help() {
# Help text printing function
cat "${sh_path}" 2>/dev/null \
| grep -E '^#[>]' \
| sed 's/^..//'
return $?
}
function ttcUpdate_report() {
# Runtime reporting function
function ttcUpdate_report_proctime() {
# Calculate the total script execution time
local out_prefix=$(echo "${time_done} - ${time_start}" | bc)
local out_string="${out_prefix} seconds"
## print the final result and exit the function
printf '%s\n' "${out_string}"
return $?
}
if [[ "${vbs_ops}" -ne 0 ]]; then
## skip the final runtime reporting if running in silent mode
return 0
fi
export time_done="$(date '+%s.%N')"
local time_proc="$(ttcUpdate_report_proctime)"
ttcUpdate_vbs "Final runtime report:"
echo " Operations Begin (UNIX): ${time_start}" &>${vbs_out}
echo " Operations End (UNIX): ${time_done}" &>${vbs_out}
echo " Execution Time (seconds): ${time_proc}" &>${vbs_out}
unset time_done
unset time_start
unset time_proc
return $?
}
function ttcUpdate_run() {
# Main script operations function
## Declare sub-functions
function ttcUpdate_run_extract() {
# Extract the newly downloaded TTC data archive
local e_c=0
cd "${out_dir}" 2>/dev/null
local e_c="$?"
if [[ "${e_c}" -ne 0 ]]; then
## kill the function if we can't get into the target directory for some unknown reason
return $e_c
fi
if [[ "${vbs_ops}" -eq 0 ]]; then
## verbosity is currently ON
unzip -o "${out_file}"
local e_c="$?"
else
## verbosity is currently OFF
unzip -o -q "${out_file}"
local e_c="$?"
fi
cd "${here_now}"
return $e_c
}
function ttcUpdate_run_get() {
# Fetch the latest TTC data from the remote server
local e_c=""
if [[ "${vbs_ops}" -eq 0 ]]; then
## display a progress bar in the terminal while the data is being downloaded
curl --xattr -o "${out_file}" "${api_target}"
local e_c=$?
else
## suppress verbosity if the script was invoked with the "silent" parameter
curl -s --xattr -o "${out_file}" "${api_target}"
local e_c=$?
fi
return $e_c
}
## Execute operations
local e_c=""
ttcUpdate_vbs "Updating reference data for the Tamriel Trade Centre addon"
ttcUpdate_vbs "Destination directory: ${out_dir}"
if [[ "${force_ops}" -ne 0 ]]; then
## print a warning if running with "force"
ttcUpdate_vbs "WARNING: Running in forced execution mode! (This could result in corrupted data)"
fi
ttcUpdate_vbs "Beginning download ..."
ttcUpdate_run_get
local e_c="$?"
if [[ "${e_c}" -ne 0 ]]; then
## download process failed
ttcUpdate_vbs "ERROR: An unknown error occurred while communicating with the remote host: ${api_target} (exit code: ${e_c})"
ttcUpdate_vbs "ERROR: Check your network connection and try again."
ttcUpdate_vbs "WARNING: Terminating operations (no changes were made to the system files)"
rm "${out_file}" 2>/dev/null
return $e_c
fi
ttcUpdate_vbs "Download complete (exit code: ${e_c})"
ttcUpdate_vbs "New archive location: ${out_file}"
ttcUpdate_vbs "Extracting new data archive ..."
ttcUpdate_run_extract
local e_c="$?"
if [[ "${e_c}" -ne 0 ]]; then
## archive extraction failed
ttcUpdate_vbs "ERROR: An unknown error occurred while attempting to extract the archive: ${out_file} (exit code: ${e_c})"
ttcUpdate_vbs "ERROR: The downloaded data might be corrupted. Check your network connection and try again."
ttcUpdate_vbs "WARNING: Terminating operations (no changes were made to the system files)"
rm "${out_file}" 2>/dev/null
return $e_c
fi
ttcUpdate_vbs "Archive extraction complete (exit code: ${e_c})"
ttcUpdate_vbs "All update operations are COMPLETE"
return $e_c
}
function ttcUpdate_vbs() {
# Verbosity handling function
local vbs_i_n="$@"
local vbs_d_t="$(date '+%s.%N')"
local vbs_p_x="${sh_ppid}"
local vbs_n_m="${sh_name}"
echo "${vbs_d_t}|${vbs_p_x}|${vbs_n_m}|${vbs_i_n}" &>${vbs_out}
return $?
}
function ttcUpdate_version() {
# Script version printing function
local e_c=""
local out_string=""
local ver_string=$(cat "${sh_path}" 2>/dev/null | grep -E '^#[>]' | grep -E ' *VERSION. *1([0-9]){9} ?$' | sed 's/[^0-9]*//g' | tail -n 1)
## construct the output string
if [[ "x${ver_string}" == "x" ]]; then
local out_string="${sh_file} (version not detected)"
local e_c=1
else
local out_string="${sh_file} v.${ver_string}"
local e_c=0
fi
## print the output string and kill the function
printf '%s\n' "${out_string}" &>/dev/stdout
return $e_c
}
#_______________________________________________________________________________
# Declare variables and arrays
_q1="'"
_q2='"'
_cba='{'
_cbb='}'
time_start="$(date '+%s.%N')"
time_done=""
d_t="${time_start}"
d_t_string="${d_t//.}"
d_t_local=$(ttcParse_unixtime2date --iso "${d_t_string}")
d_t_weekday=$(ttcParse_unixtime2date --day "${d_t_string}")
here_now="${PWD}"
e_c_main=0
vbs_out="/dev/stderr"
vbs_ops=0
out_ops=0
force_ops=0
sh_path=$(readlink -f "$0")
sh_dir="${sh_path%\/*}"
sh_file="${sh_path//*\/}"
sh_name="${sh_file%.*}"
sh_ppid="${PPID}"
eso_active=$(pgrep -l eso | grep --color=never -E 'eso((6|3)(4|2))[.]exe' &>/dev/null; echo $?)
api_target="https://us.tamrieltradecentre.com/download/PriceTable"
out_dir="${sh_dir}"
out_dir_parse="${out_dir}/ttcParse"
out_dir_parse_archive="${out_dir_parse}/prices_archive"
out_file="${sh_dir}/PriceTable.zip"
out_file_notfound="${out_dir_parse}/notfound.csv"
out_file_runhistory="${out_dir_parse}/runhistory.csv"
out_columns_notfound="${_q2}runTime${_q2},${_q2}missingKey${_q2},${_q2}message${_q2}"
out_columns_runhistory="${_q2}unixTime${_q2},${_q2}localTime${_q2},${_q2}weekDay${_q2}"
declare -a parse_keys=( Avg Max Min EntryCount AmountCount )
#_______________________________________________________________________________
# Execute operations
## catch inputs and parameters
while [[ "$#" -gt 0 ]]; do
sh_in="${1}"
sh_in_mod="${sh_in//-}"
case "${sh_in_mod}" in
f|F|force)
## set the script to force-run (NOT RECOMMENDED)
force_ops=1
shift 1
;;
h|H|help)
## print the help text and kill the script
ttcUpdate_help
exit $?
;;
s|silent|q|quiet)
## suppress verbosity messages
vbs_out="/dev/null"
vbs_ops=1
shift 1
;;
v|verbose)
## enable verbosity messages (default)
vbs_out="/dev/stderr"
vbs_ops=0
shift 1
;;
V|version)
## print the script version number and kill the script
ttcUpdate_version
exit $?
;;
*)
## check if the input is a valid directory reference
sh_in_dir=$(readlink -f "${sh_in}")
if [[ -d "${sh_in_dir}" ]]; then
## reconfigure the output directory locations to the location specified and set related ops code
out_ops=1
out_dir="${sh_in_dir}"
out_dir_parse="${out_dir}/ttcParse"
out_dir_parse_archive="${out_dir_parse}/prices_archive"
out_file="${out_dir}/PriceTable.zip"
out_file_notfound="${out_dir_parse}/notfound.csv"
out_file_runhistory="${out_dir_parse}/runhistory.csv"
else
## input was not a valid directory reference
## warn the user about an unknown input and kill the script
ttcUpdate_vbs "WARNING: Unrecognized input parameter detected: ${1} (terminating operations)"
ttcUpdate_vbs "Use '--help' for more information"
exit 1
fi
shift 1
;;
esac
done
## check if ESO is currently running and process "force" parameters if necessary
if [[ "${eso_active}" -eq 0 ]]; then
## ESO IS ACTIVE
## determine if we should force the script to run anyway ...
if [[ "${force_ops}" -eq 0 ]]; then
## user didn't request to force-run the script
## exit the script to avoid data corruption
ttcUpdate_vbs "ERROR: Cannot update addon data while ESO is running"
ttcUpdate_vbs "Exit the game and try again (or force execution with '--force')"
exit 1
fi
## the user wants to force the script to run, so let's keep going ...
fi
## run the core script operations
ttcUpdate_run
e_c_main="$?"
ttcParse_run
ttcUpdate_report
exit $e_c_main
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment