Last active
April 23, 2022 14:15
-
-
Save martyngigg/3830532f8a689dc6c29bb6226bdee813 to your computer and use it in GitHub Desktop.
Tool to simlink CMake external data files on Windows
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""Tools for dealing with data registered with | |
CMake external data: | |
- Symlink from external data cache to build binary location. | |
Currently assumes the files have been downloaded and does not deal with | |
references that change. | |
""" | |
from itertools import chain | |
from pathlib import Path | |
from typing import Iterator, Sequence | |
# Constants. | |
# todo: Make them script arguments | |
CMAKE_SOURCE_DIR = Path.home() / Path('Code/mantid') | |
CMAKE_BINARY_DIR = CMAKE_SOURCE_DIR / 'build' | |
CMAKE_EXTERNALDATA_BINARY_SUBDIR = 'ExternalData' | |
CHECKSUM_ALG = 'md5' | |
CMAKE_EXTERNALDATA_CACHE_DIR = Path('C:/MantidExternalData') | |
def find_checksum_files(base_dirs: Sequence[Path], checksum_alg: str) -> Iterator[Path]: | |
""" | |
Find files named *.checksum_alg, starting from base_dirs | |
""" | |
return chain.from_iterable([base_dir.glob(f'**/*.{checksum_alg}') for base_dir in base_dirs]) | |
def link_cache_file(checksum_filepath: Path, cmake_source_dir:Path , | |
cmake_binary_dir: Path, externaldata_cache_dir: Path): | |
""" | |
Create a symlink from an external data cache, where files are named after their | |
checksum value, to a file with the real name given by the basename of the checksum | |
file. The contents of the checksum file given the checksum value. | |
""" | |
with open(checksum_filepath, 'r') as checksum_file_handle: | |
checksum_value = checksum_file_handle.readline().strip() | |
# The final suffix denotes the checksum algorithm, e.g Testing\\Data\\DocTest\\164198.nxs.md5 | |
# contains an MD5 checksum value and the final real filename is 164198.nxs in the 'Testing\\Data\\DocTest' | |
# directory. | |
checksum_suffixes = checksum_filepath.suffixes | |
checksum_alg = checksum_suffixes[-1][1:] # final suffix and remove the leading '.' character | |
cache_filepath = externaldata_cache_dir / checksum_alg.upper() / checksum_value | |
real_filename = checksum_filepath.stem | |
real_subbdir = checksum_filepath.parent.relative_to(cmake_source_dir) | |
real_filepath = cmake_binary_dir / CMAKE_EXTERNALDATA_BINARY_SUBDIR / real_subbdir / real_filename | |
if real_filepath.exists(): | |
print(f"'{real_filepath}' exists. Skipping.") | |
return | |
print(f"Linking '{real_filepath}' to '{cache_filepath}'") | |
real_filepath.parent.mkdir(parents=True, exist_ok=True) | |
real_filepath.symlink_to(cache_filepath) | |
def main() -> int: | |
"""Main entry point for the program""" | |
for path in find_checksum_files([CMAKE_SOURCE_DIR / 'Testing'], CHECKSUM_ALG): | |
try: | |
link_cache_file(path, CMAKE_SOURCE_DIR, CMAKE_BINARY_DIR, CMAKE_EXTERNALDATA_CACHE_DIR) | |
except OSError as exc: | |
print(f"Error linking cached data for '{path}': {str(exc)}") | |
return 0 | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment