Created
January 11, 2019 01:41
-
-
Save johnliu55tw/56ce5686d4ed6cc0bb5dc32272c91151 to your computer and use it in GitHub Desktop.
Utility functions for testing Python importability
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
import shutil | |
import tempfile | |
from os.path import dirname as p_dirname | |
from os.path import join as p_join | |
from types import GeneratorType | |
import unittest | |
import mock | |
from . import utils | |
THIS_DIR = p_dirname(__file__) | |
UPPER_DIR = p_dirname(THIS_DIR) | |
FAKE_MODULE_DIR = ('{}/some_module_structure'.format(THIS_DIR)) | |
class PathToModulePathTestCase(unittest.TestCase): | |
def test_abs_path(self): | |
result = utils.path_to_module_name('/this/is/abs/path') | |
self.assertEqual(result, 'this.is.abs.path') | |
def test_abs_path_with_relative_to(self): | |
result = utils.path_to_module_name('/this/is/abs/path', relative_to='/this') | |
self.assertEqual(result, 'is.abs.path') | |
def test_abs_path_with_relative_import(self): | |
with self.assertRaises(ValueError): | |
utils.path_to_module_name('/this/is/abs/path', relative_to='/that/foo/bar') | |
def test_rel_path(self): | |
result = utils.path_to_module_name('this/is/rel/path') | |
self.assertEqual(result, 'this.is.rel.path') | |
def test_rel_path_with_relative_to(self): | |
result = utils.path_to_module_name('this/is/rel/path', relative_to='this') | |
self.assertEqual(result, 'is.rel.path') | |
def test_rel_path_with_relative_import(self): | |
with self.assertRaises(ValueError): | |
utils.path_to_module_name('this/is/abs/path', relative_to='that/foo/bar') | |
def test_abs_path_with_ext(self): | |
result = utils.path_to_module_name('/this/is/abs/file.foo') | |
self.assertEqual(result, 'this.is.abs.file') | |
def test_rel_path_with_ext(self): | |
result = utils.path_to_module_name('this/is/rel/file.foo') | |
self.assertEqual(result, 'this.is.rel.file') | |
def test_abs_path_with_rel_relative_to(self): | |
with self.assertRaises(ValueError): | |
utils.path_to_module_name('/this/is/abs/path', 'this/is/rel') | |
def test_rel_path_with_abs_relative_to(self): | |
with self.assertRaises(ValueError): | |
utils.path_to_module_name('this/is/rel/path', '/this/is/abs') | |
class FindPythonModuleTestCase(unittest.TestCase): | |
def setUp(self): | |
pass | |
def tearDown(self): | |
pass | |
def test_path_is_not_directory(self): | |
with self.assertRaises(ValueError): | |
utils.find_python_module('/some/not/exists/dir') | |
def test_path(self): | |
result = utils.find_python_module(FAKE_MODULE_DIR) | |
self.assertEqual(len(result), 7) | |
# Order does not matter, so use assertIn | |
self.assertIn('some_module_structure', result) | |
self.assertIn('some_module_structure.module_1', result) | |
self.assertIn('some_module_structure.module_1.module_1_1', result) | |
self.assertIn('some_module_structure.module_1.module_1_2', result) | |
self.assertIn('some_module_structure.module_1.module_1_3', result) | |
self.assertIn('some_module_structure.module_2', result) | |
self.assertIn('some_module_structure.module_2.module_2_1', result) | |
def test_path_with_trailing_slash(self): | |
result = utils.find_python_module(FAKE_MODULE_DIR + '/') | |
self.assertEqual(len(result), 7) | |
# Order does not matter, so use assertIn | |
self.assertIn('some_module_structure', result) | |
self.assertIn('some_module_structure.module_1', result) | |
self.assertIn('some_module_structure.module_1.module_1_1', result) | |
self.assertIn('some_module_structure.module_1.module_1_2', result) | |
self.assertIn('some_module_structure.module_1.module_1_3', result) | |
self.assertIn('some_module_structure.module_2', result) | |
self.assertIn('some_module_structure.module_2.module_2_1', result) | |
def test_relative_to_upper_level_of_path(self): | |
result = utils.find_python_module(FAKE_MODULE_DIR, | |
relative_to=UPPER_DIR) | |
self.assertEqual(len(result), 7) | |
# Order does not matter, so use assertIn | |
self.assertIn('test_importability.some_module_structure', result) | |
self.assertIn('test_importability.some_module_structure.module_1', result) | |
self.assertIn('test_importability.some_module_structure.module_1.module_1_1', result) | |
self.assertIn('test_importability.some_module_structure.module_1.module_1_2', result) | |
self.assertIn('test_importability.some_module_structure.module_1.module_1_3', result) | |
self.assertIn('test_importability.some_module_structure.module_2', result) | |
self.assertIn('test_importability.some_module_structure.module_2.module_2_1', result) | |
def test_relative_to_cause_relative_import(self): | |
with self.assertRaises(ValueError): | |
utils.find_python_module(FAKE_MODULE_DIR, | |
relative_to=p_join(FAKE_MODULE_DIR, 'module_1')) | |
def test_directory_is_not_module(self): | |
temp_dir_path = tempfile.mkdtemp() | |
result = utils.find_python_module(temp_dir_path) | |
self.assertEqual(len(result), 0) | |
shutil.rmtree(temp_dir_path) | |
class TryImportTestCase(unittest.TestCase): | |
def setUp(self): | |
self.orig_sys_path = list(sys.path) | |
# Adding the circular_imported_module into sys.path for importing | |
sys.path.append(THIS_DIR) | |
def tearDown(self): | |
# Restore the sys.path | |
sys.path = self.orig_sys_path | |
def test_builtin_module(self): | |
result = utils.try_import('os.path') | |
self.assertEqual(result, (True, None)) | |
def test_not_exists_module(self): | |
result = utils.try_import('some.weird.module') | |
self.assertEqual(result[0], False) | |
self.assertIn('ImportError: No module named some.weird.module', result[1]) | |
def test_circular_import_module(self): | |
result = utils.try_import('circular_imported_module.a') | |
self.assertEqual(result[0], False) | |
self.assertIn('ImportError: cannot import name a', result[1]) | |
class TryImportAllTestCase(unittest.TestCase): | |
def setUp(self): | |
pass | |
def tearDown(self): | |
pass | |
def test_all_valid_modules(self): | |
results = utils.try_import_all(('os.path', 'sys', 'logging')) | |
self.assertIsInstance(results, GeneratorType) | |
self.assertEqual(list(results), [('os.path', True, None), | |
('sys', True, None), | |
('logging', True, None)]) | |
def test_some_invalid_modules(self): | |
results = list(utils.try_import_all(('os.path', 'some.weird.module', 'logging'))) | |
self.assertEqual(results[0], ('os.path', True, None)) | |
self.assertEqual(results[1][0], 'some.weird.module') | |
self.assertEqual(results[1][1], False) | |
self.assertIn('ImportError', results[1][2]) | |
self.assertEqual(results[2], ('logging', True, None)) | |
class LogImportAllMessagesTestCase(unittest.TestCase): | |
def setUp(self): | |
pass | |
def tearDown(self): | |
pass | |
@mock.patch.object(utils, 'print') | |
def test_log_results(self, m_print): | |
utils.log_import_all_messages([('module_1', True, None), | |
('module_2', False, 'Some reason'), | |
('module_3', True, None)]) | |
m_print.assert_has_calls( | |
[mock.call('Import module_1 succeeded', file=sys.stderr), | |
mock.call('FAILED to import module_2: Some reason', file=sys.stderr), | |
mock.call('Import module_3 succeeded', file=sys.stderr)]) | |
def test_log_results_to_file(self): | |
file_obj = tempfile.TemporaryFile() | |
utils.log_import_all_messages([('module_1', True, None), | |
('module_2', False, 'Some reason')], | |
stream=file_obj) | |
file_obj.flush() | |
file_obj.seek(0) | |
data = file_obj.read() | |
self.assertEqual(data, | |
'Import module_1 succeeded\nFAILED to import module_2: Some reason\n') | |
file_obj.close() | |
@mock.patch.object(utils, 'print') | |
def test_failed_only(self, m_print): | |
utils.log_import_all_messages([('module_1', True, None), | |
('module_2', False, 'Some reason'), | |
('module_3', True, None)], | |
failed_only=True) | |
m_print.assert_has_calls( | |
[mock.call('FAILED to import module_2: Some reason', file=sys.stderr)]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
import os | |
import sys | |
import traceback | |
from fnmatch import fnmatch | |
def path_to_module_name(path, relative_to=None): | |
if os.path.isabs(path): | |
relative_to = '/' if relative_to is None else relative_to | |
if not os.path.isabs(relative_to): | |
raise ValueError('Absolute path with relative relative_to') | |
# XXX: In Python 2.6.6, relpath() does NOT function correctly on absolute path: | |
# >>> os.path.relpath('/this/is/abs', '/') | |
# '../this/is/abs' | |
# So manually changing the path from abs to rel is required | |
rel_path = os.path.normpath(path)[1:] | |
rel_relative_to = os.path.normpath(relative_to)[1:] | |
return path_to_module_name(rel_path, rel_relative_to) | |
else: | |
relative_to = '.' if relative_to is None else relative_to | |
if os.path.isabs(relative_to): | |
raise ValueError('Relative path with absolute relative_to') | |
# The transformation happens here | |
# XXX: The result returned by relpath() is normalized! | |
final_path = os.path.relpath(path, relative_to) | |
if final_path.find('..') != -1: | |
raise ValueError('Relative import syntax is not supported') | |
no_ext_final_path = os.path.splitext(final_path)[0] | |
assert no_ext_final_path.find('.') == -1, "Path contains '.': {0}".format(no_ext_final_path) | |
assert not no_ext_final_path.startswith('/'), "Path starts with '/': {0}".format(no_ext_final_path) | |
assert not no_ext_final_path.endswith('/'), "Path ends with '/': {0}".format(no_ext_final_path) | |
return no_ext_final_path.replace('/', '.') | |
def find_python_module(path, relative_to=None): | |
"""Find Python module from a directory. The returned module path is relative | |
to the directory itself, not the whole path.""" | |
if not os.path.isdir(path): | |
raise ValueError('{0} is not a directory'.format(path)) | |
relative_to = (os.path.dirname(os.path.normpath(path)) if relative_to is None | |
else relative_to) | |
result = list() | |
for dirpath, _, filenames in os.walk(path): | |
if os.path.isfile(os.path.join(dirpath, '__init__.py')): | |
# The dir itself is a Python module | |
result.append(path_to_module_name(dirpath, relative_to)) | |
# Find all '*.py' files, except '__init__.py' | |
python_files = (os.path.join(dirpath, filename) | |
for filename in filenames | |
if fnmatch(filename, '*.py') and filename != '__init__.py') | |
module_names = (path_to_module_name(f, relative_to) for f in python_files) | |
result.extend(module_names) | |
return result | |
def try_import(name): | |
"""Try importing the name. Returning a tuple (result, traceback_message). | |
If import succeeded, returns (True, None). Else return (False, traceback_message) | |
where reason is a string. | |
""" | |
try: | |
imported_module = __import__(name, globals(), locals(), [], -1) | |
except BaseException: | |
return (False, traceback.format_exc()) | |
else: | |
return (True, None) | |
def try_import_all(names): | |
"""Try import multiple names. Returning a list of tuple (name, result, reason). | |
If importing the name succeeds, the tuple will be (name, True, None). | |
Else the tuple will be (name, False, reason), where reason is a string | |
""" | |
each_import_results = (try_import(name) for name in names) | |
all_result = ((name, r[0], r[1]) for name, r | |
in zip(names, each_import_results)) | |
return all_result | |
def log_import_all_messages(results, stream=None, failed_only=False): | |
"""Helper function for logging the import results.""" | |
if stream is None: | |
stream = sys.stderr | |
for module_name, succeeded, fail_reason in results: | |
if succeeded is False: | |
print('FAILED to import {0}: {1}'.format(module_name, fail_reason), | |
file=stream) | |
elif succeeded is True and failed_only is False: | |
print('Import {0} succeeded'.format(module_name), | |
file=stream) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment