"""distutils.filelist Provides the FileList class, used for poking about the filesystem and building lists of files. """ import fnmatch import functools import os import re from ._log import log from .errors import DistutilsInternalError, DistutilsTemplateError from .util import convert_path class FileList: """A list of files built by on exploring the filesystem and filtered by applying various patterns to what we find there. Instance attributes: dir directory from which files will be taken -- only used if 'allfiles' not supplied to constructor files list of filenames currently being built/filtered/manipulated allfiles complete list of files under consideration (ie. without any filtering applied) """ def __init__(self, warn=None, debug_print=None): # ignore argument to FileList, but keep them for backwards # compatibility self.allfiles = None self.files = [] def set_allfiles(self, allfiles): self.allfiles = allfiles def findall(self, dir=os.curdir): self.allfiles = findall(dir) def debug_print(self, msg): """Print 'msg' to stdout if the global DEBUG (taken from the DISTUTILS_DEBUG environment variable) flag is true. """ from distutils.debug import DEBUG if DEBUG: print(msg) # Collection methods def append(self, item): self.files.append(item) def extend(self, items): self.files.extend(items) def sort(self): # Not a strict lexical sort! sortable_files = sorted(map(os.path.split, self.files)) self.files = [] for sort_tuple in sortable_files: self.files.append(os.path.join(*sort_tuple)) # Other miscellaneous utility methods def remove_duplicates(self): # Assumes list has been sorted! for i in range(len(self.files) - 1, 0, -1): if self.files[i] == self.files[i - 1]: del self.files[i] # "File template" methods def _parse_template_line(self, line): words = line.split() action = words[0] patterns = dir = dir_pattern = None if action in ('include', 'exclude', 'global-include', 'global-exclude'): if len(words) < 2: raise DistutilsTemplateError( f"'{action}' expects <pattern1> <pattern2> ..." ) patterns = [convert_path(w) for w in words[1:]] elif action in ('recursive-include', 'recursive-exclude'): if len(words) < 3: raise DistutilsTemplateError( f"'{action}' expects <dir> <pattern1> <pattern2> ..." ) dir = convert_path(words[1]) patterns = [convert_path(w) for w in words[2:]] elif action in ('graft', 'prune'): if len(words) != 2: raise DistutilsTemplateError( f"'{action}' expects a single <dir_pattern>" ) dir_pattern = convert_path(words[1]) else: raise DistutilsTemplateError(f"unknown action '{action}'") return (action, patterns, dir, dir_pattern) def process_template_line(self, line): # noqa: C901 # Parse the line: split it up, make sure the right number of words # is there, and return the relevant words. 'action' is always # defined: it's the first word of the line. Which of the other # three are defined depends on the action; it'll be either # patterns, (dir and patterns), or (dir_pattern). (action, patterns, dir, dir_pattern) = self._parse_template_line(line) # OK, now we know that the action is valid and we have the # right number of words on the line for that action -- so we # can proceed with minimal error-checking. if action == 'include': self.debug_print("include " + ' '.join(patterns)) for pattern in patterns: if not self.include_pattern(pattern, anchor=True): log.warning("warning: no files found matching '%s'", pattern) elif action == 'exclude': self.debug_print("exclude " + ' '.join(patterns)) for pattern in patterns: if not self.exclude_pattern(pattern, anchor=True): log.warning( ( "warning: no previously-included files " "found matching '%s'" ), pattern, ) elif action == 'global-include': self.debug_print("global-include " + ' '.join(patterns)) for pattern in patterns: if not self.include_pattern(pattern, anchor=False): log.warning( ( "warning: no files found matching '%s' " "anywhere in distribution" ), pattern, ) elif action == 'global-exclude': self.debug_print("global-exclude " + ' '.join(patterns)) for pattern in patterns: if not self.exclude_pattern(pattern, anchor=False): log.warning( ( "warning: no previously-included files matching " "'%s' found anywhere in distribution" ), pattern, ) elif action == 'recursive-include': self.debug_print("recursive-include {} {}".format(dir, ' '.join(patterns))) for pattern in patterns: if not self.include_pattern(pattern, prefix=dir): msg = "warning: no files found matching '%s' under directory '%s'" log.warning(msg, pattern, dir) elif action == 'recursive-exclude': self.debug_print("recursive-exclude {} {}".format(dir, ' '.join(patterns))) for pattern in patterns: if not self.exclude_pattern(pattern, prefix=dir): log.warning( ( "warning: no previously-included files matching " "'%s' found under directory '%s'" ), pattern, dir, ) elif action == 'graft': self.debug_print("graft " + dir_pattern) if not self.include_pattern(None, prefix=dir_pattern): log.warning("warning: no directories found matching '%s'", dir_pattern) elif action == 'prune': self.debug_print("prune " + dir_pattern) if not self.exclude_pattern(None, prefix=dir_pattern): log.warning( ("no previously-included directories found matching '%s'"), dir_pattern, ) else: raise DistutilsInternalError( f"this cannot happen: invalid action '{action}'" ) # Filtering/selection methods def include_pattern(self, pattern, anchor=True, prefix=None, is_regex=False): """Select strings (presumably filenames) from 'self.files' that match 'pattern', a Unix-style wildcard (glob) pattern. Patterns are not quite the same as implemented by the 'fnmatch' module: '*' and '?' match non-special characters, where "special" is platform- dependent: slash on Unix; colon, slash, and backslash on DOS/Windows; and colon on Mac OS. If 'anchor' is true (the default), then the pattern match is more stringent: "*.py" will match "foo.py" but not "foo/bar.py". If 'anchor' is false, both of these will match. If 'prefix' is supplied, then only filenames starting with 'prefix' (itself a pattern) and ending with 'pattern', with anything in between them, will match. 'anchor' is ignored in this case. If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and 'pattern' is assumed to be either a string containing a regex or a regex object -- no translation is done, the regex is just compiled and used as-is. Selected strings will be added to self.files. Return True if files are found, False otherwise. """ # XXX docstring lying about what the special chars are? files_found = False pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) self.debug_print(f"include_pattern: applying regex r'{pattern_re.pattern}'") # delayed loading of allfiles list if self.allfiles is None: self.findall() for name in self.allfiles: if pattern_re.search(name): self.debug_print(" adding " + name) self.files.append(name) files_found = True return files_found def exclude_pattern(self, pattern, anchor=True, prefix=None, is_regex=False): """Remove strings (presumably filenames) from 'files' that match 'pattern'. Other parameters are the same as for 'include_pattern()', above. The list 'self.files' is modified in place. Return True if files are found, False otherwise. """ files_found = False pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) self.debug_print(f"exclude_pattern: applying regex r'{pattern_re.pattern}'") for i in range(len(self.files) - 1, -1, -1): if pattern_re.search(self.files[i]): self.debug_print(" removing " + self.files[i]) del self.files[i] files_found = True return files_found # Utility functions def _find_all_simple(path): """ Find all files under 'path' """ all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True)) results = ( os.path.join(base, file) for base, dirs, files in all_unique for file in files ) return filter(os.path.isfile, results) class _UniqueDirs(set): """ Exclude previously-seen dirs from walk results, avoiding infinite recursion. Ref https://bugs.python.org/issue44497. """ def __call__(self, walk_item): """ Given an item from an os.walk result, determine if the item represents a unique dir for this instance and if not, prevent further traversal. """ base, dirs, files = walk_item stat = os.stat(base) candidate = stat.st_dev, stat.st_ino found = candidate in self if found: del dirs[:] self.add(candidate) return not found @classmethod def filter(cls, items): return filter(cls(), items) def findall(dir=os.curdir): """ Find all files under 'dir' and return the list of full filenames. Unless dir is '.', return full filenames with dir prepended. """ files = _find_all_simple(dir) if dir == os.curdir: make_rel = functools.partial(os.path.relpath, start=dir) files = map(make_rel, files) return list(files) def glob_to_re(pattern): """Translate a shell-like glob pattern to a regular expression; return a string containing the regex. Differs from 'fnmatch.translate()' in that '*' does not match "special characters" (which are platform-specific). """ pattern_re = fnmatch.translate(pattern) # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, # and by extension they shouldn't match such "special characters" under # any OS. So change all non-escaped dots in the RE to match any # character except the special characters (currently: just os.sep). sep = os.sep if os.sep == '\\': # we're using a regex to manipulate a regex, so we need # to escape the backslash twice sep = r'\\\\' escaped = rf'\1[^{sep}]' pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re) return pattern_re def translate_pattern(pattern, anchor=True, prefix=None, is_regex=False): """Translate a shell-like wildcard pattern to a compiled regular expression. Return the compiled regex. If 'is_regex' true, then 'pattern' is directly compiled to a regex (if it's a string) or just returned as-is (assumes it's a regex object). """ if is_regex: if isinstance(pattern, str): return re.compile(pattern) else: return pattern # ditch start and end characters start, _, end = glob_to_re('_').partition('_') if pattern: pattern_re = glob_to_re(pattern) assert pattern_re.startswith(start) and pattern_re.endswith(end) else: pattern_re = '' if prefix is not None: prefix_re = glob_to_re(prefix) assert prefix_re.startswith(start) and prefix_re.endswith(end) prefix_re = prefix_re[len(start) : len(prefix_re) - len(end)] sep = os.sep if os.sep == '\\': sep = r'\\' pattern_re = pattern_re[len(start) : len(pattern_re) - len(end)] pattern_re = rf'{start}\A{prefix_re}{sep}.*{pattern_re}{end}' else: # no prefix -- respect anchor flag if anchor: pattern_re = rf'{start}\A{pattern_re[len(start) :]}' return re.compile(pattern_re)