From d54dff79a9568c25551092711c7ebc28422006a4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Jun 2026 10:46:34 -0400 Subject: [PATCH] Normalize Unicode form when matching MANIFEST.in patterns FileList matched MANIFEST.in patterns against on-disk names byte-for-byte with no Unicode normalization. On macOS APFS/HFS+, a file stored NFD and a pattern authored NFC denote the same file but differ byte-for-byte, so an exclude/global-exclude/recursive-exclude/prune rule could silently fail to drop a non-ASCII-named file, publishing it in the sdist despite the rule. Normalize both the pattern and the candidate path to NFC before matching, via a new unicode_utils.normalize() helper and a _NormalizedMatcher wrapper around the compiled pattern in translate_pattern. Fixes GHSA-h35f-9h28-mq5c. CVE: CVE-2026-59890 Upstream-Status: Backport [https://github.com/pypa/setuptools/commit/dd9f436a36486b4cb8a4c70a2321548b0be09b8f] Co-Authored-By: Claude Opus 4.8 (cherry picked from commit dd9f436a36486b4cb8a4c70a2321548b0be09b8f) Signed-off-by: Darsh Kelaiya --- newsfragments/+ghsa-h35f-9h28-mq5c.bugfix.rst | 7 +++ setuptools/command/egg_info.py | 28 +++++++++++- setuptools/tests/test_manifest.py | 45 +++++++++++++++++++ setuptools/unicode_utils.py | 14 ++++++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 newsfragments/+ghsa-h35f-9h28-mq5c.bugfix.rst diff --git a/newsfragments/+ghsa-h35f-9h28-mq5c.bugfix.rst b/newsfragments/+ghsa-h35f-9h28-mq5c.bugfix.rst new file mode 100644 index 000000000..42d6c4cfe --- /dev/null +++ b/newsfragments/+ghsa-h35f-9h28-mq5c.bugfix.rst @@ -0,0 +1,7 @@ +``MANIFEST.in`` matching (via ``FileList``) is now insensitive to Unicode +normalization form. A pattern authored in one form (e.g. NFC, as typically +saved by editors) now matches a file whose name is stored on disk in another +(e.g. NFD, as produced by macOS APFS/HFS+). Previously an ``exclude``, +``global-exclude``, ``recursive-exclude``, or ``prune`` rule could silently +fail to drop a non-ASCII-named file from the source distribution, publishing +it despite the exclusion -- see GHSA-h35f-9h28-mq5c. diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 62d2feea9..e858708ee 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -34,6 +34,27 @@ from ..warnings import SetuptoolsDeprecationWarning PY_MAJOR = '{}.{}'.format(*sys.version_info) +class _NormalizedMatcher: + """ + Wrap a compiled pattern so that matching is insensitive to Unicode + normalization form. + + File names walked from disk (NFD on macOS APFS/HFS+) and patterns from + ``MANIFEST.in`` (typically NFC) can denote the same file while differing + byte-for-byte. Normalizing both sides before matching keeps an exclusion + (or inclusion) from silently failing. See GHSA-h35f-9h28-mq5c. + """ + + def __init__(self, pattern: re.Pattern) -> None: + self._pattern = pattern + + def match(self, path): + return self._pattern.match(unicode_utils.normalize(path)) + + def search(self, path): + return self._pattern.search(unicode_utils.normalize(path)) + + def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME """ Translate a file path glob like '*.txt' in to a regular expression. @@ -43,6 +64,11 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME """ pat = '' + # Normalize the pattern so it matches paths regardless of the Unicode + # normalization form used on disk (GHSA-h35f-9h28-mq5c). Candidate paths + # are normalized to the same form by ``_NormalizedMatcher``. + glob = unicode_utils.normalize(glob) + # This will split on '/' within [character classes]. This is deliberate. chunks = glob.split(os.path.sep) @@ -114,7 +140,7 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME pat += sep pat += r'\Z' - return re.compile(pat, flags=re.MULTILINE | re.DOTALL) + return _NormalizedMatcher(re.compile(pat, flags=re.MULTILINE | re.DOTALL)) class InfoCommon: diff --git a/setuptools/tests/test_manifest.py b/setuptools/tests/test_manifest.py index fbd21b197..1a7441fc3 100644 --- a/setuptools/tests/test_manifest.py +++ b/setuptools/tests/test_manifest.py @@ -10,6 +10,7 @@ import io import logging from distutils import log from distutils.errors import DistutilsTemplateError +import unicodedata from setuptools.command.egg_info import FileList, egg_info, translate_pattern from setuptools.dist import Distribution @@ -158,6 +159,21 @@ def test_translated_pattern_mismatch(pattern_mismatch): assert not translate_pattern(pattern).match(target) +def test_translate_pattern_unicode_normalization(): + """ + Matching is insensitive to Unicode normalization form: a pattern authored + in one form matches a path stored on disk in another (and vice versa), so + that an exclusion cannot be bypassed by an NFC/NFD mismatch. + + Regression test for GHSA-h35f-9h28-mq5c. + """ + nfc = unicodedata.normalize('NFC', 'café.txt') # 'café.txt' composed + nfd = unicodedata.normalize('NFD', 'café.txt') # 'café.txt' decomposed + assert nfc != nfd # the two byte forms genuinely differ + assert translate_pattern(nfc).match(nfd) + assert translate_pattern(nfd).match(nfc) + + class TempDirTestCase: def setup_method(self, method): self.temp_dir = tempfile.mkdtemp() @@ -331,6 +347,35 @@ class TestManifestTest(TempDirTestCase): files = default_files | set([ml('app/a.txt'), ml('app/b.txt'), ml('app/c.rst')]) assert files == self.get_files() + def test_global_exclude_unicode_normalization(self): + """ + A ``global-exclude`` authored NFC must drop a file whose on-disk name + is NFD: on macOS APFS/HFS+ the two are the same file, and even on + case/normalization-exact filesystems the decomposed name can be + committed and reach the build. Otherwise the file is published in the + sdist despite the exclusion. + + Regression test for GHSA-h35f-9h28-mq5c. + """ + nfc_name = unicodedata.normalize('NFC', 'café.txt') + nfd_name = unicodedata.normalize('NFD', 'café.txt') + assert nfc_name != nfd_name + # write the file under its decomposed (NFD) name ... + touch(os.path.join(self.temp_dir, 'app', nfd_name)) + # ... and exclude it with the composed (NFC) form. + self.make_manifest( + f""" + global-include *.txt + global-exclude {nfc_name} + """ + ) + leaked = { + f + for f in self.get_files() + if unicodedata.normalize('NFC', os.path.basename(f)) == nfc_name + } + assert not leaked, f"excluded file leaked into manifest: {leaked}" + class TestFileListTest(TempDirTestCase): """ diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py index d43dcc11f..311d4075a 100644 --- a/setuptools/unicode_utils.py +++ b/setuptools/unicode_utils.py @@ -15,6 +15,20 @@ def decompose(path): return path +def normalize(text): + """ + Return *text* in a canonical Unicode form (NFC) so that names which are + visually identical but encoded differently compare equal. + + macOS APFS/HFS+ store file names in decomposed form (NFD), while patterns + in ``MANIFEST.in`` are typically authored composed (NFC). The two denote + the same file but differ byte-for-byte, so matching them directly lets an + exclusion silently fail. Normalizing both the walked path and the pattern + to a single form before matching avoids that (GHSA-h35f-9h28-mq5c). + """ + return unicodedata.normalize('NFC', text) if isinstance(text, str) else text + + def filesys_decode(path): """ Ensure that the given path is decoded, -- 2.44.4