From 0f3ed4c8174b1abfc33c53a63b0f1ceb9c52bc4c Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 16 Dec 2019 18:24:08 -0700 Subject: [PATCH 1/8] bpo-31046: ensurepip does not honour the value of $(prefix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When cross-compiling, the local Python interpreter that is used to run `ensurepip` may not have the same value of `sys.prefix` as the value of the 'prefix' variable that is set in the Makefile. With the following values used to install Python locally for a later copy to the files hierarchy owned by the 'termux' application on an Android device: DESTDIR=/tmp/android prefix=/data/data/com.termux/files/usr/local 'make install' causes ensurepip to install pip in $(DESTDIR)/usr/local instead of the expected $(DESTDIR)/$(prefix) where is installed the standard library. The attached patch fixes the problem. The patch was implemented assuming that pip uses distutils for the installation (note that setup.py also uses the --prefix option in the Makefile), but I know nothing about pip so forgive me if the patch is wrong and please just assume it is just a way to demonstrate the problem. Fixes: https://github.com/python/cpython/issues/75229 Fixes: https://bugs.python.org/issue31046 Co-authored-by: Pradyun Gedam Co-authored-by: Erlend E. Aasland Co-authored-by: Zackery Spytz References: https://github.com/python/cpython/pull/17634 Signed-off-by: Matěj Cepl --- Doc/library/ensurepip.rst | 14 +++++++++++-- Lib/ensurepip/__init__.py | 20 ++++++++++++++----- Lib/test/test_ensurepip.py | 15 ++++++++++++++ Makefile.pre.in | 4 ++-- .../2019-12-16-17-50-42.bpo-31046.XA-Qfr.rst | 1 + 5 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-12-16-17-50-42.bpo-31046.XA-Qfr.rst diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index e0d77229b11802..8d3b2021d83d75 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -65,7 +65,11 @@ is at least as recent as the one available in ``ensurepip``, pass the By default, ``pip`` is installed into the current virtual environment (if one is active) or into the system site packages (if there is no active virtual environment). The installation location can be controlled -through two additional command line options: +through some additional command line options: + +.. option:: --prefix + + Installs ``pip`` using the given directory prefix. .. option:: --root @@ -108,7 +112,7 @@ Module API .. function:: bootstrap(root=None, upgrade=False, user=False, \ altinstall=False, default_pip=False, \ - verbosity=0) + verbosity=0, prefix=None) Bootstraps ``pip`` into the current or designated environment. @@ -136,6 +140,12 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. + *prefix* specifies the directory prefix to use when installing. + + .. versionadded:: 3.14 + + The *prefix* parameter. + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap .. note:: diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 5a55525d6bd235..3a2ed2d5d4bf74 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -109,25 +109,25 @@ def _disable_pip_configuration_settings(): def bootstrap(*, root=None, upgrade=False, user=False, altinstall=False, default_pip=False, - verbosity=0): + verbosity=0, prefix=None): """ Bootstrap pip into the current Python installation (or the given root - directory). + and directory prefix). Note that calling this function will alter both sys.path and os.environ. """ # Discard the return value _bootstrap(root=root, upgrade=upgrade, user=user, altinstall=altinstall, default_pip=default_pip, - verbosity=verbosity) + verbosity=verbosity, prefix=prefix) def _bootstrap(*, root=None, upgrade=False, user=False, altinstall=False, default_pip=False, - verbosity=0): + verbosity=0, prefix=None): """ Bootstrap pip into the current Python installation (or the given root - directory). Returns pip command status code. + and directory prefix). Returns pip command status code. Note that calling this function will alter both sys.path and os.environ. """ @@ -140,6 +140,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, "to install pip." ) from None + if root is not None and prefix is not None: + raise ValueError("Cannot use 'root' and 'prefix' together") if altinstall and default_pip: raise ValueError("Cannot use altinstall and default_pip together") @@ -172,6 +174,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] if root: args += ["--root", root] + if prefix: + args += ["--prefix", prefix] if upgrade: args += ["--upgrade"] if user: @@ -249,6 +253,11 @@ def _main(argv=None): default=None, help="Install everything relative to this alternate root directory.", ) + parser.add_argument( + "--prefix", + default=None, + help="Install everything using this prefix.", + ) parser.add_argument( "--altinstall", action="store_true", @@ -268,6 +277,7 @@ def _main(argv=None): return _bootstrap( root=args.root, + prefix=args.prefix, upgrade=args.upgrade, user=args.user, verbosity=args.verbosity, diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 20a56ed715d8ab..ed4a8e92ae416c 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -121,6 +121,21 @@ def test_bootstrapping_with_root(self): unittest.mock.ANY, ) + def test_bootstrapping_with_prefix(self): + ensurepip.bootstrap(prefix="/foo/bar/") + self.run_pip.assert_called_once_with( + [ + "install", "--no-cache-dir", "--no-index", "--find-links", + unittest.mock.ANY, "--prefix", "/foo/bar/", "pip", + ], + unittest.mock.ANY, + ) + + def test_root_and_prefix_mutual_exclusive(self): + with self.assertRaises(ValueError): + ensurepip.bootstrap(root="", prefix="") + self.assertFalse(self.run_pip.called) + def test_bootstrapping_with_user(self): ensurepip.bootstrap(user=True) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2ce53c6a816212..a45382ca2f27b2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2443,7 +2443,7 @@ install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ install|*) ensurepip="" ;; \ esac; \ $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ - $$ensurepip --root=$(DESTDIR)/ ; \ + $$ensurepip --prefix=$(prefix) ; \ fi .PHONY: altinstall @@ -2454,7 +2454,7 @@ altinstall: commoninstall install|*) ensurepip="--altinstall" ;; \ esac; \ $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ - $$ensurepip --root=$(DESTDIR)/ ; \ + $$ensurepip --prefix=$(prefix) ; \ fi .PHONY: commoninstall diff --git a/Misc/NEWS.d/next/Library/2019-12-16-17-50-42.bpo-31046.XA-Qfr.rst b/Misc/NEWS.d/next/Library/2019-12-16-17-50-42.bpo-31046.XA-Qfr.rst new file mode 100644 index 00000000000000..07eb89d4d23e50 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-12-16-17-50-42.bpo-31046.XA-Qfr.rst @@ -0,0 +1 @@ +A directory prefix can now be specified when using :mod:`ensurepip`. From f309776a6a31a405d5f49c490802a5be1e252bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Fri, 13 Jun 2025 22:58:14 +0200 Subject: [PATCH 2/8] bpo-31046: Fix ensurepip script shebangs with --root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using `python -m ensurepip` with the `--root` option for staged installations, the generated pip script contained an incorrect shebang that pointed into the staging directory. This made the installation unusable once the staging directory was removed. This commit fixes the issue by using the internal pip `--executable` option to force the shebang to point to the correct, final interpreter path. It also corrects related pathing issues: - Removes the check that incorrectly disallowed using --root and --prefix together. - Defaults the installation prefix to `/` when --root is used alone, ensuring installation occurs at the base of the staging directory. References: https://github.com/python/cpython/pull/17634#discussion_r1622453325 Signed-off-by: Matěj Cepl --- Lib/ensurepip/__init__.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 3a2ed2d5d4bf74..6d470c2746e449 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -140,8 +140,6 @@ def _bootstrap(*, root=None, upgrade=False, user=False, "to install pip." ) from None - if root is not None and prefix is not None: - raise ValueError("Cannot use 'root' and 'prefix' together") if altinstall and default_pip: raise ValueError("Cannot use altinstall and default_pip together") @@ -172,19 +170,40 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] - if root: - args += ["--root", root] - if prefix: - args += ["--prefix", prefix] if upgrade: args += ["--upgrade"] - if user: - args += ["--user"] if verbosity: args += ["-" + "v" * verbosity] if sys.implementation.cache_tag is None: args += ["--no-compile"] +        if user: +            # --user is mutually exclusive with --root/--prefix, +            # pip will enforce this. +            args += ["--user"] +        else: +            # Handle installation paths. +            # If --root is given but not --prefix, we default to a prefix of "/" +            # so that the install happens at the root of the --root directory. +            # Otherwise, pip would use the configured sys.prefix, e.g. +            # /usr/local, and install into ${root}/usr/local/. +            effective_prefix = prefix +            if root and not prefix: +                effective_prefix = "/" + +            if root: +                args += ["--root", root] + +            if effective_prefix: +                args += ["--prefix", effective_prefix] + +                # Force the script shebang to point to the correct, final +                # executable path. This is necessary when --root is used. +                executable_path = ( +                    Path(effective_prefix) / "bin" / Path(sys.executable).name +                ) +                args += ["--executable", os.fsdecode(executable_path)] + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) From 00bf1e79040fbc365b0e8aa8fefd5a686af81ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Mon, 7 Jul 2025 18:40:15 +0200 Subject: [PATCH 3/8] don't throw away `--root` parameter of ensurepip Don't break the default installation on the current system with good `sys.prefix`. --- Makefile.pre.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a45382ca2f27b2..2c12f7dcd44f4b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2443,7 +2443,7 @@ install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ install|*) ensurepip="" ;; \ esac; \ $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ - $$ensurepip --prefix=$(prefix) ; \ + $$ensurepip --root=$(DESTDIR)/ --prefix=$(prefix) ; \ fi .PHONY: altinstall @@ -2454,7 +2454,7 @@ altinstall: commoninstall install|*) ensurepip="--altinstall" ;; \ esac; \ $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ - $$ensurepip --prefix=$(prefix) ; \ + $$ensurepip --root=$(DESTDIR)/ --prefix=$(prefix) ; \ fi .PHONY: commoninstall From 182d8da75cfa1d2733c0f5a3039398b7adef3f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Tue, 8 Jul 2025 13:56:44 +0200 Subject: [PATCH 4/8] fix: remove test_root_and_prefix_mutual_exclusive It is made incorrect by the previous commit. --- Lib/test/test_ensurepip.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index ed4a8e92ae416c..fe9fff1bb0c759 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -131,11 +131,6 @@ def test_bootstrapping_with_prefix(self): unittest.mock.ANY, ) - def test_root_and_prefix_mutual_exclusive(self): - with self.assertRaises(ValueError): - ensurepip.bootstrap(root="", prefix="") - self.assertFalse(self.run_pip.called) - def test_bootstrapping_with_user(self): ensurepip.bootstrap(user=True) From ecfbdeb0979a8cc2eb1d2d598b2c7fe56dc1bb09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Fri, 25 Jul 2025 13:02:40 +0200 Subject: [PATCH 5/8] Remove   characters. --- Lib/ensurepip/__init__.py | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 6d470c2746e449..63586623ad906e 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -177,32 +177,32 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if sys.implementation.cache_tag is None: args += ["--no-compile"] -        if user: -            # --user is mutually exclusive with --root/--prefix, -            # pip will enforce this. -            args += ["--user"] -        else: -            # Handle installation paths. -            # If --root is given but not --prefix, we default to a prefix of "/" -            # so that the install happens at the root of the --root directory. -            # Otherwise, pip would use the configured sys.prefix, e.g. -            # /usr/local, and install into ${root}/usr/local/. -            effective_prefix = prefix -            if root and not prefix: -                effective_prefix = "/" - -            if root: -                args += ["--root", root] - -            if effective_prefix: -                args += ["--prefix", effective_prefix] - -                # Force the script shebang to point to the correct, final -                # executable path. This is necessary when --root is used. -                executable_path = ( -                    Path(effective_prefix) / "bin" / Path(sys.executable).name -                ) -                args += ["--executable", os.fsdecode(executable_path)] + if user: + # --user is mutually exclusive with --root/--prefix, + # pip will enforce this. + args += ["--user"] + else: + # Handle installation paths. + # If --root is given but not --prefix, we default to a prefix of "/" + # so that the install happens at the root of the --root directory. + # Otherwise, pip would use the configured sys.prefix, e.g. + # /usr/local, and install into ${root}/usr/local/. + effective_prefix = prefix + if root and not prefix: + effective_prefix = "/" + + if root: + args += ["--root", root] + + if effective_prefix: + args += ["--prefix", effective_prefix] + + # Force the script shebang to point to the correct, final + # executable path. This is necessary when --root is used. + executable_path = ( + Path(effective_prefix) / "bin" / Path(sys.executable).name + ) + args += ["--executable", os.fsdecode(executable_path)] return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) From bc377535bffbfba499c7eb50718eeb3156e09ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Thu, 7 Aug 2025 20:07:50 +0200 Subject: [PATCH 6/8] Fix ensurepip tests to match new --executable argument behavior Update test expectations in test_ensurepip.py to account for the new --executable argument that gets added when using --prefix or --root options. The implementation now correctly sets the executable path when installing with a custom prefix, but the tests weren't updated to expect this additional argument. - test_bootstrapping_with_root: expect --prefix "/" and --executable - test_bootstrapping_with_prefix: expect --executable argument --- Lib/test/test_ensurepip.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index fe9fff1bb0c759..aa472b82638b93 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -115,7 +115,8 @@ def test_bootstrapping_with_root(self): self.run_pip.assert_called_once_with( [ "install", "--no-cache-dir", "--no-index", "--find-links", - unittest.mock.ANY, "--root", "/foo/bar/", *COMPILE_OPT, + unittest.mock.ANY, "--root", "/foo/bar/", "--prefix", "/", + "--executable", unittest.mock.ANY, *COMPILE_OPT, "pip", ], unittest.mock.ANY, @@ -126,7 +127,8 @@ def test_bootstrapping_with_prefix(self): self.run_pip.assert_called_once_with( [ "install", "--no-cache-dir", "--no-index", "--find-links", - unittest.mock.ANY, "--prefix", "/foo/bar/", "pip", + unittest.mock.ANY, "--prefix", "/foo/bar/", + "--executable", unittest.mock.ANY, "pip", ], unittest.mock.ANY, ) From 60417f37bb39dfb8c7951cf90587c86e7d340192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Thu, 30 Apr 2026 01:43:49 +0200 Subject: [PATCH 7/8] gh-75229: limit ensurepip prefix overrides to explicit requests Keep root-only installs using the interpreter's default prefix while preserving the cross-build fix for Makefile-driven installs that pass an explicit prefix. --- Lib/ensurepip/__init__.py | 32 ++++++++++---------------------- Lib/test/test_ensurepip.py | 16 +++++++++++++--- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 63586623ad906e..27a44d32b5cf51 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -177,32 +177,20 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if sys.implementation.cache_tag is None: args += ["--no-compile"] + if root: + args += ["--root", root] + if user: # --user is mutually exclusive with --root/--prefix, # pip will enforce this. args += ["--user"] - else: - # Handle installation paths. - # If --root is given but not --prefix, we default to a prefix of "/" - # so that the install happens at the root of the --root directory. - # Otherwise, pip would use the configured sys.prefix, e.g. - # /usr/local, and install into ${root}/usr/local/. - effective_prefix = prefix - if root and not prefix: - effective_prefix = "/" - - if root: - args += ["--root", root] - - if effective_prefix: - args += ["--prefix", effective_prefix] - - # Force the script shebang to point to the correct, final - # executable path. This is necessary when --root is used. - executable_path = ( - Path(effective_prefix) / "bin" / Path(sys.executable).name - ) - args += ["--executable", os.fsdecode(executable_path)] + elif prefix: + args += ["--prefix", prefix] + + # Force the script shebang to point to the correct, final + # executable path. This is necessary when --root is used. + executable_path = Path(prefix) / "bin" / Path(sys.executable).name + args += ["--executable", os.fsdecode(executable_path)] return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index aa472b82638b93..e19a118d45e046 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -115,9 +115,7 @@ def test_bootstrapping_with_root(self): self.run_pip.assert_called_once_with( [ "install", "--no-cache-dir", "--no-index", "--find-links", - unittest.mock.ANY, "--root", "/foo/bar/", "--prefix", "/", - "--executable", unittest.mock.ANY, *COMPILE_OPT, - "pip", + unittest.mock.ANY, "--root", "/foo/bar/", "pip", *COMPILE_OPT, ], unittest.mock.ANY, ) @@ -133,6 +131,18 @@ def test_bootstrapping_with_prefix(self): unittest.mock.ANY, ) + def test_bootstrapping_with_root_and_prefix(self): + ensurepip.bootstrap(root="/foo/root/", prefix="/foo/prefix/") + self.run_pip.assert_called_once_with( + [ + "install", "--no-cache-dir", "--no-index", "--find-links", + unittest.mock.ANY, "--root", "/foo/root/", + "--prefix", "/foo/prefix/", "--executable", + unittest.mock.ANY, "pip", + ], + unittest.mock.ANY, + ) + def test_bootstrapping_with_user(self): ensurepip.bootstrap(user=True) From 559230109cb181622dc0af02c4908af74989819a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Wed, 13 May 2026 00:20:12 +0200 Subject: [PATCH 8/8] fix ensurepip shebangs via in-process pip monkey-patching Replace the unsupported --executable pip argument with an in-process monkey-patch of pip's internal script generation logic. The bundled version of pip does not support the --executable flag, causing integration failures (e.g., during 'make install'). This change intercepts the --executable argument in _run_pip and applies it by patching pip._internal.operations.install.wheel and pip._vendor.distlib.scripts.ScriptMaker before the installation starts. This ensures that scripts installed via ensurepip (with --prefix or --root) use the correct target interpreter shebang even when cross-compiling or performing staged installs. --- Lib/ensurepip/__init__.py | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 27a44d32b5cf51..20285463b070cb 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -70,8 +70,53 @@ def _run_pip(args, additional_paths=None): code = f""" import runpy import sys +import os + +# Extract --executable from args if present to avoid "unknown option" error from pip +# while still honoring the requested shebang. +pip_args = {args} +executable_path = None +if "--executable" in pip_args: + try: + idx = pip_args.index("--executable") + executable_path = pip_args[idx + 1] + del pip_args[idx:idx + 2] + except (ValueError, IndexError): + pass + sys.path = {additional_paths or []} + sys.path -sys.argv[1:] = {args} + +if executable_path: + try: + import pip._internal.operations.install.wheel as w + from pip._vendor.distlib.scripts import ScriptMaker + + def patched_fix_script(path): + with open(path, "rb") as script: + firstline = script.readline() + if not firstline.startswith(b"#!python"): + return False + exename = executable_path.encode(sys.getfilesystemencoding()) + firstline = b"#!" + exename + os.linesep.encode("ascii") + rest = script.read() + with open(path, "wb") as script: + script.write(firstline) + script.write(rest) + return True + + w.fix_script = patched_fix_script + + orig_init = ScriptMaker.__init__ + def patched_init(self, *a, **kw): + orig_init(self, *a, **kw) + self.executable = executable_path + ScriptMaker.__init__ = patched_init + except ImportError: + # If pip internals changed, we might not be able to patch. + # But we still want to run pip. + pass + +sys.argv[1:] = pip_args runpy.run_module("pip", run_name="__main__", alter_sys=True) """