From b4f39345ffeba1b28f016a76d9527114422e9d45 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sat, 19 Apr 2025 12:55:39 -0500 Subject: [PATCH 1/7] Support profiling modules that import __main___ --- Lib/cProfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index e7c868b8d55543..8503ccbe57f83c 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -6,6 +6,7 @@ import _lsprof import importlib.machinery +import importlib.util import io import profile as _pyprofile @@ -169,17 +170,22 @@ def main(): else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) - with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) + loader = importlib.machinery.SourceFileLoader("__main__", progname) + spec.loader = loader + module = importlib.util.module_from_spec(spec) globs = { '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, + 'module': module } + + sys.modules["__main__"] = module + code = "__spec__.loader.exec_module(module)" try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: From e068beafaffe3e1fb045fb368d80206a96e803f1 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 19 Apr 2025 18:07:35 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst new file mode 100644 index 00000000000000..5bdd54a3fbae17 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -0,0 +1 @@ +Support profiling modules that import __main___, such as modules that use to pickle. The github issue has an example repro that throws an exception without this change, and succeeds with it. From ca66a0e3e2668711a3cc7e40aae58dc163c708c4 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:12:23 -0500 Subject: [PATCH 3/7] replace __main__'s namespace instead of creating a new module --- Lib/cProfile.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 8503ccbe57f83c..a0138201573f55 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -98,13 +98,25 @@ def run(self, cmd): return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): + # cmd has to run in __main__ namespace (or imports from __main__ will + # break). Clear __main__ and replace with the globals provided. + import __main__ + # Save a reference to the current __main__ namespace so that we can + # restore it after cmd completes. + original_main = __main__.__dict__.copy() + __main__.__dict__.clear() + __main__.__dict__.update(globals) + self.enable() try: - exec(cmd, globals, locals) + exec(cmd, __main__.__dict__, locals) finally: self.disable() + __main__.__dict__.clear() + __main__.__dict__.update(original_main) return self + # This method is more useful to profile a single function call. def runcall(self, func, /, *args, **kw): self.enable() @@ -170,22 +182,19 @@ def main(): else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) + with io.open_code(progname) as fp: + code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) - loader = importlib.machinery.SourceFileLoader("__main__", progname) - spec.loader = loader - module = importlib.util.module_from_spec(spec) globs = { '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, - 'module': module + '__builtins__': __builtins__, } - sys.modules["__main__"] = module - code = "__spec__.loader.exec_module(module)" try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: From 1c621a0154480168886b62bfa9d5a611f4f6f319 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:14:29 -0500 Subject: [PATCH 4/7] Update Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst index 5bdd54a3fbae17..892e3ba4b57c81 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -1 +1 @@ -Support profiling modules that import __main___, such as modules that use to pickle. The github issue has an example repro that throws an exception without this change, and succeeds with it. +Support profiling modules that import __main___, such as modules that use to pickle. From 923cb6e5f22115a7ad6e698c84681784b82d7749 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:15:37 -0500 Subject: [PATCH 5/7] quote __main__ and fix typo --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst index 892e3ba4b57c81..bc2731d797b421 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -1 +1 @@ -Support profiling modules that import __main___, such as modules that use to pickle. +Support profiling modules that import ``__main__``, such as modules that use pickle. From 75a542e93cf5b253ad8dcaf0451907cf76d012dc Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 13:41:50 -0500 Subject: [PATCH 6/7] Add regression test --- Lib/cProfile.py | 6 ++---- Lib/test/test_cprofile.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index a0138201573f55..13e4dd239a5554 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -175,10 +175,8 @@ def main(): if len(args) > 0: if options.module: code = "run_module(modname, run_name='__main__')" - globs = { - 'run_module': runpy.run_module, - 'modname': args[0] - } + globs = globals().copy() + globs.update({"run_module": runpy.run_module, "modname": args[0]}) else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index b46edf66bf09f8..b7389adf30c3d0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -5,8 +5,10 @@ # rip off all interesting stuff from test_profile import cProfile +import tempfile +import textwrap from test.test_profile import ProfileTest, regenerate_expected_output -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test import support @@ -155,6 +157,22 @@ def test_sort(self): self.assertIn(b"option -s: invalid choice: 'demo'", err) +class TestProfilingScript(unittest.TestCase): + def test_profile_script_importing_main(self): + """Check that scripts that reference __main__ see their own namespace + when being profiled.""" + with tempfile.NamedTemporaryFile("w+") as f: + f.write(textwrap.dedent("""\ + class Foo: + pass + + import __main__ + assert Foo == __main__.Foo + """)) + f.flush() + assert_python_ok('-m', "cProfile", f.name) + + def main(): if '-r' not in sys.argv: unittest.main() From 3665a7f3f676b6f9abe477a680c507d019c9f0bd Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 13:45:48 -0500 Subject: [PATCH 7/7] Only modify __main__ in CLI invocation --- Lib/cProfile.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 13e4dd239a5554..e07ad711f89e24 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -98,25 +98,13 @@ def run(self, cmd): return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): - # cmd has to run in __main__ namespace (or imports from __main__ will - # break). Clear __main__ and replace with the globals provided. - import __main__ - # Save a reference to the current __main__ namespace so that we can - # restore it after cmd completes. - original_main = __main__.__dict__.copy() - __main__.__dict__.clear() - __main__.__dict__.update(globals) - self.enable() try: - exec(cmd, __main__.__dict__, locals) + exec(cmd, globals, locals) finally: self.disable() - __main__.__dict__.clear() - __main__.__dict__.update(original_main) return self - # This method is more useful to profile a single function call. def runcall(self, func, /, *args, **kw): self.enable() @@ -192,10 +180,19 @@ def main(): '__cached__': None, '__builtins__': __builtins__, } + # cmd has to run in __main__ namespace (or imports from __main__ will + # break). Clear __main__ and replace with the globals provided. + import __main__ + # Save a reference to the current __main__ namespace so that we can + # restore it after cmd completes. + original_main = __main__.__dict__.copy() + __main__.__dict__.update(globs) try: - runctx(code, globs, None, options.outfile, options.sort) + runctx(code, __main__.__dict__, None, options.outfile, options.sort) except BrokenPipeError as exc: + __main__.__dict__.clear() + __main__.__dict__.update(original_main) # Prevent "Exception ignored" during interpreter shutdown. sys.stdout = None sys.exit(exc.errno)