From 56ce4f97bfbfcdfcff9c0c88933944eae02687ed Mon Sep 17 00:00:00 2001 From: stepan Date: Tue, 19 May 2026 10:22:29 +0200 Subject: [PATCH 1/2] Implement C_CALL, C_RETURN, C_EXCEPTION Python hooks without Bytecode DSL support --- .../src/tests/unittest_tags/test_cprofile.txt | 7 +- .../src/tests/unittest_tags/test_profile.txt | 7 +- .../modules/MarshalModuleBuiltins.java | 4 +- .../builtins/objects/code/CodeBuiltins.java | 9 +- .../bytecode_dsl/RootNodeCompiler.java | 151 +++++++++++- .../bytecode_dsl/BytecodeDSLCodeUnit.java | 6 +- .../bytecode_dsl/PBytecodeDSLRootNode.java | 232 ++++++++++++++++-- .../lib-python/3/test/test_sys_setprofile.py | 108 ++++++++ 8 files changed, 478 insertions(+), 46 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cprofile.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cprofile.txt index 94b78fb6c0..d2b0a89322 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cprofile.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cprofile.txt @@ -2,11 +2,8 @@ test.test_cprofile.CProfileTest.test_output_file_when_changing_directory @ darwi test.test_cprofile.CProfileTest.test_run_profile_as_module @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_cprofile.CProfileTest.test_throw @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_cprofile.TestCommandLine.test_sort @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -# GR-71916 -!test.test_profile.ProfileTest.test_calling_conventions @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64 -!test.test_profile.ProfileTest.test_cprofile @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64 +test.test_profile.ProfileTest.test_calling_conventions @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64 +test.test_profile.ProfileTest.test_cprofile @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64 test.test_profile.ProfileTest.test_output_file_when_changing_directory @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_profile.ProfileTest.test_run @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -# This test times out in the gate even though it succeeds locally and in the retagger. Race condition? -!test.test_profile.ProfileTest.test_run_profile_as_module test.test_profile.ProfileTest.test_runctx @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_profile.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_profile.txt index d3d63c1134..2ab5a66cd0 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_profile.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_profile.txt @@ -1,8 +1,5 @@ -# GR-71916 -!test.test_profile.ProfileTest.test_calling_conventions @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64,win32-AMD64-github -!test.test_profile.ProfileTest.test_cprofile @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64,win32-AMD64-github +test.test_profile.ProfileTest.test_calling_conventions @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64,win32-AMD64-github +test.test_profile.ProfileTest.test_cprofile @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64,win32-AMD64-github test.test_profile.ProfileTest.test_output_file_when_changing_directory @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_profile.ProfileTest.test_run @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_profile.ProfileTest.test_run_profile_as_module @ linux-aarch64-github,linux-x86_64-github -!test.test_profile.ProfileTest.test_run_profile_as_module @ darwin-arm64,linux-aarch64,linux-x86_64,win32-AMD64,win32-AMD64-github test.test_profile.ProfileTest.test_runctx @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index d71bf7174c..5eb9452cc3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -1449,10 +1449,11 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() { int selfIndex = readInt(); int yieldFromGeneratorIndex = readInt(); int instrumentationDataIndex = readInt(); + int maxProfileCEventStackSize = readInt(); BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, bytecodeOffset, bytecodeSize, cacheKey); return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, cell2arg, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, provider); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, provider); } private void writeCodeUnit(CodeUnit code) throws IOException { @@ -1533,6 +1534,7 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti writeInt(code.selfIndex); writeInt(code.yieldFromGeneratorIndex); writeInt(code.instrumentationDataIndex); + writeInt(code.maxProfileCEventStackSize); } private PCode readCode() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java index 3e663b23ce..83830b644a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java @@ -448,12 +448,12 @@ private static List convertTripleBcisToInstructionIndices(BytecodeNode b int startInstructionIndex = 0; int instructionIndex = 0; - boolean wasLastInstructionInstrumentation = false; + boolean rangeHasInstruction = false; int lastTripleLine = -1; for (Instruction instruction : bytecodeNode.getInstructions()) { if (instruction.getBytecodeIndex() == triple[1] /* end bci */) { if (lastTripleLine != triple[2]) { - if (!wasLastInstructionInstrumentation) { + if (rangeHasInstruction) { result.add(PFactory.createTuple(language, new int[]{startInstructionIndex, instructionIndex, triple[2]})); lastTripleLine = triple[2]; } @@ -461,14 +461,13 @@ private static List convertTripleBcisToInstructionIndices(BytecodeNode b } triple = triples.get(++tripleIndex); assert triple[0] == instruction.getBytecodeIndex() : "bytecode ranges should be consecutive"; + rangeHasInstruction = false; } if (!instruction.isInstrumentation()) { // Emulate CPython's fixed 2-word instructions. instructionIndex += 2; - wasLastInstructionInstrumentation = false; - } else { - wasLastInstructionInstrumentation = true; + rangeHasInstruction = true; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index cc7bf0b7af..b6779ced5a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -268,6 +268,8 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor futureFeatures) { this(ctx, parent, null, rootNode, rootNode, futureFeatures); @@ -470,6 +472,7 @@ flags, orderedTruffleStringArray(names), selfIndex, yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, instrumentationDataLocal.getLocalIndex(), + maxProfileCEventStackSize, new BytecodeSupplier(nodes)); ctx.codeUnits.put(key, codeUnit); } @@ -2090,8 +2093,20 @@ private static boolean isAttributeLoad(ExprTy node) { private static final int NUM_ARGS_MAX_FIXED = 4; + private void enterProfileCEventCall() { + profileCEventStackSize++; + maxProfileCEventStackSize = Math.max(maxProfileCEventStackSize, profileCEventStackSize); + } + + private void exitProfileCEventCall() { + assert profileCEventStackSize > 0; + profileCEventStackSize--; + } + private void beginCallNAry(int numArgs) { assert numArgs <= NUM_ARGS_MAX_FIXED; + enterProfileCEventCall(); + b.beginInstrumentCallReturn(); switch (numArgs) { case 0 -> b.beginCallNilaryMethod(); case 1 -> b.beginCallUnaryMethod(); @@ -2110,16 +2125,64 @@ private void endCallNAry(int numArgs) { case 3 -> b.endCallTernaryMethod(); case 4 -> b.endCallQuaternaryMethod(); } + b.endInstrumentCallReturn(); + exitProfileCEventCall(); + } + + private void beginCallNilaryMethod() { + enterProfileCEventCall(); + b.beginInstrumentCallReturn(); + b.beginCallNilaryMethod(); + } + + private void endCallNilaryMethod() { + b.endCallNilaryMethod(); + b.endInstrumentCallReturn(); + exitProfileCEventCall(); + } + + private void beginCallUnaryMethod() { + enterProfileCEventCall(); + b.beginInstrumentCallReturn(); + b.beginCallUnaryMethod(); + } + + private void endCallUnaryMethod() { + b.endCallUnaryMethod(); + b.endInstrumentCallReturn(); + exitProfileCEventCall(); + } + + private void beginCallVarargsMethod() { + enterProfileCEventCall(); + b.beginInstrumentCallReturn(); + b.beginCallVarargsMethod(); + } + + private void endCallVarargsMethod() { + b.endCallVarargsMethod(); + b.endInstrumentCallReturn(); + exitProfileCEventCall(); } private void visitArguments(ExprTy func, ExprTy[] args, int numArgs) { + visitArguments(func, args, numArgs, true); + } + + private void visitArguments(ExprTy func, ExprTy[] args, int numArgs, boolean instrumentCall) { if (numArgs > 0) { for (int i = 0; i < numArgs - 1; i++) { args[i].accept(this); } + if (instrumentCall) { + b.beginInstrumentCall(); + } beginTraceLineChecked(b); args[numArgs - 1].accept(this); endTraceLineChecked(func, b); + if (instrumentCall) { + b.endInstrumentCall(); + } } } @@ -2139,7 +2202,7 @@ private void emitCall(ExprTy func, ExprTy[] args, KeywordTy[] keywords) { // @formatter:off if (useVariadic) { - b.beginCallVarargsMethod(); + beginCallVarargsMethod(); } else { beginCallNAry(numArgs); } @@ -2155,29 +2218,43 @@ private void emitCall(ExprTy func, ExprTy[] args, KeywordTy[] keywords) { function = beginTemporaryLocal(); b.beginBlock(); b.beginStoreLocal(function); + b.beginInstrumentCallable(); emitGetMethod(func, receiver); + b.endInstrumentCallable(); b.endStoreLocal(); b.emitLoadLocal(function); b.endBlock(); } else { + b.beginInstrumentCallable(); emitGetMethod(func, receiver); + b.endInstrumentCallable(); } b.beginCollectToObjectArray(); emitUnstar(() -> loadAndEndTemporaryLocal(receiver), args, null, func); b.endCollectToObjectArray(); + b.beginInstrumentCall(); if (hasKeywords) { emitNonEmptyKeywords(keywordGroups, function); // function local cleared in emitNonEmptyKeywords } else { emitEmptyKeywords(); } + b.endInstrumentCall(); } else { assert len(keywords) == 0; + b.beginInstrumentCallable(); emitGetMethod(func, receiver); - loadAndEndTemporaryLocal(receiver); // callable - visitArguments(func, args, numArgs - 1); + b.endInstrumentCallable(); + if (numArgs == 1) { + b.beginInstrumentCall(); + loadAndEndTemporaryLocal(receiver); // callable + b.endInstrumentCall(); + } else { + loadAndEndTemporaryLocal(receiver); // callable + visitArguments(func, args, numArgs - 1); + } } } else { if (useVariadic) { @@ -2188,23 +2265,31 @@ private void emitCall(ExprTy func, ExprTy[] args, KeywordTy[] keywords) { b.beginStoreLocal(function); func.accept(this); b.endStoreLocal(); + b.beginInstrumentCallable(); b.emitLoadLocal(function); + b.endInstrumentCallable(); b.endBlock(); } else if (hasKeywords) { + b.beginInstrumentCallable(); func.accept(this); + b.endInstrumentCallable(); } else { + b.beginInstrumentCallable(); func.accept(this); + b.endInstrumentCallable(); } b.beginCollectToObjectArray(); emitUnstar(null, args, null, func); b.endCollectToObjectArray(); + b.beginInstrumentCall(); if (hasKeywords) { emitNonEmptyKeywords(keywordGroups, function); // function local cleared in emitNonEmptyKeywords } else { emitEmptyKeywords(); } + b.endInstrumentCall(); } else { assert len(keywords) == 0; @@ -2214,7 +2299,15 @@ private void emitCall(ExprTy func, ExprTy[] args, KeywordTy[] keywords) { b.beginTag(DebuggerTags.AlwaysHalt.class); } + if (numArgs == 0) { + b.beginInstrumentCall(); + } + b.beginInstrumentCallable(); func.accept(this); // callable + b.endInstrumentCallable(); + if (numArgs == 0) { + b.endInstrumentCall(); + } visitArguments(func, args, numArgs); if (isBreakpoint) { @@ -2225,7 +2318,7 @@ private void emitCall(ExprTy func, ExprTy[] args, KeywordTy[] keywords) { // @formatter:off if (useVariadic) { - b.endCallVarargsMethod(); + endCallVarargsMethod(); } else { endCallNAry(numArgs); } @@ -2809,7 +2902,7 @@ private void emitUnstar(Runnable initialElementsProducer, ExprTy[] args, Runnabl initialElementsProducer.run(); } if (func != null) { - visitArguments(func, args, args.length); + visitArguments(func, args, args.length, false); } else { visitSequence(args); } @@ -3882,10 +3975,14 @@ public Void visit(StmtTy.ClassDef node) { BytecodeDSLCompilerResult classBody = classBodyCompiler.compileClassDefBody(node); BytecodeDSLCompilerResult typeParamsFun = typeParamsCompiler.compileClassTypeParams(node, classBody.codeUnit()); - b.beginCallNilaryMethod(); + beginCallNilaryMethod(); String typeParamsName = ""; + b.beginInstrumentCall(); + b.beginInstrumentCallable(); emitMakeFunction(typeParamsFun.codeUnit(), node.typeParams, typeParamsName, null, null); - b.endCallNilaryMethod(); + b.endInstrumentCallable(); + b.endInstrumentCall(); + endCallNilaryMethod(); } else { BytecodeDSLCompilerResult classBody = createRootNodeCompilerFor(node).compileClassDefBody(node); emitBuildClass(classBody.codeUnit(), node); @@ -3927,12 +4024,14 @@ private void emitBuildClass(BytecodeDSLCodeUnit body, ClassDef node) { b.endStoreLocal(); } - b.beginCallVarargsMethod(); + beginCallVarargsMethod(); + b.beginInstrumentCallable(); if (hasEmptyKeywords) { b.emitLoadBuildClass(); } else { b.emitLoadLocal(buildClassFunction); } + b.endInstrumentCallable(); Runnable finalElements = null; if (node.isGeneric()) { @@ -3961,14 +4060,16 @@ private void emitBuildClass(BytecodeDSLCodeUnit body, ClassDef node) { b.endCollectToObjectArray(); // keyword args + b.beginInstrumentCall(); if (hasEmptyKeywords) { emitEmptyKeywords(); } else { validateKeywords(node.keywords); emitNonEmptyKeywords(node.keywords, buildClassFunction); } + b.endInstrumentCall(); - b.endCallVarargsMethod(); + endCallVarargsMethod(); b.endBlock(); } @@ -4184,13 +4285,29 @@ public void emitFunctionDef(StmtTy node, String name, ArgumentsTy args, StmtTy[] BytecodeDSLCompilerResult typeParamsFunUnit = typeParamsCompiler.compileFunctionTypeParams(funBodyUnit.codeUnit(), node, name, args, returns, typeParams); String typeParamsName = ""; + if (argsCount == 0) { + b.beginInstrumentCall(); + } + b.beginInstrumentCallable(); emitMakeFunction(typeParamsFunUnit.codeUnit(), typeParams, typeParamsName, null, null); + b.endInstrumentCallable(); + if (argsCount == 0) { + b.endInstrumentCall(); + } if (hasDefaultArgs(args)) { + if (!hasDefaultKwargs(args)) { + b.beginInstrumentCall(); + } emitDefaultArgsArray(args); + if (!hasDefaultKwargs(args)) { + b.endInstrumentCall(); + } } if (hasDefaultKwargs(args)) { + b.beginInstrumentCall(); emitDefaultKwargsArray(args); + b.endInstrumentCall(); } endCallNAry(argsCount); @@ -4284,15 +4401,18 @@ public void beginWrapWithDecorators(ExprTy[] decorators) { for (int i = 0; i < decorators.length; i++) { // Attribute the eventual decorator call to the decorator expression, not the def/class line. beginSourceSectionInner(b, decorators[i].getSourceRange()); - b.beginCallUnaryMethod(); + beginCallUnaryMethod(); // evaluation of the decorator expression b.beginTraceLineWithArgument(); + b.beginInstrumentCallable(); decorators[i].accept(this); + b.endInstrumentCallable(); // trace line for the decorator expression, must be executed before the next // decorator expression is evaluated and before the function declaration itself b.endTraceLineWithArgument(decorators[i].getSourceRange().startLine); // trace the call to the decorator function (Python 3.12+) + b.beginInstrumentCall(); b.beginTraceLineWithArgument(); } } @@ -4306,7 +4426,8 @@ public void endWrapWithDecorators(ExprTy[] decorators) { // they will "flip" w.r.t. original decorator ordering, but tracings won't, so we // need to flip them manually b.endTraceLineWithArgument(decorators[decorators.length - 1 - i].getSourceRange().startLine); - b.endCallUnaryMethod(); + b.endInstrumentCall(); + endCallUnaryMethod(); b.endSourceSection(); } } @@ -6741,9 +6862,13 @@ public Void visit(TypeAlias node) { BytecodeDSLCompilerResult typeParamsFun = typeParamsCompiler.compileTypeAliasTypeParameters(name, body.codeUnit(), node); String typeParamsName = ""; - b.beginCallNilaryMethod(); + beginCallNilaryMethod(); + b.beginInstrumentCall(); + b.beginInstrumentCallable(); emitMakeFunction(typeParamsFun.codeUnit(), node.typeParams, typeParamsName, null, null); - b.endCallNilaryMethod(); + b.endInstrumentCallable(); + b.endInstrumentCall(); + endCallNilaryMethod(); } else { BytecodeDSLCompilerResult body = createRootNodeCompilerFor(node).compileTypeAliasBody(node); emitBuildTypeAlias(body.codeUnit(), node); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java index c719c0fd2a..0c0c5fc199 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java @@ -53,17 +53,19 @@ public final class BytecodeDSLCodeUnit extends CodeUnit { public final int selfIndex; public final int yieldFromGeneratorIndex; public final int instrumentationDataIndex; + public final int maxProfileCEventStackSize; private final BytecodeSupplier supplier; public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCount, int kwOnlyArgCount, int positionalOnlyArgCount, int flags, TruffleString[] names, TruffleString[] varnames, TruffleString[] cellvars, TruffleString[] freevars, int[] cell2arg, Object[] constants, int startLine, int startColumn, int endLine, int endColumn, - int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, BytecodeSupplier supplier) { + int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, BytecodeSupplier supplier) { super(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, cell2arg, constants, startLine, startColumn, endLine, endColumn); this.classcellIndex = classcellIndex; this.selfIndex = selfIndex; this.supplier = supplier; this.yieldFromGeneratorIndex = yieldFromGeneratorIndex; this.instrumentationDataIndex = instrumentationDataIndex; + this.maxProfileCEventStackSize = maxProfileCEventStackSize; } public abstract static class BytecodeSupplier { @@ -75,7 +77,7 @@ public abstract static class BytecodeSupplier { public BytecodeDSLCodeUnit withFlags(int flags) { return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, cell2arg, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, supplier); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, supplier); } @TruffleBoundary diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index 745a33e299..37bf559250 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -91,6 +91,7 @@ import com.oracle.graal.python.builtins.objects.exception.PBaseExceptionGroup; import com.oracle.graal.python.builtins.objects.frame.PFrame; import com.oracle.graal.python.builtins.objects.function.PArguments; +import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction; import com.oracle.graal.python.builtins.objects.function.PFunction; import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.function.Signature; @@ -103,6 +104,8 @@ import com.oracle.graal.python.builtins.objects.iterator.PLongSequenceIterator; import com.oracle.graal.python.builtins.objects.iterator.PObjectSequenceIterator; import com.oracle.graal.python.builtins.objects.list.PList; +import com.oracle.graal.python.builtins.objects.method.PBuiltinMethod; +import com.oracle.graal.python.builtins.objects.method.PMethod; import com.oracle.graal.python.builtins.objects.module.ModuleBuiltins; import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.object.ObjectBuiltins; @@ -205,6 +208,7 @@ import com.oracle.graal.python.nodes.bytecode.PrintExprNode; import com.oracle.graal.python.nodes.bytecode.RaiseNode; import com.oracle.graal.python.nodes.bytecode.SetupAnnotationsNode; +import com.oracle.graal.python.nodes.call.BoundDescriptor; import com.oracle.graal.python.nodes.call.CallDispatchers; import com.oracle.graal.python.nodes.call.CallDispatchers.FunctionIndirectInvokeNode; import com.oracle.graal.python.nodes.call.CallNode; @@ -383,8 +387,10 @@ public abstract class PBytecodeDSLRootNode extends PRootNode implements Bytecode addInstrumentation(TraceLine.class).// addInstrumentation(TraceLineAtLoopHeader.class).// addInstrumentation(ClearTraceLine.class).// + addInstrumentation(InstrumentCallable.class).// + addInstrumentation(InstrumentCall.class).// + addInstrumentation(InstrumentCallReturn.class).// addInstrumentation(TraceOrProfileReturn.class).// - addInstrumentation(TraceException.class).// addInstrumentation(TraceLineWithArgument.class).// addInstrumentation(EnterInstrumentedRoot.class).// addInstrumentation(ResumeYieldGenerator.class).// @@ -412,6 +418,7 @@ private static final class TracingNodes extends Node { @CompilationFinal protected transient int classcellIndex; @CompilationFinal protected transient int instrumentationDataIndex; @CompilationFinal protected transient int yieldFromGeneratorIndex = -1; + @CompilationFinal protected transient int maxProfileCEventStackSize; @CompilationFinal(dimensions = 1) protected transient Assumption[] cellEffectivelyFinalAssumptions; /* @@ -478,6 +485,7 @@ public void setMetadata(BytecodeDSLCodeUnit co, ParserCallbacksImpl parserErrorC } instrumentationDataIndex = co.instrumentationDataIndex; yieldFromGeneratorIndex = co.yieldFromGeneratorIndex; + maxProfileCEventStackSize = co.maxProfileCEventStackSize; PythonOptions.setUncachedInterpreterThreshold(getLanguage(), getBytecodeNode()); } @@ -522,7 +530,7 @@ private InstrumentationData getInstrumentationData(VirtualFrame frame, BytecodeN // This should only happen when this root was on stack when the config was updated. It // should have been deoptimized anyway when the stack was unwound back to it. CompilerDirectives.transferToInterpreterAndInvalidate(); - current = new InstrumentationData(); + current = new InstrumentationData(maxProfileCEventStackSize); bytecode.setLocalValue(0, frame, instrumentationDataIndex, current); } return current; @@ -531,7 +539,7 @@ private InstrumentationData getInstrumentationData(VirtualFrame frame, BytecodeN private void resetInstrumentationData(VirtualFrame frame, BytecodeNode bytecode) { InstrumentationData current = (InstrumentationData) bytecode.getLocalValue(0, frame, instrumentationDataIndex); if (current == null) { - current = new InstrumentationData(); + current = new InstrumentationData(maxProfileCEventStackSize); bytecode.setLocalValue(0, frame, instrumentationDataIndex, current); } current.reset(); @@ -548,7 +556,7 @@ public static final class EnterInstrumentedRoot { public static void doEnter(VirtualFrame frame, @Bind PBytecodeDSLRootNode root, @Bind BytecodeNode bytecode) { - bytecode.setLocalValue(0, frame, root.instrumentationDataIndex, new InstrumentationData()); + root.resetInstrumentationData(frame, bytecode); } } @@ -583,16 +591,45 @@ public static void doExit(VirtualFrame frame, AbstractTruffleException ate, } } - /* - * Data for tracing, profiling and instrumentation + /** + * Data for tracing, profiling and instrumentation. This data is stored in a dedicated frame + * slot when instrumentation is enabled. + * + *

+ * Builtin C profile events are implemented by wrapping calls with Bytecode DSL + * {@code @Instrumentation} operations using this structure: + * + * 

{@code
+     * InstrumentCallReturn    // report C_RETURN and pop profileCEventCallables
+     *     Call
+     *         InstrumentCallable    // push the callable before the call consumes it
+     *             callable-expression
+     *         arg1-expression
+     *         ...
+     *         InstrumentCall        // report C_CALL and set profileCEventCallStarted
+     *             argN-expression
+     * }
+     *
+ * + * In {@code interceptTruffleException} we check whether a {@code C_CALL} is underway. If so, + * we report the {@code C_EXCEPTION} event. + * + *

+ * A simpler implementation should be possible if GR-71168 adds support for replacing call + * operations directly, e.g. {@code @Instrumentation(replaces = Call.class)}. */ public static final class InstrumentationData { private int pastLine; // Sometimes, we need to use pastLine value after it has been cleared. Implicit returns in // combination with loops are one such scenario. private int nonClearingPastLine; + // null entries are non-builtin calls that must still preserve nested C-call stack shape + private final Object[] profileCEventCallables; + private int profileCEventStackTop; + private boolean profileCEventCallStarted; - public InstrumentationData() { + public InstrumentationData(int maxProfileCEventStackSize) { + this.profileCEventCallables = new Object[maxProfileCEventStackSize]; reset(); } @@ -619,6 +656,72 @@ void setNonClearingPastLine(int value) { public void reset() { this.pastLine = -1; this.nonClearingPastLine = -1; + clearProfileCEventCallableStack(); + } + + boolean hasProfileCEventCallables() { + return profileCEventStackTop > 0; + } + + Object exitProfileCall() { + if (profileCEventStackTop == 0) { + return null; + } + boolean callStarted = profileCEventCallStarted; + Object result = popProfileCEventCallable(); + return callStarted ? result : null; + } + + void pushProfileCEventCallable(Object callable) { + assert profileCEventStackTop < profileCEventCallables.length; + profileCEventCallables[profileCEventStackTop] = callable; + profileCEventStackTop++; + profileCEventCallStarted = false; + } + + private Object popProfileCEventCallable() { + profileCEventStackTop--; + Object result = profileCEventCallables[profileCEventStackTop]; + profileCEventCallables[profileCEventStackTop] = null; + profileCEventCallStarted = false; + return result; + } + + void startProfileCEventCall(PBytecodeDSLRootNode root, VirtualFrame frame, BytecodeNode location) { + if (profileCEventStackTop > 0) { + Object profileArg = profileCEventCallables[profileCEventStackTop - 1]; + profileCEventCallStarted = profileArg != null && root.profileCEvent(frame, location, profileArg, ProfileEvent.C_CALL); + } + } + + void clearProfileCEventCallableStack() { + if (profileCEventCallables != null) { + for (int i = 0; i < profileCEventStackTop; i++) { + profileCEventCallables[i] = null; + } + } + profileCEventStackTop = 0; + profileCEventCallStarted = false; + } + + // called when we intercept an exception; report C_EXCEPTION only for a C call that actually started + void profileCEventCallablesForException(PBytecodeDSLRootNode root, VirtualFrame frame, BytecodeNode location) { + int oldStackTop = profileCEventStackTop; + boolean callStarted = profileCEventCallStarted; + profileCEventStackTop = 0; + try { + if (callStarted && oldStackTop > 0) { + Object profileArg = profileCEventCallables[oldStackTop - 1]; + if (profileArg != null) { + root.profileCEvent(frame, location, profileArg, ProfileEvent.C_EXCEPTION); + } + } + } finally { + for (int i = oldStackTop - 1; i >= 0; i--) { + profileCEventCallables[i] = null; + } + profileCEventCallStarted = false; + } } } @@ -677,6 +780,45 @@ private void invokeProfileFunction(VirtualFrame virtualFrame, BytecodeNode locat } } + private static Object getBuiltinProfileArg(Object callable) { + Object unwrappedCallable = unwrapBoundDescriptor(callable); + if (isBuiltin(unwrappedCallable)) { + return unwrappedCallable; + } + if (unwrappedCallable instanceof PMethod method && isBuiltin(method.getFunction())) { + return method.getFunction(); + } + return null; + } + + private static Object unwrapBoundDescriptor(Object callable) { + return callable instanceof BoundDescriptor boundDescriptor ? boundDescriptor.descriptor : callable; + } + + private static boolean isBuiltin(Object callable) { + return callable instanceof PBuiltinFunction || callable instanceof PBuiltinMethod; + } + + @InliningCutoff + private boolean profileCEvent(VirtualFrame virtualFrame, BytecodeNode location, Object arg, PythonContext.ProfileEvent event) { + PythonThreadState threadState = getThreadState(); + Object profileFun = threadState.getProfileFun(); + if (profileFun != null && !threadState.isProfiling()) { + invokeProfileFunction(virtualFrame, location, profileFun, threadState, event, arg); + return true; + } + return false; + } + + @InliningCutoff + private void profilePendingCExceptions(VirtualFrame frame, BytecodeNode location) { + getInstrumentationData(frame, location).profileCEventCallablesForException(this, frame, location); + } + + private void clearPendingCExceptions(VirtualFrame frame, BytecodeNode location) { + getInstrumentationData(frame, location).clearProfileCEventCallableStack(); + } + @InliningCutoff private void invokeTraceFunction(VirtualFrame virtualFrame, BytecodeNode location, Object traceFun, PythonContext.PythonThreadState threadState, PythonContext.TraceEvent event, Object arg, int line) { @@ -825,6 +967,7 @@ private PException traceException(VirtualFrame frame, BytecodeNode bytecode, int PException result = pe; PythonThreadState threadState = getThreadState(); + profilePendingCExceptions(frame, bytecode); // We should only trace the exception if tracing is enabled. if (threadState.getTraceFun() != null && !pe.getShouldTrace()) { PFrame pyFrame = ensurePyFrame(frame, bytecode); @@ -899,6 +1042,70 @@ public static void perform(VirtualFrame frame, } } + @Instrumentation(storeBytecodeIndex = true) + public static final class InstrumentCallable { + @Specialization + public static Object perform(VirtualFrame frame, Object callable, + @Bind BytecodeNode location, + @Bind PBytecodeDSLRootNode root) { + PythonThreadState threadState = root.getThreadState(); + if (threadState.isProfiling()) { + return callable; + } + InstrumentationData instrumentationData = root.getInstrumentationData(frame, location); + if (threadState.getProfileFun() == null) { + if (instrumentationData.hasProfileCEventCallables()) { + instrumentationData.pushProfileCEventCallable(null); + } + return callable; + } + Object profileArg = getBuiltinProfileArg(callable); + if (profileArg != null) { + instrumentationData.pushProfileCEventCallable(profileArg); + } else if (instrumentationData.hasProfileCEventCallables()) { + instrumentationData.pushProfileCEventCallable(null); + } + return callable; + } + } + + @Instrumentation(storeBytecodeIndex = true) + public static final class InstrumentCall { + @Specialization + public static Object perform(VirtualFrame frame, Object value, + @Bind BytecodeNode location, + @Bind PBytecodeDSLRootNode root) { + PythonThreadState threadState = root.getThreadState(); + if (!threadState.isProfiling()) { + root.getInstrumentationData(frame, location).startProfileCEventCall(root, frame, location); + } + return value; + } + } + + @Instrumentation(storeBytecodeIndex = true) + public static final class InstrumentCallReturn { + @Specialization + public static Object perform(VirtualFrame frame, Object value, + @Bind BytecodeNode location, + @Bind PBytecodeDSLRootNode root) { + PythonThreadState threadState = root.getThreadState(); + if (threadState.isProfiling()) { + return value; + } + InstrumentationData instrumentationData = root.getInstrumentationData(frame, location); + if (threadState.getProfileFun() == null) { + instrumentationData.exitProfileCall(); + return value; + } + Object profileArg = instrumentationData.exitProfileCall(); + if (profileArg != null) { + root.profileCEvent(frame, location, profileArg, ProfileEvent.C_RETURN); + } + return value; + } + } + @Instrumentation(storeBytecodeIndex = true) public static final class TraceOrProfileReturn { @Specialization @@ -924,16 +1131,11 @@ public static Object perform(VirtualFrame frame, Object value, int line, } } - @Instrumentation - public static final class TraceException { - @Specialization - public static void perform() { - throw new UnsupportedOperationException("trace exception not implemented"); - } - } - @Override public Throwable interceptInternalException(Throwable throwable, VirtualFrame frame, BytecodeNode bytecodeNode, int bci) { + if (needsTraceAndProfileInstrumentation()) { + clearPendingCExceptions(frame, bytecodeNode); + } PythonLanguage language = getLanguage(); if (language.getEngineOption(PythonOptions.CatchAllExceptions) && (throwable instanceof Exception || throwable instanceof AssertionError)) { return ExceptionUtils.wrapJavaException(throwable, this, PFactory.createBaseException(language, SystemError, ErrorMessages.M, new Object[]{throwable})); diff --git a/graalpython/lib-python/3/test/test_sys_setprofile.py b/graalpython/lib-python/3/test/test_sys_setprofile.py index f77036962c..97f58a97d6 100644 --- a/graalpython/lib-python/3/test/test_sys_setprofile.py +++ b/graalpython/lib-python/3/test/test_sys_setprofile.py @@ -492,6 +492,114 @@ class A: # The last c_call is the call to sys.setprofile self.assertEqual(events, ['c_call', 'c_return', 'c_call']) + def test_nested_c_call_argument_exception(self): + events = [] + + def profile(frame, event, arg): + arg_name = getattr(arg, "__name__", None) + if frame.f_code.co_name == "f" or (event.startswith("c_") and arg_name != "setprofile"): + events.append((event, arg_name)) + + def f(): + try: + id(len(1)) + except TypeError: + pass + + sys.setprofile(profile) + f() + sys.setprofile(None) + + self.assertEqual(events, [ + ("call", None), + ("c_call", "len"), + ("c_exception", "len"), + ("return", None), + ]) + + def test_c_call_after_python_argument_evaluation(self): + events = [] + + def profile(frame, event, arg): + arg_name = getattr(arg, "__name__", None) + if frame.f_code.co_name in {"f", "make_arg"} or (event.startswith("c_") and arg_name != "setprofile"): + events.append((event, frame.f_code.co_name, arg_name)) + + def make_arg(): + return [] + + def f(): + id(make_arg()) + + sys.setprofile(profile) + f() + sys.setprofile(None) + + self.assertEqual(events, [ + ("call", "f", None), + ("call", "make_arg", None), + ("return", "make_arg", None), + ("c_call", "f", "id"), + ("c_return", "f", "id"), + ("return", "f", None), + ]) + + def test_nested_c_call_after_inner_return(self): + events = [] + + def profile(frame, event, arg): + arg_name = getattr(arg, "__name__", None) + if frame.f_code.co_name == "f" or (event.startswith("c_") and arg_name != "setprofile"): + events.append((event, arg_name)) + + def f(): + id(len([])) + + sys.setprofile(profile) + f() + sys.setprofile(None) + + self.assertEqual(events, [ + ("call", None), + ("c_call", "len"), + ("c_return", "len"), + ("c_call", "id"), + ("c_return", "id"), + ("return", None), + ]) + + def test_nested_c_call_callback_exception(self): + events = [] + + def profile(frame, event, arg): + arg_name = getattr(arg, "__name__", None) + if frame.f_code.co_name in {"f", "key"} or (event.startswith("c_") and arg_name != "setprofile"): + events.append((event, frame.f_code.co_name, arg_name)) + + def key(obj): + return len(1) + + def f(): + try: + sorted([1], key=key) + except TypeError: + pass + + sys.setprofile(profile) + f() + sys.setprofile(None) + + self.assertEqual(events, [ + ("call", "f", None), + ("c_call", "f", "sorted"), + ("call", "key", None), + ("c_call", "key", "len"), + ("c_exception", "key", "len"), + ("return", "key", None), + ("c_exception", "f", "sorted"), + ("return", "f", None), + ]) + if __name__ == "__main__": unittest.main() From 123f674b9ba0a61c893c0d831e63502ef5d89c5e Mon Sep 17 00:00:00 2001 From: stepan Date: Fri, 22 May 2026 09:16:14 +0200 Subject: [PATCH 2/2] Add tests for async tracing ignored for now --- .../src/tests/test_sys_settrace.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py b/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py index ee862e408f..a4bb8236e7 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py @@ -519,6 +519,150 @@ def func(): self.assert_events(self.events, events) +class AsyncTracingEventsUnitTest(TracingEventsUnitTest): + def async_trace(self, frame, event, arg): + code = frame.f_code + name = code.co_name + if name in self.names: + self.events.append((frame.f_lineno - self.first_line, name, event)) + return self.async_trace + + async def async_wrapper(self, afunc, names): + self.names = names + sys.settrace(self.async_trace) + try: + await afunc() + finally: + sys.settrace(None) + self.names = None + + def trace_async_function(self, afunc, names): + self.first_line = afunc.__code__.co_firstlineno + self.events = [] + asyncio.run(self.async_wrapper(afunc, names)) + + @unittest.skipIf( + sys.implementation.name == "graalpy", + "GR-65570: GraalPy does not report await exception trace events yet.", + ) + def test_01_await(self): + async def helper(): # line -3 + return 42 + + async def afunc(): + value = await helper() + return value + + self.trace_async_function(afunc, {"afunc", "helper"}) + + events = [ + (0, 'afunc', 'call'), + (1, 'afunc', 'line'), + (-3, 'helper', 'call'), + (-2, 'helper', 'line'), + (-2, 'helper', 'return'), + (1, 'afunc', 'exception'), + (2, 'afunc', 'line'), + (2, 'afunc', 'return'), + ] + + self.assert_events(self.events, events) + + @unittest.skipIf( + sys.implementation.name == "graalpy", + "GR-65570: GraalPy does not report async with exception trace events yet.", + ) + def test_02_async_with(self): + class A: + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc, tb): + pass + + async def afunc(): + async with A(): + pass + + self.trace_async_function(afunc, {"afunc", "__aenter__", "__aexit__"}) + + events = [ + (0, 'afunc', 'call'), + (1, 'afunc', 'line'), + (-5, '__aenter__', 'call'), + (-4, '__aenter__', 'line'), + (-4, '__aenter__', 'return'), + (1, 'afunc', 'exception'), + (2, 'afunc', 'line'), + (1, 'afunc', 'line'), + (-3, '__aexit__', 'call'), + (-2, '__aexit__', 'line'), + (-2, '__aexit__', 'return'), + (1, 'afunc', 'exception'), + (1, 'afunc', 'return'), + ] + + self.assert_events(self.events, events) + + @unittest.skipIf( + sys.implementation.name == "graalpy", + "GR-65570: GraalPy does not match CPython async for trace events yet.", + ) + def test_03_async_for(self): + class AsyncIterator: + def __init__(self, items): + self.items = iter(items) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.items) + except StopIteration: + raise StopAsyncIteration + + async def afunc(): + total = 0 + async for item in AsyncIterator([1, 2]): + total += item + return total + + self.trace_async_function(afunc, {"afunc", "__aiter__", "__anext__"}) + + events = [ + (0, 'afunc', 'call'), + (1, 'afunc', 'line'), + (2, 'afunc', 'line'), + (-8, '__aiter__', 'call'), + (-7, '__aiter__', 'line'), + (-7, '__aiter__', 'return'), + (-6, '__anext__', 'call'), + (-5, '__anext__', 'line'), + (-4, '__anext__', 'line'), + (-4, '__anext__', 'return'), + (2, 'afunc', 'exception'), + (3, 'afunc', 'line'), + (2, 'afunc', 'line'), + (-6, '__anext__', 'call'), + (-5, '__anext__', 'line'), + (-4, '__anext__', 'line'), + (-4, '__anext__', 'return'), + (2, 'afunc', 'exception'), + (3, 'afunc', 'line'), + (2, 'afunc', 'line'), + (-6, '__anext__', 'call'), + (-5, '__anext__', 'line'), + (-4, '__anext__', 'line'), + (-4, '__anext__', 'exception'), + (-3, '__anext__', 'line'), + (-2, '__anext__', 'line'), + (-2, '__anext__', 'exception'), + (-2, '__anext__', 'return'), + (2, 'afunc', 'exception'), + (4, 'afunc', 'line'), + (4, 'afunc', 'return'), + ] + + self.assert_events(self.events, events) + class MultilineCallsTraceTest(TracingEventsUnitTest): class A: def m_basic(self, a1, a2, a3):