Stackless issue #275: Support PEP-523 eval_frame-hook

Support the PEP-523 eval_frame-hook for one particular use case: byte
code instrumentation. An example is the pydev-debugger. It can use the
hook to inject "break-points". This is much more efficient, than
traditional trace functions.

Author: akruis

Include/internal/pycore_stackless.h

@@ -814,6 +814,9 @@ long slp_parse_thread_id(PyObject *thread_id, unsigned long *id);
     ((frame_)->f_executing >= SLP_FRAME_EXECUTING_VALUE && \
      (frame_)->f_executing <= SLP_FRAME_EXECUTING_YIELD_FROM)
 
+/* Frame is executing, ignore value in retval.
+ * This is used, if the eval_frame hook is in use. */
+#define SLP_FRAME_EXECUTING_HOOK ((char)100)
 
 /* Defined in slp_transfer.c */
 int

Python/ceval.c

@@ -4171,34 +4171,67 @@ handle_unwinding(int lineno, PyFrameObject *f,
 }
 
 PyObject*
-PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval_arg)
+PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval)
 {
     PyThreadState *tstate = _PyThreadState_GET();
     int executing = f->f_executing;
     if (executing == SLP_FRAME_EXECUTING_INVALID) {
         --tstate->recursion_depth;
-        return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval_arg);
-    } else if (executing == SLP_FRAME_EXECUTING_NO) {
+        return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval);
+    } else if (executing == SLP_FRAME_EXECUTING_NO ||
+               executing == SLP_FRAME_EXECUTING_HOOK) {
         /* Processing of a frame starts here */
 
         /* Check, if an extension module has changed tstate->interp->eval_frame.
          * PEP 523 defines this function pointer as an API to customise the frame
-         * evaluation. Stackless can not support this API. In order to prevent
-         * undefined behavior, we terminate the interpreter.
+         * evaluation. Stackless support this API and calls it, if it starts the
+         * evaluation of a frame.
          */
-        if (tstate->interp->eval_frame != _PyEval_EvalFrameDefault)
+        if (executing == SLP_FRAME_EXECUTING_NO &&
+                tstate->interp->eval_frame != _PyEval_EvalFrameDefault) {
+            Py_XDECREF(retval);
+            f->f_executing = SLP_FRAME_EXECUTING_HOOK;
+            retval = tstate->interp->eval_frame(f, throwflag);
+            /* There are two possibilities:
+             *  - Either the hook-functions delegates to _PyEval_EvalFrameDefault
+             *      Then the frame transfer protocol is observed, SLP_STORE_NEXT_FRAME(tstate, f->f_back)
+             *      has been called.
+             *  - Or the the hook function does not call _PyEval_EvalFrameDefault
+             *      Then the frame transfer protocol is (probably) violated and the code just
+             *      assigned tstate->frame = f->f_back.
+             *
+             * To distinguish both cases, we look a f->f_executing. If the value is still
+             * SLP_FRAME_EXECUTING_HOOK, then _PyEval_EvalFrameDefault wasn't called for frame f.
+             */
+            if (f->f_executing != SLP_FRAME_EXECUTING_HOOK)
+                /* _PyEval_EvalFrameDefault was called */
+                return retval;
+            /* Try to repair the frame reference count.
+             * It is possible in case of a simple tstate->frame = f->f_back */
+            if (tstate->frame == f->f_back) {
+                SLP_STORE_NEXT_FRAME(tstate, f->f_back);
+                return retval;
+            }
+            /* Game over */
             Py_FatalError("An extension module has set a custom frame evaluation function (see PEP 523).\n"
-                          "Stackless Python does not support the frame evaluation API defined by PEP 523.\n"
+                          "Stackless Python does not completely support the frame evaluation API defined by PEP 523.\n"
                           "The programm now terminates to prevent undefined behavior.\n");
+        } else if (executing == SLP_FRAME_EXECUTING_HOOK) {
+            f->f_executing = SLP_FRAME_EXECUTING_NO;
+        }
 
         if (SLP_CSTACK_SAVE_NOW(tstate, f)) {
             /* Setup the C-stack and recursively call PyEval_EvalFrameEx_slp with the same arguments.
              * SLP_CSTACK_SAVE_NOW(tstate, f) will be false then.
              */
-            return slp_eval_frame_newstack(f, throwflag, retval_arg);
+            return slp_eval_frame_newstack(f, throwflag, retval);
         }
     }
-    return slp_eval_frame_value(f, throwflag, retval_arg);
+
+    /* This is the only call of static slp_eval_frame_value.
+     * An optimizing compiler will eliminate this call
+     */
+    return slp_eval_frame_value(f, throwflag, retval);
 }
 
 
@@ -4261,6 +4294,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
     }
     if (PyFrame_Check(f)) {
         if (!(f->f_executing == SLP_FRAME_EXECUTING_NO ||
+              f->f_executing == SLP_FRAME_EXECUTING_HOOK ||
                 SLP_FRAME_IS_EXECUTING(f))) {
             PyErr_BadInternalCall();
             return NULL;
@@ -4282,6 +4316,10 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
         retval = Py_None;
     }
 
+    if (f->f_executing == SLP_FRAME_EXECUTING_HOOK)
+        /* Used, if a eval_frame-hook is installed */
+        return PyEval_EvalFrameEx_slp(f, throwflag, retval);
+
     /* test, if the stackless system has been initialized. */
     if (tstate->st.main == NULL) {
         /* Call from extern. Same logic as PyStackless_Call_Main */

Stackless/changelog.txt

@@ -9,6 +9,11 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://github.com/stackless-dev/stackless/issues/275
+  Add limited support for the eval_frame-hook introduced by PEP-523.
+  Stackless can't fully support all use-cases. Currently the hook can be used
+  by a debugger (i.e. pydevd) to modify the Python byte-code.
+
 - https://github.com/stackless-dev/stackless/issues/297
   Stackless now works on Linux ARM64 (architecture "aarch64").
 

Stackless/module/_teststackless.c

@@ -450,6 +450,92 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec
     return result;
 }
 
+static PyObject *pep523_frame_hook_throw = NULL;
+static PyObject *pep523_frame_hook_store_args = NULL;
+static PyObject *pep523_frame_hook_result = NULL;
+
+static PyObject *
+pep523_frame_hook_eval_frame(PyFrameObject *f, int throwflag) {
+    PyThreadState *tstate = PyThreadState_GET();
+    if (pep523_frame_hook_store_args) {
+        PyObject *ret = PyObject_CallFunction(pep523_frame_hook_store_args, "((Oi))", (PyObject *)f, throwflag);
+        if (!ret) {
+            tstate->frame = f->f_back;
+            return NULL;
+        }
+        Py_DECREF(ret);
+    }
+    if (pep523_frame_hook_throw) {
+        PyErr_SetObject((PyObject *)Py_TYPE(pep523_frame_hook_throw), pep523_frame_hook_throw);
+        Py_CLEAR(pep523_frame_hook_throw);
+        tstate->frame = f->f_back;
+        return NULL;
+    }
+    PyObject *retval = _PyEval_EvalFrameDefault(f, throwflag);
+
+    /* After _PyEval_EvalFrameDefault the current frame is invalid.
+     * We must not enter the interpreter and we can't use the frame transfer macros,
+     * because they are a Py_BUILD_CORE API only.
+     * In practice entering the interpreter would work. Only if you compile
+     * Stackless with SLP_WITH_FRAME_REF_DEBUG defined you get assertion failures.
+     */
+    if (retval != NULL ) {
+        Py_INCREF(retval);
+        Py_XSETREF(pep523_frame_hook_result, retval);
+    } else {
+        Py_INCREF(Py_NotImplemented);  /* anything else but None */
+        Py_XSETREF(pep523_frame_hook_result, Py_NotImplemented);
+    }
+    return retval;
+}
+
+/*
+ * The code below uses Python internal APIs
+ */
+#include "pycore_pystate.h"
+
+PyDoc_STRVAR(test_install_PEP523_eval_frame_hook__doc__,
+    "test_install_PEP523_eval_frame_hook(*, store_args, store_result, reset=False) -- a test function.\n\
+This function tests the PEP-523 eval_frame-hook. Usually it is not used by Stackless Python.");
+
+static PyObject* test_install_PEP523_eval_frame_hook(PyObject *self, PyObject *args, PyObject *kwds) {
+    static char *kwlist[] = { "throw", "store_args", "reset", NULL };
+    PyThreadState *tstate = PyThreadState_GET();
+    PyObject *throw = NULL;
+    PyObject *store_args = NULL;
+    PyObject *reset = Py_False;
+    PyObject *retval;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$O!OO!:test_install_PEP523_eval_frame_hook", kwlist,
+        PyExc_Exception, &throw, &store_args, &PyBool_Type, &reset))
+        return NULL;
+
+    if (PyObject_IsTrue(reset)) {
+        tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
+        Py_CLEAR(pep523_frame_hook_throw);
+        Py_CLEAR(pep523_frame_hook_store_args);
+    } else {
+
+        if (throw) {
+            Py_INCREF(throw);
+            Py_XSETREF(pep523_frame_hook_throw, throw);
+        }
+        if (store_args) {
+            Py_INCREF(store_args);
+            Py_XSETREF(pep523_frame_hook_store_args, store_args);
+        }
+
+        tstate->interp->eval_frame = pep523_frame_hook_eval_frame;
+    }
+    retval = pep523_frame_hook_result;
+    pep523_frame_hook_result = NULL;
+    if (!retval) {
+        retval = Py_Ellipsis;  /* just something different from None and NotImplemented */
+        Py_INCREF(retval);
+    }
+    return retval;
+}
+
 
 /* ---------- */
 
@@ -465,6 +551,8 @@ static PyMethodDef _teststackless_methods[] = {
     test_cstate__doc__ },
     { "test_PyEval_EvalFrameEx", (PyCFunction)(void(*)(void))test_PyEval_EvalFrameEx, METH_VARARGS | METH_KEYWORDS,
     test_PyEval_EvalFrameEx__doc__ },
+    { "test_install_PEP523_eval_frame_hook", (PyCFunction)(void(*)(void))test_install_PEP523_eval_frame_hook, METH_VARARGS | METH_KEYWORDS,
+    test_install_PEP523_eval_frame_hook__doc__ },
     {NULL,              NULL}           /* sentinel */
 };
 

Stackless/unittests/test_capi.py

@@ -33,6 +33,7 @@
 import subprocess
 import glob
 import pickle
+import types
 import _teststackless
 from support import test_main  # @UnusedImport
 from support import (StacklessTestCase, withThreads, require_one_thread,
@@ -375,6 +376,85 @@ def _test_soft_switchable_extension(self):
                 sys.stdout.buffer.write(output)
 
 
+class Test_pep523_frame_hook(StacklessTestCase):
+    def setUp(self):
+        super().setUp()
+        self.is_done = False
+
+    def tearDown(self):
+        super().tearDown()
+        _teststackless.test_install_PEP523_eval_frame_hook(reset=True)
+        self.is_done = None
+
+    def record_done(self, fail=False):
+        self.is_done = sys._getframe()
+        if fail:
+            4711 / 0
+        return id(self)
+
+    def filter_audit_frames(self, l):
+        # the test runner installs an audit callback. We have no control
+        # over this callback, but we can observe its frames.
+        return list(i for i in l if "audit" not in i[0].f_code.co_name)
+
+    def test_hook_delegates_to__PyEval_EvalFrameDefault(self):
+        # test, that the hook function delegates to _PyEval_EvalFrameDefault
+        args = []
+        r1 = _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append)
+        try:
+            r2 = self.record_done()
+        finally:
+            r3 = _teststackless.test_install_PEP523_eval_frame_hook(reset=True)
+
+        self.assertIs(r1, ...)
+        self.assertIsInstance(self.is_done, types.FrameType)
+        self.assertEqual(r2, id(self))
+        self.assertEqual(r3, r2)
+        args = self.filter_audit_frames(args)
+        self.assertListEqual(args, [(self.is_done, 0)])
+
+    def test_hook_delegates_to__PyEval_EvalFrameDefault_exc(self):
+        # test, that the hook function delegates to _PyEval_EvalFrameDefault
+        args = []
+        r1 = _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append)
+        try:
+            try:
+                self.record_done(fail=True)
+            finally:
+                r2 = _teststackless.test_install_PEP523_eval_frame_hook(reset=True)
+        except ZeroDivisionError as e:
+            self.assertEqual(str(e), "division by zero")
+        else:
+            self.fail("Expected exception")
+
+        self.assertIs(r1, ...)
+        self.assertIsInstance(self.is_done, types.FrameType)
+        self.assertEqual(r2, NotImplemented)
+        args = self.filter_audit_frames(args)
+        self.assertListEqual(args, [(self.is_done, 0)])
+
+    def test_hook_raises_exception(self):
+        # test, that the hook function delegates to _PyEval_EvalFrameDefault
+        args = []
+        _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append,
+                                                           throw=ZeroDivisionError("0/0"))
+        try:
+            try:
+                self.record_done()
+            finally:
+                r = _teststackless.test_install_PEP523_eval_frame_hook(reset=True)
+        except ZeroDivisionError as e:
+            self.assertEqual(str(e), "0/0")
+        else:
+            self.fail("Expected exception")
+
+        self.assertIs(self.is_done, False)
+        self.assertEqual(r, ...)
+        self.assertEqual(len(args), 1)
+        self.assertIsInstance(args[0][0], types.FrameType)
+        self.assertEqual(args[0][1], 0)
+
+
 if __name__ == "__main__":
     if not sys.argv[1:]:
         sys.argv.append('-v')