Merge with default (PEP 492)

Unfortunately the outcome of this merge is not yet functional.

Author: Anselm Kruis

Doc/includes/typestruct.h

@@ -9,7 +9,7 @@ typedef struct _typeobject {
     printfunc tp_print;
     getattrfunc tp_getattr;
     setattrfunc tp_setattr;
-    void *tp_reserved;
+    PyAsyncMethods *tp_as_async;
     reprfunc tp_repr;
 
     /* Method suites for standard classes */

Doc/library/dis.rst

@@ -508,6 +508,38 @@ the original TOS1.
    Implements ``del TOS1[TOS]``.
 
 
+**Coroutines opcodes**
+
+.. opcode:: GET_AWAITABLE
+
+   Implements ``TOS = get_awaitable(TOS)``; where ``get_awaitable(o)``
+   returns ``o`` if ``o`` is a coroutine object; or resolved ``o.__await__``.
+
+
+.. opcode:: GET_AITER
+
+   Implements ``TOS = get_awaitable(TOS.__aiter__())``.  See ``GET_AWAITABLE``
+   for details about ``get_awaitable``
+
+
+.. opcode:: GET_ANEXT
+
+   Implements ``PUSH(get_awaitable(TOS.__anext__()))``.  See ``GET_AWAITABLE``
+   for details about ``get_awaitable``
+
+
+.. opcode:: BEFORE_ASYNC_WITH
+
+   Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the
+   stack.  Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack.
+
+
+.. opcode:: SETUP_ASYNC_WITH
+
+   Creates a new frame object.
+
+
+
 **Miscellaneous opcodes**
 
 .. opcode:: PRINT_EXPR
@@ -612,7 +644,7 @@ iterations of the loop.
    :opcode:`UNPACK_SEQUENCE`).
 
 
-.. opcode:: WITH_CLEANUP
+.. opcode:: WITH_CLEANUP_START
 
    Cleans up the stack when a :keyword:`with` statement block exits.  TOS is the
    context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values
@@ -624,7 +656,13 @@ iterations of the loop.
    * (SECOND, THIRD, FOURTH) = exc_info()
 
    In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise
-   ``TOS(None, None, None)``.  In addition, TOS is removed from the stack.
+   ``TOS(None, None, None)``.  Pushes SECOND and result of the call
+   to the stack.
+
+
+.. opcode:: WITH_CLEANUP_FINISH
+
+   Pops exception type and result of 'exit' function call from the stack.
 
    If the stack represents an exception, *and* the function call returns a
    'true' value, this information is "zapped" and replaced with a single

Grammar/Grammar

@@ -21,8 +21,11 @@ eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
+
 parameters: '(' [typedargslist] ')'
 typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
        ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
@@ -65,7 +68,8 @@ global_stmt: 'global' NAME (',' NAME)*
 nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -99,7 +103,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: atom_expr ['**' factor]
+atom_expr: [AWAIT] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |

Include/Python-ast.h

@@ -63,12 +63,13 @@ struct _mod {
     } v;
 };
 
-enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
-                  Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7,
-                  While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
-                  Try_kind=12, Assert_kind=13, Import_kind=14,
-                  ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17,
-                  Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
+enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
+                  Return_kind=4, Delete_kind=5, Assign_kind=6,
+                  AugAssign_kind=7, For_kind=8, AsyncFor_kind=9, While_kind=10,
+                  If_kind=11, With_kind=12, AsyncWith_kind=13, Raise_kind=14,
+                  Try_kind=15, Assert_kind=16, Import_kind=17,
+                  ImportFrom_kind=18, Global_kind=19, Nonlocal_kind=20,
+                  Expr_kind=21, Pass_kind=22, Break_kind=23, Continue_kind=24};
 struct _stmt {
     enum _stmt_kind kind;
     union {
@@ -80,6 +81,14 @@ struct _stmt {
             expr_ty returns;
         } FunctionDef;
 
+        struct {
+            identifier name;
+            arguments_ty args;
+            asdl_seq *body;
+            asdl_seq *decorator_list;
+            expr_ty returns;
+        } AsyncFunctionDef;
+        
         struct {
             identifier name;
             asdl_seq *bases;
@@ -114,6 +123,13 @@ struct _stmt {
             asdl_seq *orelse;
         } For;
 
+        struct {
+            expr_ty target;
+            expr_ty iter;
+            asdl_seq *body;
+            asdl_seq *orelse;
+        } AsyncFor;
+        
         struct {
             expr_ty test;
             asdl_seq *body;
@@ -131,6 +147,11 @@ struct _stmt {
             asdl_seq *body;
         } With;
 
+        struct {
+            asdl_seq *items;
+            asdl_seq *body;
+        } AsyncWith;
+        
         struct {
             expr_ty exc;
             expr_ty cause;
@@ -178,11 +199,11 @@ struct _stmt {
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
-                  Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
-                  Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
-                  NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
-                  Subscript_kind=22, Starred_kind=23, Name_kind=24,
-                  List_kind=25, Tuple_kind=26};
+                  Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
+                  Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
+                  Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
+                  Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
+                  Name_kind=25, List_kind=26, Tuple_kind=27};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -243,6 +264,10 @@ struct _expr {
             asdl_seq *generators;
         } GeneratorExp;
 
+        struct {
+            expr_ty value;
+        } Await;
+        
         struct {
             expr_ty value;
         } Yield;
@@ -402,6 +427,10 @@ mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
                         asdl_seq * decorator_list, expr_ty returns, int lineno,
                         int col_offset, PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
+                             body, asdl_seq * decorator_list, expr_ty returns,
+                             int lineno, int col_offset, PyArena *arena);
 #define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
                      asdl_seq * body, asdl_seq * decorator_list, int lineno,
@@ -420,6 +449,9 @@ stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
 #define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
                 orelse, int lineno, int col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
+                     orelse, int lineno, int col_offset, PyArena *arena);
 #define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
 stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
                   int col_offset, PyArena *arena);
@@ -429,6 +461,9 @@ stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
 #define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
 stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
                  PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
+                      col_offset, PyArena *arena);
 #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
 stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
                   PyArena *arena);
@@ -491,6 +526,8 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
 #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
                          col_offset, PyArena *arena);
+#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
 expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)

Include/ceval.h

@@ -23,6 +23,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj,
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
 PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) PyEval_SetCoroutineWrapper(PyObject *wrapper);
+PyAPI_FUNC(PyObject *) PyEval_GetCoroutineWrapper();
 #endif
 
 struct _frame; /* Avoid including frameobject.h */

Include/code.h

@@ -51,6 +51,11 @@ typedef struct {
 */
 #define CO_NOFREE       0x0040
 
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+   ``async def`` keywords) */
+#define CO_COROUTINE            0x0080
+#define CO_ITERABLE_COROUTINE   0x0100
+
 /* These are no longer used. */
 #if 0
 #define CO_GENERATOR_ALLOWED    0x1000

Include/genobject.h

@@ -38,6 +38,12 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 #define PyGen_Check(op) PyObject_TypeCheck(op, &PyGen_Type)
 #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
 
+#define PyGen_CheckCoroutineExact(op) (PyGen_CheckExact(op) && \
+                                       (((PyCodeObject*) \
+                                           ((PyGenObject*)op)->gi_code) \
+                                         ->co_flags & (CO_ITERABLE_COROUTINE | \
+                                                       CO_COROUTINE)))
+
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
 PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *,
     PyObject *name, PyObject *qualname);
@@ -46,6 +52,7 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyObject *_PyGen_Send(PyGenObject *, PyObject *);
 PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
 
+PyObject *_PyGen_GetAwaitableIter(PyObject *o);
 
 #ifdef __cplusplus
 }