Support new opcodes

ASTNode.h

@@ -18,24 +18,29 @@ class ASTNode {
         NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE,
         NODE_FORMATTEDVALUE, NODE_JOINEDSTR, NODE_CONST_MAP,
         NODE_ANNOTATED_VAR, NODE_CHAINSTORE, NODE_TERNARY,
-        NODE_KW_NAMES_MAP,
+        NODE_KW_NAMES_MAP, NODE_CALL_INTRINSIC_1, NODE_CALL_INTRINSIC_2,
 
         // Empty node types
         NODE_LOCALS,
     };
 
-    ASTNode(int type = NODE_INVALID) : m_refs(), m_type(type), m_processed() { }
+    ASTNode(int type = NODE_INVALID, bool unpacked = false) : m_refs(), m_type(type), m_processed(), m_unpacked(unpacked) { }
     virtual ~ASTNode() { }
 
     int type() const { return internalGetType(this); }
 
     bool processed() const { return m_processed; }
     void setProcessed() { m_processed = true; }
 
+    bool unpacked() const { return m_unpacked; }
+    void setUnpacked() { m_unpacked = true; }
+
 private:
     int m_refs;
     int m_type;
     bool m_processed;
+    // unpack this node into constituent values
+    bool m_unpacked;
 
     // Hack to make clang happy :(
     static int internalGetType(const ASTNode *node)
@@ -757,4 +762,26 @@ class ASTTernary : public ASTNode
     PycRef<ASTNode> m_else_expr;
 };
 
+class ASTCallIntrinsic1: public ASTNode
+{
+public:
+    enum Function {
+        INTRINSIC_1_INVALID, INTRINSIC_PRINT, INTRINSIC_IMPORT_STAR,
+        INTRINSIC_STOPITERATION_ERROR, INTRINSIC_ASYNC_GEN_WRAP,
+        INTRINSIC_UNARY_POSITIVE, INTRINSIC_LIST_TO_TUPLE, INTRINSIC_TYPEVAR,
+        INTRINSIC_PARAMSPEC, INTRINSIC_TYPEVARTUPLE,
+        INTRINSIC_SUBSCRIPT_GENERIC, INTRINSIC_TYPEALIAS,
+    };
+};
+
+class ASTCallIntrinsic2: public ASTNode
+{
+public:
+    enum Function {
+        INTRINSIC_2_INVALID, INTRINSIC_PREP_RERAISE_STAR,
+        INTRINSIC_TYPEVAR_WITH_BOUND, INTRINSIC_TYPEVAR_WITH_CONSTRAINTS,
+        INTRINSIC_SET_FUNCTION_TYPE_PARAMS, INTRINSIC_SET_TYPEPARAM_DEFAULT,
+    };
+};
+
 #endif

ASTree.cpp

@@ -5,6 +5,7 @@
 #include "FastStack.h"
 #include "pyc_numeric.h"
 #include "bytecode.h"
+#include <iostream>
 
 // This must be a triple quote (''' or """), to handle interpolated string literals containing the opposite quote style.
 // E.g. f'''{"interpolated "123' literal"}'''    -> valid.
@@ -1443,29 +1444,46 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             break;
         case Pyc::LIST_EXTEND_A:
             {
+                if (operand != 1) {
+                    fprintf(stderr, "LIST_EXTEND operand list is not at the top of the stack\n");
+                    break;
+                }
+
                 PycRef<ASTNode> rhs = stack.top();
                 stack.pop();
                 PycRef<ASTList> lhs = stack.top().cast<ASTList>();
                 stack.pop();
 
-                if (rhs.type() != ASTNode::NODE_OBJECT) {
-                    fprintf(stderr, "Unsupported argument found for LIST_EXTEND\n");
-                    break;
-                }
+                if (rhs.type() == ASTNode::NODE_OBJECT) {
 
-                // I've only ever seen this be a SMALL_TUPLE, but let's be careful...
-                PycRef<PycObject> obj = rhs.cast<ASTObject>()->object();
-                if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) {
-                    fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n");
-                    break;
-                }
+                    // I've only ever seen this be a SMALL_TUPLE, but let's be careful...
+                    PycRef<PycObject> obj = rhs.cast<ASTObject>()->object();
+                    if (obj->type() != PycObject::TYPE_TUPLE && obj->type() != PycObject::TYPE_SMALL_TUPLE) {
+                        fprintf(stderr, "Unsupported argument type found for LIST_EXTEND\n");
+                        break;
+                    }
 
-                ASTList::value_t result = lhs->values();
-                for (const auto& it : obj.cast<PycTuple>()->values()) {
-                    result.push_back(new ASTObject(it));
+                    ASTList::value_t result = lhs->values();
+                    for (const auto& it : obj.cast<PycTuple>()->values()) {
+                        result.push_back(new ASTObject(it));
+                    }
+
+                    stack.push(new ASTList(result));
                 }
+                else if (rhs.type() == ASTNode::NODE_NAME) {
+                    ASTList::value_t result = lhs->values();
+
+                    // rhs is a variable, so to extend the list
+                    // we need to unpack rhs
+                    PycRef<ASTNode> unpacked_ref = rhs;
+                    unpacked_ref.setUnpacked();
 
-                stack.push(new ASTList(result));
+                    result.push_back(unpacked_ref);
+                    stack.push(new ASTList(result));
+                }
+                else {
+                    fprintf(stderr, "Unsupported argument %i found for LIST_EXTEND\n", rhs.type());
+                }
             }
             break;
         case Pyc::LOAD_ATTR_A:
@@ -1515,6 +1533,7 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             stack.push(new ASTName(code->getCellVar(mod, operand)));
             break;
         case Pyc::LOAD_FAST_A:
+        case Pyc::LOAD_FAST_CHECK_A:
             if (mod->verCompare(1, 3) < 0)
                 stack.push(new ASTName(code->getName(operand)));
             else
@@ -2577,6 +2596,76 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
                 stack.push(value);
             }
             break;
+        case Pyc::CALL_INTRINSIC_1_A:
+            {
+                PycRef<ASTNode> arg = stack.top();
+                stack.pop();
+
+                if (operand != ASTCallIntrinsic1::INTRINSIC_LIST_TO_TUPLE) {
+                    fprintf(stderr, "Unimplemented function %i", operand);
+                    break;
+                }
+
+                if (arg.type() != ASTNode::NODE_LIST) {
+                    fprintf(stderr, "Unexpected argument type %i\n", arg.type());
+                    break;
+                }
+
+                PycRef<ASTList> list = arg.cast<ASTList>();
+                ASTTuple::value_t values;
+                for (PycRef<ASTNode> val : list->values()) {
+                    values.push_back(val);
+                }
+                stack.push(new ASTTuple(values));
+            }
+            break;
+        case Pyc::CALL_FUNCTION_EX_A:
+            {
+                int has_kwmap = operand & 1;
+                ASTCall::kwparam_t kwparamList;
+                ASTCall::pparam_t pparamList;
+
+                // callable, iterable object & kwmap object (if present)
+
+                if (has_kwmap) {
+                    PycRef<ASTNode> object_or_map = stack.top();
+                    if (object_or_map.type() == ASTNode::NODE_KW_NAMES_MAP) {
+                        stack.pop();
+                        PycRef<ASTKwNamesMap> kwparams_map = object_or_map.cast<ASTKwNamesMap>();
+                        for (ASTKwNamesMap::map_t::const_iterator it = kwparams_map->values().begin(); it != kwparams_map->values().end(); it++) {
+                            kwparamList.push_front(std::make_pair(it->first, it->second));
+                        }
+                    }
+                    else {
+                        fprintf(stderr, "Unexpected object type %i\n", object_or_map.type());
+                    }
+                }
+
+                PycRef<ASTNode> iterable = stack.top();
+                stack.pop();
+
+                if (iterable.type() == ASTNode::NODE_LIST) {
+                    PycRef<ASTList> list = iterable.cast<ASTList>();
+                    for (PycRef<ASTNode> n: list->values()) {
+                        pparamList.push_back(n);
+                    }
+                }
+                else if (iterable.type() == ASTNode::NODE_TUPLE) {
+                    PycRef<ASTTuple> tuple = iterable.cast<ASTTuple>();
+                    for (PycRef<ASTNode> n: tuple->values()) {
+                        pparamList.push_back(n);
+                    }
+                }
+                else {
+                    fprintf(stderr, "Unsupported iterable type %i\n", iterable.type());
+                }
+
+                PycRef<ASTNode> func = stack.top();
+                stack.pop();
+
+                stack.push(new ASTCall(func, pparamList, kwparamList));
+            }
+            break;
         default:
             fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode);
             cleanBuild = false;
@@ -2773,6 +2862,7 @@ void print_formatted_value(PycRef<ASTFormattedValue> formatted_value, PycModule*
     pyc_output << "}";
 }
 
+// TODO: Handle m_unpack for node correctly here.
 void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
 {
     if (node == NULL) {
@@ -2891,6 +2981,10 @@ void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
         break;
     case ASTNode::NODE_LIST:
         {
+            if (node.isUnpacked()) {
+                pyc_output << "*";
+            }
+
             pyc_output << "[";
             bool first = true;
             cur_indent++;
@@ -2984,6 +3078,9 @@ void print_src(PycRef<ASTNode> node, PycModule* mod, std::ostream& pyc_output)
         }
         break;
     case ASTNode::NODE_NAME:
+        if (node.isUnpacked()) {
+            pyc_output << "*";
+        }
         pyc_output << node.cast<ASTName>()->name()->value();
         break;
     case ASTNode::NODE_NODELIST:

FastStack.h

@@ -61,6 +61,25 @@ class FastStack {
         return m_ptr == -1;
     }
 
+    void debug_print(PycModule* mod, std::ostream& pyc_output)
+    {
+        pyc_output << "---- STACK CONTENTS ----\n";
+        if (empty()) {
+            pyc_output << "empty stack\n";
+        }
+        else {
+            for (int i = m_ptr; i >= 0; i--) {
+                print_src(m_stack[i], mod, pyc_output);
+                if (i == m_ptr) {
+                    pyc_output << " <- STACK TOP";
+                }
+                pyc_output << "\n";
+            }
+        }
+        pyc_output << "------------------------\n";
+    }
+
+
 private:
     std::vector<PycRef<ASTNode>> m_stack;
     int m_ptr;

pyc_object.h

@@ -6,21 +6,21 @@
 template <class _Obj>
 class PycRef {
 public:
-    PycRef() noexcept : m_obj() { }
+    PycRef() noexcept : m_obj(), m_unpack(false) { }
 
-    PycRef(_Obj* obj) noexcept : m_obj(obj)
+    PycRef(_Obj* obj) noexcept : m_obj(obj), m_unpack(false)
     {
         if (m_obj)
             m_obj->addRef();
     }
 
-    PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj)
+    PycRef(const PycRef<_Obj>& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack)
     {
         if (m_obj)
             m_obj->addRef();
     }
 
-    PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj)
+    PycRef(PycRef<_Obj>&& obj) noexcept : m_obj(obj.m_obj), m_unpack(obj.m_unpack)
     {
         obj.m_obj = nullptr;
     }
@@ -31,6 +31,8 @@ class PycRef {
             m_obj->delRef();
     }
 
+    // Most operators should deal with m_unpack but we leave it as is for now
+
     PycRef<_Obj>& operator=(_Obj* obj)
     {
         if (obj)
@@ -75,16 +77,29 @@ class PycRef {
     template <class _Cast>
     PycRef<_Cast> cast() const
     {
-        _Cast* result = dynamic_cast<_Cast*>(m_obj);
-        if (!result)
+        _Cast* casted_obj = dynamic_cast<_Cast*>(m_obj);
+        if (!casted_obj)
             throw std::bad_cast();
+
+        PycRef<_Cast> result = casted_obj;
+        if (m_unpack) {
+            result.setUnpacked();
+        }
         return result;
     }
 
     bool isIdent(const _Obj* obj) const { return m_obj == obj; }
 
+    bool isUnpacked() const { return m_unpack; }
+    void setUnpacked() { m_unpack = true; }
+
 private:
     _Obj* m_obj;
+
+    // References to an object can be either packed or unpacked
+    // Usually unpacked references will be used with variables but
+    // they may arise in other places as well.
+    bool m_unpack;
 };
 
 

tests/input/list_extend_2.py

@@ -0,0 +1,2 @@
+def get(l):
+    return [*l]

tests/input/test_unpack.py

@@ -0,0 +1,6 @@
+import struct
+
+def wtob(w):
+    return struct.pack('<'+'I'*len(w), *w)
+
+wtob([12,3])

tests/tokenized/list_extend_2.txt

@@ -0,0 +1,3 @@
+def get ( l ) : <EOL>
+<INDENT>
+return [ * l ] <EOL>

tests/tokenized/test_unpack.txt

@@ -0,0 +1,6 @@
+import struct <EOL>
+def wtob ( w ) : <EOL>
+<INDENT>
+return struct . pack ( '<' + 'I' * len ( w ) , * w ) <EOL>
+<OUTDENT>
+wtob ( [ 12 , 3 ] ) <EOL>