[unladen-swallow] r1168 committed - This adds two opcodes that the compiler uses when it sees an ast with...

2 views

Skip to first unread message

unladen...@googlecode.com

unread,

Aug 8, 2010, 10:50:16 PM8/8/10

to unladen...@googlegroups.com

Revision: 1168
Author: reid.kleckner
Date: Sun Aug 8 19:48:05 2010
Log: This adds two opcodes that the compiler uses when it sees an ast with
foo.bar(...). Instead of compiling down to LOAD_ATTR and CALL_FUNCTION, it
uses
LOAD_METHOD and CALL_METHOD, which try to avoid bound method allocations by
putting 'self' on the stack.

Most of the changes involve adding PyMethodDescr to the PyCFunction fast
paths.

Benchmarks below.

=====

Report on Darwin bitsaw 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15
16:55:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_I386 i386 i386
Total CPU cores: 2

### 2to3 ###
Min: 23.673999 -> 23.024160: 1.0282x faster
Avg: 23.686317 -> 23.039097: 1.0281x faster
Significant (t=81.103059)
Stddev: 0.01176 -> 0.01342: 1.1414x larger
Timeline: http://tinyurl.com/2ap5yc3

### call_method ###
Min: 0.661787 -> 0.361642: 1.8300x faster
Avg: 0.677248 -> 0.366301: 1.8489x faster
Significant (t=258.726159)
Stddev: 0.01545 -> 0.01395: 1.1081x smaller
Timeline: http://tinyurl.com/2f994yz

### call_method_slots ###
Min: 0.531741 -> 0.369789: 1.4380x faster
Avg: 0.555819 -> 0.371695: 1.4954x faster
Significant (t=199.683798)
Stddev: 0.01251 -> 0.00993: 1.2594x smaller
Timeline: http://tinyurl.com/38a2q36

### call_method_unknown ###
Min: 0.640354 -> 0.463809: 1.3806x faster
Avg: 0.645816 -> 0.470659: 1.3722x faster
Significant (t=133.058916)
Stddev: 0.01611 -> 0.01614: 1.0017x larger
Timeline: http://tinyurl.com/2bzo9py

### call_simple ###
Min: 0.357481 -> 0.340356: 1.0503x faster
Avg: 0.359481 -> 0.341971: 1.0512x faster
Significant (t=28.857147)
Stddev: 0.00748 -> 0.00738: 1.0138x smaller
Timeline: http://tinyurl.com/35amql3

### django ###
Min: 0.705525 -> 0.676860: 1.0424x faster
Avg: 0.709704 -> 0.682694: 1.0396x faster
Significant (t=22.400801)
Stddev: 0.00739 -> 0.00953: 1.2890x larger
Timeline: http://tinyurl.com/2w7e44r

### nbody ###
Min: 0.263536 -> 0.254410: 1.0359x faster
Avg: 0.270854 -> 0.259636: 1.0432x faster
Significant (t=3.134935)
Stddev: 0.02599 -> 0.02460: 1.0566x smaller
Timeline: http://tinyurl.com/397m8ho

### slowpickle ###
Min: 0.507721 -> 0.485639: 1.0455x faster
Avg: 0.522787 -> 0.501101: 1.0433x faster
Not significant
Stddev: 0.07781 -> 0.07948: 1.0215x larger
Timeline: http://tinyurl.com/237dgyg

### slowspitfire ###
Min: 0.634327 -> 0.626623: 1.0123x faster
Avg: 0.640595 -> 0.628389: 1.0194x faster
Significant (t=53.841896)
Stddev: 0.00201 -> 0.00104: 1.9312x smaller
Timeline: http://tinyurl.com/39lll2d

### slowunpickle ###
Min: 0.248880 -> 0.244499: 1.0179x faster
Avg: 0.256967 -> 0.253378: 1.0142x faster
Not significant
Stddev: 0.03629 -> 0.03854: 1.0622x larger
Timeline: http://tinyurl.com/39c8kfn

### spambayes ###
Min: 0.270303 -> 0.257670: 1.0490x faster
Avg: 0.327131 -> 0.314532: 1.0401x faster
Not significant
Stddev: 0.22207 -> 0.23178: 1.0437x larger
Timeline: http://tinyurl.com/3xd97ga

http://code.google.com/p/unladen-swallow/source/detail?r=1168

Modified:
/trunk/Include/classobject.h
/trunk/Include/descrobject.h
/trunk/Include/frameobject.h
/trunk/Include/methodobject.h
/trunk/Include/object.h
/trunk/Include/opcode.h
/trunk/JIT/PyBytecodeDispatch.cc
/trunk/JIT/PyBytecodeDispatch.h
/trunk/JIT/PyTypeBuilder.h
/trunk/JIT/RuntimeFeedback.cc
/trunk/JIT/RuntimeFeedback.h
/trunk/JIT/llvm_compile.cc
/trunk/JIT/llvm_fbuilder.h
/trunk/JIT/llvm_inline_functions.c
/trunk/JIT/opcodes/attributes.cc
/trunk/JIT/opcodes/attributes.h
/trunk/JIT/opcodes/call.cc
/trunk/JIT/opcodes/call.h
/trunk/Lib/opcode.py
/trunk/Lib/test/test_llvm.py
/trunk/Modules/_lsprof.c
/trunk/Objects/classobject.c
/trunk/Objects/descrobject.c
/trunk/Objects/methodobject.c
/trunk/Objects/object.c
/trunk/Python/compile.c
/trunk/Python/eval.cc
/trunk/Python/opcode_targets.h

=======================================
--- /trunk/Include/classobject.h Mon Feb 1 12:44:12 2010
+++ /trunk/Include/classobject.h Sun Aug 8 19:48:05 2010
@@ -51,6 +51,7 @@
PyAPI_FUNC(PyObject *) PyMethod_Function(PyObject *);
PyAPI_FUNC(PyObject *) PyMethod_Self(PyObject *);
PyAPI_FUNC(PyObject *) PyMethod_Class(PyObject *);
+PyAPI_FUNC(int) _PyMethod_ShouldBind(PyObject *, PyObject *);

/* Look up attribute with name (a string) on instance object pinst, using
* only the instance and base class dicts. If a descriptor is found in
=======================================
--- /trunk/Include/descrobject.h Tue Jan 12 14:13:47 2010
+++ /trunk/Include/descrobject.h Sun Aug 8 19:48:05 2010
@@ -46,7 +46,7 @@
PyDescr_COMMON;
} PyDescrObject;

-typedef struct {
+typedef struct PyMethodDescrObject {
PyDescr_COMMON;
PyMethodDef *d_method;
} PyMethodDescrObject;
@@ -85,10 +85,12 @@
PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *);
PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *);

-
+#define PyMethodDescr_Check(d) (Py_TYPE(d) == &PyMethodDescr_Type)
+#define PyWrapperDescr_Check(d) (Py_TYPE(d) == &PyWrapperDescr_Type)
+
+PyAPI_DATA(PyTypeObject) PyMethodDescr_Type;
PyAPI_DATA(PyTypeObject) PyProperty_Type;
#ifdef __cplusplus
}
#endif
#endif /* !Py_DESCROBJECT_H */
-
=======================================
--- /trunk/Include/frameobject.h Wed Mar 17 14:43:50 2010
+++ /trunk/Include/frameobject.h Sun Aug 8 19:48:05 2010
@@ -96,6 +96,8 @@
_PYGUARD_CFUNC,
_PYGUARD_BRANCH,
_PYGUARD_STORE_SUBSCR,
+ _PYGUARD_LOAD_METHOD,
+ _PYGUARD_CALL_METHOD,
};

/* Standard object interface */
=======================================
--- /trunk/Include/methodobject.h Mon Dec 7 16:18:11 2009
+++ /trunk/Include/methodobject.h Sun Aug 8 19:48:05 2010
@@ -66,6 +66,13 @@

PyAPI_FUNC(PyObject *) Py_FindMethod(PyMethodDef[], PyObject *, const char
*);

+/* Make sure a method definition is ready by performing checks and
updating old
+ method protocols to new ones. This must be called before the method
+ definition can be used to make a call. */
+PyAPI_FUNC(int) PyMethodDef_Ready(PyMethodDef *ml);
+PyAPI_FUNC(PyObject *) PyMethodDef_Call(PyMethodDef *, PyObject *,
PyObject *,
+ PyObject *);
+
#define PyCFunction_New(ML, SELF) PyCFunction_NewEx((ML), (SELF), NULL)
PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
PyObject *);
=======================================
--- /trunk/Include/object.h Mon Dec 7 13:42:55 2009
+++ /trunk/Include/object.h Sun Aug 8 19:48:05 2010
@@ -489,6 +489,8 @@
PyAPI_FUNC(int) PyCallable_Check(PyObject *);
PyAPI_FUNC(int) PyNumber_Coerce(PyObject **, PyObject **);
PyAPI_FUNC(int) PyNumber_CoerceEx(PyObject **, PyObject **);
+PyAPI_FUNC(int) _PyObject_ShouldBindMethod(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyObject_GetMethod(PyObject *, PyObject *);

PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *);

=======================================
--- /trunk/Include/opcode.h Fri Nov 13 17:46:58 2009
+++ /trunk/Include/opcode.h Sun Aug 8 19:48:05 2010
@@ -115,8 +115,9 @@
BUILD_MAP = 104, /* Always zero for now */
LOAD_ATTR = 105, /* Index in name list */
COMPARE_OP = 106, /* Comparison operator */
-/* IMPORT_FROM = 108, Replaced by #@import_from builtin. */
-
+
+/* IMPORT_FROM = 108, Replaced by #@import_from builtin. */
+ LOAD_METHOD = 109, /* Index in name list */
JUMP_FORWARD = 110, /* Number of bytes to skip */
JUMP_IF_FALSE_OR_POP = 111, /* Target byte offset from beginning
of code */
@@ -138,7 +139,7 @@
/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
CALL_FUNCTION = 131, /* #args + (#kwargs<<8) */
/* MAKE_FUNCTION = 132, Replaced by #@make_function() builtin. */
-
+ CALL_METHOD = 133, /* #args + (#kwargs<<8) */
MAKE_CLOSURE = 134, /* #free vars */
LOAD_CLOSURE = 135, /* Load free variable from closure */
LOAD_DEREF = 136, /* Load and dereference from closure cell */
=======================================
--- /trunk/JIT/PyBytecodeDispatch.cc Mon May 24 13:49:39 2010
+++ /trunk/JIT/PyBytecodeDispatch.cc Sun Aug 8 19:48:05 2010
@@ -86,6 +86,13 @@
OpcodeAttributes attr(fbuilder_);
attr.LOAD_ATTR(names_index);
}
+
+void
+PyBytecodeDispatch::LOAD_METHOD(int names_index)
+{
+ OpcodeAttributes attr(fbuilder_);
+ attr.LOAD_METHOD(names_index);
+}

void
PyBytecodeDispatch::STORE_ATTR(int names_index)
@@ -135,6 +142,13 @@
OpcodeCall call(fbuilder_);
call.CALL_FUNCTION(oparg);
}
+
+void
+PyBytecodeDispatch::CALL_METHOD(int oparg)
+{
+ OpcodeCall call(fbuilder_);
+ call.CALL_METHOD(oparg);
+}

void
PyBytecodeDispatch::CALL_FUNCTION_VAR(int oparg)
=======================================
--- /trunk/JIT/PyBytecodeDispatch.h Mon May 24 13:49:39 2010
+++ /trunk/JIT/PyBytecodeDispatch.h Sun Aug 8 19:48:05 2010
@@ -138,6 +138,7 @@
void COMPARE_OP(int cmp_op);

void LOAD_ATTR(int index);
+ void LOAD_METHOD(int index);
void STORE_ATTR(int index);
void DELETE_ATTR(int index);

=======================================
--- /trunk/JIT/PyTypeBuilder.h Thu Apr 1 17:29:10 2010
+++ /trunk/JIT/PyTypeBuilder.h Sun Aug 8 19:48:05 2010
@@ -564,6 +564,22 @@
DEFINE_FIELD(PyCFunctionObject, m_module)
};

+template<> class TypeBuilder<PyMethodDescrObject, false> {
+public:
+ static const StructType *get(llvm::LLVMContext &context) {
+ static const StructType *const result =
+
cast<StructType>(PyGlobalLlvmData::Get()->module()->getTypeByName(
+ // Clang's name for the PyMethodDescrObject struct.
+ "struct.PyMethodDescrObject"));
+ return result;
+ }
+
+ DEFINE_OBJECT_HEAD_FIELDS(PyMethodDescrObject)
+ DEFINE_FIELD(PyMethodDescrObject, d_type)
+ DEFINE_FIELD(PyMethodDescrObject, d_name)
+ DEFINE_FIELD(PyMethodDescrObject, d_method)
+};
+
template<> class TypeBuilder<PyMethodDef, false> {
public:
static const StructType *get(llvm::LLVMContext &context) {
@@ -640,6 +656,7 @@
typedef PyTypeBuilder<PyFrameObject> FrameTy;
typedef PyTypeBuilder<PyThreadState> ThreadStateTy;
typedef PyTypeBuilder<PyCFunctionObject> CFunctionTy;
+typedef PyTypeBuilder<PyMethodDescrObject> MethodDescrTy;
typedef PyTypeBuilder<PyMethodDef> MethodDefTy;
} // namespace py

=======================================
--- /trunk/JIT/RuntimeFeedback.cc Thu Apr 1 17:29:10 2010
+++ /trunk/JIT/RuntimeFeedback.cc Sun Aug 8 19:48:05 2010
@@ -13,12 +13,6 @@
using llvm::SmallPtrSet;
using llvm::SmallVector;

-
-static bool
-is_duplicate_method(PyObject *a, PyMethodDef *b)
-{
- return PyCFunction_Check(a) && PyCFunction_GET_FUNCTION(a) ==
b->ml_meth;
-}

PyLimitedFeedback::PyLimitedFeedback()
{
@@ -112,8 +106,9 @@
bool object_mode = this->InObjectMode();

for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
- if (object_mode)
+ if (object_mode) {
Py_XDECREF((PyObject *)this->data_[i].getPointer());
+ }
this->data_[i].setPointer(NULL);
this->data_[i].setInt(0);
}
@@ -173,41 +168,48 @@
this->SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
return;
}
- if (!PyCFunction_Check(obj))
- return;
-
- for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
- PyMethodDef *value = (PyMethodDef *)this->data_[i].getPointer();
- if (value == NULL) {
- this->data_[i].setPointer((void
*)PyCFunction_GET_METHODDEF(obj));
- return;
- }
- // Deal with the fact that "for x in y: l.append(x)" results in
- // multiple method objects for l.append.
- if (is_duplicate_method(obj, value))
- return;
- }
- // Record overflow.
- this->SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
+
+ // Record the type of the function, and the methoddef if it's a call
to a C
+ // function.
+ PyTypeObject *type = Py_TYPE(obj);
+ PyMethodDef *ml = NULL;
+ if (PyCFunction_Check(obj)) {
+ ml = PyCFunction_GET_METHODDEF(obj);
+ } else if (PyMethodDescr_Check(obj)) {
+ ml = ((PyMethodDescrObject *)obj)->d_method;
+ }
+
+ PyTypeObject *old_type = (PyTypeObject *)this->data_[0].getPointer();
+ PyMethodDef *old_ml = (PyMethodDef *)this->data_[1].getPointer();
+ if (old_type == NULL) {
+ // Record this method.
+ Py_INCREF(type);
+ this->data_[0].setPointer((void*)type);
+ this->data_[1].setPointer((void*)ml);
+ } else if (old_type != type || old_ml != ml) {
+ // We found something else here already. Set this flag to
indicate the
+ // call site is polymorphic, even if we haven't seen more than
three
+ // objects.
+ this->SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
+ }
+ // The call site is monomorphic, so we leave it as is.
}

void
PyLimitedFeedback::GetSeenFuncsInto(
- SmallVector<PyMethodDef*, 3> &result) const
+ SmallVector<PyTypeMethodPair, 3> &result) const
{
assert(this->InFuncMode());

result.clear();
if (this->GetFlagBit(SAW_A_NULL_OBJECT_BIT)) {
// Saw a NULL value, so add NULL to the result.
- result.push_back(NULL);
- }
- for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
- PyMethodDef *value = (PyMethodDef *)this->data_[i].getPointer();
- if (value == NULL)
- return;
- result.push_back(value);
- }
+ result.push_back(
+ std::make_pair<PyTypeObject*, PyMethodDef*>(NULL, NULL));
+ }
+ PyTypeObject *type = (PyTypeObject *)this->data_[0].getPointer();
+ PyMethodDef *ml = (PyMethodDef *)this->data_[1].getPointer();
+ result.push_back(std::make_pair<PyTypeObject*, PyMethodDef*>(type,
ml));
}

@@ -234,6 +236,13 @@

PyFullFeedback::~PyFullFeedback()
{
+ if (this->InFuncMode()) {
+ // We have to free these pairs if we're in func mode.
+ for (ObjSet::const_iterator it = this->data_.begin(),
+ end = this->data_.end(); it != end; ++it) {
+ delete (PyTypeMethodPair*)*it;
+ }
+ }
this->Clear();
}

@@ -304,34 +313,40 @@
assert(this->InFuncMode());
this->usage_ = FuncMode;

- // We only record C functions for now.
- if (!PyCFunction_Check(obj))
- return;
- if (obj == NULL)
- this->data_.insert(NULL);
- else if (!this->data_.count(obj)) {
- // Deal with the fact that "for x in y: l.append(x)" results in
- // multiple method objects for l.append.
- for (ObjSet::const_iterator it = this->data_.begin(),
- end = this->data_.end(); it != end; ++it) {
- if (is_duplicate_method(obj, (PyMethodDef *)*it))
- return;
- }
-
- this->data_.insert((void *)PyCFunction_GET_METHODDEF(obj));
- }
+ PyMethodDef *ml = NULL;
+ PyTypeObject *type = NULL;
+ if (obj != NULL) {
+ type = Py_TYPE(obj);
+
+ // We only record a methoddef if this is a C function.
+ if (PyCFunction_Check(obj)) {
+ ml = PyCFunction_GET_METHODDEF(obj);
+ } else if (PyMethodDescr_Check(obj)) {
+ ml = ((PyMethodDescrObject *)obj)->d_method;
+ }
+ }
+
+ for (ObjSet::const_iterator it = this->data_.begin(),
+ end = this->data_.end(); it != end; ++it) {
+ PyTypeMethodPair *pair = (PyTypeMethodPair*)*it;
+ if (pair->first == type && pair->second == ml)
+ return;
+ }
+
+ PyTypeMethodPair *pair = new PyTypeMethodPair(type, ml);
+ this->data_.insert((void *)pair);
}

void
PyFullFeedback::GetSeenFuncsInto(
- SmallVector<PyMethodDef*, /*in-object elems=*/3> &result) const
+ SmallVector<PyTypeMethodPair, 3> &result) const
{
assert(this->InFuncMode());

result.clear();
for (ObjSet::const_iterator it = this->data_.begin(),
end = this->data_.end(); it != end; ++it) {
- result.push_back((PyMethodDef *)*it);
+ result.push_back(*((PyTypeMethodPair *)*it));
}
}

=======================================
--- /trunk/JIT/RuntimeFeedback.h Thu Apr 1 17:29:10 2010
+++ /trunk/JIT/RuntimeFeedback.h Sun Aug 8 19:48:05 2010
@@ -33,12 +33,17 @@
namespace llvm {
template<typename, unsigned> class SmallVector;
}
+
+typedef std::pair<PyTypeObject*, PyMethodDef*> PyTypeMethodPair;

// These are the counters used for feedback in the JUMP_IF opcodes.
// The number of boolean inputs can be computed as (PY_FDO_JUMP_TRUE +
// PY_FDO_JUMP_FALSE - PY_FDO_JUMP_NON_BOOLEAN).
enum { PY_FDO_JUMP_TRUE = 0, PY_FDO_JUMP_FALSE, PY_FDO_JUMP_NON_BOOLEAN };

+// These are the counters used for feedback in the LOAD_METHOD opcode.
+enum { PY_FDO_LOADMETHOD_METHOD = 0, PY_FDO_LOADMETHOD_OTHER };
+
class PyLimitedFeedback {
public:
PyLimitedFeedback();
@@ -55,8 +60,9 @@

// Record that a given function was called.
void AddFuncSeen(PyObject *obj);
- // Clears result and fills it with the set of observed PyMethodDefs.
- void GetSeenFuncsInto(llvm::SmallVector<PyMethodDef*, 3> &result)
const;
+ // Clears result and fills it with the set of observed types and
+ // PyMethodDefs.
+ void GetSeenFuncsInto(llvm::SmallVector<PyTypeMethodPair, 3> &result)
const;
bool FuncsOverflowed() const {
return GetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT);
}
@@ -133,8 +139,9 @@

// Record that a given function was called.
void AddFuncSeen(PyObject *obj);
- // Clears result and fills it with the set of seen function objects.
- void GetSeenFuncsInto(llvm::SmallVector<PyMethodDef*, 3> &result)
const;
+ // Clears result and fills it with the set of observed types and
+ // PyMethodDefs.
+ void GetSeenFuncsInto(llvm::SmallVector<PyTypeMethodPair, 3> &result)
const;
bool FuncsOverflowed() const { return false; }

void IncCounter(unsigned counter_id);
@@ -148,8 +155,8 @@

private:
// Assume three pointers in the set to start with. We store either
- // PyObject *s (when in object mode) or PyMethodDef *s (when in
function
- // mode).
+ // PyObject *s (when in object mode) or pairs of types and PyMethodDef
*s
+ // (when in function mode).
typedef llvm::SmallPtrSet<void*, 3> ObjSet;

void Swap(PyFullFeedback *other);
=======================================
--- /trunk/JIT/llvm_compile.cc Wed Jul 28 23:50:04 2010
+++ /trunk/JIT/llvm_compile.cc Sun Aug 8 19:48:05 2010
@@ -463,11 +463,13 @@
OPCODE_WITH_ARG(BUILD_MAP)
OPCODE_WITH_ARG(LOAD_ATTR)
OPCODE_WITH_ARG(COMPARE_OP)
+ OPCODE_WITH_ARG(LOAD_METHOD)
OPCODE_WITH_ARG(LOAD_GLOBAL)
OPCODE_WITH_ARG(LOAD_FAST)
OPCODE_WITH_ARG(STORE_FAST)
OPCODE_WITH_ARG(DELETE_FAST)
OPCODE_WITH_ARG(CALL_FUNCTION)
+ OPCODE_WITH_ARG(CALL_METHOD)
OPCODE_WITH_ARG(MAKE_CLOSURE)
OPCODE_WITH_ARG(LOAD_CLOSURE)
OPCODE_WITH_ARG(LOAD_DEREF)
=======================================
--- /trunk/JIT/llvm_fbuilder.h Sun Aug 1 14:12:15 2010
+++ /trunk/JIT/llvm_fbuilder.h Sun Aug 8 19:48:05 2010
@@ -48,7 +48,8 @@
PyCodeObject *code_object() const { return this->code_object_; }
PyGlobalLlvmData *llvm_data() const { return this->llvm_data_; }
llvm::LLVMContext& context() { return this->context_; }
- bool& uses_delete_fast() { return this->uses_delete_fast_; }
+ bool &uses_delete_fast() { return this->uses_delete_fast_; }
+ std::vector<bool> &loads_optimized() { return this->loads_optimized_; }

llvm::BasicBlock *unreachable_block() const
{
@@ -356,6 +357,13 @@

llvm::SmallPtrSet<PyTypeObject*, 5> types_used_;

+ // A stack that corresponds to LOAD_METHOD/CALL_METHOD pairs. For
every
+ // load, we push on a boolean for whether or not the load was
optimized.
+ // The call uses this value to decide whether to expect an extra "self"
+ // argument. The stack is necessary if the user wrote code with nested
+ // method calls, like this: f.foo(b.bar()).
+ std::vector<bool> loads_optimized_;
+
// True if something went wrong and we need to stop compilation without
// aborting the process. If this is true, a Python error has already
// been set.
=======================================
--- /trunk/JIT/llvm_inline_functions.c Thu Apr 1 17:29:10 2010
+++ /trunk/JIT/llvm_inline_functions.c Sun Aug 8 19:48:05 2010
@@ -173,8 +173,8 @@
/* Keep this in sync with _PyObject_GetDictPtr. We need it inlined in
order
* for constant propagation to work.
*/
-PyObject ** __attribute__((always_inline))
-_PyLlvm_Object_GetDictPtr(PyObject *obj, PyTypeObject *tp, long dictoffset)
+static inline PyObject **
+get_dict_ptr(PyObject *obj, PyTypeObject *tp, long dictoffset)
{
if (dictoffset == 0)
return NULL;
@@ -196,6 +196,28 @@
}
return (PyObject **) ((char *)obj + dictoffset);
}
+
+/* Try to get an attribute from the attribute dictionary of obj, or return
NULL
+ * on failure.
+ */
+static inline PyObject *
+get_attr_from_dict(PyObject *obj, PyTypeObject *tp, long dictoffset,
+ PyObject *name)
+{
+ PyObject **dictptr = get_dict_ptr(obj, tp, dictoffset);
+ PyObject *dict = dictptr == NULL ? NULL : *dictptr;
+ PyObject *attr = NULL;
+
+ /* If the object has a dict, and the attribute is in it, return it. */
+ if (dict != NULL) {
+ /* TODO(rnk): @reviewer: Are these refcounts necessary? */
+ Py_INCREF(dict);
+ attr = PyDict_GetItem(dict, name);
+ Py_DECREF(dict);
+ }
+
+ return attr;
+}

/* Keep this in sync with PyObject_GenericGetAttr. The reason we take so
many
* extra arguments is to allow LLVM optimizers to notice that all of these
@@ -203,37 +225,27 @@
* function, we ensure that they will benefit from constant propagation.
*/
PyObject * __attribute__((always_inline))
-_PyLlvm_Object_GenericGetAttr(PyObject *obj, PyTypeObject *type,
+_PyLlvm_Object_GenericGetAttr(PyObject *obj, PyTypeObject *tp,
PyObject *name, long dictoffset, PyObject
*descr,
descrgetfunc descr_get, char is_data_descr)
{
- PyObject *res = NULL;
- PyObject **dictptr;
- PyObject *dict;
+ PyObject *dict_attr;

/* If it's a data descriptor, that has the most precedence, so we just
call
* the getter. */
if (is_data_descr) {
- return descr_get(descr, obj, (PyObject *)type);
+ return descr_get(descr, obj, (PyObject *)tp);
}

- dictptr = _PyLlvm_Object_GetDictPtr(obj, type, dictoffset);
- dict = dictptr == NULL ? NULL : *dictptr;
-
- /* If the object has a dict, and the attribute is in it, return it. */
- if (dict != NULL) {
- Py_INCREF(dict);
- res = PyDict_GetItem(dict, name);
- Py_DECREF(dict);
- if (res != NULL) {
- Py_INCREF(res);
- return res;
- }
+ dict_attr = get_attr_from_dict(obj, tp, dictoffset, name);
+ if (dict_attr != NULL) {
+ Py_INCREF(dict_attr);
+ return dict_attr;
}

/* Otherwise, try calling the descriptor getter. */
if (descr_get != NULL) {
- return descr_get(descr, obj, (PyObject *)type);
+ return descr_get(descr, obj, (PyObject *)tp);
}

/* If the descriptor has no getter, it's probably a vanilla PyObject
@@ -245,14 +257,92 @@

PyErr_Format(PyExc_AttributeError,
"'%.50s' object has no attribute '%.400s'",
- type->tp_name, PyString_AS_STRING(name));
+ tp->tp_name, PyString_AS_STRING(name));
return NULL;
}
+
+/* A stripped down version of PyObject_GenericGetAttr that only gets
methods or
+ * returns NULL if the object it would return is not a method. Rather
than try
+ * to handle those possible alternative cases, we focus on drastically
reducing
+ * the final generated code size. */
+PyObject *
+_PyLlvm_Object_GetUnknownMethod(PyObject *obj, PyObject *name)
+{
+ PyObject *descr;
+ PyObject *dict_attr;
+ PyTypeObject *tp = Py_TYPE(obj);
+
+ /* We only support string names. */
+ /* TODO(rnk): Uncomment when NDEBUG works.
+ *assert(PyString_Check(name));
+ */
+
+ /* Bail if the type isn't ready. We could call PyType_Ready, but that
+ * would add unecessary code. */
+ if (tp->tp_dict == NULL) {
+ return NULL;
+ }
+
+ /* Bail for objects that have overridden tp_getattro. */
+ if (tp->tp_getattro != &PyObject_GenericGetAttr) {
+ return NULL;
+ }
+
+ descr = _PyType_Lookup(tp, name);
+
+ /* Bail in any of the following cases:
+ * - There is no descriptor on the type.
+ * - The descriptor type does not have Py_TPFLAGS_HAVE_CLASS.
+ * - The descriptor is a data descriptor.
+ */
+ if (descr == NULL ||
+ !PyType_HasFeature(descr->ob_type, Py_TPFLAGS_HAVE_CLASS) ||
+ PyDescr_IsData(descr)) {
+ return NULL;
+ }
+
+ /* Bail if there is a dict attribute shadowing the method. */
+ dict_attr = get_attr_from_dict(obj, tp, tp->tp_dictoffset, name);
+ if (dict_attr != NULL) {
+ return NULL;
+ }
+
+ /* Bail if this isn't a method that we should be binding to obj. */
+ if (!_PyObject_ShouldBindMethod((PyObject*)tp, descr)) {
+ return NULL;
+ }
+
+ /* Otherwise, we've found ourselves a real method! */
+ Py_INCREF(descr);
+ return descr;
+}
+
+PyObject * __attribute__((always_inline))
+_PyLlvm_Object_GetKnownMethod(PyObject *obj, PyTypeObject *tp,
+ PyObject *name, long dictoffset,
+ PyObject *method)
+{
+ PyObject *dict_attr;
+
+ /* Bail if there is a dict attribute shadowing the method. */
+ dict_attr = get_attr_from_dict(obj, tp, dictoffset, name);
+ if (dict_attr != NULL) {
+ printf("shadowed dict attr!\n");
+ return NULL;
+ }
+
+ /* Otherwise, return the method descriptor unmodified. */
+ /* TODO(rnk): Uncomment when NDEBUG works.
+ *assert(_PyObject_ShouldBindMethod(tp, method));
+ */
+ Py_INCREF(method);
+ return method;
+}

/* Keep this in sync with PyObject_GenericSetAttr. */
int __attribute__((always_inline))
_PyLlvm_Object_GenericSetAttr(PyObject *obj, PyObject *value,
- PyTypeObject *type, PyObject *name,
+ PyTypeObject *tp, PyObject *name,
long dictoffset, PyObject *descr,
descrsetfunc descr_set, char is_data_descr)
{
@@ -266,7 +356,7 @@
return descr_set(descr, obj, value);
}

- dictptr = _PyLlvm_Object_GetDictPtr(obj, type, dictoffset);
+ dictptr = get_dict_ptr(obj, tp, dictoffset);

/* If the object has a dict slot, store it in there. */
if (dictptr != NULL) {
@@ -298,13 +388,13 @@
if (descr == NULL) {
PyErr_Format(PyExc_AttributeError,
"'%.100s' object has no attribute '%.200s'",
- type->tp_name, PyString_AS_STRING(name));
+ tp->tp_name, PyString_AS_STRING(name));
return -1;
}

PyErr_Format(PyExc_AttributeError,
"'%.50s' object attribute '%.400s' is read-only",
- type->tp_name, PyString_AS_STRING(name));
+ tp->tp_name, PyString_AS_STRING(name));
return -1;
}

@@ -777,6 +867,8 @@
PyUnicodeObject *_dummy_UnicodeObject;
/* PyCFunctionObject, */
PyCFunctionObject *_dummy_CFunctionObject;
+/* PyMethodDescrObject, */
+PyMethodDescrObject *_dummy_MethodDescrObject;
/* PyIntObject, */
PyIntObject *_dummy_IntObject;
/* PyLongObject, */
=======================================
--- /trunk/JIT/opcodes/attributes.cc Wed Jul 28 23:50:04 2010
+++ /trunk/JIT/opcodes/attributes.cc Sun Aug 8 19:48:05 2010
@@ -78,9 +78,32 @@

static llvm::ManagedStatic<AccessAttrStats> access_attr_stats;

+class MethodStats {
+public:
+ ~MethodStats() {
+ errs() << "\nLOAD/CALL_METHOD optimization:\n";
+ errs() << "Total load opcodes: " << this->total << "\n";
+ errs() << "Optimized opcodes: "
+ << (this->known + this->unknown) << "\n";
+ errs() << "Predictable methods: " << this->known << "\n";
+ errs() << "Unpredictable methods: " << this->unknown << "\n";
+ }
+
+ // Total number of LOAD_METHOD opcodes compiled.
+ unsigned total;
+ // Number of monomorphic method call sites.
+ unsigned known;
+ // Number of polymorphic method call sites that were still optimized.
+ unsigned unknown;
+};
+
+static llvm::ManagedStatic<MethodStats> method_stats;
+
#define ACCESS_ATTR_INC_STATS(field) access_attr_stats->field++
+#define METHOD_INC_STATS(field) method_stats->field++
#else
#define ACCESS_ATTR_INC_STATS(field)
+#define METHOD_INC_STATS(field)
#endif /* Py_WITH_INSTRUMENTATION */

namespace py {
@@ -235,6 +258,148 @@
this->fbuilder_->PropagateExceptionOnNonZero(result);
return true;
}
+
+void
+OpcodeAttributes::LOAD_METHOD(int names_index)
+{
+ const PyRuntimeFeedback *counters = this->fbuilder_->GetFeedback(1);
+ int method_count = 0;
+ int nonmethod_count = 0;
+ if (counters != NULL) {
+ method_count = counters->GetCounter(PY_FDO_LOADMETHOD_METHOD);
+ nonmethod_count = counters->GetCounter(PY_FDO_LOADMETHOD_OTHER);
+ }
+
+ METHOD_INC_STATS(total);
+
+ bool optimized = false;
+ if (method_count > 0 && nonmethod_count == 0) {
+ // We have data, and all loads have turned out to be methods.
+ optimized = this->LOAD_METHOD_known(names_index);
+ // If we can't use type feedback, fall back to a simpler
optimization.
+ if (!optimized) {
+ optimized = this->LOAD_METHOD_unknown(names_index);
+ }
+ }
+
+ if (!optimized) {
+ // No data, conflicting data, or we couldn't do the optimization.
Emit
+ // the unoptimized, safe code.
+ this->LOAD_ATTR(names_index);
+ Value *attr = this->fbuilder_->Pop();
+ this->fbuilder_->Push(this->state_->GetNull<PyObject*>());
+ this->fbuilder_->Push(attr);
+ } else {
+ // We currently count LOAD_METHODs as attribute loads. LOAD_ATTR
will
+ // automatically count the opcode, but we won't call it if we took
the
+ // optimized path.
+ ACCESS_ATTR_INC_STATS(loads);
+ }
+
+ this->fbuilder_->loads_optimized().push_back(optimized);
+}
+
+bool
+OpcodeAttributes::LOAD_METHOD_unknown(int names_index)
+{
+ PyObject *name =
+ PyTuple_GET_ITEM(this->fbuilder_->code_object()->co_names,
names_index);
+
+ // This optimization only supports string names.
+ if (!PyString_Check(name)) {
+ return false;
+ }
+
+ METHOD_INC_STATS(unknown);
+
+ // Call the inline function that deals with the lookup. LLVM
propagates
+ // these constant arguments through the body of the function.
+ Value *obj_v = this->fbuilder_->Pop();
+ Value *name_v = this->state_->EmbedPointer<PyObject*>(name);
+ Value *getattr_func = this->state_->GetGlobalFunction<
+ PyObject *(PyObject *obj, PyObject *name)>(
+ "_PyLlvm_Object_GetUnknownMethod");
+ Value *method = this->state_->CreateCall(getattr_func, obj_v, name_v);
+
+ // Bail if method is NULL.
+ BasicBlock *bail_block =
+ this->state_->CreateBasicBlock("LOAD_METHOD_unknown_bail_block");
+ BasicBlock *push_result =
+ this->state_->CreateBasicBlock("LOAD_METHOD_unknown_push_result");
+ this->builder_.CreateCondBr(this->state_->IsNull(method),
+ bail_block, push_result);
+
+ // Fill in the bail bb.
+ this->builder_.SetInsertPoint(bail_block);
+ this->fbuilder_->Push(obj_v);
+ this->fbuilder_->CreateGuardBailPoint(_PYGUARD_LOAD_METHOD);
+
+ // Put the method and self on the stack. We bail instead of raising
+ // exceptions.
+ this->builder_.SetInsertPoint(push_result);
+ this->fbuilder_->Push(method);
+ this->fbuilder_->Push(obj_v);
+ return true;
+}
+
+bool
+OpcodeAttributes::LOAD_METHOD_known(int names_index)
+{
+ // Do an optimized LOAD_ATTR with the optimized LOAD_METHOD.
+ PyObject *name =
+ PyTuple_GET_ITEM(this->fbuilder_->code_object()->co_names,
names_index);
+ AttributeAccessor accessor(this->fbuilder_, name, ATTR_ACCESS_LOAD);
+
+ // Check that we can optimize this load.
+ if (!accessor.CanOptimizeAttrAccess()) {
+ return false;
+ }
+
+ // Check that the descriptor is in fact a method. The only way this
could
+ // fail is if between recording feedback and optimizing this code, the
type
+ // is modified and the method replaced.
+ if (!_PyObject_ShouldBindMethod((PyObject*)accessor.guard_type_,
+ accessor.descr_)) {
+ return false;
+ }
+
+ METHOD_INC_STATS(known);
+ ACCESS_ATTR_INC_STATS(optimized_loads);
+
+ // Emit the appropriate guards.
+ Value *obj_v = this->fbuilder_->Pop();
+ BasicBlock *do_load = state_->CreateBasicBlock("LOAD_METHOD_do_load");
+ accessor.GuardAttributeAccess(obj_v, do_load);
+
+ // Call the inline function that deals with the lookup. LLVM
propagates
+ // these constant arguments through the body of the function.
+ this->builder_.SetInsertPoint(do_load);
+ Value *getattr_func = state_->GetGlobalFunction<
+ PyObject *(PyObject *obj, PyTypeObject *tp, PyObject *name,
+ long dictoffset, PyObject *method)>(
+ "_PyLlvm_Object_GetKnownMethod");
+ Value *args[] = {
+ obj_v,
+ accessor.guard_type_v_,
+ accessor.name_v_,
+ accessor.dictoffset_v_,
+ accessor.descr_v_,
+ };
+ Value *method_v = state_->CreateCall(getattr_func, args,
array_endof(args));
+
+ // Bail if method_v is NULL.
+ BasicBlock *push_result =
+ state_->CreateBasicBlock("LOAD_METHOD_push_result");
+ this->builder_.CreateCondBr(state_->IsNull(method_v),
+ accessor.bail_block_, push_result);
+
+ // Put the method and self on the stack. We bail instead of raising
+ // exceptions.
+ this->builder_.SetInsertPoint(push_result);
+ this->fbuilder_->Push(method_v);
+ this->fbuilder_->Push(obj_v);
+ return true;
+}

bool
AttributeAccessor::CanOptimizeAttrAccess()
@@ -308,12 +473,6 @@
ACCESS_ATTR_INC_STATS(no_opt_no_mcache);
return false;
}
-
- // Now that we know for sure that we are going to optimize this
lookup, add
- // the type to the list of types we need to listen for modifications
from
- // and make the llvm::Values.
- this->fbuilder_->WatchType(type);
- MakeLlvmValues();

return true;
}
@@ -359,9 +518,16 @@
BuilderT &builder = this->fbuilder_->builder();
LlvmFunctionState *state = this->fbuilder_->state();

+ // Now that we know for sure that we are going to optimize this
lookup, add
+ // the type to the list of types we need to listen for modifications
from
+ // and make the llvm::Values.
+ fbuilder->WatchType(this->guard_type_);
+ this->MakeLlvmValues();
+
BasicBlock *bail_block = state->CreateBasicBlock("ATTR_bail_block");
BasicBlock *guard_type = state->CreateBasicBlock("ATTR_check_valid");
BasicBlock *guard_descr = state->CreateBasicBlock("ATTR_check_descr");
+ this->bail_block_ = bail_block;

// Make sure that the code object is still valid. This may fail if the
// code object is invalidated inside of a call to the code object.
=======================================
--- /trunk/JIT/opcodes/attributes.h Wed Jul 28 23:50:04 2010
+++ /trunk/JIT/opcodes/attributes.h Sun Aug 8 19:48:05 2010
@@ -56,7 +56,8 @@
name_v_(0),
dictoffset_v_(0),
descr_v_(0),
- is_data_descr_v_(0) { }
+ is_data_descr_v_(0),
+ bail_block_(0) { }

// This helper method returns false if a LOAD_ATTR or STORE_ATTR opcode
// cannot be optimized. If the opcode can be optimized, it fills in
@@ -102,6 +103,8 @@
llvm::Value *descr_v_;
llvm::Value *is_data_descr_v_;

+ llvm::BasicBlock *bail_block_;
+
private:
typedef llvm::IRBuilder<true, llvm::TargetFolder> BuilderT;

@@ -126,6 +129,7 @@
OpcodeAttributes(LlvmFunctionBuilder *fbuilder);

void LOAD_ATTR(int index);
+ void LOAD_METHOD(int index);
void STORE_ATTR(int index);
void DELETE_ATTR(int index);

@@ -137,6 +141,8 @@
bool LOAD_ATTR_fast(int names_index);
void STORE_ATTR_safe(int names_index);
bool STORE_ATTR_fast(int names_index);
+ bool LOAD_METHOD_known(int names_index);
+ bool LOAD_METHOD_unknown(int names_index);

typedef llvm::IRBuilder<true, llvm::TargetFolder> BuilderT;

=======================================
--- /trunk/JIT/opcodes/call.cc Wed Jul 28 23:50:04 2010
+++ /trunk/JIT/opcodes/call.cc Sun Aug 8 19:48:05 2010
@@ -35,6 +35,7 @@
errs() << "Inlined: " << this->inlined << "\n";
errs() << "No opt: callsite kwargs: " << this->no_opt_kwargs
<< "\n";
errs() << "No opt: function params: " << this->no_opt_params
<< "\n";
+ errs() << "No opt: not C function: " << this->no_opt_not_cfunc
<< "\n";
errs() << "No opt: no data: " << this->no_opt_no_data << "\n";
errs() << "No opt: polymorphic: " << this->no_opt_polymorphic
<< "\n";
}
@@ -56,6 +57,8 @@
unsigned no_opt_no_data;
// We only optimize monomorphic callsites so far.
unsigned no_opt_polymorphic;
+ // We only optimize direct calls to C functions.
+ unsigned no_opt_not_cfunc;
};

static llvm::ManagedStatic<CallFunctionStats> call_function_stats;
@@ -75,22 +78,28 @@
{
}

-void
-OpcodeCall::CALL_FUNCTION_fast(int oparg,
- const PyRuntimeFeedback *feedback)
-{
- CF_INC_STATS(total);
-
+bool
+OpcodeCall::CALL_FUNCTION_fast(int oparg)
+{
// Check for keyword arguments; we only optimize callsites with
positional
// arguments.
if ((oparg >> 8) & 0xff) {
CF_INC_STATS(no_opt_kwargs);
- this->CALL_FUNCTION_safe(oparg);
- return;
+ return false;
}

// Only optimize monomorphic callsites.
- llvm::SmallVector<PyMethodDef*, 3> fdo_data;
+ const PyRuntimeFeedback *feedback = this->fbuilder_->GetFeedback();
+ if (!feedback) {
+ CF_INC_STATS(no_opt_no_data);
+ return false;
+ }
+ if (feedback->FuncsOverflowed()) {
+ CF_INC_STATS(no_opt_polymorphic);
+ return false;
+ }
+
+ llvm::SmallVector<PyTypeMethodPair, 3> fdo_data;
feedback->GetSeenFuncsInto(fdo_data);
if (fdo_data.size() != 1) {
#ifdef Py_WITH_INSTRUMENTATION
@@ -99,11 +108,20 @@
else
CF_INC_STATS(no_opt_polymorphic);
#endif
- this->CALL_FUNCTION_safe(oparg);
- return;
+ return false;
}

- PyMethodDef *func_record = fdo_data[0];
+ PyMethodDef *func_record = fdo_data[0].second;
+ PyTypeObject *type_record = (PyTypeObject *)fdo_data[0].first;
+ // We embed a pointer to type_record but we don't incref it because it
can
+ // only be PyCFunction_Type or PyMethodDescr_Type, which are statically
+ // allocated anyway.
+ if (type_record != &PyCFunction_Type &&
+ type_record != &PyMethodDescr_Type) {
+ CF_INC_STATS(no_opt_not_cfunc);
+ return false;
+ }
+ bool func_is_cfunc = (type_record == &PyCFunction_Type);

// Only optimize calls to C functions with a known number of
parameters,
// where the number of arguments we have is in that range.
@@ -114,8 +132,7 @@
if (!(flags & METH_ARG_RANGE &&
min_arity <= num_args && num_args <= max_arity)) {
CF_INC_STATS(no_opt_params);
- this->CALL_FUNCTION_safe(oparg);
- return;
+ return false;
}
assert(num_args <= PY_MAX_ARITY);

@@ -161,26 +178,36 @@
Type::getInt64Ty(this->fbuilder_->context()),
-num_args - 1)));

- // Make sure it's a PyCFunction; if not, bail.
- Value *is_cfunction = this->state_->CreateCall(
- this->state_->GetGlobalFunction<int(PyObject *)>(
- "_PyLlvm_WrapCFunctionCheck"),
- actual_func,
- "is_cfunction");
- Value *is_cfunction_guard = this->builder_.CreateICmpEQ(
- is_cfunction, ConstantInt::get(is_cfunction->getType(), 1),
- "is_cfunction_guard");
- this->builder_.CreateCondBr(is_cfunction_guard, check_is_same_func,
+ // Make sure it's the right type; if not, bail.
+ Value *actual_type = this->builder_.CreateLoad(
+ ObjectTy::ob_type(this->builder_, actual_func));
+ Value *right_type =
this->state_->EmbedPointer<PyTypeObject*>(type_record);
+ Value *is_right_type = this->builder_.CreateICmpEQ(
+ actual_type, right_type, "is_right_type");
+ this->builder_.CreateCondBr(is_right_type, check_is_same_func,
invalid_assumptions);

// Make sure we got the same underlying function pointer; if not, bail.
this->builder_.SetInsertPoint(check_is_same_func);
- Value *actual_as_pycfunc = this->builder_.CreateBitCast(
- actual_func,
- PyTypeBuilder<PyCFunctionObject
*>::get(this->fbuilder_->context()));
- Value *actual_method_def = this->builder_.CreateLoad(
- CFunctionTy::m_ml(this->builder_, actual_as_pycfunc),
- "CALL_FUNCTION_actual_method_def");
+ Value *actual_as_righttype, *actual_method_def;
+ if (func_is_cfunc) {
+ const Type *pycfunction_ty =
+ PyTypeBuilder<PyCFunctionObject
*>::get(this->fbuilder_->context());
+ actual_as_righttype = this->builder_.CreateBitCast(
+ actual_func, pycfunction_ty);
+ actual_method_def = this->builder_.CreateLoad(
+ CFunctionTy::m_ml(this->builder_, actual_as_righttype),
+ "CALL_FUNCTION_actual_method_def");
+ } else {
+ actual_as_righttype = this->builder_.CreateBitCast(
+ actual_func,
+ PyTypeBuilder<PyMethodDescrObject *>::get(
+ this->fbuilder_->context()));
+ actual_method_def = this->builder_.CreateLoad(
+ MethodDescrTy::d_method(this->builder_, actual_as_righttype),
+ "CALL_FUNCTION_actual_method_def");
+ }
+
Value *actual_func_ptr = this->builder_.CreateLoad(
MethodDefTy::ml_meth(this->builder_, actual_method_def),
"CALL_FUNCTION_actual_func_ptr");
@@ -220,7 +247,7 @@
invalid_assumptions,
function_name);
CF_INC_STATS(inlined);
- return;
+ return true;
}
}

@@ -231,14 +258,16 @@
// functions like len() and C-level methods like list.append(), we
pull the
// invocant (called m_self) from the PyCFunction object we popped
// off the stack. Once the function returns, we patch up the stack
pointer.
- Value *self = this->builder_.CreateLoad(
- CFunctionTy::m_self(this->builder_, actual_as_pycfunc),
- "CALL_FUNCTION_actual_self");
llvm::SmallVector<Value*, PY_MAX_ARITY + 1> args; // +1 for self.
- args.push_back(self);
- if (num_args == 0 && max_arity == 0) {
- args.push_back(this->state_->GetNull<PyObject *>());
- }
+ // If the function is a PyCFunction, pass its self member.
+ if (func_is_cfunc) {
+ Value *self = this->builder_.CreateLoad(
+ CFunctionTy::m_self(this->builder_, actual_as_righttype),
+ "CALL_FUNCTION_actual_self");
+ args.push_back(self);
+ }
+
+ // Pass the arguments that are on the stack.
for (int i = num_args; i >= 1; --i) {
args.push_back(
this->builder_.CreateLoad(
@@ -247,9 +276,16 @@
ConstantInt::getSigned(
Type::getInt64Ty(this->fbuilder_->context()),
-i))));
}
- for(int i = 0; i < (max_arity - num_args); ++i) {
+
+ // Pad optional arguments with NULLs.
+ for (int i = args.size(); i < max_arity + 1; ++i) {
args.push_back(this->state_->GetNull<PyObject *>());
}
+
+ // Pass a single NULL after self for METH_NOARGS functions.
+ if (args.size() == 1 && max_arity == 0) {
+ args.push_back(this->state_->GetNull<PyObject *>());
+ }

#ifdef WITH_TSC
this->state_->LogTscEvent(CALL_ENTER_C);
@@ -257,11 +293,16 @@
Value *result =
this->state_->CreateCall(llvm_func, args.begin(), args.end());

+ // Decref and the function and all of its arguments.
this->state_->DecRef(actual_func);
- // Decrefing args[0] will cause self to be double-decrefed, so avoid
that.
- for (int i = 1; i <= num_args; ++i) {
- this->state_->DecRef(args[i]);
- }
+ // If func is a cfunc, decrefing args[0] will cause self to be
+ // double-decrefed, so avoid that.
+ for (unsigned i = (func_is_cfunc ? 1 : 0); i < args.size(); ++i) {
+ // If LLVM knows that args[i] is NULL, it will delete this code.
+ this->state_->XDecRef(args[i]);
+ }
+
+ // Adjust the stack pointer to pop the function and its arguments.
Value *new_stack_pointer = this->builder_.CreateGEP(
stack_pointer,
ConstantInt::getSigned(
@@ -274,7 +315,9 @@

// Check signals and maybe switch threads after each function call.
this->fbuilder_->CheckPyTicker();
+
CF_INC_STATS(direct_calls);
+ return true;
}

void
@@ -350,11 +393,74 @@
void
OpcodeCall::CALL_FUNCTION(int oparg)
{
- const PyRuntimeFeedback *feedback = this->fbuilder_->GetFeedback();
- if (feedback == NULL || feedback->FuncsOverflowed())
+ CF_INC_STATS(total);
+ if (!this->CALL_FUNCTION_fast(oparg)) {
this->CALL_FUNCTION_safe(oparg);
- else
- this->CALL_FUNCTION_fast(oparg, feedback);
+ }
+}
+
+void
+OpcodeCall::CALL_METHOD(int oparg)
+{
+ // We only want to generate code to handle one case, but we need to be
+ // robust in the face of malformed code objects, which might cause
there to
+ // be mismatched LOAD_METHOD/CALL_METHOD opcodes. Therefore the value
we
+ // get from the loads_optimized_ stack is only a guess, but it should
be
+ // accurate for all code objects with matching loads and calls.
+ bool load_optimized = false;
+ std::vector<bool> &loads_optimized =
this->fbuilder_->loads_optimized();
+ if (!loads_optimized.empty()) {
+ load_optimized = loads_optimized.back();
+ loads_optimized.pop_back();
+ }
+
+ BasicBlock *call_block = state_->CreateBasicBlock("CALL_METHOD_call");
+ BasicBlock *bail_block = state_->CreateBasicBlock("CALL_METHOD_bail");
+
+ int num_args = (oparg & 0xff);
+ int num_kwargs = (oparg>>8) & 0xff;
+ // +1 for the actual function object, +1 for self.
+ int num_stack_slots = num_args + 2 * num_kwargs + 1 + 1;
+
+ // Look down the stack for the cell that is either padding or a method.
+ Value *stack_pointer =
+ this->builder_.CreateLoad(this->fbuilder_->stack_pointer_addr());
+ Value *stack_idx =
+
ConstantInt::getSigned(Type::getInt32Ty(this->fbuilder_->context()),
+ -num_stack_slots);
+ Value *padding_ptr = this->builder_.CreateGEP(stack_pointer,
+ stack_idx);
+ Value *method_or_padding = this->builder_.CreateLoad(padding_ptr);
+
+ // Depending on how we optimized the load, we either expect it to be
NULL
+ // or we expect it to be non-NULL. We bail if it's not what we expect.
+ Value *bail_cond = state_->IsNull(method_or_padding);
+ if (!load_optimized) {
+ bail_cond = this->builder_.CreateNot(bail_cond);
+ }
+ this->builder_.CreateCondBr(bail_cond, bail_block, call_block);
+
+ // Restore the stack in the bail bb.
+ this->builder_.SetInsertPoint(bail_block);
+ this->fbuilder_->CreateGuardBailPoint(_PYGUARD_CALL_METHOD);
+
+ this->builder_.SetInsertPoint(call_block);
+ if (load_optimized) {
+ // Increment the argument count in oparg and do a regular
CALL_FUNCTION.
+ assert((num_args + 1) <= 0xff &&
+ "TODO(rnk): Deal with oparg overflow.");
+ oparg = (oparg & ~0xff) | ((num_args + 1) & 0xff);
+ }
+
+ this->CALL_FUNCTION(oparg);
+
+ if (!load_optimized) {
+ // Pop off the padding cell.
+ Value *result = fbuilder_->Pop();
+ Value *padding = fbuilder_->Pop();
+ state_->Assert(state_->IsNull(padding), "Padding was non-NULL!");
+ fbuilder_->Push(result);
+ }
}

// Keep this in sync with eval.cc
=======================================
--- /trunk/JIT/opcodes/call.h Wed Jul 28 23:50:04 2010
+++ /trunk/JIT/opcodes/call.h Sun Aug 8 19:48:05 2010
@@ -6,7 +6,6 @@
#error This header expects to be included only in C++ source
#endif

-#include "JIT/RuntimeFeedback.h"
#include "llvm/Support/IRBuilder.h"
#include "llvm/Support/TargetFolder.h"

@@ -22,6 +21,7 @@
OpcodeCall(LlvmFunctionBuilder *fbuilder);

void CALL_FUNCTION(int num_args);
+ void CALL_METHOD(int num_args);
void CALL_FUNCTION_VAR(int num_args);
void CALL_FUNCTION_KW(int num_args);
void CALL_FUNCTION_VAR_KW(int num_args);
@@ -38,7 +38,7 @@
// work, while CALL_FUNCTION_fast takes advantage of data gathered
while
// running through the eval loop to omit as much flexibility as
possible.
void CALL_FUNCTION_safe(int num_args);
- void CALL_FUNCTION_fast(int num_args, const PyRuntimeFeedback *);
+ bool CALL_FUNCTION_fast(int num_args);

// Specialized version of CALL_FUNCTION for len() on certain types.
void CALL_FUNCTION_fast_len(llvm::Value *actual_func,
=======================================
--- /trunk/Lib/opcode.py Fri Nov 13 17:46:58 2009
+++ /trunk/Lib/opcode.py Sun Aug 8 19:48:05 2010
@@ -150,8 +150,9 @@
name_op('LOAD_ATTR', 105) # Index in name list
def_op('COMPARE_OP', 106) # Comparison operator
hascompare.append(106)
-# name_op('IMPORT_FROM', 108) # Replaced by #@import_from.
-
+
+# name_op('IMPORT_FROM', 108) # Replaced by #@import_from.
+name_op('LOAD_METHOD', 109) # Index in name list
jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip
jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning
of code
jabs_op('JUMP_IF_TRUE_OR_POP', 112) # ""
@@ -175,6 +176,7 @@

def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8)
# def_op('MAKE_FUNCTION', 132) Replaced by #@make_function calls.
+def_op('CALL_METHOD', 133) # #args + (#kwargs << 8)

def_op('MAKE_CLOSURE', 134)
def_op('LOAD_CLOSURE', 135)
=======================================
--- /trunk/Lib/test/test_llvm.py Sun Jul 11 08:07:54 2010
+++ /trunk/Lib/test/test_llvm.py Sun Aug 8 19:48:05 2010
@@ -4124,6 +4124,153 @@
self.assertFalse(foo.__code__.co_use_jit)

+class LoadMethodTests(LlvmTestCase, ExtraAssertsTestCase):
+
+ """Tests for the LOAD_METHOD/CALL_METHOD opcode optimization."""
+
+ def test_basic(self):
+ receiver_cell = [None] # The method self arg goes here.
+ class C(object):
+ def meth(self):
+ receiver_cell[0] = self
+ c = C()
+ foo = compile_for_llvm("foo", "def foo(c): c.meth()",
+ optimization_level=None)
+
+ foo(c)
+ self.assertEqual(c, receiver_cell[0])
+ spin_until_hot.__code__.co_use_jit = True
+ spin_until_hot(foo, [c])
+ receiver_cell[0] = None # Reset the cell.
+
+ foo(c)
+ self.assertEqual(c, receiver_cell[0])
+
+ def test_c_method_descr(self):
+ foo = compile_for_llvm("foo", "def foo(l): l.append(1)",
+ optimization_level=None)
+ spin_until_hot(foo, [[]])
+ l = []
+ foo(l)
+ self.assertEqual(l, [1])
+
+ def test_c_method_descr_arg_range(self):
+ # This test used to raise a SystemError for failed function
+ # verification because we weren't passing an extra NULL for the
+ # optional argument to pop.
+ foo = compile_for_llvm("foo", """\
+def foo(l):
+ l.append(0) # Just so we'll have something to pop.
+ l.pop()
+""", optimization_level=None)
+ spin_until_hot(foo, [[]])
+ foo([])
+
+ def test_module_method(self):
+ tracer = sys.gettrace()
+ foo = compile_for_llvm("foo", "def foo(): return sys.gettrace()",
+ optimization_level=None)
+ self.assertEqual(foo(), tracer)
+ spin_until_hot(foo, [])
+ self.assertEqual(foo(), tracer)
+
+ def test_nested_method_calls(self):
+ class C(object):
+ def bar(self, arg):
+ return arg
+ def baz(self):
+ return 1
+ c = C()
+ foo = compile_for_llvm("foo", "def foo(c): return c.bar(c.baz())",
+ optimization_level=None)
+ self.assertEqual(foo(c), 1)
+ spin_until_hot(foo, [c])
+ self.assertEqual(foo(c), 1)
+
+ def test_other_class_methods(self):
+ # This test only tests the interpreter, but it's testing the
+ # _PyObject_ShouldBindMethod function that we also use in the JIT
+ # compiler.
+ receiver_cell = [None]
+ class C(object):
+ def bar(self):
+ receiver_cell[0] = self
+ class D(object):
+ def bar(self):
+ pass
+ D.bar = C.bar # This won't work.
+ d = D()
+ self.assertRaises(TypeError, d.bar)
+
+ # This will, however, grab the underlying function, and work.
+ D.bar = C.__dict__["bar"]
+ d.bar() # This should not raise.
+
+ def test_object_attrs(self):
+ # Test that we don't optimize method access to an object attribute,
+ # even though it looks like a method access.
+ class C(object):
+ def bar(self):
+ return 2
+ c = C()
+ c.bar = lambda: 1
+ foo = compile_for_llvm("foo", "def foo(c): return c.bar()",
+ optimization_level=None)
+ self.assertEqual(foo(c), 1)
+ spin_until_hot(foo, [c])
+ self.assertEqual(foo(c), 1)
+
+ # This should not bail, because LOAD_METHOD should fall back to
+ # LOAD_ATTR when the feedback says we're not loading methods.
+ del c.bar
+ self.assertEqual(foo(c), 2)
+
+ def test_already_bound(self):
+ # Test that we don't optimize method access to an already bound
method,
+ # even though it looks like a method access.
+ receiver_cell = [None]
+ class C(object):
+ pass
+ # We need two classes so we don't optimize the load attribute,
which
+ # will invalidate the machine code if we change C.
+ class D(object):
+ def baz(self):
+ receiver_cell[0] = self
+ c = C()
+ d = D()
+ C.baz = d.baz # Put this bound method on the class.
+ foo = compile_for_llvm("foo", "def foo(c): c.baz()",
+ optimization_level=None)
+ spin_until_hot(foo, [c], [d])
+
+ # Check that c.baz() sets receiver_cell[0] to d. If we didn't
check if
+ # a method were already bound, we might have rebound D.baz to c.
+ receiver_cell[0] = None
+ foo(c)
+ self.assertEqual(receiver_cell[0], d)
+
+ def test_unknown_method(self):
+ # Train the function on two different types so that the classic
+ # LOAD_ATTR optimization doesn't apply, forcing us to use a
different
+ # code path.
+ receiver_cell = [None]
+ class C(object):
+ def meth(self):
+ receiver_cell[0] = self
+ class D(object):
+ def meth(self):
+ receiver_cell[0] = self
+ foo = compile_for_llvm("foo", "def foo(o): o.meth()",
+ optimization_level=None)
+ c = C()
+ d = D()
+ spin_until_hot(foo, [c], [d])
+ foo(c)
+ self.assertEqual(receiver_cell[0], c)
+ foo(d)
+ self.assertEqual(receiver_cell[0], d)
+
+
class InliningTests(LlvmTestCase, ExtraAssertsTestCase):

def test_manual_optimization(self):
@@ -4350,7 +4497,7 @@
OperatorTests, LiteralsTests, BailoutTests, InliningTests,
LlvmRebindBuiltinsTests, OptimizationTests,
SetJitControlTests, TypeBasedAnalysisTests,
- CrashRegressionTests]
+ CrashRegressionTests, LoadMethodTests]
if sys.flags.optimize >= 1:
print >>sys.stderr, "test_llvm -- skipping some tests due to -O
flag."
sys.stderr.flush()
=======================================
--- /trunk/Modules/_lsprof.c Fri Dec 5 15:10:10 2008
+++ /trunk/Modules/_lsprof.c Sun Aug 8 19:48:05 2010
@@ -165,17 +165,12 @@
static PyObject *
normalizeUserObj(PyObject *obj)
{
- PyCFunctionObject *fn;
- if (!PyCFunction_Check(obj)) {
- Py_INCREF(obj);
- return obj;
- }
/* Replace built-in function objects with a descriptive string
because of built-in methods -- keeping a reference to
__self__ is probably not a good idea. */
- fn = (PyCFunctionObject *)obj;
-
- if (fn->m_self == NULL) {
+ PyCFunctionObject *fn = (PyCFunctionObject *)obj;
+
+ if (PyCFunction_Check(obj) && fn->m_self == NULL) {
/* built-in function: look up the module name */
PyObject *mod = fn->m_module;
char *modname;
@@ -200,16 +195,27 @@
return PyString_FromFormat("<%s>",
fn->m_ml->ml_name);
}
- else {
+ else if (PyCFunction_Check(obj) || PyMethodDescr_Check(obj)) {
/* built-in method: try to return
repr(getattr(type(__self__), __name__))
*/
- PyObject *self = fn->m_self;
- PyObject *name = PyString_FromString(fn->m_ml->ml_name);
- if (name != NULL) {
- PyObject *mo = _PyType_Lookup(Py_TYPE(self), name);
+ PyTypeObject *type;
+ const char *name;
+ PyObject *name_obj;
+ if (PyCFunction_Check(obj)) {
+ type = Py_TYPE(fn->m_self);
+ name = fn->m_ml->ml_name;
+ }
+ else {
+ PyMethodDescrObject *descr = (PyMethodDescrObject *)obj;
+ type = descr->d_type;
+ name = descr->d_method->ml_name;
+ }
+ name_obj = PyString_FromString(name);
+ if (name_obj != NULL) {
+ PyObject *mo = _PyType_Lookup(type, name_obj);
Py_XINCREF(mo);
- Py_DECREF(name);
+ Py_DECREF(name_obj);
if (mo != NULL) {
PyObject *res = PyObject_Repr(mo);
Py_DECREF(mo);
@@ -218,8 +224,11 @@
}
}
PyErr_Clear();
- return PyString_FromFormat("<built-in method %s>",
- fn->m_ml->ml_name);
+ return PyString_FromFormat("<built-in method %s>", name);
+ }
+ else {
+ Py_INCREF(obj);
+ return obj;
}
}

@@ -454,11 +463,15 @@
/* the Python function 'frame' is issuing a call to the built-in
function 'arg' */
case PyTrace_C_CALL:
- if ((((ProfilerObject *)self)->flags & POF_BUILTINS)
- && PyCFunction_Check(arg)) {
- ptrace_enter_call(self,
- ((PyCFunctionObject *)arg)->m_ml,
- arg);
+ if (((ProfilerObject *)self)->flags & POF_BUILTINS) {
+ PyMethodDef *ml = NULL;
+ if (PyCFunction_Check(arg)) {
+ ml = ((PyCFunctionObject *)arg)->m_ml;
+ }
+ else if (PyMethodDescr_Check(arg)) {
+ ml = ((PyMethodDescrObject*)arg)->d_method;
+ }
+ ptrace_enter_call(self, ml, arg);
}
break;

@@ -466,10 +479,15 @@
caller 'frame' */
case PyTrace_C_RETURN: /* ...normally */
case PyTrace_C_EXCEPTION: /* ...with an exception set */
- if ((((ProfilerObject *)self)->flags & POF_BUILTINS)
- && PyCFunction_Check(arg)) {
- ptrace_leave_call(self,
- ((PyCFunctionObject *)arg)->m_ml);
+ if (((ProfilerObject *)self)->flags & POF_BUILTINS) {
+ PyMethodDef *ml = NULL;
+ if (PyCFunction_Check(arg)) {
+ ml = ((PyCFunctionObject *)arg)->m_ml;
+ }
+ else if (PyMethodDescr_Check(arg)) {
+ ml = ((PyMethodDescrObject*)arg)->d_method;
+ }
+ ptrace_leave_call(self, ml);
}
break;
#endif
=======================================
--- /trunk/Objects/classobject.c Fri Dec 5 15:10:10 2008
+++ /trunk/Objects/classobject.c Sun Aug 8 19:48:05 2010
@@ -2581,30 +2581,46 @@
return result;
}

-static PyObject *
-instancemethod_descr_get(PyObject *meth, PyObject *obj, PyObject *cls)
-{
- /* Don't rebind an already bound method, or an unbound method
- of a class that's not a base class of cls. */
-
- if (PyMethod_GET_SELF(meth) != NULL) {
- /* Already bound */
- Py_INCREF(meth);
- return meth;
- }
- /* No, it is an unbound method */
+/* Return true if we should bind the method to obj when getting as an
attribute
+ of obj. We do not rebind it if it is already bound, or if it is an
unbound
+ method of a class that's not a base class of cls. We return -1 if
there is
+ an error.
+*/
+
+int
+_PyMethod_ShouldBind(PyObject *meth, PyObject *cls)
+{
+ if (PyMethod_GET_SELF(meth) != NULL)
+ return 0; /* Already bound */
+
+ /* meth is an unbound method */
if (PyMethod_GET_CLASS(meth) != NULL && cls != NULL) {
- /* Do subclass test. If it fails, return meth unchanged. */
+ /* Do subclass test. If it fails, we should not bind it. */
int ok = PyObject_IsSubclass(cls, PyMethod_GET_CLASS(meth));
if (ok < 0)
- return NULL;
- if (!ok) {
- Py_INCREF(meth);
- return meth;
- }
- }
- /* Bind it to obj */
- return PyMethod_New(PyMethod_GET_FUNCTION(meth), obj, cls);
+ return -1;
+ if (!ok)
+ return 0;
+ }
+
+ /* If either of those checks fail, we should bind the method. */
+ return 1;
+}
+
+static PyObject *
+instancemethod_descr_get(PyObject *meth, PyObject *obj, PyObject *cls)
+{
+ int r = _PyMethod_ShouldBind(meth, cls);
+ if (r < 0) {
+ return NULL;
+ } else if (r > 0) {
+ /* Bind it to obj */
+ return PyMethod_New(PyMethod_GET_FUNCTION(meth), obj, cls);
+ } else {
+ /* Return meth unchanged. */
+ Py_INCREF(meth);
+ return meth;
+ }
}

PyTypeObject PyMethod_Type = {
=======================================
--- /trunk/Objects/descrobject.c Tue Jan 12 14:13:47 2010
+++ /trunk/Objects/descrobject.c Sun Aug 8 19:48:05 2010
@@ -206,11 +206,14 @@
return -1;
}

+/* When calling a method descriptor or wrapper descriptor, check the
arguments
+ * this way. Return self or NULL on exception.
+ */
static PyObject *
-methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject
*kwds)
+descr_call_check(PyDescrObject *descr, PyObject *args)
{
Py_ssize_t argc;
- PyObject *self, *func, *result;
+ PyObject *self;

/* Make sure that the first argument is acceptable as 'self' */
assert(PyTuple_Check(args));
@@ -234,18 +237,27 @@
self->ob_type->tp_name);
return NULL;
}
-
- func = PyCFunction_New(descr->d_method, self);
- if (func == NULL)
+ return self;
+}
+
+static PyObject *
+methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject
*kwds)
+{
+ PyObject *self, *result;
+
+ self = descr_call_check((PyDescrObject*)descr, args);
+ if (self == NULL) {
return NULL;
- args = PyTuple_GetSlice(args, 1, argc);
+ }
+
+ /* This is an allocation, but we can probably avoid it best by special
+ * casing descriptor objects in _PyEval_CallFunction. */
+ args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args));
if (args == NULL) {
- Py_DECREF(func);
return NULL;
}
- result = PyEval_CallObjectWithKeywords(func, args, kwds);
+ result = PyMethodDef_Call(descr->d_method, self, args, kwds);
Py_DECREF(args);
- Py_DECREF(func);
return result;
}

@@ -264,48 +276,8 @@
return result;
}

-static PyObject *
-wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject
*kwds)
-{
- Py_ssize_t argc;
- PyObject *self, *func, *result;
-
- /* Make sure that the first argument is acceptable as 'self' */
- assert(PyTuple_Check(args));
- argc = PyTuple_GET_SIZE(args);
- if (argc < 1) {
- PyErr_Format(PyExc_TypeError,
- "descriptor '%.300s' of '%.100s' "
- "object needs an argument",
- descr_name((PyDescrObject *)descr),
- descr->d_type->tp_name);
- return NULL;
- }
- self = PyTuple_GET_ITEM(args, 0);
- if (!PyObject_IsInstance(self, (PyObject *)(descr->d_type))) {
- PyErr_Format(PyExc_TypeError,
- "descriptor '%.200s' "
- "requires a '%.100s' object "
- "but received a '%.100s'",
- descr_name((PyDescrObject *)descr),
- descr->d_type->tp_name,
- self->ob_type->tp_name);
- return NULL;
- }
-
- func = PyWrapper_New((PyObject *)descr, self);
- if (func == NULL)
- return NULL;
- args = PyTuple_GetSlice(args, 1, argc);
- if (args == NULL) {
- Py_DECREF(func);
- return NULL;
- }
- result = PyEval_CallObjectWithKeywords(func, args, kwds);
- Py_DECREF(args);
- Py_DECREF(func);
- return result;
-}
+static PyObject *wrapperdescr_call(PyWrapperDescrObject *descr, PyObject
*args,
+ PyObject *kwds);

static PyObject *
method_get_doc(PyMethodDescrObject *descr, void *closure)
@@ -381,7 +353,7 @@
return 0;
}

-static PyTypeObject PyMethodDescr_Type = {
+PyTypeObject PyMethodDescr_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"method_descriptor",
sizeof(PyMethodDescrObject),
@@ -590,6 +562,9 @@
{
PyMethodDescrObject *descr;

+ if (PyMethodDef_Ready(method) < 0)
+ return NULL;
+
descr = (PyMethodDescrObject *)descr_new(&PyMethodDescr_Type,
type, method->ml_name);
if (descr != NULL)
@@ -602,6 +577,9 @@
{
PyMethodDescrObject *descr;

+ if (PyMethodDef_Ready(method) < 0)
+ return NULL;
+
descr = (PyMethodDescrObject *)descr_new(&PyClassMethodDescr_Type,
type, method->ml_name);
if (descr != NULL)
@@ -990,6 +968,31 @@
}
return (*wrapper)(self, args, wp->descr->d_wrapped);
}
+
+static PyTypeObject wrappertype;
+
+static PyObject *
+wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject
*kwds)
+{
+ PyObject *self, *result;
+ /* Stack allocate the wrapper to avoid heap allocations. */
+ wrapperobject func = {PyObject_HEAD_INIT(&wrappertype) NULL, NULL};
+
+ self = descr_call_check((PyDescrObject*)descr, args);
+ if (self == NULL) {
+ return NULL;
+ }
+
+ func.descr = descr;
+ func.self = self;
+ args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args));
+ if (args == NULL) {
+ return NULL;
+ }
+ result = wrapper_call(&func, args, kwds);
+ Py_DECREF(args);
+ return result;
+}

static int
wrapper_traverse(PyObject *self, visitproc visit, void *arg)
=======================================
--- /trunk/Objects/methodobject.c Mon Jan 25 16:39:14 2010
+++ /trunk/Objects/methodobject.c Sun Aug 8 19:48:05 2010
@@ -13,21 +13,47 @@
#define PyCFunction_MAXFREELIST 256
#endif

-PyObject *
-PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module)
-{
- PyCFunctionObject *op;
-
- /* Sanity check early, to avoid having to clean up from the mixed
- free list/allocation scheme below. */
+int
+PyMethodDef_Ready(PyMethodDef *ml)
+{
+ /* Rewrite the old METH_O/METH_NOARGS flags to the new METH_ARG_RANGE
+ so we only have to implement METH_ARG_RANGE. */
+ if (ml->ml_flags & METH_NOARGS) {
+ ml->ml_flags &= ~METH_NOARGS;
+ ml->ml_flags |= METH_ARG_RANGE;
+ ml->ml_min_arity = 0;
+ ml->ml_max_arity = 0;
+ }
+ else if (ml->ml_flags & METH_O) {
+ ml->ml_flags &= ~METH_O;
+ ml->ml_flags |= METH_ARG_RANGE;
+ ml->ml_min_arity = 1;
+ ml->ml_max_arity = 1;
+ }
+
+ /* Check that METH_ARG_RANGE methods have valid arities. */
if (ml->ml_flags & METH_ARG_RANGE) {
if (ml->ml_min_arity < 0 || ml->ml_min_arity > PY_MAX_ARITY ||
ml->ml_max_arity < 0 || ml->ml_max_arity > PY_MAX_ARITY ||
ml->ml_max_arity < ml->ml_min_arity) {
PyErr_BadInternalCall();
- return NULL;
+ return -1;
}
}
+
+ return 0;
+}
+
+PyObject *
+PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module)
+{
+ PyCFunctionObject *op;
+
+ /* Sanity check early, to avoid having to clean up from the mixed
+ free list/allocation scheme below. */
+ if (PyMethodDef_Ready(ml) < 0) {
+ return NULL;
+ }

op = free_list;
if (op != NULL) {
@@ -40,21 +66,6 @@
if (op == NULL)
return NULL;
}
-
- /* Rewrite the old METH_O/METH_NOARGS flags to the new METH_ARG_RANGE
- so we only have to implement METH_ARG_RANGE. */
- if (ml->ml_flags & METH_NOARGS) {
- ml->ml_flags &= ~METH_NOARGS;
- ml->ml_flags |= METH_ARG_RANGE;
- ml->ml_min_arity = 0;
- ml->ml_max_arity = 0;
- }
- else if (ml->ml_flags & METH_O) {
- ml->ml_flags &= ~METH_O;
- ml->ml_flags |= METH_ARG_RANGE;
- ml->ml_min_arity = 1;
- ml->ml_max_arity = 1;
- }

op->m_ml = ml;
Py_XINCREF(self);
@@ -98,12 +109,17 @@
PyObject *
PyCFunction_Call(PyObject *func, PyObject *arg, PyObject *kw)
{
- PyCFunctionObject* f = (PyCFunctionObject*)func;
- PyCFunction meth = PyCFunction_GET_FUNCTION(func);
- PyObject *self = PyCFunction_GET_SELF(func);
+ return PyMethodDef_Call(PyCFunction_GET_METHODDEF(func),
+ PyCFunction_GET_SELF(func), arg, kw);
+}
+
+PyObject *
+PyMethodDef_Call(PyMethodDef *ml, PyObject *self, PyObject *arg, PyObject
*kw)
+{
+ PyCFunction meth = ml->ml_meth;
Py_ssize_t size;

- switch (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC |
METH_COEXIST)) {
+ switch (ml->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST)) {
case METH_VARARGS:
if (kw == NULL || PyDict_Size(kw) == 0)
return (*meth)(self, arg);
@@ -115,8 +131,8 @@
{
if (kw == NULL || PyDict_Size(kw) == 0) {
PyObject *args[PY_MAX_ARITY] = {NULL};
- int min_arity = PyCFunction_GET_MIN_ARITY(func);
- int max_arity = PyCFunction_GET_MAX_ARITY(func);
+ int min_arity = ml->ml_min_arity;
+ int max_arity = ml->ml_max_arity;
size = PyTuple_GET_SIZE(arg);
switch (size) {
default:
@@ -139,13 +155,13 @@
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly %d argument(s)"
" (%zd given)",
- f->m_ml->ml_name, max_arity, size);
+ ml->ml_name, max_arity, size);
else
PyErr_Format(PyExc_TypeError,
"%.200s() takes %d-%d arguments"
" (%zd given)",
- f->m_ml->ml_name,
- min_arity, max_arity, size);
+ ml->ml_name, min_arity, max_arity,
+ size);

return NULL;
}
@@ -170,7 +186,7 @@
return NULL;
}
PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
- f->m_ml->ml_name);
+ ml->ml_name);
return NULL;
}

=======================================
--- /trunk/Objects/object.c Mon Jan 25 16:39:14 2010
+++ /trunk/Objects/object.c Sun Aug 8 19:48:05 2010
@@ -1272,7 +1272,7 @@
}

/* Helper to get a pointer to an object's __dict__ slot, if any. Keep in
sync
- with _PyLlvm_Object_GetDictPtr. */
+ * with get_dict_ptr in llvm_inline_functions.c. */

PyObject **
_PyObject_GetDictPtr(PyObject *obj)
@@ -1307,6 +1307,166 @@
Py_INCREF(obj);
return obj;
}
+
+/* This does the first part of generic attribute lookup, which is when data
+ * descriptors get called and dict attributes returned.
+ */
+static inline int
+generic_getattr_part1(PyObject *obj, PyObject *name, PyTypeObject *tp,
+ PyObject **descr, descrgetfunc *f, PyObject **res)
+{
+ PyObject **dictptr;
+
+ /* Make the type dict ready if it isn't already. */
+ if (tp->tp_dict == NULL) {
+ if (PyType_Ready(tp) < 0)
+ return -1;
+ }
+
+ *descr = _PyType_Lookup(tp, name);
+ Py_XINCREF(*descr);
+
+ /* Data descriptors have highest precedence. */
+ *f = NULL;
+ if (*descr != NULL &&
+ PyType_HasFeature((*descr)->ob_type, Py_TPFLAGS_HAVE_CLASS)) {
+ *f = (*descr)->ob_type->tp_descr_get;
+ if (*f != NULL && PyDescr_IsData(*descr)) {
+ *res = (*f)(*descr, obj, (PyObject *)tp);
+ Py_DECREF(*descr);
+ return 1;
+ }
+ }
+
+ /* Instance attributes have the next level of precedence. */
+ dictptr = _PyObject_GetDictPtr(obj);
+ if (dictptr != NULL) {
+ PyObject *dict = *dictptr;
+ if (dict != NULL) {
+ Py_INCREF(dict);
+ *res = PyDict_GetItem(dict, name);
+ if (*res != NULL) {
+ Py_INCREF(*res);
+ Py_XDECREF(*descr);
+ Py_DECREF(dict);
+ return 1;
+ }
+ Py_DECREF(dict);
+ }
+ }
+
+ /* We are not done doing attribute lookup yet. */
+ return 0;
+}
+
+/* This does the second part of generic attribute lookup, which is when
+ * non-data descriptors (method descriptors) get called or class
attributes get
+ * returned.
+ */
+static inline void
+generic_getattr_part2(PyObject *obj, PyObject *name, PyTypeObject *tp,
+ PyObject *descr, descrgetfunc f, PyObject **res)
+{
+ /* Non-data descriptors (methods) come next. */
+ if (f != NULL) {
+ *res = f(descr, obj, (PyObject *)Py_TYPE(obj));
+ Py_DECREF(descr);
+ return;
+ }
+
+ /* Class attributes have lowest precedence. */
+ if (descr != NULL) {
+ *res = descr;
+ /* descr was already increfed */
+ return;
+ }
+
+ PyErr_Format(PyExc_AttributeError,
+ "'%.50s' object has no attribute '%.400s'",
+ tp->tp_name, PyString_AS_STRING(name));
+ return;
+}
+
+/* Return true if descr is a function or method that should be bound when
+ * loaded as an attribute of obj. */
+int
+_PyObject_ShouldBindMethod(PyObject *tp, PyObject *descr)
+{
+ if (PyMethod_Check(descr)) {
+ int r = _PyMethod_ShouldBind(descr, tp);
+ if (r < 0) {
+ /* Swallow exceptions that shouldn't occur. */
+ PyErr_Clear();
+ }
+ else {
+ return r;
+ }
+ }
+ else if (PyFunction_Check(descr)) {
+ /* PyFunctionObjects should always be bound to the
+ * instance. */
+ return 1;
+ }
+ else if (PyMethodDescr_Check(descr) ||
+ PyWrapperDescr_Check(descr)) {
+ /* Method and wrapper descriptors are always unbound, so we
+ * just have to check that the types match. */
+ PyObject *d_type = (PyObject*)((PyDescrObject*)descr)->d_type;
+ return (tp == d_type || PyObject_IsSubclass(tp, d_type));
+ }
+ /* You might think that we should also special case PyCFunctionObjects,
+ * but those don't actually have a tp_descr_get slot. */
+ return 0;
+}
+
+/* Similar to PyObject_GetAttr, except that it attempts to return method
+ * objects without allocating a bound method for them by calling the
+ * descriptor. If we can return the method unbound, we set the lowest bit
of
+ * the result to 1, and otherwise we fall back to the behavior of
+ * PyObject_GetAttr and leave the bit 0. Keep this in sync with
+ * PyObject_GenericGetAttr. */
+PyObject *
+PyObject_GetMethod(PyObject *obj, PyObject *name)
+{
+ PyTypeObject *tp = Py_TYPE(obj);
+ PyObject *descr = NULL;
+ PyObject *res = NULL;
+ descrgetfunc f;
+
+ assert(PyString_Check(name));
+ Py_INCREF(name);
+
+ /* We fall back to normal lookup for objects that have overridden
+ * tp_getattro. */
+ if (tp->tp_getattro != &PyObject_GenericGetAttr) {
+ res = PyObject_GetAttr(obj, name);
+ goto done;
+ }
+
+ /* Do the first part of generic attribute lookup. */
+ if (generic_getattr_part1(obj, name, tp, &descr, &f, &res))
+ goto done;
+
+ if (descr != NULL) {
+ /* Instead of binding unbound pure Python methods, or method
+ * descriptors, set their low bit and return them directly so
+ * we can avoid calling tp_descr_get, which does an allocation.
+ */
+ if (_PyObject_ShouldBindMethod((PyObject*)tp, descr)) {
+ res = descr;
+ res = (PyObject*)(((Py_uintptr_t)res) | 1);
+ /* descr was already increfed above */
+ goto done;
+ }
+ }
+
+ /* Do the second part of generic attribute lookup. */
+ generic_getattr_part2(obj, name, tp, descr, f, &res);
+
+ done:
+ Py_DECREF(name);
+ return res;
+}

/* Generic GetAttr functions - put these in your tp_[gs]etattro slot.
Keep in
* sync with _PyLlvm_Object_Generic[GS]etAttr. */
@@ -1318,8 +1478,6 @@
PyObject *descr = NULL;
PyObject *res = NULL;
descrgetfunc f;
- Py_ssize_t dictoffset;
- PyObject **dictptr;

if (!PyString_Check(name)){
#ifdef Py_USING_UNICODE
@@ -1343,75 +1501,13 @@
else
Py_INCREF(name);

- if (tp->tp_dict == NULL) {
- if (PyType_Ready(tp) < 0)
- goto done;
- }
-
- /* TODO(rnk): This function uses a cache. Someone should try using
- partial inlining for the cache hits. */
- descr = _PyType_Lookup(tp, name);
-
- Py_XINCREF(descr);
-
- f = NULL;
- if (descr != NULL &&
- PyType_HasFeature(descr->ob_type, Py_TPFLAGS_HAVE_CLASS)) {
- f = descr->ob_type->tp_descr_get;
- if (f != NULL && PyDescr_IsData(descr)) {
- res = f(descr, obj, (PyObject *)obj->ob_type);
- Py_DECREF(descr);
- goto done;
- }
- }
-
- /* Inline _PyObject_GetDictPtr */
- dictoffset = tp->tp_dictoffset;
- if (dictoffset != 0) {
- PyObject *dict;
- if (dictoffset < 0) {
- Py_ssize_t tsize;
- size_t size;
-
- tsize = ((PyVarObject *)obj)->ob_size;
- if (tsize < 0)
- tsize = -tsize;
- size = _PyObject_VAR_SIZE(tp, tsize);
-
- dictoffset += (long)size;
- assert(dictoffset > 0);
- assert(dictoffset % SIZEOF_VOID_P == 0);
- }
- dictptr = (PyObject **) ((char *)obj + dictoffset);
- dict = *dictptr;
- if (dict != NULL) {
- Py_INCREF(dict);
- res = PyDict_GetItem(dict, name);
- if (res != NULL) {
- Py_INCREF(res);
- Py_XDECREF(descr);
- Py_DECREF(dict);
- goto done;
- }
- Py_DECREF(dict);
- }
- }
-
- if (f != NULL) {
- res = f(descr, obj, (PyObject *)Py_TYPE(obj));
- Py_DECREF(descr);
+ /* Do the first part of generic attribute lookup. */
+ if (generic_getattr_part1(obj, name, tp, &descr, &f, &res))
goto done;
- }
-
- if (descr != NULL) {
- res = descr;
- /* descr was already increfed above */
- goto done;
- }
-
- PyErr_Format(PyExc_AttributeError,
- "'%.50s' object has no attribute '%.400s'",
- tp->tp_name, PyString_AS_STRING(name));
+
+ /* Do the second part of generic attribute lookup. */
+ generic_getattr_part2(obj, name, tp, descr, f, &res);
+
done:
Py_DECREF(name);
return res;
=======================================
--- /trunk/Python/compile.c Thu Jul 29 23:33:28 2010
+++ /trunk/Python/compile.c Sun Aug 8 19:48:05 2010
@@ -816,6 +816,8 @@
return 1;
case LOAD_ATTR:
return 0;
+ case LOAD_METHOD:
+ return 1; /* Maybe set top, push one. */
case COMPARE_OP:
return -1;
case IMPORT_NAME:
@@ -860,6 +862,8 @@
#define NARGS(o) (((o) % 256) + 2*((o) / 256))
case CALL_FUNCTION:
return -NARGS(oparg);
+ case CALL_METHOD:
+ return -NARGS(oparg)-1;
case CALL_FUNCTION_VAR:
case CALL_FUNCTION_KW:
return -NARGS(oparg)-1;
@@ -2662,8 +2666,20 @@
{
int n, code = 0;
int n_positional_args, n_keyword_args = 0;
-
- VISIT(c, expr, e->v.Call.func);
+ expr_ty func = e->v.Call.func;
+
+ /* If this looks like a method call, emit specialized opcodes that
+ * avoid bound method allocation. */
+ if (!e->v.Call.starargs && !e->v.Call.kwargs &&
+ func->kind == Attribute_kind &&
+ func->v.Attribute.ctx == Load) {
+ VISIT(c, expr, func->v.Attribute.value);
+ ADDOP_NAME(c, LOAD_METHOD, func->v.Attribute.attr, names);
+ code = -1;
+ } else {
+ VISIT(c, expr, func);
+ }
+
n_positional_args = asdl_seq_LEN(e->v.Call.args);
VISIT_SEQ(c, expr, e->v.Call.args);
if (e->v.Call.keywords) {
@@ -2680,6 +2696,9 @@
code |= 2;
}
switch (code) {
+ case -1:
+ ADDOP_I(c, CALL_METHOD, n);
+ break;
case 0:
ADDOP_I(c, CALL_FUNCTION, n);
break;
=======================================
--- /trunk/Python/eval.cc Thu Jul 29 23:33:28 2010
+++ /trunk/Python/eval.cc Sun Aug 8 19:48:05 2010
@@ -350,7 +350,7 @@
static void record_type(PyCodeObject *, int, int, int, PyObject *);
static void record_func(PyCodeObject *, int, int, int, PyObject *);
static void record_object(PyCodeObject *, int, int, int, PyObject *);
-static void inc_feedback_counter(PyCodeObject *, int, int, int);
+static void inc_feedback_counter(PyCodeObject *, int, int, int, int);
#endif /* WITH_LLVM */

int _Py_ProfilingPossible = 0;
@@ -433,7 +433,7 @@
PyObject *
PyEval_GetCallStats(PyObject *self)
{
- return Py_BuildValue("iiiiiiiiiii",
+ return Py_BuildValue("iiiiiiiiiiiii",
pcall[0], pcall[1], pcall[2], pcall[3],
pcall[4], pcall[5], pcall[6], pcall[7],
pcall[8], pcall[9], pcall[10]);
@@ -968,18 +968,24 @@
if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);}
#define RECORD_FUNC(obj) \
if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);}
+#define INC_COUNTER(arg_index, counter_id) \
+ if (rec_feedback) { \
+ inc_feedback_counter(co, opcode, f->f_lasti, arg_index, \
+ counter_id); \
+ }
#define RECORD_TRUE() \
- if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti,
PY_FDO_JUMP_TRUE);}
+ INC_COUNTER(0, PY_FDO_JUMP_TRUE)
#define RECORD_FALSE() \
- if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti,
PY_FDO_JUMP_FALSE);}
+ INC_COUNTER(0, PY_FDO_JUMP_FALSE)
#define RECORD_NONBOOLEAN() \
- if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti,
PY_FDO_JUMP_NON_BOOLEAN);}
+ INC_COUNTER(0, PY_FDO_JUMP_NON_BOOLEAN)
#define UPDATE_HOTNESS_JABS() \
do { if (oparg <= f->f_lasti) ++co->co_hotness; } while (0)
#else
#define RECORD_TYPE(arg_index, obj)
#define RECORD_OBJECT(arg_index, obj)
#define RECORD_FUNC(obj)
+#define INC_COUNTER(arg_index, counter_id)
#define RECORD_TRUE()
#define RECORD_FALSE()
#define RECORD_NONBOOLEAN()
@@ -1128,18 +1134,21 @@
bail_count_stats->RecordBail(f, bail_reason);
#endif
if (_Py_BailError) {
- PyErr_SetString(PyExc_RuntimeError,
- "bailed to the interpreter");
+ /* When we bail, we set f_lasti to the current opcode
+ * minus 1, so we add one back. */
+ int lasti = f->f_lasti + 1;
+ PyErr_Format(PyExc_RuntimeError, "bailed to the "
+ "interpreter at opcode index %d", lasti);
goto exit_eval_frame;
}
}

- // Create co_runtime_feedback now that we're about to use it. You
- // might think this would cause a problem if the user flips
- // Py_JitControl from "never" to "whenhot", but since the value of
- // rec_feedback is constant for the duration of this frame's execution,
- // we will not accidentally try to record feedback without initializing
- // co_runtime_feedback.
+ /* Create co_runtime_feedback now that we're about to use it. You
+ * might think this would cause a problem if the user flips
+ * Py_JitControl from "never" to "whenhot", but since the value of
+ * rec_feedback is constant for the duration of this frame's execution,
+ * we will not accidentally try to record feedback without initializing
+ * co_runtime_feedback. */
if (rec_feedback && co->co_runtime_feedback == NULL) {
#if Py_WITH_INSTRUMENTATION
feedback_map_counter->IncCounter();
@@ -2469,6 +2478,39 @@
}
DISPATCH();

+ TARGET(LOAD_METHOD)
+ w = GETITEM(names, oparg);
+ v = TOP();
+ RECORD_TYPE(0, v);
+ x = PyObject_GetMethod(v, w);
+ if (((long)x) & 1) {
+ /* Record that this was a regular method. */
+ INC_COUNTER(1, PY_FDO_LOADMETHOD_METHOD);
+
+ /* Set up the stack as if self were the first
+ * argument to the unbound method. */
+ x = (PyObject*)(((Py_uintptr_t)x) & ~1);
+ SET_TOP(x);
+ PUSH(v);
+ }
+ else {
+ /* Record that this was not a regular method.
+ */
+ INC_COUNTER(1, PY_FDO_LOADMETHOD_OTHER);
+
+ /* Set up the stack as if there were no self
+ * argument. Pad the stack with a NULL so
+ * CALL_METHOD knows the method is bound. */
+ Py_DECREF(v);
+ SET_TOP(NULL);
+ PUSH(x);
+ }
+ if (x == NULL) {
+ why = UNWIND_EXCEPTION;
+ break;
+ }
+ DISPATCH();
+
TARGET(COMPARE_OP)
w = POP();
v = TOP();
@@ -2844,6 +2886,48 @@
}
DISPATCH();
}
+
+ TARGET(CALL_METHOD)
+ {
+ int num_args, num_kwargs, num_stack_slots;
+ PyObject *method;
+ PY_LOG_TSC_EVENT(CALL_START_EVAL);
+ PCALL(PCALL_ALL);
+ num_args = oparg & 0xff;
+ num_kwargs = (oparg>>8) & 0xff;
+ /* +1 for the actual function object, +1 for self. */
+ num_stack_slots = num_args + 2 * num_kwargs + 1 + 1;
+ method = stack_pointer[-num_stack_slots];
+ if (method != NULL) {
+ /* We loaded an unbound method. Adjust
+ * num_args to include the self argument pushed
+ * on the stack after the method. */
+ num_args++;
+ }
+#ifdef WITH_LLVM
+ else {
+ /* The method is really in the next slot. */
+ method = stack_pointer[-num_stack_slots+1];
+ }
+ /* We'll focus on these simple calls with only
+ * positional args for now (since they're easy to
+ * implement). */
+ if (num_kwargs == 0) {
+ RECORD_FUNC(method);
+ }
+#endif
+ x = _PyEval_CallFunction(stack_pointer,
+ num_args, num_kwargs);
+ /* Clear the stack of the function object and
+ * arguments. */
+ stack_pointer -= num_stack_slots;
+ PUSH(x);
+ if (x == NULL) {
+ why = UNWIND_EXCEPTION;
+ break;
+ }
+ DISPATCH();
+ }

TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw)
TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw)
@@ -4217,29 +4301,24 @@
}

static void
-err_args(PyObject *func, int flags, int min_arity, int max_arity, int
nargs)
+err_args(PyMethodDef *ml, int flags, int min_arity, int max_arity, int
nargs)
{
if (min_arity != max_arity)
PyErr_Format(PyExc_TypeError,
"%.200s() takes %d-%d arguments (%d given)",
- ((PyCFunctionObject *)func)->m_ml->ml_name,
- min_arity, max_arity, nargs);
+ ml->ml_name, min_arity, max_arity, nargs);
else if (min_arity == 0)
PyErr_Format(PyExc_TypeError,
"%.200s() takes no arguments (%d given)",
- ((PyCFunctionObject *)func)->m_ml->ml_name,
- nargs);
+ ml->ml_name, nargs);
else if (min_arity == 1)
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly one argument (%d given)",
- ((PyCFunctionObject *)func)->m_ml->ml_name,
- nargs);
+ ml->ml_name, nargs);
else
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly %d arguments (%d given)",
- ((PyCFunctionObject *)func)->m_ml->ml_name,
- min_arity,
- nargs);
+ ml->ml_name, min_arity, nargs);
}

#ifdef WITH_LLVM
@@ -4413,20 +4492,38 @@
PyObject **pfunc = stack_pointer - n - 1;
PyObject *func = *pfunc;
PyObject *x, *w;
-
- /* Always dispatch PyCFunction first, because these are
- presumed to be the most frequent callable object.
- */
- if (PyCFunction_Check(func) && nk == 0) {
- int flags = PyCFunction_GET_FLAGS(func);
+ PyMethodDef *ml = NULL;
+ PyObject *self = NULL;
+
+ /* Always dispatch PyCFunction and PyMethodDescr first, because these
+ both represent methods written in C, and are presumed to be the most
+ frequently called objects.
+ */
+ if (nk == 0) {
+ if (PyCFunction_Check(func)) {
+ ml = PyCFunction_GET_METHODDEF(func);
+ self = PyCFunction_GET_SELF(func);
+ }
+ else if (PyMethodDescr_Check(func)) {
+ ml = ((PyMethodDescrObject*)func)->d_method;
+ /* The first argument on the stack (the one immediately
+ after func) is self. We borrow the reference from
+ the stack, which gets cleaned off and decrefed at
+ the end of the function.
+ */
+ self = pfunc[1];
+ na--;
+ }
+ }
+ if (ml != NULL) {
+ int flags = ml->ml_flags;
PyThreadState *tstate = PyThreadState_GET();

PCALL(PCALL_CFUNCTION);
if (flags & METH_ARG_RANGE) {
- PyCFunction meth = PyCFunction_GET_FUNCTION(func);
- PyObject *self = PyCFunction_GET_SELF(func);
- int min_arity = PyCFunction_GET_MIN_ARITY(func);
- int max_arity = PyCFunction_GET_MAX_ARITY(func);
+ PyCFunction meth = ml->ml_meth;
+ int min_arity = ml->ml_min_arity;
+ int max_arity = ml->ml_max_arity;
PyObject *args[PY_MAX_ARITY] = {NULL};

switch (na) {
@@ -4444,20 +4541,20 @@
Yes, but C allows this. Go C. */
if (min_arity <= na && na <= max_arity) {
C_TRACE(x, (*(PyCFunctionThreeArgs)meth)
- (self, args[0], args[1], args[2]));
- Py_XDECREF(args[0]);
- Py_XDECREF(args[1]);
- Py_XDECREF(args[2]);
+ (self, args[0], args[1], args[2]));
}
else {
- err_args(func, flags, min_arity, max_arity, na);
+ err_args(ml, flags, min_arity, max_arity, na);
x = NULL;
}
+ Py_XDECREF(args[0]);
+ Py_XDECREF(args[1]);
+ Py_XDECREF(args[2]);
}
else {
PyObject *callargs;
callargs = load_args(&stack_pointer, na);
- C_TRACE(x, PyCFunction_Call(func,callargs,NULL));
+ C_TRACE(x, PyMethodDef_Call(ml, self, callargs, NULL));
Py_XDECREF(callargs);
}
} else {
@@ -4468,13 +4565,12 @@
PCALL(PCALL_BOUND_METHOD);
Py_INCREF(self);
func = PyMethod_GET_FUNCTION(func);
- Py_INCREF(func);
Py_DECREF(*pfunc);
*pfunc = self;
na++;
n++;
- } else
- Py_INCREF(func);
+ }
+ Py_INCREF(func);
if (PyFunction_Check(func))
x = fast_function(func, &stack_pointer, n, na, nk);
else
@@ -4483,9 +4579,9 @@
}

/* Clear the stack of the function object. Also removes
- the arguments in case they weren't consumed already
- (fast_function() and err_args() leave them on the stack).
- */
+ the arguments in case they weren't consumed already
+ (fast_function() and err_args() leave them on the stack).
+ */
while (stack_pointer > pfunc) {
w = EXT_POP(stack_pointer);
Py_DECREF(w);
@@ -4695,6 +4791,18 @@
PyObject *callargs = NULL;
PyObject *kwdict = NULL;
PyObject *result = NULL;
+ PyMethodDef *ml = NULL;
+ PyObject *self = NULL;
+
+ if (PyCFunction_Check(func)) {
+ ml = PyCFunction_GET_METHODDEF(func);
+ self = PyCFunction_GET_SELF(func);
+ }
+ else if (PyMethodDescr_Check(func)) {
+ ml = ((PyMethodDescrObject*)func)->d_method;
+ self = (*pp_stack)[-(na + 2 * nk)];
+ na--;
+ }

if (nk > 0) {
kwdict = update_keyword_args(NULL, nk, pp_stack, func);
@@ -4720,9 +4828,9 @@
else
PCALL(PCALL_OTHER);
#endif
- if (PyCFunction_Check(func)) {
+ if (ml) {
PyThreadState *tstate = PyThreadState_GET();
- C_TRACE(result, PyCFunction_Call(func, callargs, kwdict));
+ C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict));
}
else
result = PyObject_Call(func, callargs, kwdict);
@@ -4740,6 +4848,22 @@
PyObject *stararg = NULL;
PyObject *kwdict = NULL;
PyObject *result = NULL;
+ PyMethodDef *ml = NULL;
+ PyObject *self = NULL;
+
+ if (PyCFunction_Check(func)) {
+ ml = PyCFunction_GET_METHODDEF(func);
+ self = PyCFunction_GET_SELF(func);
+ }
+ else if (PyMethodDescr_Check(func)) {
+ /* Only apply C calling optimization if self is on the stack
+ * and not the first element of callargs. */
+ if (na > 0) {
+ ml = ((PyMethodDescrObject*)func)->d_method;
+ self = (*pp_stack)[-(na + 2 * nk)];
+ na--;
+ }
+ }

if (flags & CALL_FLAG_KW) {
kwdict = EXT_POP(*pp_stack);
@@ -4815,9 +4939,9 @@
else
PCALL(PCALL_OTHER);
#endif
- if (PyCFunction_Check(func)) {
+ if (ml) {
PyThreadState *tstate = PyThreadState_GET();
- C_TRACE(result, PyCFunction_Call(func, callargs, kwdict));
+ C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict));
}
else
result = PyObject_Call(func, callargs, kwdict);
@@ -4830,7 +4954,7 @@

#ifdef WITH_LLVM
void inc_feedback_counter(PyCodeObject *co, int expected_opcode,
- int opcode_index, int counter_id)
+ int opcode_index, int arg_index, int counter_id)
{
#ifndef NDEBUG
unsigned char actual_opcode =
@@ -4841,7 +4965,7 @@
#endif /* NDEBUG */
PyRuntimeFeedback &feedback =
co->co_runtime_feedback->GetOrCreateFeedbackEntry(
- opcode_index, 0);
+ opcode_index, arg_index);
feedback.IncCounter(counter_id);
}

=======================================
--- /trunk/Python/opcode_targets.h Fri Nov 13 17:46:58 2009
+++ /trunk/Python/opcode_targets.h Sun Aug 8 19:48:05 2010
@@ -108,7 +108,7 @@
&&TARGET_COMPARE_OP,
&&_unknown_opcode,
&&_unknown_opcode,
- &&_unknown_opcode,
+ &&TARGET_LOAD_METHOD,
&&TARGET_JUMP_FORWARD,
&&TARGET_JUMP_IF_FALSE_OR_POP,
&&TARGET_JUMP_IF_TRUE_OR_POP,
@@ -132,7 +132,7 @@
&&_unknown_opcode,
&&TARGET_CALL_FUNCTION,
&&_unknown_opcode,
- &&_unknown_opcode,
+ &&TARGET_CALL_METHOD,
&&TARGET_MAKE_CLOSURE,
&&TARGET_LOAD_CLOSURE,
&&TARGET_LOAD_DEREF,

Reply all

Reply to author

Forward

0 new messages