Descriptor/Decorator challenge
Raymond Hettinger
Perhaps the under-under name mangling trick
can be replaced (in Py3.0) with a suitably designed decorator.
Your challenge is to write the decorator.
Any trick in the book (metaclasses, descriptors, etc) is fair game.
Raymond
-------- how we currently localize method access with name mangling
------
class A:
def __m(self):
print 'A.__m'
def am(self):
self.__m()
class B(A):
def __m(self):
print 'B.__m'
def bm(self):
self.__m()
m = B()
m.am() # prints 'A.__m'
m.bm() # prints 'B.__m'
-------- how I would like to localize method access with a decorator
------
class A:
@localmethod
def m(self):
print 'A.m'
def am(self):
self.m()
class B(A):
@localmethod
def m(self):
print 'B.m'
def bm(self):
self.m()
m = B()
m.am() # prints 'A.m'
m.bm() # prints 'B.m'
---------------------
P.S. Here's a link to the descriptor how-to:
http://users.rcn.com/python/download/Descriptor.htm
Arnaud Delobelle
> I had an idea but no time to think it through.
> Perhaps the under-under name mangling trick
> can be replaced (in Py3.0) with a suitably designed decorator.
> Your challenge is to write the decorator.
> Any trick in the book (metaclasses, descriptors, etc) is fair game.
I had some time this lunchtimes and I needed to calm my nerves so I
took up your challenge :)
Here is my poor effort. I'm sure lots of things are wrong with it but
I'm not sure I'll look at it again.
from types import MethodType, FunctionType
# The suggested localmethod decorator
class localmethod(object):
def __init__(self, f):
self.f = f
self.defclass = None
self.nextmethod = None
def __get__(self, obj, objtype=None):
callobj = obj or objtype
if callobj.callerclass == self.defclass:
return MethodType(self.f, obj, objtype)
elif self.nextmethod:
return self.nextmethod.__get__(obj, objtype)
else:
raise AttributeError
class BoundMethod(object):
def __init__(self, meth, callobj, callerclass):
self.meth = meth
self.callobj = callobj
self.callerclass = callerclass
def __call__(self, *args, **kwargs):
callobj = self.callobj
try:
callobj.callerclass = self.callerclass
return self.meth(*args, **kwargs)
finally:
callobj.callerclass = None
# A 'mormal' method decorator is needed as well
class method(object):
def __init__(self, f):
self.f = f
self.defclass = None
def __get__(self, obj, objtype=None):
callobj = obj or objtype
return BoundMethod(MethodType(self.f, obj, objtype), callobj,
self.defclass)
class Type(type):
def __new__(self, name, bases, attrs):
for attr, val in attrs.items():
if type(val) == FunctionType:
attrs[attr] = method(val)
return type.__new__(self, name, bases, attrs)
def __init__(self, name, bases, attrs):
for attr, val in attrs.iteritems():
if type(val) == localmethod:
val.defclass = self
for base in self.mro()[1:]:
if attr in base.__dict__:
nextmethod = base.__dict__[attr]
val.nextmethod = nextmethod
break
elif type(val) == method:
val.defclass = self
class Object(object):
__metaclass__ = Type
# Note: any class or object has to have a callerclass attribute
for this to work.
# That makes it thread - incompatible I guess.
callerclass = None
# Here is your example code
class A(Object):
@localmethod
def m(self):
print 'A.m'
def am(self):
self.m()
class B(A):
@localmethod
def m(self):
print 'B.m'
def bm(self):
self.m()
# Untested beyond this particular example!
--
Arnaud
Arnaud Delobelle
> On 5 Mar, 07:31, "Raymond Hettinger" <pyt...@rcn.com> wrote:
>
> > I had an idea but no time to think it through.
> > Perhaps the under-under name mangling trick
> > can be replaced (in Py3.0) with a suitably designed decorator.
> > Your challenge is to write the decorator.
> > Any trick in the book (metaclasses, descriptors, etc) is fair game.
>
> I had some time this lunchtimes and I needed to calm my nerves so I
> took up your challenge :)
> Here is my poor effort. I'm sure lots of things are wrong with it but
> I'm not sure I'll look at it again.
Well in fact I couldn't help but try to improve it a bit. Objects now
don't need a callerclass attribute, instead all necessary info is
stored in a global __callerclass__. Bits that didn't work now do.
So here is my final attempt, promised. The awkward bits are:
* how to find out where a method is called from
* how to resume method resolution once it has been established a
local method has to be bypassed, as I don't know how to interfere
directly with mro.
Feedback of any form is welcome (though I prefer when it's polite :)
--------------------
from types import MethodType, FunctionType
class IdDict(object):
def __init__(self):
self.objects = {}
def __getitem__(self, obj):
return self.objects.get(id(obj), None)
def __setitem__(self, obj, callerclass):
self.objects[id(obj)] = callerclass
def __delitem__(self, obj):
del self.objects[id(obj)]
# This stores the information about from what class an object is
calling a method
# It is decoupled from the object, better than previous version
# Maybe that makes it easier to use with threads?
__callerclass__ = IdDict()
# The purpose of this class is to update __callerclass__ just before
and after a method is called
class BoundMethod(object):
def __init__(self, meth, callobj, callerclass):
self.values = meth, callobj, callerclass
def __call__(self, *args, **kwargs):
meth, callobj, callerclass = self.values
if callobj is None and args:
callobj = args[0]
try:
__callerclass__[callobj] = callerclass
return meth(*args, **kwargs)
finally:
del __callerclass__[callobj]
# A 'normal' method decorator is needed as well
class method(object):
def __init__(self, f):
self.f = f
def __get__(self, obj, objtype=None):
return BoundMethod(MethodType(self.f, obj, objtype), obj,
self.defclass)
class LocalMethodError(AttributeError):
pass
# The suggested localmethod decorator
class localmethod(method):
def __get__(self, obj, objtype=None):
callobj = obj or objtype
defclass = self.defclass
if __callerclass__[callobj] is defclass:
return MethodType(self.f, obj, objtype)
else:
# The caller method is from a different class, so look for
the next candidate.
mro = iter((obj and type(obj) or objtype).mro())
for c in mro: # Skip all classes up to the localmethod's
class
if c == defclass: break
name = self.name
for base in mro:
if name in base.__dict__:
try:
return base.__dict__[name].__get__(obj,
objtype)
except LocalMethodError:
continue
raise LocalMethodError, "localmethod '%s' is not accessible
outside object '%s'" % (self.name, self.defclass.__name__)
class Type(type):
def __new__(self, name, bases, attrs):
# decorate all function attributes with 'method'
for attr, val in attrs.items():
if type(val) == FunctionType:
attrs[attr] = method(val)
return type.__new__(self, name, bases, attrs)
def __init__(self, name, bases, attrs):
for attr, val in attrs.iteritems():
# Inform methods of what class they are created in
if isinstance(val, method):
val.defclass = self
# Inform localmethod of their name (in case they have to
be bypassed)
if isinstance(val, localmethod):
val.name = attr
class Object(object):
__metaclass__ = Type
# Here is your example code
class A(Object):
@localmethod
def m(self):
print 'A.m'
def am(self):
self.m()
class B(A):
@localmethod
def m(self):
print 'B.m'
def bm(self):
self.m()
# Added:
B.am(m) # prints 'A.m'
B.bm(m) # prints 'B.m'
m.m() # LocalMethodError (which descends from AttributeError)
# Untested beyond this particular example!
--------------------
--
Arnaud
George Sakkis
What would the semantics be if m is decorated as local only in A or
only in B ?
George
Arnaud Delobelle
[snip]
> Well in fact I couldn't help but try to improve it a bit. Objects now
> don't need a callerclass attribute, instead all necessary info is
> stored in a global __callerclass__. Bits that didn't work now do.
OK that wasn't really thought through. Because I changed the design
mid-way through writing it __callerclass__ wasn't doing the right
thing. I've sorted the issues I could see and made it (hopefully)
thread-safe. I'm not going to pollute this list again with my code so
I've put it at the following address:
http://marooned.org.uk/local.py
The problem is that all normal functions need to be decorated with
'@function' for it to work completely: if I understand correctly the
snippet below should raise an exception. It only does so if 'f' is
decorated with '@function' as below.
----------
@function
def f(x):
x.l()
class C(Object):
@localmethod
def l(self):
print "Shouldn't get here"
def v(self):
return f(self)
C().v() # Raises LocalMethod exception
----------
PS: in fact I'm not sure it's a good idea to decorate local methods:
what about local attributes which are not methods? They have to be
treated differently as only functions can be decorated.
What about functions / classes which are local to a module?
--
Arnaud
Raymond Hettinger
> What would the semantics be if m is decorated as local only in A or
> only in B ?
The goal is to as closely as possible emulate the sematics of under-
under name mangling.
Raymond
Jack Diederich
> On 5 Mar, 18:59, "Arnaud Delobelle" <arno...@googlemail.com> wrote:
> [snip]
> > Well in fact I couldn't help but try to improve it a bit. Objects now
> > don't need a callerclass attribute, instead all necessary info is
> > stored in a global __callerclass__. Bits that didn't work now do.
>
> OK that wasn't really thought through. Because I changed the design
> mid-way through writing it __callerclass__ wasn't doing the right
> thing. I've sorted the issues I could see and made it (hopefully)
> thread-safe. I'm not going to pollute this list again with my code so
> I've put it at the following address:
>
> http://marooned.org.uk/local.py
>
I fooled around with this a bit and even when using different techniques
than Arnaud (namely stack inspection and/or class decorators) it ends up
looking the same. In order to pull this off you need to
1) mark the localmethods as special
(@localmethod works here)
2) mark all non-localmethods as not special
(metaclasses, class decorators, or module-level stack inspection)
3) keep track of the call stack
(can be implemented as part of #1 and #2 but adds overhead regardless)
Double underscore names take care of #1 at compile time and by definition
anything not name-manged falls into the non-special class of #2. After
that the special/non-special calls are given different names so the
native call semantics take care of the call stack for you.
With some bytecode manipulation it should be possible to fake the same
name mangling by using just @localmethod decorators and adding some
overhead to always checking the current class (eg A.m() for the function
am()) and falling back on doing the normal call of self.m(). This could
be made more exact by doing inspection after the fact with any of
metaclasses/class decorators/module inspection because then we could
inspect what is a @localmethod and what isn't all the way through the class
tree.
I could be wrong on the byte-hack part as I've only recently learned
to read the chicken bones that are byte codes (thanks to PyCon sprints
I got to spend time picking python-devs's brains over burritos). It
seem plausible if fragile.
-Jack
Michele Simionato
> Any trick in the book (metaclasses, descriptors, etc) is fair game.
So you are asking for a serious hack, right?
As soon as I saw your challenge I thought "That's difficult. Very
difficult.
No way I can solve that with a simple descriptor/decorator. I need
more POWER.
Time to look at the byteplay module".
The byteplay module by Noam Raphael (http://byteplay.googlecode.com/
svn/trunk/byteplay.py) seems to exist just to make possible
spectacular hacks. So I took
your challenge as an opportunity to explore a bit its secrets. In
doing so,
I have also decided to break the rules and not to solve your problem,
but
a different one, which is the one I am more interested in ;)
Basically, I am happy with the double underscores, but I am unhappy
with
the fact that a method does not know the class where it is defined, so
that you have to repeat the name of the class in ``super``.
With some bytecode + metaclass hackery it is possible to make the
methods
smart enough to recognize the class where they are defined, so your
example could be solved as follows:
from currentclass import CurrentClassEnabled
class A(CurrentClassEnabled):
def m(self):
print 'A.m'
def am(self):
CurrentClass.m(self) # the same as A.m(self)
class B(A):
def m(self):
print 'B.m'
def bm(self):
CurrentClass.m(self) # the same as B.m(self)
def superm(self):
super(CurrentClass, self).m() # the same as super(B, self).m()
m.superm() # prints 'A.m'
As you see, as a byproduct, double underscores are not needed anymore,
since methods are called directly from the CurrentClass. The approach
also works for ordinary attributes which are not methods.
The code to enable recognition of CurrentClass is short enough to be
includede here, but I will qualify it as a five star-level hackery:
$ cat currentclass.py
# requires byteplay by Noam Raphael
# see http://byteplay.googlecode.com/svn/trunk/byteplay.py
from types import FunctionType
from byteplay import Code, LOAD_GLOBAL, STORE_FAST, LOAD_FAST
def addlocalvar(f, locname, globname):
if locname not in f.func_code.co_names:
return f # do nothing
c = Code.from_code(f.func_code)
c.code[1:1] = [(LOAD_GLOBAL, globname), (STORE_FAST, locname)]
for i, (opcode, value) in enumerate(c.code[2:]):
if opcode == LOAD_GLOBAL and value == locname:
c.code[i+2] = (LOAD_FAST, locname)
f.func_code = c.to_code()
return f
class _CurrentClassEnabled(type):
def __new__(mcl, name, bases, dic):
for n, v in dic.iteritems():
if isinstance(v, FunctionType):
dic[n] = addlocalvar(v, 'CurrentClass', name)
return super(_CurrentClassEnabled, mcl).__new__(mcl, name,
bases, dic)
class CurrentClassEnabled:
__metaclass__ = _CurrentClassEnabled
Enjoy!
Michele Simionato
Gabriel Genellina
<michele....@gmail.com> escribió:
> The code to enable recognition of CurrentClass is short enough to be
> includede here, but I will qualify it as a five star-level hackery:
You forgot the standard disclaimer: "This is extremely dangerous stuff,
only highly trained professionals can do that! Kids, never try this at
home!"
--
Gabriel Genellina
Michele Simionato
wrote:
> You forgot the standard disclaimer: "This is extremely dangerous stuff,
> only highly trained professionals can do that! Kids, never try this at
> home!"
;)
Yep, the only good use case for this kind of games is for prototyping
in current Python
features that are candidate for inclusion in future versions of
Python, but this was
what Raymond asked. However, there is also the fun of it ;)
Michele Simionato