Google Groups no longer supports new Usenet posts or subscriptions. Historical content remains viewable.
Dismiss
working with ctypes and complex data structures
671 views
Skip to first unread message
Michael Felt
Oct 2, 2016, 1:51:06 PM10/2/16
to
I am trying to understand the documentation re: ctypes and interfacing
with existing libraries.
I am reading the documentation, and also other sites that have largely
just copied the documentation - as well as the "free chapter" at
O'Reilly (Python Cookbook).
I am missing anything on CFields.
My test (template) is:
"""Internal ctypes definitions for AIX libperfstat"""
from sys import maxsize
from ctypes import Structure, Union
from ctypes import c_bool, c_char, c_wchar, c_byte, c_ubyte, c_short,
c_ushort
from ctypes import c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong
from ctypes import c_float, c_double, c_longdouble
from ctypes import c_char_p, c_wchar_p, c_void_p
from ctypes import sizeof
IDENTIFIER_LENGTH = 64
class time_t(Structure):
"""
#ifndef _TIME_T
#define _TIME_T
#ifdef _LINUX_SOURCE_COMPAT
typedef long int time_t;
#else /* !_LINUX_SOURCE_COMPAT */
#ifdef __64BIT__
typedef long time_t;
#else
typedef int time_t;
#endif
#endif /* !_LINUX_SOURCE_COMPAT */
#endif
"""
if (maxsize > 2**32):
_fields_ = [("v", c_long)]
else:
_fields_ = [("v", c_int)]
class perfstat_cpu_total_t(Structure):
"""
typedef struct { /* global cpu information */
int ncpus; /* number of active logical processors */
int ncpus_cfg; /* number of configured processors */
char description[IDENTIFIER_LENGTH]; /* processor description
(type/official name) */
u_longlong_t buffer1[15]; /* several variables being skipped
for now */
time_t lbolt; /* number of ticks since last reboot */
u_longlong_t loadavg[3]; /* (1<<SBITS) times the average
number of runnables processes during the last
1, 5 and 15 minutes. */
u_longlong_t buffer2[29]; /* several variables being skipped
for now */
int ncpus_high; /* index of highest processor online */
u_longlong_t puser; /* raw number of physical processor
tics in user mode */
u_longlong_t psys; /* raw number of physical processor
tics in system mode */
u_longlong_t pidle; /* raw number of physical processor
tics idle */
u_longlong_t pwait; /* raw number of physical processor
tics waiting for I/O */
u_longlong_t buffer2[29]; /* several variables being skipped
for now */
} perfstat_cpu_total_t;
"""
_fields_ = [
("ncpus", c_int),
("ncpus_cfg", c_int),
("description", c_char * IDENTIFIER_LENGTH),
("buffer1", c_ulonglong * 15),
("lbolt", time_t),
("loadavg", c_ulonglong * 3),
("buffer2", c_ulonglong * 29),
("ncpus_high", c_int),
("puser", c_ulonglong),
("psys", c_ulonglong),
("pidle", c_ulonglong),
("pwait", c_ulonglong),
("buffer3", c_ulonglong * 12)
]
xxx = time_t
print xxx
print xxx.v
print sizeof(xxx)
# print sizeof(xxx.v)
xxx = perfstat_cpu_total_t
print xxx
print xxx.lbolt
print sizeof(xxx)
# print sizeof(xxx.lbolt)
xxx = xxx.lbolt
print xxx
print xxx.value
result:
michael@x071:[/data/prj/python/cloud-init/aix-cloud-init-0.7.8.1/new]python
test*
<class '__main__.time_t'>
<Field type=c_long, ofs=0, size=8>
8
<class '__main__.perfstat_cpu_total_t'>
<Field type=time_t, ofs=192, size=8>
592
<Field type=time_t, ofs=192, size=8>
Traceback (most recent call last):
File "test_1.py", line 79, in <module>
print xxx.value
AttributeError: '_ctypes.CField' object has no attribute 'value'
Question:
a) where is documentation on "CField"'s? I have not been successful with
google - :(
b) what I am not understanding - as the basic documentation shows
FOO.value as the way to set/get the value of a _field_
+++
I assume I am not understanding something from the base documentation at
https://docs.python.org/2/library/ctypes.html#Structured-data-types
Hints, improved links, etc are appreciated!
with existing libraries.
I am reading the documentation, and also other sites that have largely
just copied the documentation - as well as the "free chapter" at
O'Reilly (Python Cookbook).
I am missing anything on CFields.
My test (template) is:
"""Internal ctypes definitions for AIX libperfstat"""
from sys import maxsize
from ctypes import Structure, Union
from ctypes import c_bool, c_char, c_wchar, c_byte, c_ubyte, c_short,
c_ushort
from ctypes import c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong
from ctypes import c_float, c_double, c_longdouble
from ctypes import c_char_p, c_wchar_p, c_void_p
from ctypes import sizeof
IDENTIFIER_LENGTH = 64
class time_t(Structure):
"""
#ifndef _TIME_T
#define _TIME_T
#ifdef _LINUX_SOURCE_COMPAT
typedef long int time_t;
#else /* !_LINUX_SOURCE_COMPAT */
#ifdef __64BIT__
typedef long time_t;
#else
typedef int time_t;
#endif
#endif /* !_LINUX_SOURCE_COMPAT */
#endif
"""
if (maxsize > 2**32):
_fields_ = [("v", c_long)]
else:
_fields_ = [("v", c_int)]
class perfstat_cpu_total_t(Structure):
"""
typedef struct { /* global cpu information */
int ncpus; /* number of active logical processors */
int ncpus_cfg; /* number of configured processors */
char description[IDENTIFIER_LENGTH]; /* processor description
(type/official name) */
u_longlong_t buffer1[15]; /* several variables being skipped
for now */
time_t lbolt; /* number of ticks since last reboot */
u_longlong_t loadavg[3]; /* (1<<SBITS) times the average
number of runnables processes during the last
1, 5 and 15 minutes. */
u_longlong_t buffer2[29]; /* several variables being skipped
for now */
int ncpus_high; /* index of highest processor online */
u_longlong_t puser; /* raw number of physical processor
tics in user mode */
u_longlong_t psys; /* raw number of physical processor
tics in system mode */
u_longlong_t pidle; /* raw number of physical processor
tics idle */
u_longlong_t pwait; /* raw number of physical processor
tics waiting for I/O */
u_longlong_t buffer2[29]; /* several variables being skipped
for now */
} perfstat_cpu_total_t;
"""
_fields_ = [
("ncpus", c_int),
("ncpus_cfg", c_int),
("description", c_char * IDENTIFIER_LENGTH),
("buffer1", c_ulonglong * 15),
("lbolt", time_t),
("loadavg", c_ulonglong * 3),
("buffer2", c_ulonglong * 29),
("ncpus_high", c_int),
("puser", c_ulonglong),
("psys", c_ulonglong),
("pidle", c_ulonglong),
("pwait", c_ulonglong),
("buffer3", c_ulonglong * 12)
]
xxx = time_t
print xxx
print xxx.v
print sizeof(xxx)
# print sizeof(xxx.v)
xxx = perfstat_cpu_total_t
print xxx
print xxx.lbolt
print sizeof(xxx)
# print sizeof(xxx.lbolt)
xxx = xxx.lbolt
print xxx
print xxx.value
result:
michael@x071:[/data/prj/python/cloud-init/aix-cloud-init-0.7.8.1/new]python
test*
<class '__main__.time_t'>
<Field type=c_long, ofs=0, size=8>
8
<class '__main__.perfstat_cpu_total_t'>
<Field type=time_t, ofs=192, size=8>
592
<Field type=time_t, ofs=192, size=8>
Traceback (most recent call last):
File "test_1.py", line 79, in <module>
print xxx.value
AttributeError: '_ctypes.CField' object has no attribute 'value'
Question:
a) where is documentation on "CField"'s? I have not been successful with
google - :(
b) what I am not understanding - as the basic documentation shows
FOO.value as the way to set/get the value of a _field_
+++
I assume I am not understanding something from the base documentation at
https://docs.python.org/2/library/ctypes.html#Structured-data-types
Hints, improved links, etc are appreciated!
eryk sun
Oct 2, 2016, 5:45:25 PM10/2/16
to
On Sun, Oct 2, 2016 at 5:50 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>
> a) where is documentation on "CField"'s?
It's undocumented. A CField is a data descriptor that accesses a
>
> a) where is documentation on "CField"'s?
struct field with the given type, size, and offset. Like most
descriptors, it's meant to be accessed as an attribute of an instance,
not as an attribute of the type.
When accessed as an attribute of a struct type, the value returned is
the CField descriptor itself. One reason to reference the descriptor
is for its "offset" attribute, since there's no offsetof() in ctypes.
At least in this regard it should be documented.
> b) what I am not understanding - as the basic documentation shows FOO.value
> as the way to set/get the value of a _field_
and c_char_p. These simple types have getter and setter functions that
get called automatically whenever the type is used as a function
result, array index, or struct field. For example:
class Foo(ctypes.Structure):
_fields_ = (('v', ctypes.c_int),)
>>> foo = Foo()
>>> foo.v = 5
>>> foo.v
5
You can use a subclass to avoid the getfunc's automatic conversion. For example;
class my_int(ctypes.c_int):
pass
class Bar(ctypes.Structure):
_fields_ = (('v', my_int),)
>>> bar = Bar()
>>> bar.v = 5
>>> bar.v
<my_int object at 0x7f7385e8aae8>
>>> bar.v.value
5
> class time_t(Structure):
> ...
> if (maxsize > 2**32):
> _fields_ = [("v", c_long)]
> else:
> _fields_ = [("v", c_int)]
I'd alias the type instead of defining a struct, e.g. `time_t =
> _fields_ = [("v", c_long)]
> else:
> _fields_ = [("v", c_int)]
c_long`. This preserves automatic conversion of the simple type.
Also, sys.maxsize is based on the platform's size_t type. That's
generally the same size as a pointer, but C doesn't require this.
Instead use sizeof(c_void_p), which will be 8 on a 64-bit platform and
4 on a 32-bit platform.
Also, while it's unrelated to your current problem, bear in mind that
int and long are both always 32-bit on Windows. c_int64 is a
cross-platform 64-bit integer.
Lawrence D’Oliveiro
Oct 2, 2016, 6:41:07 PM10/2/16
to
On Monday, October 3, 2016 at 6:51:06 AM UTC+13, Michael Felt wrote:
> from ctypes import Structure, Union
> from ctypes import c_bool, c_char, c_wchar, c_byte, c_ubyte, c_short,
> c_ushort
> from ctypes import c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong
> from ctypes import c_float, c_double, c_longdouble
> from ctypes import c_char_p, c_wchar_p, c_void_p
> from ctypes import sizeof
Let me suggest replacing all this with
> from ctypes import Structure, Union
> from ctypes import c_bool, c_char, c_wchar, c_byte, c_ubyte, c_short,
> c_ushort
> from ctypes import c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong
> from ctypes import c_float, c_double, c_longdouble
> from ctypes import c_char_p, c_wchar_p, c_void_p
> from ctypes import sizeof
import ctypes as ct
Then you can use “ct.c_int”, “ct.c_uint” etc, without falling to the temptation of wildcard imports.
Michael Felt
Oct 3, 2016, 6:48:24 AM10/3/16
to
On 02-Oct-16 23:44, eryk sun wrote:
> On Sun, Oct 2, 2016 at 5:50 PM, Michael Felt<mic...@felt.demon.nl> wrote:
>> >
>> >a) where is documentation on "CField"'s?
Not using maxsize will be good, also in a different patch - also
specific to AIX.
This "thing" I am working is to:
a) document stuff (for a novice like myself)
b) have a better way to get system low-level statistics, comparable to
Linux, where additions have been made to libc to get them.
Michael Felt
Oct 3, 2016, 10:36:18 AM10/3/16
to
On 02-Oct-16 23:44, eryk sun wrote:
> On Sun, Oct 2, 2016 at 5:50 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>> a) where is documentation on "CField"'s?
So I do not feel so bad about not finding anything :)
> A CField is a data descriptor that accesses a
> struct field with the given type, size, and offset. Like most
> descriptors, it's meant to be accessed as an attribute of an instance,
> not as an attribute of the type.
>
> When accessed as an attribute of a struct type, the value returned is
> the CField descriptor itself. One reason to reference the descriptor
> is for its "offset" attribute, since there's no offsetof() in ctypes.
> At least in this regard it should be documented.
>
>> b) what I am not understanding - as the basic documentation shows FOO.value
>> as the way to set/get the value of a _field_
> You may be surprised when accessing simple-type fields such as c_int
> and c_char_p. These simple types have getter and setter functions that
> get called automatically whenever the type is used as a function
> result, array index, or struct field. For example:
OK - so lucky me - it "does not work" like earlier examples because I am
> struct field with the given type, size, and offset. Like most
> descriptors, it's meant to be accessed as an attribute of an instance,
> not as an attribute of the type.
>
> When accessed as an attribute of a struct type, the value returned is
> the CField descriptor itself. One reason to reference the descriptor
> is for its "offset" attribute, since there's no offsetof() in ctypes.
> At least in this regard it should be documented.
>
>> b) what I am not understanding - as the basic documentation shows FOO.value
>> as the way to set/get the value of a _field_
> You may be surprised when accessing simple-type fields such as c_int
> and c_char_p. These simple types have getter and setter functions that
> get called automatically whenever the type is used as a function
> result, array index, or struct field. For example:
referencing, generally, c_ulonglong - and these do not have a automatic
getter/setter function? If not, how do I go about making one (preferably
without needing to right a "method" for each and every _field_ in a class.
> class Foo(ctypes.Structure):
> _fields_ = (('v', ctypes.c_int),)
>
> >>> foo = Foo()
> >>> foo.v = 5
> >>> foo.v
> 5
>
> You can use a subclass to avoid the getfunc's automatic conversion. For example;
>
> class my_int(ctypes.c_int):
> pass
>
> class Bar(ctypes.Structure):
> _fields_ = (('v', my_int),)
>
> >>> bar = Bar()
> >>> bar.v = 5
> >>> bar.v
> <my_int object at 0x7f7385e8aae8>
> >>> bar.v.value
> 5
I'll try it also with my c_int/c_uint fields - maybe these just work. If
> _fields_ = (('v', ctypes.c_int),)
>
> >>> foo = Foo()
> >>> foo.v = 5
> >>> foo.v
> 5
>
> You can use a subclass to avoid the getfunc's automatic conversion. For example;
>
> class my_int(ctypes.c_int):
> pass
>
> class Bar(ctypes.Structure):
> _fields_ = (('v', my_int),)
>
> >>> bar = Bar()
> >>> bar.v = 5
> >>> bar.v
> <my_int object at 0x7f7385e8aae8>
> >>> bar.v.value
> 5
so, again - how do I get a c_ulonglong?
>
>> class time_t(Structure):
>> ...
>> if (maxsize > 2**32):
>> _fields_ = [("v", c_long)]
>> else:
>> _fields_ = [("v", c_int)]
> I'd alias the type instead of defining a struct, e.g. `time_t =
> c_long`. This preserves automatic conversion of the simple type.
The reason for the not using alias is because a) I was trying to be more
>> class time_t(Structure):
>> ...
>> if (maxsize > 2**32):
>> _fields_ = [("v", c_long)]
>> else:
>> _fields_ = [("v", c_int)]
> I'd alias the type instead of defining a struct, e.g. `time_t =
> c_long`. This preserves automatic conversion of the simple type.
inline with the text of the include file.
I will have to check the sizeof c_long (i.e., sizeof(long) in both 32
and 64-bit modes
> Also, sys.maxsize is based on the platform's size_t type. That's
> generally the same size as a pointer, but C doesn't require this.
> Instead use sizeof(c_void_p), which will be 8 on a 64-bit platform and
> 4 on a 32-bit platform.
Thanks!
> generally the same size as a pointer, but C doesn't require this.
> Instead use sizeof(c_void_p), which will be 8 on a 64-bit platform and
> 4 on a 32-bit platform.
eryk sun
Oct 3, 2016, 11:54:33 AM10/3/16
to
On Mon, Oct 3, 2016 at 2:35 PM, Michael Felt <mic...@felt.demon.nl> wrote:
> On 02-Oct-16 23:44, eryk sun wrote:
>> On Sun, Oct 2, 2016 at 5:50 PM, Michael Felt <mic...@felt.demon.nl>
>> wrote:
>>
>>> b) what I am not understanding - as the basic documentation shows
>>> FOO.value as the way to set/get the value of a _field_
>>
>> You may be surprised when accessing simple-type fields such as c_int
>> and c_char_p. These simple types have getter and setter functions that
>> get called automatically whenever the type is used as a function
>> result, array index, or struct field. For example:
>
> OK - so lucky me - it "does not work" like earlier examples because I am
> referencing, generally, c_ulonglong - and these do not have a automatic
> getter/setter function? If not, how do I go about making one (preferably
> without needing to right a "method" for each and every _field_ in a class.
No, c_ulonglong is a simple (fundamental) type, which behaves just
> On 02-Oct-16 23:44, eryk sun wrote:
>> On Sun, Oct 2, 2016 at 5:50 PM, Michael Felt <mic...@felt.demon.nl>
>> wrote:
>>
>>> b) what I am not understanding - as the basic documentation shows
>>> FOO.value as the way to set/get the value of a _field_
>>
>> You may be surprised when accessing simple-type fields such as c_int
>> and c_char_p. These simple types have getter and setter functions that
>> get called automatically whenever the type is used as a function
>> result, array index, or struct field. For example:
>
> OK - so lucky me - it "does not work" like earlier examples because I am
> referencing, generally, c_ulonglong - and these do not have a automatic
> getter/setter function? If not, how do I go about making one (preferably
> without needing to right a "method" for each and every _field_ in a class.
like other simple types such as c_int or c_char_p.
On platform's with a 64-bit long, c_ulonglong is an alias for c_ulong
(i.e. type "L"). On the other hand, on 64-bit Windows, c_ulonglong is
an unsigned quad word (i.e. type "Q").
All simple types subclass ctypes._SimpleCData and define a `_type_`
code such as "c" for c_char. On 64-bit Linux the simple types are
defined as follows:
?: c_bool
c: c_char
z: c_char_p
u: c_wchar
Z: c_wchar_p
P: c_void_p
b: c_int8, c_byte
B: c_uint8, c_ubyte
h: c_int16, c_short
H: c_uint16, c_ushort
i: c_int32, c_int
I: c_uint32, c_uint
l: c_int64, c_long, c_longlong, c_ssize_t
L: c_uint64, c_ulong, c_ulonglong, c_size_t
f: c_float
d: c_double
g: c_longdouble
For convenience, simple types are automatically converted when
accessed as a function result, struct field, or array index. As I
mentioned previously, the only way around this behavior is to use a
subclass. A subclass doesn't get automatically converted because it
might define custom methods and attributes that need to be preserved.
>> I'd alias the type instead of defining a struct, e.g. `time_t =
>> c_long`. This preserves automatic conversion of the simple type.
>
> The reason for the not using alias is because a) I was trying to be more
> inline with the text of the include file. I will have to check the sizeof
> c_long (i.e., sizeof(long) in both 32 and 64-bit modes
you could just use those types directly. You have to check the pointer
size, but then it's up to you what assumptions to make about the
target platform's integer sizes. Currently on a 64-bit system you're
assuming a Unix-style LP64 data model [1], in which a long is 64-bit.
That should be fine if you're writing Unix-specific code that doesn't
care about LLP64 Windows systems.
Wrapping the type in a struct provides more type safety, but if I
wanted that I'd create my own simple subclass. For example, assuming
time_t should be a signed integer that's the same size as a pointer:
class time_t(ctypes._SimpleCData):
if ctypes.sizeof(ctypes.c_void_p) == ctypes.sizeof(ctypes.c_longlong):
_type_ = ctypes.c_longlong._type_
elif ctypes.sizeof(ctypes.c_void_p) == ctypes.sizeof(ctypes.c_long):
_type_ = ctypes.c_long._type_
elif ctypes.sizeof(ctypes.c_void_p) == ctypes.sizeof(ctypes.c_int):
_type_ = ctypes.c_int._type_
# else raise AttributeError for missing _type_
[1]: https://en.wikipedia.org/wiki/64-bit_computing#64-bit_data_models
Michael Felt
Oct 3, 2016, 2:07:29 PM10/3/16
to
On 03-Oct-16 16:35, Michael Felt wrote:
>> I'd alias the type instead of defining a struct, e.g. `time_t =
>> c_long`. This preserves automatic conversion of the simple type.
> The reason for the not using alias is because a) I was trying to be
> more inline with the text of the include file.
> I will have to check the sizeof c_long (i.e., sizeof(long) in both 32
> and 64-bit modes
>> Also, sys.maxsize is based on the platform's size_t type. That's
>> generally the same size as a pointer, but C doesn't require this.
>> Instead use sizeof(c_void_p), which will be 8 on a 64-bit platform and
>> 4 on a 32-bit platform.
I had checked this before - I shall have to try a 32-bit python to see
>> generally the same size as a pointer, but C doesn't require this.
>> Instead use sizeof(c_void_p), which will be 8 on a 64-bit platform and
>> 4 on a 32-bit platform.
what _ctypes is doing with c_long on AIX - is it 4, or is it 8.
michael@x071:[/data/prj/python/cloud-init/aix-cloud-init-0.7.8.1/new]cat
xxx.c
#include <libperfstat.h>
time_t
lbolt()
{
perfstat_cpu_total_t cpu_totals;
void *p1, *p2;
printf("int_size:%d\n", sizeof(int));
printf("long_size:%d\n", sizeof(long));
printf("timet_size:%d\n", sizeof(time_t));
printf("void_ptr_size:%d\n", sizeof(void *));
printf("lbolt_size:%d\n", sizeof(cpu_totals.lbolt));
p1 = &cpu_totals;
p2 = &cpu_totals.lbolt;
printf("lbolt offset:%d\n", p2 - p1);
}
main()
{
lbolt();
}
michael@x071:[/data/prj/python/cloud-init/aix-cloud-init-0.7.8.1/new]cc
-q32 >
int_size:4
long_size:4
timet_size:4
void_ptr_size:4
lbolt_size:4
lbolt offset:192
michael@x071:[/data/prj/python/cloud-init/aix-cloud-init-0.7.8.1/new]cc
-q64 >
int_size:4
long_size:8
timet_size:8
void_ptr_size:8
lbolt_size:8
lbolt offset:192
I think 4 and 8 because of problems I have with this block when not in
the right "mode", i.e. SIZEOF_LONG changes
#if LONG_BIT != 8 * SIZEOF_LONG
/* 04-Oct-2000 LONG_BIT is apparently (mis)defined as 64 on some recent
* 32-bit platforms using gcc. We try to catch that here at compile-time
* rather than waiting for integer multiplication to trigger bogus
* overflows.
*/
#error "LONG_BIT definition appears wrong for platform (bad gcc/glibc
config?)."
#endif
Michael Felt
Oct 3, 2016, 5:28:31 PM10/3/16
to
a) use ctypes.CDLL to open libperfstat.a(shr_64.o)
b) call a number for functions, among them
The prototype template (in C) is:
The common signature used by all of the global interfaces is as follows:
int perfstat_subsystem_total(
perfstat_id_t *name,
perfstat_subsystem_total_t *userbuff,
int sizeof_struct,
int desired_number);
The usage of the parameters for all of the interfaces is as follows:
perfstat_id_t *name Reserved for future use, should be NULL
perfstat_subsystem_total_t *userbuff A pointer to a memory area with
enough space for the returned structure
int sizeof_struct Should be set to sizeof(perfstat_subsystem_t)
int desired_number Reserved for future use, must be set to 0 or 1
The return value will be -1 in case of errors. Otherwise, the number of
structures copied is returned. This
is always 1.
So, in C the call looks something like this - for cpu_total
perfstat_cpu_total(NULL, (struct perfstat_cpu_total_t *) &xxx, sizeof(perfstat_cpu_total_t), 1)
I am struggling with the declarations needed to make the call
I have several iterations - the most simple case (no CFUNCTYPE declaration) gives
+79 class cpu_total:
+80 def __init__(self):
+81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+82 xxx = perfstat_cpu_total_t
+83 cpu_total = __perfstat__.perfstat_cpu_total
+84 ret = cpu_total(None, xxx, sizeof(xxx), 1)
+85 print xxx.ncpus
+86 print xxx.lbolt.v
+87
+88 a = cpu_total()
!cat test_1.py | python
Traceback (most recent call last):
File "<stdin>", line 88, in <module>
File "<stdin>", line 84, in __init__
ctypes.ArgumentError: argument 2: <type 'exceptions.TypeError'>: Don't know how to convert parameter 2
I have tried some experiments with CFUNCTYPE - basically
prototype = CFUNCTYPE(c_int, c_void_p, c_int, c_int, c_int)
but I do not understand how (I think some of the details are missing from the example) how the following works:
I have also tried:
+80 def __init__(self):
+81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+82 xxx = perfstat_cpu_total_t
+83 xptr = POINTER(xxx)
+84 x2 = cast(xptr,POINTER(c_void_p))
+85 cpu_total = __perfstat__.perfstat_cpu_total
+86 ret = cpu_total(None, x2, sizeof(xxx), 1)
+87 print xxx.ncpus
+88 print xxx.lbolt.v
+89
+90 a = cpu_total()
+91 print sizeof(a)
!cat test_1.py | python
Traceback (most recent call last):
File "<stdin>", line 90, in <module>
File "<stdin>", line 84, in __init__
File "/opt/lib/python2.7/ctypes/__init__.py", line 509, in cast
return _cast(obj, obj, typ)
ctypes.ArgumentError: argument 1: <type 'exceptions.TypeError'>: wrong type
+79 class cpu_total:
+80 def __init__(self):
+81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+82 xxx = perfstat_cpu_total_t
+83 xptr = POINTER(xxx)
+84 cpu_total = __perfstat__.perfstat_cpu_total
+85 ret = cpu_total(None, xptr, sizeof(xxx), 1)
+86 print xxx.ncpus
+87 print xxx.lbolt.v
+88
+89 a = cpu_total()
+90 print sizeof(a)
+91
!cat test_1.py | python
Traceback (most recent call last):
File "<stdin>", line 89, in <module>
File "<stdin>", line 85, in __init__
ctypes.ArgumentError: argument 2: <type 'exceptions.TypeError'>: Don't know how to convert parameter 2
and
+79 class cpu_total:
+80 def __init__(self):
+81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+82 xxx = perfstat_cpu_total_t
+83 xptr = c_void_p(xxx)
+84 cpu_total = __perfstat__.perfstat_cpu_total
+85 ret = cpu_total(None, xptr, sizeof(xxx), 1)
+86 print xxx.ncpus
+87 print xxx.lbolt.v
+88
+89 a = cpu_total()
+90 print sizeof(a)
+91
!cat test_1.py | python
Traceback (most recent call last):
File "<stdin>", line 89, in <module>
File "<stdin>", line 83, in __init__
TypeError: cannot be converted to pointer
I know they are "stabs in the dark" as I really do not grasp the documentation. Maybe it is clear to someone well versed in python, or just smarter than I.
Your expert assistance is appreciated!
eryk sun
Oct 3, 2016, 10:49:37 PM10/3/16
to
On Mon, Oct 3, 2016 at 9:27 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>
> int perfstat_subsystem_total(
> perfstat_id_t *name,
> perfstat_subsystem_total_t *userbuff,
> int sizeof_struct,
> int desired_number);
> ...
>
> int perfstat_subsystem_total(
> perfstat_id_t *name,
> perfstat_subsystem_total_t *userbuff,
> int sizeof_struct,
> int desired_number);
> +79 class cpu_total:
> +80 def __init__(self):
> +81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
Move loading the library and defining function prototypes to either
> +80 def __init__(self):
> +81 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
the module or class body. Also, don't use dunder names. The convention
for a private attribute is a single underscore. For type safety,
define each function's argtypes (and restype where required, the
default is c_int). For a more Pythonic interface, define a ctypes
errcheck function that encapsulates raising an appropriate exception
when a C call fails. For example:
import ctypes
# If the following types are used generally, define them at
# the module level, else define them in the class namespace
# that uses them.
perfstat = ctypes.CDLL("libperfstat.a(shr_64.o)")
class perfstat_id_t(ctypes.Structure):
pass
IDENTIFIER_LENGTH = 64
class time64_t(ctypes._SimpleCData):
_type_ = ctypes.c_int64._type_
time_t = time64_t
class perfstat_cpu_total_t(ctypes.Structure):
_fields_ = (("ncpus", ctypes.c_int),
("ncpus_cfg", ctypes.c_int),
("description", ctypes.c_char * IDENTIFIER_LENGTH),
("buffer1", ctypes.c_ulonglong * 15),
("lbolt", time_t),
("loadavg", ctypes.c_ulonglong * 3),
("buffer2", ctypes.c_ulonglong * 29),
("ncpus_high", ctypes.c_int),
("puser", ctypes.c_ulonglong),
("psys", ctypes.c_ulonglong),
("pidle", ctypes.c_ulonglong),
("pwait", ctypes.c_ulonglong),
("buffer3", ctypes.c_ulonglong * 12))
def _perfstat_errcheck(result, func, args):
if result == -1:
# get error info and
raise SomeException()
return args
class CPUTotal(object):
# These may be defined here or just referenced here.
_perfstat = perfstat
_perfstat_id_t = perfstat_id_t
_perfstat_cpu_total_t = perfstat_cpu_total_t
_cpu_total = _perfstat.perfstat_cpu_total
_cpu_total.errcheck = _perfstat_errcheck
_cpu_total.argtypes = (
ctypes.POINTER(_perfstat_id_t),
ctypes.POINTER(_perfstat_cpu_total_t),
ctypes.c_int, ctypes.c_int)
def __init__(self):
self._tcpu = self._perfstat_cpu_total_t()
self._cpu_total(None,
ctypes.byref(self._tcpu),
ctypes.sizeof(self._tcpu), 1)
Michael Felt
Oct 5, 2016, 4:07:43 PM10/5/16
to
On 02-Oct-16 19:50, Michael Felt wrote:
> I am trying to understand the documentation re: ctypes and interfacing
> with existing libraries.
>
> I am reading the documentation, and also other sites that have largely
> just copied the documentation - as well as the "free chapter" at
> O'Reilly (Python Cookbook).
>
Using O'Reilly and the standard documentation I now have these two
> with existing libraries.
>
> I am reading the documentation, and also other sites that have largely
> just copied the documentation - as well as the "free chapter" at
> O'Reilly (Python Cookbook).
>
excerpts. The second def works, the first does not.
Someone helping me understand what I am not reading properly in this bit
of documentation - is much appreciated.
>>>from ctypes import c_int, WINFUNCTYPE, windll
>>>from ctypes.wintypes import HWND, LPCSTR, UINT
>>>prototype = WINFUNCTYPE(c_int, HWND, LPCSTR, LPCSTR, UINT)
>>>paramflags = (1, "hwnd", 0), (1, "text", "Hi"), (1, "caption", None), (1, "flags", 0)
>>>MessageBox = prototype(("MessageBoxA", windll.user32), paramflags)
>>>
The MessageBox foreign function can now be called in these ways:
>>>
>>>MessageBox()
>>>MessageBox(text="Spam, spam, spam")
>>>MessageBox(flags=2, text="foo bar")
>>>
Note that MessageBox() may give two arguments.
My code: note both def init() are meant to do the same thing, just different call syntax.
+76 class cpu_total:
+77 def __init__(self):
+78 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+79 prototype = CFUNCTYPE(c_int, c_void_p, c_void_p, c_int, c_int)
+80 args = (1, "name", None), (2, "buff", None), (1, "size", 0), (1, "count", 1)
+81 cpu_total = prototype(("perfstat_cpu_total", __perfstat__), args)
+82
+83 buff = perfstat_cpu_total_t()
+84 cpu_total(buff=buff, size=sizeof(buff))
+85
+86 class perfstat_cpu_total:
+87 def __init__(self):
+88 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
+89 cpu_total = __perfstat__.perfstat_cpu_total
+90
+91 _perfstat_cpu_total = perfstat_cpu_total_t()
+92 ret = cpu_total(None,
+93 byref(_perfstat_cpu_total),
+94 sizeof(perfstat_cpu_total_t), 1)
+95 self.boot = _perfstat_cpu_total.lbolt.v / 100
+96 self.ncpus = _perfstat_cpu_total.ncpus
+97 self._v = _perfstat_cpu_total
When I call the second I can print values and even call functions, reload the _v, etc..
The first class fails with this message:
a = cpu_total()
Traceback (most recent call last):
File "<stdin>", line 135, in <module>
File "<stdin>", line 84, in __init__
TypeError: call takes exactly 1 arguments (2 given)
Thanks for your attention.
Emile van Sebille
Oct 5, 2016, 4:30:23 PM10/5/16
to
> +84 cpu_total(buff=buff, size=sizeof(buff))
because the expectation is defined as:
> +77 def __init__(self):
HTH,
Emile
Michael Felt
Oct 5, 2016, 5:03:16 PM10/5/16
to
On 05-Oct-16 22:29, Emile van Sebille wrote:
Thanks for the reply!
After a shirt coffeebreak - back into the fray - and I found the following:
> +76 class cpu_total:
> +77 def __init__(self):
> +78 __perfstat__ = CDLL("libperfstat.a(shr_64.o)")
> +79 prototype = CFUNCTYPE(c_int, c_void_p, c_void_p, c_int,
> c_int)
> +80 args = (1, "name", None), (2, "buff", None), (1, "size",
> 0), (1, "count", 1)
error #1. paramater type 2 (the buffer might be where data is being put,
> 0), (1, "count", 1)
but for the call, the pointer is INPUT)
> +81 cpu_total = prototype(("perfstat_cpu_total",
> __perfstat__), args)
> +82
> +83 buff = perfstat_cpu_total_t()
> +84 cpu_total(buff=buff, size=sizeof(buff))
> +85
The error #2 is #2, because only after I corrected the inputs did I get
> __perfstat__), args)
> +82
> +83 buff = perfstat_cpu_total_t()
> +84 cpu_total(buff=buff, size=sizeof(buff))
> +85
Type Exceptions. I had hoped that c_void_p was a "typeless" type, mainly
the right size, but the ctypes interface is more particular when you use
the CFUNCTYPE() - so now I have my boilerplate for what I hope
can be a tutorial for interfacing - by hand - with complex functions,
i.e., using one class as the data aka typedef defintion, and a second
class to work
on it.
In template form:
class perfstat_xxx_t:
_fields_ = ( ... )
class perfstat_xxx:
def init(self):
_perflib = ctypes.CDLL("libperfstat.a(shr_64.o)") # AIX member
in an archive
_fn_xxx = _perflib.xxx
# ALL AIX perfstat routines have 4 arguments: (name, buff,
sizeof_buff, count)
# the buff is a pointer to where the information will be stored
_fn = CFUNCTYPE(c_int, c_void_p, POINTER(xxx_t), c_int, c_int)
_args = (1, "name", None), (1, "buff", None), (1, "size", 0),
(1, "count", 1)
_xxx = _fn(("xxx", _fn_xxx), _args)
_buff = perfstat_cpu_total_t()
_xxx(buff=_buff, size=sizeof(_buff))
self._v = _buff
So, the next question/test will be to move the _fields_ into the
"second" class definition. I just need to find what "self" attribute
that is.
Michael Felt
Oct 5, 2016, 5:12:04 PM10/5/16
to
Never said thank you - so, thanks!
What I need to do was add the .v at the end so I was accessing the value
of the structure.
Unlilke Linux, AIX - for reasons unknown to all, they have the time_t
definition that is specific to the ABI size, at least for these
performance libraries that probably originated before 64-bit was a
concern. So my guess is that the dual size in the struct, depending on
the application size (not the kernel) is to maintain compatibility with
32-bit applications that had been built against/on a 32-bit kernel.
So, lucky for me it did not work intiallly - because it made me pause
and understand better what I had written.
And now the real thankyou for the detail you shared!
M
On 03-Oct-16 17:53, eryk sun wrote:
> On Mon, Oct 3, 2016 at 2:35 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>> On 02-Oct-16 23:44, eryk sun wrote:
What I need to do was add the .v at the end so I was accessing the value
of the structure.
Unlilke Linux, AIX - for reasons unknown to all, they have the time_t
definition that is specific to the ABI size, at least for these
performance libraries that probably originated before 64-bit was a
concern. So my guess is that the dual size in the struct, depending on
the application size (not the kernel) is to maintain compatibility with
32-bit applications that had been built against/on a 32-bit kernel.
So, lucky for me it did not work intiallly - because it made me pause
and understand better what I had written.
And now the real thankyou for the detail you shared!
M
On 03-Oct-16 17:53, eryk sun wrote:
> On Mon, Oct 3, 2016 at 2:35 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>> On 02-Oct-16 23:44, eryk sun wrote:
>>> I'd alias the type instead of defining a struct, e.g. `time_t =
>>> c_long`. This preserves automatic conversion of the simple type.
>> The reason for the not using alias is because a) I was trying to be more
>> inline with the text of the include file. I will have to check the sizeof
>> c_long (i.e., sizeof(long) in both 32 and 64-bit modes
>>> c_long`. This preserves automatic conversion of the simple type.
>> The reason for the not using alias is because a) I was trying to be more
>> inline with the text of the include file. I will have to check the sizeof
>> c_long (i.e., sizeof(long) in both 32 and 64-bit modes
eryk sun
Oct 5, 2016, 6:16:37 PM10/5/16
to
On Wed, Oct 5, 2016 at 9:03 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>
>> +80 args = (1, "name", None), (2, "buff", None), (1, "size",
>> 0), (1, "count", 1)
>
> error #1. paramater type 2 (the buffer might be where data is being put, but
> for the call, the pointer is INPUT)
An output parameter (type 2) has ctypes implicitly create the buffer
>
>> +80 args = (1, "name", None), (2, "buff", None), (1, "size",
>> 0), (1, "count", 1)
>
> error #1. paramater type 2 (the buffer might be where data is being put, but
> for the call, the pointer is INPUT)
and insert it in the argument list. It gets returned as the call's
result. Here's a contrived example:
class Buf(ctypes.Structure):
_fields_ = (('value', ctypes.c_char * 100),)
LP_Buf = ctypes.POINTER(Buf)
proto = ctypes.CFUNCTYPE(ctypes.c_int,
LP_Buf, ctypes.c_char_p, ctypes.c_int)
flags = ((2, 'buf'), (1, 'fmt'), (1, 'arg', 42))
sprintf = proto(('sprintf', ctypes.CDLL(None)), flags)
>>> r = sprintf(b'The answer is %d.')
>>> r.value
b'The answer is 42.'
This works best when combined with an errcheck function. This lets you
see the actual arguments that were passed in the call and the original
result:
def errcheck(result, func, args):
print('result:', result)
print('args:', args)
return args
sprintf.errcheck = errcheck
>>> r = sprintf(b'The answer is %d.')
result: 17
args: (<__main__.Buf object at 0x7fede840bb70>, b'The answer is %d.', 42)
When the errcheck function returns "args", this tells ctypes to
continue with its normal post-processing, so instead of the actual 17
result, it returns the "buf" output parameter:
>>> r.value
b'The answer is 42.'
Notice that I declared the output parameter's type as a pointer type,
but ctypes is smart enough to create a Buf instance for me instead of
just a pointer. Also, because the argument type is a pointer type, its
from_param method implicitly passes the Buf instance by reference.
> class perfstat_xxx:
> def init(self):
> # AIX member in an archive
> _perflib = ctypes.CDLL("libperfstat.a(shr_64.o)")
> _fn_xxx = _perflib.xxx
> # ALL AIX perfstat routines have 4 arguments:
> # (name, buff, sizeof_buff, count)
> _perflib = ctypes.CDLL("libperfstat.a(shr_64.o)")
> _fn_xxx = _perflib.xxx
> # ALL AIX perfstat routines have 4 arguments:
> # the buff is a pointer to where the information will be stored
> _fn = CFUNCTYPE(c_int, c_void_p, POINTER(xxx_t), c_int, c_int)
> _args = ((1, "name", None), (1, "buff", None), (1, "size", 0),
> _fn = CFUNCTYPE(c_int, c_void_p, POINTER(xxx_t), c_int, c_int)
> (1, "count", 1))
> _xxx = _fn(("xxx", _fn_xxx), _args)
You should really do this work at the class or module level. Every
time this init() method is called, you're needlessly incrementing the
reference count on the "libperfstat.a(shr_64.o)" shared library and
needlessly creating and instantiating the function pointer type.
Michael Felt
Oct 6, 2016, 6:15:21 AM10/6/16
to
On 04-Oct-16 04:48, eryk sun wrote:
> On Mon, Oct 3, 2016 at 9:27 PM, Michael Felt <mic...@felt.demon.nl> wrote:
>> int perfstat_subsystem_total(
>> perfstat_id_t *name,
>> perfstat_subsystem_total_t *userbuff,
>> int sizeof_struct,
>> int desired_number);
>> ...
>> +79 class cpu_total:
>> +80 def __init__(self):
Thanks for the model below - a lot to think about. Some was expected,
some is a gift!
Python is "a lot to learn". This helps a great deal.
0 new messages