Google Groups no longer supports new Usenet posts or subscriptions. Historical content remains viewable.
Dismiss

Threaded file system benchmark

2 views
Skip to first unread message

Ryo ONODERA

unread,
Jul 12, 2015, 5:28:28 AM7/12/15
to
Hi,

I have added sched(3) support for fio, Flexible IO Tester.
(See: https://github.com/ryo-on/fio/tree/netbsd-cpu-affinity
and included patch.)
However my 'top -1' says only 1 CPU is consumed.

My sched(3) support code is wrong? Or is there another reason?
If there is another resason, could you explain me the reason?
If my code is wrong, could you give me some advices?

Thank you.

My machine:
NetBSD/amd64 current on 4 CPUs laptop.
My kernel is GENERIC with dtrace support.
$ uname -a
NetBSD angelcake.elements.tetera.org 7.99.19 NetBSD 7.99.19 (DTRACE7) #7: Thu Jul 9 21:58:33 JST 2015 ryo...@angelcake.elements.tetera.org:/usr/world/7.99/amd64/obj/sys/arch/amd64/compile/DTRACE7 amd64

On one terminal:
$ cat random-write-test.fio
; random read of 128mb of data

[random-write]
rw=randwrite
size=1G
directory=/home/ryo_on/tmp2/fio-test

$ fio --numjobs 4 random-write-test.fio

Another terminal:
$ top -1

=== === patch === ===
diff --git a/Makefile b/Makefile
index 7fe7d4d..f795c74 100644
--- a/Makefile
+++ b/Makefile
@@ -284,7 +284,7 @@ override CFLAGS += -DFIO_VERSION='"$(FIO_VERSION)"'
@$(CC) -MM $(CFLAGS) $(CPPFLAGS) $(SRCDIR)/$*.c > $*.d
@mv -f $*.d $*.d.tmp
@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
- @sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \
+ @sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -g 1 | \
sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
@rm -f $*.d.tmp

diff --git a/backend.c b/backend.c
index 3eafff6..384b2d3 100644
--- a/backend.c
+++ b/backend.c
@@ -1386,7 +1386,11 @@ static void *thread_main(void *data)
* thread from this job
*/
if (o->gtod_cpu)
+#if defined(__NetBSD__)
+ fio_cpu_clear(o->cpumask, o->gtod_cpu);
+#else
fio_cpu_clear(&o->cpumask, o->gtod_cpu);
+#endif

/*
* Set affinity first, in case it has an impact on the memory
@@ -1394,7 +1398,11 @@ static void *thread_main(void *data)
*/
if (fio_option_is_set(o, cpumask)) {
if (o->cpus_allowed_policy == FIO_CPUS_SPLIT) {
+#if defined(__NetBSD__)
+ ret = fio_cpus_split(o->cpumask, td->thread_number - 1);
+#else
ret = fio_cpus_split(&o->cpumask, td->thread_number - 1);
+#endif
if (!ret) {
log_err("fio: no CPUs set\n");
log_err("fio: Try increasing number of available CPUs\n");
@@ -1653,7 +1661,11 @@ err:
verify_free_state(td);

if (fio_option_is_set(o, cpumask)) {
+#if defined(__NetBSD__)
+ ret = fio_cpuset_exit(o->cpumask);
+#else
ret = fio_cpuset_exit(&o->cpumask);
+#endif
if (ret)
td_verror(td, ret, "fio_cpuset_exit");
}
diff --git a/gettime-thread.c b/gettime-thread.c
index 9bf85f0..6b02107 100644
--- a/gettime-thread.c
+++ b/gettime-thread.c
@@ -9,7 +9,11 @@
struct timeval *fio_tv = NULL;
int fio_gtod_offload = 0;
static pthread_t gtod_thread;
+#if defined(__NetBSD__)
+static os_cpu_mask_t *fio_gtod_cpumask;
+#else
static os_cpu_mask_t fio_gtod_cpumask;
+#endif

void fio_gtod_init(void)
{
@@ -96,6 +100,10 @@ err:
void fio_gtod_set_cpu(unsigned int cpu)
{
#ifdef FIO_HAVE_CPU_AFFINITY
+#if defined(__NetBSD__)
+ fio_cpu_set(fio_gtod_cpumask, cpu);
+#else
fio_cpu_set(&fio_gtod_cpumask, cpu);
#endif
+#endif
}
diff --git a/gettime.c b/gettime.c
index ac54111..395342c 100644
--- a/gettime.c
+++ b/gettime.c
@@ -483,18 +483,30 @@ static void *clock_thread_fn(void *data)
{
struct clock_thread *t = data;
struct clock_entry *c;
+#if defined(__NetBSD__)
+ os_cpu_mask_t *cpu_mask = cpuset_create();
+#else
os_cpu_mask_t cpu_mask;
+#endif
uint32_t last_seq;
int i;

- if (fio_cpuset_init(&cpu_mask)) {
+#if defined(__NetBSD__)
+ if (fio_cpuset_init(cpu_mask)) {
+#else
+ if (fio_cpuset_init(cpu_mask)) {
+#endif
int __err = errno;

log_err("clock cpuset init failed: %s\n", strerror(__err));
goto err_out;
}

+#if defined(__NetBSD__)
+ fio_cpu_set(cpu_mask, t->cpu);
+#else
fio_cpu_set(&cpu_mask, t->cpu);
+#endif

if (fio_setaffinity(gettid(), cpu_mask) == -1) {
int __err = errno;
@@ -538,10 +550,18 @@ static void *clock_thread_fn(void *data)
if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
goto err;

+#if defined(__NetBSD__)
+ fio_cpuset_exit(cpu_mask);
+#else
fio_cpuset_exit(&cpu_mask);
+#endif
return NULL;
err:
+#if defined(__NetBSD__)
+ fio_cpuset_exit(cpu_mask);
+#else
fio_cpuset_exit(&cpu_mask);
+#endif
err_out:
return (void *) 1;
}
diff --git a/hash.h b/hash.h
index 02b0614..abafa3b 100644
--- a/hash.h
+++ b/hash.h
@@ -18,6 +18,10 @@
* machines where multiplications are slow.
*/

+#if !defined(BITS_PER_LONG)
+#define BITS_PER_LONG __SIZEOF_LONG__*8
+#endif
+
#if BITS_PER_LONG == 32
/* 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */
#define GOLDEN_RATIO_PRIME 0x9e370001UL
diff --git a/idletime.c b/idletime.c
index db272fe..370dcc2 100644
--- a/idletime.c
+++ b/idletime.c
@@ -46,23 +46,39 @@ static double calibrate_unit(unsigned char *data)
static void free_cpu_affinity(struct idle_prof_thread *ipt)
{
#if defined(FIO_HAVE_CPU_AFFINITY)
+#if defined(__NetBSD__)
+ fio_cpuset_exit(ipt->cpu_mask);
+#else
fio_cpuset_exit(&ipt->cpu_mask);
#endif
+#endif
}

static int set_cpu_affinity(struct idle_prof_thread *ipt)
{
#if defined(FIO_HAVE_CPU_AFFINITY)
+#if defined(__NetBSD__)
+ if (fio_cpuset_init(ipt->cpu_mask)) {
+#else
if (fio_cpuset_init(&ipt->cpu_mask)) {
+#endif
log_err("fio: cpuset init failed\n");
return -1;
}

+#if defined(__NetBSD__)
+ fio_cpu_set(ipt->cpu_mask, ipt->cpu);
+#else
fio_cpu_set(&ipt->cpu_mask, ipt->cpu);
+#endif

if (fio_setaffinity(gettid(), ipt->cpu_mask)) {
log_err("fio: fio_setaffinity failed\n");
+#if defined(__NetBSD__)
+ fio_cpuset_exit(ipt->cpu_mask);
+#else
fio_cpuset_exit(&ipt->cpu_mask);
+#endif
return -1;
}

diff --git a/idletime.h b/idletime.h
index bd6dcef..be3a482 100644
--- a/idletime.h
+++ b/idletime.h
@@ -35,7 +35,11 @@ struct idle_prof_thread {
pthread_mutex_t init_lock;
pthread_mutex_t start_lock;

+#if defined(__NetBSD__)
+ os_cpu_mask_t *cpu_mask;
+#else
os_cpu_mask_t cpu_mask;
+#endif
};

struct idle_prof_common {
diff --git a/init.c b/init.c
index 5edd53e..2e13868 100644
--- a/init.c
+++ b/init.c
@@ -1702,7 +1702,11 @@ static int fill_def_thread(void)
{
memset(&def_thread, 0, sizeof(def_thread));

+#if defined(__NetBSD__)
+ fio_getaffinity(getpid(), def_thread.o.cpumask);
+#else
fio_getaffinity(getpid(), &def_thread.o.cpumask);
+#endif
def_thread.o.error_dump = 1;

/*
diff --git a/options.c b/options.c
index ed5d37e..f1b44ef 100644
--- a/options.c
+++ b/options.c
@@ -435,7 +435,11 @@ static int str_cpumask_cb(void *data, unsigned long long *val)
if (parse_dryrun())
return 0;

+#if defined(__NetBSD__)
+ ret = fio_cpuset_init(td->o.cpumask);
+#else
ret = fio_cpuset_init(&td->o.cpumask);
+#endif
if (ret < 0) {
log_err("fio: cpuset_init failed\n");
td_verror(td, ret, "fio_cpuset_init");
@@ -452,7 +456,11 @@ static int str_cpumask_cb(void *data, unsigned long long *val)
return 1;
}
dprint(FD_PARSE, "set cpu allowed %d\n", i);
+#if defined(__NetBSD__)
+ fio_cpu_set(td->o.cpumask, i);
+#else
fio_cpu_set(&td->o.cpumask, i);
+#endif
}
}

@@ -532,14 +540,22 @@ static int str_cpus_allowed_cb(void *data, const char *input)
if (parse_dryrun())
return 0;

+#if defined(__NetBSD__)
+ return set_cpus_allowed(td, td->o.cpumask, input);
+#else
return set_cpus_allowed(td, &td->o.cpumask, input);
+#endif
}

static int str_verify_cpus_allowed_cb(void *data, const char *input)
{
struct thread_data *td = data;

+#if defined(__NetBSD__)
+ return set_cpus_allowed(td, td->o.verify_cpumask, input);
+#else
return set_cpus_allowed(td, &td->o.verify_cpumask, input);
+#endif
}
#endif

diff --git a/os/os-netbsd.h b/os/os-netbsd.h
index 4b0269e..664edfd 100644
--- a/os/os-netbsd.h
+++ b/os/os-netbsd.h
@@ -12,6 +12,9 @@
#undef rb_node
#undef rb_left
#undef rb_right
+#define _NetBSD_SOURCE
+#include <sched.h>
+#include <unistd.h>

#include "../file.h"

@@ -20,8 +23,61 @@
#define FIO_USE_GENERIC_RAND
#define FIO_USE_GENERIC_INIT_RANDOM_STATE
#define FIO_HAVE_GETTID
+#define FIO_HAVE_CPU_AFFINITY
+#define FIO_MAX_CPUS 1024

-#undef FIO_HAVE_CPU_AFFINITY /* XXX notyet */
+typedef cpuset_t os_cpu_mask_t;
+
+static inline int fio_cpu_isset(os_cpu_mask_t *mask, const cpuid_t cpu)
+{
+ return cpuset_isset(cpu, mask);
+}
+
+static inline int fio_setaffinity(int tid, os_cpu_mask_t *cpumask)
+{
+ return _sched_getaffinity(getpid(), tid, cpuset_size(cpumask), cpumask);
+}
+
+static inline int fio_getaffinity(int tid, os_cpu_mask_t *cpumask)
+{
+ return _sched_getaffinity(getpid(), tid, cpuset_size(cpumask), cpumask);
+}
+
+static inline int fio_cpuset_init(os_cpu_mask_t *mask)
+{
+ cpuset_zero(mask);
+ return 0;
+}
+
+static inline void fio_cpu_set(os_cpu_mask_t *mask, int cpu)
+{
+ cpuset_set(cpu, mask);
+ return;
+}
+
+static inline int fio_cpuset_exit(os_cpu_mask_t *mask)
+{
+ return 0;
+}
+
+static inline int fio_cpu_count(os_cpu_mask_t *mask)
+{
+ int cpu = sysconf(_SC_NPROCESSORS_CONF);
+ int count = 0;
+
+ for (int i = 0; i < cpu; i++) {
+ if(cpuset_isset(i, mask)) {
+ count++;
+ }
+ }
+ return count;
+}
+
+static inline void fio_cpu_clear(os_cpu_mask_t *mask, int cpu)
+{
+ cpuset_clr(cpu, mask);
+ return;
+}

#define OS_MAP_ANON MAP_ANON

diff --git a/parse.c b/parse.c
index 745056b..f7f164e 100644
--- a/parse.c
+++ b/parse.c
@@ -152,7 +152,7 @@ static unsigned long long get_mult_time(const char *str, int len,

c = strdup(p);
for (int i = 0; i < strlen(c); i++)
- c[i] = tolower(c[i]);
+ c[i] = tolower((int)c[i]);

if (!strncmp("us", c, 2) || !strncmp("usec", c, 4))
mult = 1;
@@ -198,7 +198,7 @@ static unsigned long long __get_mult_bytes(const char *p, void *data,
c = strdup(p);

for (i = 0; i < strlen(c); i++) {
- c[i] = tolower(c[i]);
+ c[i] = tolower((int)c[i]);
if (is_separator(c[i])) {
c[i] = '\0';
break;
diff --git a/thread_options.h b/thread_options.h
index 6604a37..8a66824 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -163,8 +163,13 @@ struct thread_options {
unsigned int stonewall;
unsigned int new_group;
unsigned int numjobs;
+#if defined(__NetBSD__)
+ os_cpu_mask_t *cpumask;
+ os_cpu_mask_t *verify_cpumask;
+#else
os_cpu_mask_t cpumask;
os_cpu_mask_t verify_cpumask;
+#endif
unsigned int cpus_allowed_policy;
char *numa_cpunodes;
unsigned short numa_mem_mode;


--
Ryo ONODERA // ryo...@yk.rim.or.jp
PGP fingerprint = 82A2 DC91 76E0 A10A 8ABB FD1B F404 27FA C7D1 15F3

--
Posted automagically by a mail2news gateway at muc.de e.V.
Please direct questions, flames, donations, etc. to news-...@muc.de

Mindaugas Rasiukevicius

unread,
Jul 12, 2015, 10:14:16 AM7/12/15
to
Ryo ONODERA <ryo...@yk.rim.or.jp> wrote:
> ...
>
> --- a/gettime.c
> +++ b/gettime.c
> @@ -483,18 +483,30 @@ static void *clock_thread_fn(void *data)
> {
> struct clock_thread *t = data;
> struct clock_entry *c;
> +#if defined(__NetBSD__)
> + os_cpu_mask_t *cpu_mask = cpuset_create();
> +#else
> os_cpu_mask_t cpu_mask;
> +#endif

You can always use a pointer, e.g.:

#if defined(__NetBSD__)
os_cpu_mask_t *cpu_mask = cpuset_create();
#else
os_cpu_mask_t cpu_mask_store;
os_cpu_mask_t *cpu_mask = &cpu_mask_store;
#endif

This will save some #ifdef __NetBSD__ elsewhere.

> --- a/hash.h
> +++ b/hash.h
> @@ -18,6 +18,10 @@
> * machines where multiplications are slow.
> */
>
> +#if !defined(BITS_PER_LONG)
> +#define BITS_PER_LONG __SIZEOF_LONG__*8
> +#endif

I did not look how exactly is this used, but it is a good idea to use
brackets around the expression when dealing with macros. Just prevents
from any future bugs.

> +static inline int fio_setaffinity(int tid, os_cpu_mask_t *cpumask)
> +{
> + return _sched_getaffinity(getpid(), tid, cpuset_size(cpumask), cpumask);
> +}

I suppose you meant _sched_setaffinity() here? Well, that is your bug.
Also, _sched_{get,set}affinity() are internal calls which should not be
used as a public API. Instead, pthread_{get,set}affinity_np() should
be used. See affinity(3) man page for the details.

> --- a/parse.c
> +++ b/parse.c
> @@ -152,7 +152,7 @@ static unsigned long long get_mult_time(const char
> *str, int len,
> c = strdup(p);
> for (int i = 0; i < strlen(c); i++)
> - c[i] = tolower(c[i]);
> + c[i] = tolower((int)c[i]);

The value should be type casted to unsigned char rather than int.
See CAVEATS section of the ctype(3) man page for the explanation.

--
Mindaugas

David Holland

unread,
Jul 12, 2015, 1:40:06 PM7/12/15
to
On Sun, Jul 12, 2015 at 03:13:56PM +0100, Mindaugas Rasiukevicius wrote:
> > +#if !defined(BITS_PER_LONG)
> > +#define BITS_PER_LONG __SIZEOF_LONG__*8
> > +#endif
>
> I did not look how exactly is this used, but it is a good idea to use
> brackets around the expression when dealing with macros. Just prevents
> from any future bugs.

Also, that should be CHAR_BIT and not a magic 8...

--
David A. Holland
dhol...@netbsd.org

Ryo ONODERA

unread,
Jul 12, 2015, 4:38:27 PM7/12/15
to
Hi,

From: Mindaugas Rasiukevicius <rm...@netbsd.org>, Date: Sun, 12 Jul 2015 15:13:56 +0100

> Ryo ONODERA <ryo...@yk.rim.or.jp> wrote:
>> ...
>>
>> --- a/gettime.c
>> +++ b/gettime.c
>> @@ -483,18 +483,30 @@ static void *clock_thread_fn(void *data)
>> {
>> struct clock_thread *t = data;
>> struct clock_entry *c;
>> +#if defined(__NetBSD__)
>> + os_cpu_mask_t *cpu_mask = cpuset_create();
>> +#else
>> os_cpu_mask_t cpu_mask;
>> +#endif
>
> You can always use a pointer, e.g.:
>
> #if defined(__NetBSD__)
> os_cpu_mask_t *cpu_mask = cpuset_create();
> #else
> os_cpu_mask_t cpu_mask_store;
> os_cpu_mask_t *cpu_mask = &cpu_mask_store;
> #endif
>
> This will save some #ifdef __NetBSD__ elsewhere.

I will reduce #ifdef __NetBSD__.

>> --- a/hash.h
>> +++ b/hash.h
>> @@ -18,6 +18,10 @@
>> * machines where multiplications are slow.
>> */
>>
>> +#if !defined(BITS_PER_LONG)
>> +#define BITS_PER_LONG __SIZEOF_LONG__*8
>> +#endif
>
> I did not look how exactly is this used, but it is a good idea to use
> brackets around the expression when dealing with macros. Just prevents
> from any future bugs.

I will use "(__SIZEOF_LONG__*CHAR_BIT)".

>> +static inline int fio_setaffinity(int tid, os_cpu_mask_t *cpumask)
>> +{
>> + return _sched_getaffinity(getpid(), tid, cpuset_size(cpumask), cpumask);
>> +}
>
> I suppose you meant _sched_setaffinity() here? Well, that is your bug.
> Also, _sched_{get,set}affinity() are internal calls which should not be
> used as a public API. Instead, pthread_{get,set}affinity_np() should
> be used. See affinity(3) man page for the details.

It may be the real cause of my problem.
Thank you very much.

I understand that internal calls should not be used.
I read affinity(3) man page, and found Solaris/HP-UX compatible pset(3)
APIs.
I will use pset(3), I may borrow the code from Solaris's definitions.

>> --- a/parse.c
>> +++ b/parse.c
>> @@ -152,7 +152,7 @@ static unsigned long long get_mult_time(const char
>> *str, int len,
>> c = strdup(p);
>> for (int i = 0; i < strlen(c); i++)
>> - c[i] = tolower(c[i]);
>> + c[i] = tolower((int)c[i]);
>
> The value should be type casted to unsigned char rather than int.
> See CAVEATS section of the ctype(3) man page for the explanation.

I see.
I have added this to pass "gcc -Werror", however fio does not require
-Werror originally. I will remove it.

> --
> Mindaugas

Thank you very much.

Ryo ONODERA

unread,
Jul 12, 2015, 5:53:42 PM7/12/15
to
Hi,

From: David Holland <dholla...@netbsd.org>, Date: Sun, 12 Jul 2015 17:39:55 +0000

> On Sun, Jul 12, 2015 at 03:13:56PM +0100, Mindaugas Rasiukevicius wrote:
> > > +#if !defined(BITS_PER_LONG)
> > > +#define BITS_PER_LONG __SIZEOF_LONG__*8
> > > +#endif
> >
> > I did not look how exactly is this used, but it is a good idea to use
> > brackets around the expression when dealing with macros. Just prevents
> > from any future bugs.
>
> Also, that should be CHAR_BIT and not a magic 8...

Thank you.
I will use CHAR_BIT.

> --
> David A. Holland
> dhol...@netbsd.org

Ryo ONODERA

unread,
Jul 14, 2015, 10:32:34 AM7/14/15
to
Hi,

From: Ryo ONODERA <ryo...@yk.rim.or.jp>, Date: Mon, 13 Jul 2015 05:37:59 +0900 (JST)


>
> I understand that internal calls should not be used.
> I read affinity(3) man page, and found Solaris/HP-UX compatible pset(3)
> APIs.
> I will use pset(3), I may borrow the code from Solaris's definitions.

Our pset(3) has no pset_info and I cannot borrow the code from
Solaris specific file.

I will try to convert various CPU affinity code to
pthread_{set,get}affinity_np...

Thank you.
0 new messages