[lemona commit] r418 - in trunk: driver tools/basket tools/picker

[codesite...@google.com]

Author: Barratis
Date: Tue Nov 11 01:39:44 2008
New Revision: 418

Modified:
trunk/driver/init.c
trunk/driver/lemona.h
trunk/driver/lemona_net.h
trunk/driver/lemona_patch.h
trunk/driver/logging.c
trunk/driver/net.c
trunk/driver/relay.c
trunk/tools/basket/main.c
trunk/tools/picker/picker.py

Log:
- corrected lemona_patch.h an #elif had stolen an #else place
- update lemona_net to listen from interface down event to avoid
freezing while rebooting. This is not really perfomant right now
since I don't know how to differentiate between reboot or simple
shutdown. I'll look into it later
- started to try and make something cleaner of picker.py
- updated logging code to align zest size by sizeof(int)

Modified: trunk/driver/init.c
==============================================================================
--- trunk/driver/init.c (original)
+++ trunk/driver/init.c Tue Nov 11 01:39:44 2008
@@ -30,7 +30,7 @@
# define lemona_clients_wait()
#endif

-struct lemona *juice = NULL;
+struct lemona *juice = NULL;

static void lemona_cleanup(void)
{
@@ -56,7 +56,7 @@
err = lemona_relay_init();
if (err)
goto err;
- err = lemona_net_init(true);
+ err = lemona_net_init();
if (err < 0)
goto err;
lemona_printk("Done.\n");

Modified: trunk/driver/lemona.h
==============================================================================
--- trunk/driver/lemona.h (original)
+++ trunk/driver/lemona.h Tue Nov 11 01:39:44 2008
@@ -25,8 +25,13 @@
* Every single log entry is represented by a zest.
*/
struct lemona_zest {
- char magic[4];
+ /*
+ need always to be the first member. This facilitate parsing when a
+ zest can find itself cut in two
+ */
int size; /* size taken by this zest and args sz/value */
+
+ char magic[4];

int in;
struct timespec time; /* call start/end time (getnstimeofday) */

Modified: trunk/driver/lemona_net.h
==============================================================================
--- trunk/driver/lemona_net.h (original)
+++ trunk/driver/lemona_net.h Tue Nov 11 01:39:44 2008
@@ -49,7 +49,7 @@
struct socket *sock;
};

-int lemona_net_init(bool init);
+int lemona_net_init(void);
void lemona_net_log(struct lemona_zest *zest);
void lemona_net_cleanup(void);

@@ -57,7 +57,7 @@

struct lemona_net { };

-# define lemona_net_init(x) 0
+# define lemona_net_init() 0
# define lemona_net_log(x)
# define lemona_net_cleanup()

Modified: trunk/driver/lemona_patch.h
==============================================================================
--- trunk/driver/lemona_patch.h (original)
+++ trunk/driver/lemona_patch.h Tue Nov 11 01:39:44 2008
@@ -68,7 +68,7 @@
*_lemona_relay_is_ours =
(lemonarelayisoursfn)kallsyms_lookup_name("lemona_relay_is_ours");
}
# endif /* LEMONA_READ */
-# elif /* CONFIG_LEMONA */
+# else /* CONFIG_LEMONA */

# define lemona_block_end \
}

Modified: trunk/driver/logging.c
==============================================================================
--- trunk/driver/logging.c (original)
+++ trunk/driver/logging.c Tue Nov 11 01:39:44 2008
@@ -253,9 +253,10 @@
zsz = lemona_zest_get_size(mixer, in, ap);
if (zsz > 0)
{
- z = kzalloc(zsz, GFP_KERNEL);
+ /* aligned the allocation */
+ z = kzalloc(zsz + (zsz % sizeof(int)), GFP_KERNEL);
if (z != NULL)
- z->size = zsz;
+ z->size = zsz;
else
{
lemona_printk("(syscall %i) Unable to create zest with size: %i\n",

Modified: trunk/driver/net.c
==============================================================================
--- trunk/driver/net.c (original)
+++ trunk/driver/net.c Tue Nov 11 01:39:44 2008
@@ -12,9 +12,13 @@
** governing permissions and limitations under the License.
*/

+#include <linux/inetdevice.h>
+#include <linux/notifier.h> /* struct notifier_block */
#include <linux/jiffies.h> /* jiffies, time_after, HZ */
#include <linux/module.h> /* module_param */

+#include <linux/in.h> /* INADDR_LOOPBACK */
+
#include "lemona.h"

static char *net_log_addr = CONFIG_LEMONA_NET_LOG_SERV;
@@ -22,9 +26,20 @@
module_param(net_log_addr, charp, S_IRUGO);
module_param(net_log_port, int, S_IRUGO);

+static int lemona_net_activated = 0;
+
extern struct lemona *juice;

-static void __lemona_net_cleanup(void);
+static int __lemona_net_init(void);
+static void __lemona_net_cleanup(void);
+
+static int lemona_net_inetaddr_notifier(struct notifier_block *,
+ unsigned long,
+ void *);
+
+static struct notifier_block lemona_net_inetaddr_nb = {
+ .notifier_call = lemona_net_inetaddr_notifier,
+};

static inline int lemona_net_serv_get(void)
{
@@ -37,26 +52,34 @@
return (htonl((a << 24) | (b << 16) | (c << 8) | d));
}

-int lemona_net_init(bool init)
+static int lemona_net_inetaddr_notifier(struct notifier_block *nb,
+ unsigned long val,
+ void * data)
{
- int ret = 0;
- struct socket *sock = NULL;
+/* struct in_ifaddr *addr = (struct in_ifaddr *)data; */

- if (init == true)
- {
- mutex_init(&(juice->net.lock));
+ if (!lemona_net_activated)
+ return (NOTIFY_DONE);

- juice->net.sin.sin_family = AF_INET;
- juice->net.sin.sin_addr.s_addr = lemona_net_serv_get();
- juice->net.sin.sin_port = htons(net_log_port);
- }
+ /* if interface is going down... */
+ if (val == NETDEV_DOWN)
+ lemona_net_cleanup();
+
+ /*
+ TODO: Found a way to only cleanup when reboot/halt function are called
+ to avoid overhead of releasing and reopening the socket for nothing
+ */
+ return (NOTIFY_DONE);
+}

- mutex_lock(&(juice->net.lock));
+static int __lemona_net_init(void)
+{
+ int ret = 0;
+ struct socket *sock = NULL;

if (juice->net.sock != NULL
|| (juice->net.sock == NULL
- && !(ret = time_after(jiffies, juice->net.timeout))
- && init == false))
+ && !(ret = time_after(jiffies, juice->net.timeout))))
goto out;

ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
@@ -83,8 +106,34 @@
juice->net.timeout = jiffies + (NET_LOG_RETRY * HZ);
__lemona_net_cleanup();
}
+ return (ret);
+}
+
+int lemona_net_init(void)
+{
+ int ret = 0;
+
+ mutex_init(&(juice->net.lock));
+
+ juice->net.sin.sin_family = AF_INET;
+ juice->net.sin.sin_addr.s_addr = lemona_net_serv_get();
+ juice->net.sin.sin_port = htons(net_log_port);
+
+ ret = register_inetaddr_notifier(&lemona_net_inetaddr_nb);
+ if (ret != 0)
+ {
+ lemona_printk("Unable to register notifier for inetaddr changes. "
+ "Net module will be deactivated\n");
+ return (0);
+ }
+
+ mutex_lock(&(juice->net.lock));
+ /* ensure timeout don't block us */
+ juice->net.timeout = jiffies - (NET_LOG_RETRY * HZ);
+ __lemona_net_init();
+ lemona_net_activated = 1;
mutex_unlock(&(juice->net.lock));
- return (init ? 0 : ret);
+ return (0);
}

/*
@@ -93,8 +142,15 @@
*/
void lemona_net_log(struct lemona_zest *zest)
{
- if (lemona_net_init(false) != 0)
- return; /* couldn't get a socket working */
+ if (!lemona_net_activated)
+ return;
+
+ mutex_lock(&(juice->net.lock));
+ if (__lemona_net_init() != 0)
+ {
+ mutex_unlock(&(juice->net.lock));
+ return; /* couldn't get a socket working */
+ }

if (juice->net.sock)
{
@@ -102,13 +158,13 @@
int sent;
struct kvec kvec = {
.iov_base = zest,
- .iov_len = zest->size
+ /* Zest are aligned on sizeof(int) */
+ .iov_len = zest->size + (zest->size % sizeof(int))
};
struct msghdr hdr = { 0 };

hdr.msg_flags = MSG_NOSIGNAL;
- mutex_lock(&(juice->net.lock));
- for (sent = 0; sent != zest->size; sent += ret)
+ for (sent = 0; sent != kvec.iov_len; sent += ret)
{
ret = kernel_sendmsg(juice->net.sock, &hdr, &kvec, 1, kvec.iov_len);
if (ret < 0)
@@ -121,11 +177,11 @@
break;
}
}
- mutex_unlock(&(juice->net.lock));
}
+ mutex_unlock(&(juice->net.lock));
}

-static void __lemona_net_cleanup(void)
+static void __lemona_net_cleanup(void)
{
if (!juice->net.sock)
return;
@@ -133,7 +189,7 @@
juice->net.sock = NULL;
}

-void lemona_net_cleanup(void)
+void lemona_net_cleanup(void)
{
mutex_lock(&(juice->net.lock));
__lemona_net_cleanup();

Modified: trunk/driver/relay.c
==============================================================================
--- trunk/driver/relay.c (original)
+++ trunk/driver/relay.c Tue Nov 11 01:39:44 2008
@@ -143,7 +143,8 @@

int lemona_relay_log(const struct lemona_zest *zest)
{
- relay_write(juice->rchan, zest, zest->size);
+ /* Zest are aligned on sizeof(int) */
+ relay_write(juice->rchan, zest, zest->size + (zest->size % sizeof(int)));
return (0);
}

Modified: trunk/tools/basket/main.c
==============================================================================
--- trunk/tools/basket/main.c (original)
+++ trunk/tools/basket/main.c Tue Nov 11 01:39:44 2008
@@ -46,7 +46,7 @@

#define MAX_BASKET 6
/* must be a multiple of sysconf(_SC_PAGE_SIZE) */
-#define FILE_SZ 20 * 1024 * 1024
+#define FILE_SZ 50 * 1024 * 1024
#define BUF_SZ 4096

void basket_destroy(struct basket* basket)
@@ -236,6 +236,7 @@
int cur_basket;
int off;
int to_copy;
+ int new_file = 1;

if ((sock = init_socket()) == -1)
return;
@@ -288,6 +289,13 @@

while (1)
{
+ if (new_file)
+ {
+ fprintf(stdout, "Switching to file %s\n",
+ baskets[cur_basket].file);
+ new_file = 0;
+ }
+
FD_ZERO(&readfds);
FD_SET(fd, &readfds);
ret = select(fd + 1, &readfds, NULL, NULL, NULL);
@@ -305,6 +313,8 @@
++cur_basket;
if (cur_basket == MAX_BASKET)
cur_basket = 0;
+ off = 0;
+ new_file = 1;

break;
}
@@ -328,6 +338,8 @@
++cur_basket;
if (cur_basket == MAX_BASKET)
cur_basket = 0;
+ off = 0;
+ new_file = 1;

break;
}
@@ -345,6 +357,8 @@
++cur_basket;
if (cur_basket == MAX_BASKET)
cur_basket = 0;
+ off = 0;
+ new_file = 1;

break;
}
@@ -372,6 +386,7 @@
pthread_mutex_lock(&(baskets[cur_basket].lock));
write(baskets[cur_basket].fd, buf, ret - to_copy);
off = ret - to_copy;
+ new_file = 1;
}
}
}

Modified: trunk/tools/picker/picker.py
==============================================================================
--- trunk/tools/picker/picker.py (original)
+++ trunk/tools/picker/picker.py Tue Nov 11 01:39:44 2008
@@ -1,109 +1,93 @@
#!/usr/bin/env python

+from __future__ import with_statement
+from datetime import timedelta
+from Zest import *
+from cStringIO import StringIO
+
import re, os, sys

-from datetime import timedelta
-from struct import unpack, calcsize

-class Zest:
- "A Simple Zest of Lemon(a)"
- Zest_Properties = (
- [ "magic" , "4s" , calcsize("4s") ],
- [ "size" , "i" , calcsize("i") ],
- [ "inout" , "i" , calcsize("i") ],
- [ "sec" , "l" , calcsize("l") ],
- [ "usec" , "l" , calcsize("l") ],
- [ "pid" , "i" , calcsize("i") ],
- [ "tgid" , "i" , calcsize("i") ],
- [ "uid" , "i" , calcsize("i") ], #unsigned short
- [ "euid" , "i" , calcsize("i") ],
- [ "fsuid" , "i" , calcsize("i") ],
- [ "gid" , "i" , calcsize("i") ],
- [ "egid" , "i" , calcsize("i") ],
- [ "fsgid" , "i" , calcsize("i") ],
- [ "sysnr" , "i" , calcsize("i") ],
- [ "argnr" , "i" , calcsize("i") ],
- [ "pad" , "2P" , calcsize("2P") ], #void * / argsz, args
- [ "extnr" , "i" , calcsize("i") ],
- [ "pad" , "2P" , calcsize("2P") ], #void * / extsz, exts
- )
- Zest_Properties_Size = calcsize("4s 2i 2l 2i 6i 2i 2P i 2P");
-
- def __init__(self, f):
- parsed_sz = 0
- #get C struct values
- for arg, fmt, sz in self.Zest_Properties:
- buf = f.read(sz)
- if len(buf) != sz:
- print "Invalid File!!! %d <=> %d (%d)" % (len(buf),
- sz, f.tell())
- return None
- if arg != "pad":
- val = unpack(fmt, buf)[0]
- self.setattr(arg, val)
-
- #get args size
- self.setattr("argsz", [])
- for i in range(self.argnr):
- parsed_sz += calcsize("i");
- val = unpack("i", f.read(calcsize("i")))[0]
- self.argsz.append(val)
-
- #get args
- self.setattr("args", [])
- for sz in self.argsz:
- parsed_sz += calcsize(str(sz) + "s");
- val = unpack(str(sz) + "s", f.read(sz))[0]
- self.args.append(val)
-
- #get exts size
- self.setattr("extsz", [])
- for i in range(self.extnr):
- parsed_sz += calcsize("i");
- val = unpack("i", f.read(calcsize("i")))[0]
- self.extsz.append(val)
-
- #get exts
- self.setattr("exts", [])
- for sz in self.extsz:
- parsed_sz += calcsize(str(sz) + "s");
- val = unpack(str(sz) + "s", f.read(sz))[0]
- self.exts.append(val)
-
- if self.size - parsed_sz != self.Zest_Properties_Size:
- print self
- raise NameError("Corrupted file!")
-
- def __str__(self):
- return str(self.__dict__)
-
- def setattr(self, key, val):
- self.__dict__[key] = val
-
-# scripting
-
-PATH = "../basket/"
-
-files = []
-dirlist = sorted(os.listdir(PATH))
-
-for f in dirlist:
- if re.match("^\d+$", f) != None:
- files.append(f)
-
-print "Preparing to examin files", files
-
-zests = []
-
-#for filename in files:
-filename = files[0]
-path = PATH + filename
-sz = os.path.getsize(path)
-pos = 0
-f = open(path, "rb")
-while pos < sz:
- print "====PARSING NEW ZEST (Off: %d)====" % pos
- z = Zest(f)
- pos = pos + z.size
- print z
-f.close()
+def usage():
+ print >> sys.stderr, "usage: picker <directories|files>"
+ exit(2)
+
+def main():
+ #do we have our working directory/file?
+ if len(sys.argv) == 1:
+ usage()
+
+ #list of files to work on
+ files = []
+
+ #if it's a directory list it contents to find on what to work on
+ for path in sys.argv[1:]:
+ if os.path.isdir(path):
+ dir = path
+ dirFiles = sorted(os.listdir(dir))
+ for f in dirFiles:
+ if re.match("^\d+$", f) != None:
+ if dir[len(dir) - 1] == "/":
+ files.append(dir + f)
+ else:
+ files.append(dir + "/" + f)
+ else:
+ files.append(path)
+
+ #do we find something to work on?
+ if len(files) == 0:
+ print >> sys.stderr, "Sorry, I didn't find anything relevant to
me!"
+ exit(3)
+
+ #following vars help us keeping track of truncated zest
+ missing = 0
+ left = StringIO()
+
+ #Yes? Good, go on
+ for file in files:
+ try:
+ fileSize = os.path.getsize(file)
+ print ">>>Examining File", file
+ with open(file, "rb") as f:
+ pos = 0;
+ while pos != fileSize:
+ print "\t>>Parsing new entry at %s:%d" % (file, pos)
+ try:
+ if missing != 0:
+ print "\t\t>Concatening with previous half",
missing
+ left.write(f.read(missing))
+ left.seek(0, os.SEEK_SET)
+ z = Zest(left)
+ missing = 0;
+ left.truncate(0);
+ else:
+ z = Zest(f)
+ pos = f.tell()
+ #TODO, pass the zest to a backend
+ #(e.g. console, database)
+ print z
+ except ZestTruncatedFile, e:
+ missing = e.missing
+ f.seek(pos, os.SEEK_SET)
+ left.write(f.read())
+ print "\t\t>Truncated Zest, Read: %d Missing: %d"\
+ % (left.tell(), missing)
+ break
+ except ZestNullSizeError:
+ break # No more entry in this file
+ except ZestInvalidFormat:
+ print >> sys.stdout, "Invalid file format!"
+ break
+ finally:
+ print "\t<<Done"
+ print "<<<Done", file
+ except (IOError, OSError), inst:
+ print >> sys.stderr, "Error:", inst
+ except Exception, inst:
+ print >> sys.stderr, "Error:", inst
+
+#scripting
+if __name__ == "__main__":
+ main()
+ exit(0)
+