[redistricter] r205 committed - more resilient runallstates. user-agent in client

[redist...@googlecode.com]

Revision: 205
Author: brian.olson
Date: Tue Mar 1 05:23:04 2011
Log: more resilient runallstates. user-agent in client
http://code.google.com/p/redistricter/source/detail?r=205

Modified:
/trunk/client.py
/trunk/runallstates.py

=======================================
--- /trunk/client.py Tue Feb 22 18:54:10 2011
+++ /trunk/client.py Tue Mar 1 05:23:04 2011
@@ -72,7 +72,6 @@
# no change, return as was
return url

-
def getIfNewer(url, path, lastModifiedString):
try:
req = urllib2.Request(url, data=None, headers={'If-Modified-Since':
lastModifiedString})
@@ -136,6 +135,15 @@
if not self.runoptsraw:
return []
return self.runoptsraw.splitlines()
+
+ def defaultGETHeaders(self):
+ # track which client is which by a random number it picks.
+ aid = self.lastmods.get('AGENT_ID')
+ if not aid:
+ aid = rand.randint(100000, 999999999)
+ self.lastmods['AGENT_ID'] = str(aid)
+ useragent = 'bdistricting_client_%s_%s' % (os.stat(__file__).st_mtime,
aid)
+ return {'User-Agent':useragent}

def loadConfiguration(self):
"""Load config from server or cache.
@@ -169,7 +177,7 @@
self.parseHtmlConfigPage(f.read())
f.close()
return
- GET_headers = {}
+ GET_headers = self.defaultGETHeaders()
if not configCacheMissing:
if_modified_since = self.lastmods.get('.configCache')
if if_modified_since:
@@ -242,7 +250,7 @@
logging.info('fetchIfServerCopyNewer "%s" -> "%s"', remoteurl,
localpath)
return localpath
lastmod = self.lastmods.get(dataset)
- GET_headers = {}
+ GET_headers = self.defaultGETHeaders()
if lastmod:
GET_headers['If-Modified-Since'] = lastmod
try:
@@ -362,9 +370,9 @@
body.write(ddboundary)
body.write('--\r\n\r\n')
sbody = body.getvalue()
- GET_headers = {
- 'Content-Type': 'multipart/form-data; charset="utf-8"; boundary=' +
boundary,
- 'Content-Length': str(len(sbody))}
+ GET_headers = self.defaultGETHeaders()
+ GET_headers['Content-Type'] = 'multipart/form-data; charset="utf-8";
boundary=' + boundary
+ GET_headers['Content-Length'] = str(len(sbody))
print 'sending to ' + submiturl
req = urllib2.Request(submiturl, data=sbody, headers=GET_headers)
try:
=======================================
--- /trunk/runallstates.py Sun Feb 27 12:50:04 2011
+++ /trunk/runallstates.py Tue Mar 1 05:23:04 2011
@@ -397,11 +397,20 @@
if not self.readDatadirConfig():
raise ParseError('problem with datadir "%s"' % self.datadir)
elif line.startswith('weight:'):
- self.weight = float(line[7:].strip())
+ try:
+ self.weight = float(line[7:].strip())
+ except:
+ pass
elif line.startswith('kmppSendThreshold:'):
- self.kmppSendThreshold = float(line[18:].strip())
+ try:
+ self.kmppSendThreshold = float(line[18:].strip())
+ except:
+ pass
elif line.startswith('spreadSendThreshold:'):
- self.spreadSendThreshold = float(line[20:].strip())
+ try:
+ self.spreadSendThreshold = float(line[20:].strip())
+ except:
+ pass
elif line.startswith('sendAnything'):
self.sendAnything = line.lower() != 'sendanything: false'
elif line == 'enable' or line == 'enabled':
@@ -1172,7 +1181,17 @@
cmdstr = "(cd %s && %s)" % (stu, ' '.join(cmd))
print cmdstr
if not self.dry_run:
- ret = subprocess.Popen(cmd, shell=False, cwd=stu).wait()
+ p = subprocess.Popen(cmd, shell=False, cwd=stu)
+ timeout = time.time() + 60
+ ret = p.poll()
+ while ret is None:
+ if time.time() > timeout:
+ # occasional hang bugs in drend shouldn't kill the calculation. move
on.
+ p.kill()
+ ret = p.returncode
+ break
+ time.sleep(1)
+ ret = p.poll()
if ret != 0:
self.addStopReason("drend failed (%d): %s\n" % (ret, cmdstr))
sys.stderr.write(self.stopreason + '\n')