[redistricter] r202 committed - add /2010/XX/ dirs...
redist...@googlecode.com
Author: brian.olson
Date: Sun Feb 27 12:50:04 2011
Log: add /2010/XX/ dirs
rarrange buzz/reddit/tweet sharing templating.
add legislature info to states.py
kill a bunch of TODO comments that are done.
emit kmppSendThreshold, fix usage of it.
http://code.google.com/p/redistricter/source/detail?r=202
Added:
/trunk/stdir_index_pyt.html
Modified:
/trunk/analyze_submissions.py
/trunk/new_st_index_pyt.html
/trunk/notes.txt
/trunk/result_index_pyt.html
/trunk/runallstates.py
/trunk/states.py
=======================================
--- /dev/null
+++ /trunk/stdir_index_pyt.html Sun Feb 27 12:50:04 2011
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html><head><title>$statename Redistricting</title>
+<link href="${rooturl}report.css" rel="stylesheet" type="text/css" />
+<style>body{color:#000000;background-color:#ffffff;}</style></head>
+<body bgcolor="#ffffff">
+<div style="float:right;width:20em;">
+
+</div>
+<h1>$statename Redistricting</h1>
+<div class="small">From 2010 data</div>
+<table class="sp"><tr><td>
+<div><a href="/2010/">2010 Redistricting Home</a></div>
+${statenav}
+<div><a
href="http://www.google.com/buzz/post?url=${cgipageabsurl}&imageurl=${cgiimageurl}"
style="text-decoration:none" target="_blank"><img
src="http://www.gstatic.com/buzz/api/images/buzz-link.png"> Buzz</a> <a
href="http://www.reddit.com/submit?url=${cgipageabsurl}"
target="_blank"><img src="http://www.reddit.com/static/spreddit7.gif"
alt="submit to reddit" border="0" /></a> <span id="custom-tweet-button"><a
href="http://twitter.com/share?url=${cgipageabsurl}"
target="_blank">Tweet</a></span></div>
+</td><td>
+
+<table id="imt">
+$bodyrows
+</table>
+$extra
+</td></tr></table>
+${google_analytics}
+</body></html>
=======================================
--- /trunk/analyze_submissions.py Sun Feb 27 09:24:40 2011
+++ /trunk/analyze_submissions.py Sun Feb 27 12:50:04 2011
@@ -1,11 +1,4 @@
#!/usr/bin/python
-# TODO: move results into directories per config?
-# solutions/YYYYMMDD/HHMMSS_IP_XXX.tar.gz
-# ->
-# processed/XX_Config/YYYYMMDD/HHMMSS_IP_XXX.tar.gz
-# TODO: emit config information for cliens for threshold of kmpp to send
to server.
-# TODO: emit web directory of best-so-far results.
-# TODO: extract kmpp_spread.svg from these.
import cgi
import logging
@@ -100,6 +93,8 @@
def configToName(cname):
(px, body) = cname.split('_', 1)
+ legstats = states.legislatureStatsForPostalCode(px)
+ body = states.expandLegName(legstats, body)
return states.nameForPostalCode(px) + ' ' + body
@@ -176,6 +171,9 @@
if self.dbpath:
self.opendb(self.dbpath)
self.pageTemplate = None
+ self.dirTemplate = None
+ # cache for often used self.statenav(None, configs)
+ self._statenav_all = None
def getPageTemplate(self, rootdir=None):
if self.pageTemplate is None:
@@ -186,6 +184,15 @@
f.close()
return self.pageTemplate
+ def getDirTemplate(self, rootdir=None):
+ if self.dirTemplate is None:
+ if rootdir is None:
+ rootdir = srcdir_
+ f = open(os.path.join(rootdir, 'stdir_index_pyt.html'), 'r')
+ self.dirTemplate = string.Template(f.read())
+ f.close()
+ return self.dirTemplate
+
def loadDatadir(self, path=None):
if path is None:
path = self.options.datadir
@@ -359,8 +366,13 @@
data = bestconfigs[cname]
out.write('%s:sendAnything: False\n' % (cname,))
out.write('%s:weight:%f\n' % (cname, rweight(data['count'])))
- # TODO: tweak weight/kmppSendTheshold/spreadSendTheshold automatically
- #out.write('%s:disabled\n')
+ if (cname in bestconfigs) and (bestconfigs[cname]['count'] >= 10):
+ c = self.db.cursor()
+ rows = c.execute('SELECT kmpp FROM submissions WHERE config = ? ORDER
BY kmpp ASC LIMIT 10', (cname,))
+ rows = list(rows)
+ kmpplimit = float(rows[-1][0])
+ out.write('%s:kmppSendThreshold:%f' % (cname, kmpplimit))
+ # TODO: tweak spreadSendThreshold automatically ?
mpath = outpath + '_manual'
if os.path.exists(mpath):
mf = open(mpath, 'r')
@@ -424,13 +436,13 @@
self.stderr.write('error %d running "%s"\n' % (retcode, ' '.join(cmd)))
return None
- def statenav(self, current, configs=None):
+ def statenav(self, current, configs):
+ if (current is None) and self._statenav_all:
+ return self._statenav_all
statevars = {}
- if configs:
- citer = configs.iterkeys()
- else:
- citer = self.config.iterkeys()
- for cname in citer:
+ for cname, data in configs.iteritems():
+ if not data.get('kmpp'):
+ continue
(st, variation) = cname.split('_', 1)
if st not in statevars:
statevars[st] = [variation]
@@ -454,13 +466,79 @@
isCurrent = True
vlist.append('<td><b>%s</b></td>' % (v,))
else:
- vlist.append('<td><a href="%s">%s</a></td>' %
(os.path.join(self.options.rooturl, stu_v) + '/', v))
+ vlist.append('<td><a href="%s">%s</a></td>' %
(urljoin(self.options.rooturl, stu_v) + '/', v))
if isCurrent:
dclazz = 'slgC'
else:
dclazz = 'slg'
- outl.append('<tr class="%s"><td>%s</td>%s</tr>' % (dclazz,
name, ' '.join(vlist)))
- return '<table class="snl">' + ''.join(outl) + '</table>'
+ outl.append('<tr class="%s"><td><a href="%s/">%s</a></td>%s</tr>' %
(dclazz, urljoin(self.options.rooturl, stu), name, ' '.join(vlist)))
+ text = '<table class="snl">' + ''.join(outl) + '</table>'
+ if current is None:
+ self._statenav_all = text
+ return text
+
+ def statedir(self, stu, configs):
+ stu = stu.upper()
+ name = states.nameForPostalCode(stu)
+ if not name:
+ logging.error('no name for postal code %s', stu)
+ return
+ variations = []
+ for cname, data in configs.iteritems():
+ if not data.get('kmpp'):
+ continue
+ (st, variation) = cname.split('_', 1)
+ if st != stu:
+ continue
+ variations.append(variation)
+ if not variations:
+ logging.warn('no active variations for %s', stu)
+ if 'Congress' in variations:
+ variations.remove('Congress')
+ variations.sort()
+ variations.insert(0, 'Congress')
+ legstats = states.legislatureStatsForPostalCode(stu)
+ bodyrows = []
+ firstvar = None
+ for variation in variations:
+ if not firstvar:
+ firstvar = stu + '_' + variation
+ bodyname = variation
+ numd = 'X'
+ for ls in legstats:
+ if ls.shortname == variation:
+ bodyname = ls.name
+ numd = ls.count
+ #bodyname = states.expandLegName(legstats, variation)
+ configurl = '%s%s_%s/' % (self.options.rooturl, stu, variation)
+ tr = """<tr><td><div><a href="%s">%s %s</a></div><div><a href="%s"><img
src="%smap500.png" height="150"></a> %s districts</div></td></tr>""" %
(configurl, name, bodyname, configurl, configurl, numd)
+ bodyrows.append(tr)
+ outdir = self.options.outdir
+ sdir = os.path.join(outdir, stu)
+ ihtmlpath = os.path.join(sdir, 'index.html')
+ st_template = self.getDirTemplate()
+ cgipageabsurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, stu) + '/')
+ cgiimageurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, firstvar, 'map500.png'))
+
+ if not os.path.isdir(sdir):
+ os.makedirs(sdir)
+ extrahtml = ''
+ extrahtmlpath = os.path.join(sdir, 'extra.html')
+ if os.path.isfile(extrahtmlpath):
+ extrahtml = open(extrahtmlpath, 'r').read()
+
+ out = open(ihtmlpath, 'w')
+ out.write(st_template.substitute(
+ statename=name,
+ statenav=self.statenav(None, configs),
+ bodyrows='\n'.join(bodyrows),
+ extra=extrahtml,
+ rooturl=self.options.rooturl,
+ cgipageabsurl=cgipageabsurl,
+ cgiimageurl=cgiimageurl,
+ google_analytics=_google_analytics(),
+ ))
+ out.close()
def measureRace(self, cname, solution, htmlout):
config = self.config[cname]
@@ -488,7 +566,7 @@
p.stdin.write(zf.read(part1name))
p.stdin.close()
p.wait()
-
+
def processFailedSubmissions(self, configs, cname):
c = self.db.cursor()
rows = c.execute('SELECT id, path FROM submissions WHERE config = ? AND
kmpp IS NULL', (cname,))
@@ -531,7 +609,7 @@
out.xy(spread, kmpp)
out.close()
return kmpppath
-
+
def cleanupSolutionPath(self, tpath):
if tpath[0] == os.sep:
tpath = tpath[len(os.sep):]
@@ -569,7 +647,6 @@
dszout.write(tfparts['solution'])
dszout.close()
- # TODO: run `measurerace` to get demographic analysis
racehtml = os.path.join(sdir, 'race.html')
if self.options.redraw or (not os.path.exists(racehtml)):
self.measureRace(cname, solpath, racehtml)
@@ -589,7 +666,6 @@
if (kmpp is None) or (spread is None) or (std is None):
logging.error('bad statsum for %s', cname)
return
- st_template = self.getPageTemplate()
# TODO: permalink
permalink = os.path.join(self.options.rooturl, cname, str(data['id']))
+ '/'
racedata = ''
@@ -601,11 +677,10 @@
if os.path.exists(extrapath):
extrahtml = open(extrapath, 'r').read()
statename = configToName(cname)
- pageabsurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, cname) + '/')
- imageurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, cname, 'map500.png'))
- buzzurl = 'http://www.google.com/buzz/post?url=%s&imageurl=%s' %
(pageabsurl, imageurl)
- tweeturl = 'http://twitter.com/share?url=' + pageabsurl
-
+ cgipageabsurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, cname) + '/')
+ cgiimageurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, cname, 'map500.png'))
+
+ st_template = self.getPageTemplate()
out = open(ihpath, 'w')
out.write(st_template.substitute(
statename=statename,
@@ -623,8 +698,8 @@
extra=extrahtml,
racedata=racedata,
rooturl=self.options.rooturl,
- buzzurl=buzzurl,
- tweeturl=tweeturl,
+ cgipageabsurl=cgipageabsurl,
+ cgiimageurl=cgiimageurl,
google_analytics=_google_analytics(),
))
out.close()
@@ -634,22 +709,24 @@
def buildBestSoFarDirs(self, configs=None):
"""$outdir/$XX_yyy/$id/{index.html,ba_500.png,ba.png,map.png,map500.png}
With hard links from $XX_yyy/* to $XX_yyy/$id/* for the current best."""
- # TODO: build state dirs /??/ which has latest sub image and link to
full page for each
outdir = self.options.outdir
if not os.path.isdir(outdir):
os.makedirs(outdir)
if configs is None:
configs = self.getBestConfigs()
+ stutodo = set()
for cname, data in configs.iteritems():
self.buildReportDirForConfig(configs, cname, data)
+ (stu, rest) = cname.split('_', 1)
+ stutodo.add(stu)
+ for stu in stutodo:
+ self.statedir(stu, configs)
# build top level index.html
result_index_html_path = os.path.join(srcdir_, 'result_index_pyt.html')
newestconfig = self.newestWinner(configs)['config']
newestname = configToName(newestconfig)
- pageabsurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl))
- imageurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, newestconfig, 'map500.png'))
- buzzurl = 'http://www.google.com/buzz/post?url=%s&imageurl=%s' %
(pageabsurl, imageurl)
- tweeturl = 'http://twitter.com/share?url=' + pageabsurl
+ cgipageabsurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl))
+ cgiimageurl = urllib.quote_plus(urljoin(self.options.siteurl,
self.options.rooturl, newestconfig, 'map500.png'))
f = open(result_index_html_path, 'r')
result_index_html_template = string.Template(f.read())
@@ -662,8 +739,8 @@
localtime=localtime(),
nwinner=newestconfig,
nwinnername=newestname,
- buzzurl=buzzurl,
- tweeturl=tweeturl,
+ cgipageabsurl=cgipageabsurl,
+ cgiimageurl=cgiimageurl,
google_analytics=_google_analytics(),
))
index_html.close()
=======================================
--- /trunk/new_st_index_pyt.html Sun Feb 27 08:18:29 2011
+++ /trunk/new_st_index_pyt.html Sun Feb 27 12:50:04 2011
@@ -11,7 +11,7 @@
<table class="sp"><tr><td>
<div><a href="/2010/">2010 Redistricting Home</a></div>
${statenav}
-<div><a href="${buzzurl}" style="text-decoration:none"><img
src="http://www.gstatic.com/buzz/api/images/buzz-link.png"> Buzz</a> <a
href="http://www.reddit.com/submit" onclick="window.location
= 'http://www.reddit.com/submit?url=' +
encodeURIComponent(window.location); return false"><img
src="http://www.reddit.com/static/spreddit7.gif" alt="submit to reddit"
border="0" /></a> <span id="custom-tweet-button"><a href="${tweeturl}"
target="_blank">Tweet</a></span></div>
+<div><a
href="http://www.google.com/buzz/post?url=${cgipageabsurl}&imageurl=${cgiimageurl}"
style="text-decoration:none" target="_blank"><img
src="http://www.gstatic.com/buzz/api/images/buzz-link.png"> Buzz</a> <a
href="http://www.reddit.com/submit?url=${cgipageabsurl}"
target="_blank"><img src="http://www.reddit.com/static/spreddit7.gif"
alt="submit to reddit" border="0" /></a> <span id="custom-tweet-button"><a
href="http://twitter.com/share?url=${cgipageabsurl}"
target="_blank">Tweet</a></span></div>
</td><td>
<table id="imt"><tr><td align="center" colspan="3"><a
href="${ba_large}"><img src="${ba_small}" title="$statename congressional
district map, current and my way" alt="$statename congressional district
map, current and my way" border="1"></a></td></tr>
=======================================
--- /trunk/notes.txt Fri Feb 11 23:03:15 2011
+++ /trunk/notes.txt Sun Feb 27 12:50:04 2011
@@ -176,3 +176,4 @@
2011-02-10 Thursday 10:24 AM Boston
I have a first results on 2011 data! It's just 1000 generations of the
nearest neighbor solver, so I'm sure I'll do better soon. So far I've run
it for NJ and VA. Had to toss the super-fine face+edges based linking and
fall back to common-line-segment linking based on the tabblock tiger maps.
The faces seem to have lots of 'blocks' that don't exist in the
redistricting data. Probably extra detail on water and unpopulated areas.
+http://www.redistricting.wa.gov/
=======================================
--- /trunk/result_index_pyt.html Sun Feb 27 08:18:29 2011
+++ /trunk/result_index_pyt.html Sun Feb 27 12:50:04 2011
@@ -14,7 +14,7 @@
<!-- <div><a href="/">Redistricting Home</a></div> -->
$statenav
<hr width="50%" align="center">
-<div><a href="${buzzurl}" style="text-decoration:none"><img
src="http://www.gstatic.com/buzz/api/images/buzz-link.png"> Buzz</a> <a
href="http://www.reddit.com/submit" onclick="window.location
= 'http://www.reddit.com/submit?url=' +
encodeURIComponent(window.location); return false"><img
src="http://www.reddit.com/static/spreddit7.gif" alt="submit to reddit"
border="0" /></a> <span id="custom-tweet-button"><a href="${tweeturl}"
target="_blank">Tweet</a></span></div>
+<div><a
href="http://www.google.com/buzz/post?url=${cgipageabsurl}&imageurl=${cgiimageurl}"
style="text-decoration:none" target="_blank"><img
src="http://www.gstatic.com/buzz/api/images/buzz-link.png"> Buzz</a> <a
href="http://www.reddit.com/submit?url=${cgipageabsurl}"
target="_blank"><img src="http://www.reddit.com/static/spreddit7.gif"
alt="submit to reddit" border="0" /></a> <span id="custom-tweet-button"><a
href="http://twitter.com/share?url=${cgipageabsurl}"
target="_blank">Tweet</a></span></div>
<hr width="50%" align="center">
<div class="rb"><a href="/about.html#for">What is a district for?</a></div>
<div class="rb"><a href="/about.html#good">What is a good
district?</a></div>
=======================================
--- /trunk/runallstates.py Wed Feb 16 08:01:49 2011
+++ /trunk/runallstates.py Sun Feb 27 12:50:04 2011
@@ -1151,10 +1151,10 @@
self.client and
(bestPath != didSend) and
((not sconf.kmppSendThreshold)
- or (mb.they[0].kmpp >= sconf.kmppSendThreshold)) and
+ or (mb.they[0].kmpp <= sconf.kmppSendThreshold)) and
((not sconf.spreadSendThreshold) or
(mb.they[0].spread is None) or
- (mb.they[0].spread >= sconf.spreadSendThreshold))):
+ (mb.they[0].spread <= sconf.spreadSendThreshold))):
if self.dry_run:
print 'would send best dir "%s" if it has not already been sent' %
(mb.they[0], )
else:
=======================================
--- /trunk/states.py Wed Feb 9 20:10:08 2011
+++ /trunk/states.py Sun Feb 27 12:50:04 2011
@@ -1,5 +1,7 @@
#!/usr/bin/python
+import os
+
states = [
# name, abbrev, fips number
('Alabama', 'AL', 1),
@@ -99,3 +101,48 @@
if x[2] == fips:
return x[1]
return None
+
+_legpath = None
+
+# dict from postal code to [(body name, body short name, body count), ...]
+_legstats = None
+
+class LegislatureStat(object):
+ """Basic info for a state legislature or congressional delegation."""
+
+ def __init__(self, name, shortname, code, count):
+ self.name = name
+ self.shortname = shortname
+ self.code = code
+ self.count = count
+
+def legislatureStatsForPostalCode(code):
+ """Return [LegislatureStat, ...].
+ Returns None if code is bogus."""
+ global _legpath
+ global _legstats
+ if not _legstats:
+ _legstats = {}
+ if not _legpath:
+ _legpath =
os.path.join(os.path.dirname(__file__), 'legislatures2010.csv')
+ f = open(_legpath, 'r')
+ for line in f:
+ line = line.strip()
+ if (not line) or (line[0] == '#'):
+ continue
+ (state, body, count) = line.split(',')
+ count = int(count)
+ bodyshort = body.split()[0]
+ code = codeForState(state)
+ ls = LegislatureStat(body, bodyshort, code, count)
+ if code not in _legstats:
+ _legstats[code] = [ls]
+ else:
+ _legstats[code].append(ls)
+ return _legstats.get(code)
+
+def expandLegName(leglist, name):
+ for ls in leglist:
+ if ls.shortname == name:
+ return ls.name
+ return name