[SCM] watchdog branch, master, updated. 31f31f870dfdd9891211a64aef286e067f40e71b
devi
generated because a ref change was pushed to the repository containing
the project "watchdog".
The branch, master has been updated
via 31f31f870dfdd9891211a64aef286e067f40e71b (commit)
via 0c35ab7a8fea6c45572c27aefe87ceffdf02e111 (commit)
from 6770141097799c54a48cbcafdd60f5018c307883 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 31f31f870dfdd9891211a64aef286e067f40e71b
Author: Devi <asl...@gmail.com>
Date: Wed Apr 8 19:11:58 2009 +0530
pages with strict bound of 100K
commit 0c35ab7a8fea6c45572c27aefe87ceffdf02e111
Author: Devi <asl...@gmail.com>
Date: Wed Apr 8 14:39:15 2009 +0530
better testing and bug fixes in creating index
-----------------------------------------------------------------------
Summary of changes:
utils/index.py | 49 ++++++++++++++++++++++++++++++++++---------------
webapp.py | 25 +++++++------------------
2 files changed, 41 insertions(+), 33 deletions(-)
diff --git a/utils/index.py b/utils/index.py
index 607dfa4..11d7f66 100644
--- a/utils/index.py
+++ b/utils/index.py
@@ -2,17 +2,28 @@ import web
import webapp
import petition
import inspect, types, itertools, gzip, os
+from web.browser import AppBrowser
single_pages = ('/', '/about', '/about/team', '/about/api', '/about/feedback', '/about/help',
'/contribute/', '/blog/',
'/lob/c/', '/lob/f/', '/lob/o/', '/lob/pa/', '/lob/pe/',
'/b/', '/e/', '/p/', '/c/', '/writerep/')
+b = AppBrowser(webapp.app)
+
+def test(klass):
+ index = klass().index()
+ for path in do_flatten(take(2, iter(index))):
+ try:
+ b.open(path)
+ assert(b.status == 200)
+ except:
+ print b.status, path, klass
+
def get_class_index(klass, _test=False):
try:
if _test:
- print klass
- return take(5, iter(klass().index()))
+ return test(klass)
return klass().index()
except AttributeError:
return []
@@ -57,13 +68,20 @@ def take(n, seq):
for i in xrange(n):
yield seq.next()
-def group(seq, n):
- while True:
- x = list(take(n, seq))
- if x:
+def group(seq, maxsize):
+ def limit(seq, maxsize, itemlen):
+ size = 0
+ while size < maxsize:
+ x = seq.next()
+ size += itemlen(x)
yield x
- else:
- break
+
+ overhead = len('<a href=""></a>\n')
+ itemlen = lambda x: overhead+2*len(x)
+ x = 1
+ while x:
+ x = list(limit(seq, maxsize, itemlen))
+ yield x
t_sitemap = """$def with (title, items)
<h1>Index</h1>
@@ -84,11 +102,13 @@ $if title != "index": <a href="../index.html">Back to index</a>
make_sitemap = web.template.Template(t_sitemap)
make_index = web.template.Template(t_index)
-entries_per_page = 2200
+pagesize = 99*1024 #1K for overheads like <h1> and back links
+entries_per_page = pagesize/30
def write(filename, text):
f = open(filename, 'w')
f.write(text)
+ print filename, len(text)/1024
f.close()
def write_sitemap(i, seq):
@@ -97,21 +117,20 @@ def write_sitemap(i, seq):
if not os.path.exists(dir):
os.mkdir(dir)
write(filename, str(make_sitemap(filename, seq)))
- print filename
def write_sitemaps(data):
- for i, x in enumerate(group(data, entries_per_page)):
+ for i, x in enumerate(group(data, pagesize)):
write_sitemap(i, x)
def create_index(index_dir, _test=False):
if not os.path.exists(index_dir):
os.mkdir(index_dir)
-
+
data = getindex(webapp.app, _test)
write_sitemaps(data)
-
- dirs = [d for d in os.listdir('index') if os.path.isdir(os.path.join('index', d))]
- write('index/index.html', str(make_index('index', [d+'/index.html' for d in dirs])))
+
+ dirs = [d for d in os.listdir(index_dir) if os.path.isdir(os.path.join(index_dir, d))]
+ write(index_dir + '/index.html', str(make_index(index_dir, [d+'/index.html' for d in dirs])))
for d in dirs:
d = os.path.join('index', d)
diff --git a/webapp.py b/webapp.py
index d54cc54..35c786c 100755
--- a/webapp.py
+++ b/webapp.py
@@ -68,46 +68,32 @@ urls = (
)
class index:
- def index(self): return ['/']
-
def GET(self):
return render.index()
class about:
- def index(self): return ['/about/']
-
def GET(self, endslash=None):
if not endslash: raise web.seeother('/about/')
return render.about()
class aboutapi:
- def index(self): return ['/about/api']
-
def GET(self):
return render.about_api()
class aboutteam:
- def index(self): return ['/about/team']
-
def GET(self):
return render.about_team()
class abouthelp:
- def index(self): return ['/about/help']
-
def GET(self):
return render.about_help()
class contribute:
- def index(self): return ['/contribute/']
-
def GET(self, endslash=None):
if not endslash: raise web.seeother('/contribute/')
return render.contribute()
class feedback:
- def index(self): return ['/feedback']
-
def GET(self):
return render.feedback()
@@ -391,7 +377,12 @@ class occupation_committees:
class contributions:
"""from a corp to a pol"""
- def index(self): return ['/contrib/']
+ def index(self):
+ return ('/contrib/?from=%s&to=%s' % (c.frm, c.to) \
+ for c in db.query("""SELECT cn.employer_stem as frm, p.id as to
+ FROM contribution cn, committee cm, politician_fec_ids pfi, politician p
+ WHERE cn.recipient_id = cm.id AND cm.candidate_id = pfi.fec_id
+ AND pfi.politician_id = p.id"""))
def GET(self, img=None):
i = web.input()
@@ -757,11 +748,9 @@ class dproperty:
def get_number_columns(table):
return [cname for cname, c in table.columns.iteritems() if c.sql_type in ('int', 'real')]
- l = []
for prefix, table in table_map.iteritems():
table = getattr(schema, table.title())
- l.append(['/by/'.join([prefix, col]) for col in get_number_columns(table)])
- return l
+ yield ('/%s/by/%s' % (prefix, col) for col in get_number_columns(table))
def GET(self, table, what):
try:
hooks/post-receive
--
watchdog