[gpapers] 6 new revisions pushed by marcelCo...@gmail.com on 2012-03-31 21:21 GMT
codesite...@google.com
Revision: 6d669bcd8473
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 08:28:46 2012
Log: extract text with pdfminer
http://code.google.com/p/gpapers/source/detail?r=6d669bcd8473
Revision: 294f43e224e3
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:16:18 2012
Log: whitespace changes
http://code.google.com/p/gpapers/source/detail?r=294f43e224e3
Revision: 04cf905ac0e9
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:16:37 2012
Log: Extract info from PDFs after downloading them
http://code.google.com/p/gpapers/source/detail?r=04cf905ac0e9
Revision: 2ec73542bbf3
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:38:25 2012
Log: fix PaperEditGUI.toolbutton_refresh_extracted_text_from_pdf
http://code.google.com/p/gpapers/source/detail?r=2ec73542bbf3
Revision: 7c19695c8362
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 14:06:48 2012
Log: fix the search caching mechanism and simplify the use
of "user_data"
http://code.google.com/p/gpapers/source/detail?r=7c19695c8362
Revision: 434244072e66
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 14:20:15 2012
Log: list dependencies
http://code.google.com/p/gpapers/source/detail?r=434244072e66
==============================================================================
Revision: 6d669bcd8473
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 08:28:46 2012
Log: extract text with pdfminer
http://code.google.com/p/gpapers/source/detail?r=6d669bcd8473
Modified:
/gPapers.py
/gPapers/models.py
/importer/pdf_file.py
=======================================
--- /gPapers.py Sun Mar 25 14:49:56 2012
+++ /gPapers.py Sat Mar 31 08:28:46 2012
@@ -334,7 +334,7 @@
author_obj.save()
# Simple attributes
- attributes = ['title', 'abstract', 'doi']
+ attributes = ['title', 'abstract', 'doi', 'extracted_text']
for attr in attributes:
log_debug('Checking if %s is in paper_info' % attr)
@@ -1562,7 +1562,7 @@
source = liststore[rows[0]][15]
log_debug('Source: %s' % source)
- if not paper and source != 'local': # This is a search result
+ if not paper and source is not None and source != 'local': #
This is a search result
button = Gtk.ToolButton(stock_id=Gtk.STOCK_ADD)
button.set_tooltip_text('Add this paper to your
library...')
provider = self.search_providers[source]
=======================================
--- /gPapers/models.py Sun Mar 25 14:49:56 2012
+++ /gPapers/models.py Sat Mar 31 08:28:46 2012
@@ -22,9 +22,6 @@
from logger import log_debug
-p_doi = re.compile('doi *: *(10.[a-z0-9]+/[a-z0-9.]+)', re.IGNORECASE)
-
-
class Publisher(models.Model):
name = models.CharField(max_length='1024')
@@ -166,9 +163,6 @@
django.core.files.base.ContentFile(raw_contents),
save)
self.save()
- log_debug('Extracting information from PDF')
- try: self.extract_document_information_from_pdf()
- except: traceback.print_exc()
def get_authors_in_order(self):
from django.db import connection
@@ -186,53 +180,6 @@
self.read_count = self.read_count + 1
self.save()
- # FIXME: This is obsolete, see importer.pdf_file
- def extract_document_information_from_pdf(self, force_overwrite=False):
- """will overwrite the extracted_text and page_count fields, and
the title if the title is empty"""
- if self.full_text and os.path.isfile(self.full_text.path):
- content = []
-
- # Load PDF into pyPDF
- pdf = pyPdf.PdfFileReader(file(self.full_text.path, "rb"))
- doc_info = pdf.getDocumentInfo()
- content.append(str(doc_info))
- content.append('\n\n')
- if force_overwrite or not self.title:
- try: self.title = doc_info['/Title']
- except: self.title = os.path.split(self.full_text.path)[1]
- if force_overwrite or self.authors.count() == 0:
- try:
- author_text = doc_info['/Author']
- print 'author_text', author_text
- if author_text.find(';') > 0:
- author_list = author_text.split(';')
- else:
- author_list = author_text.split(',')
- if author_list:
- self.authors.clear()
- for author_name in author_list:
- author, created =
Author.objects.get_or_create(name=author_name.strip())
- if created:
- author.save()
- self.authors.add(author)
- except:
- pass
-
- # extract the actual text
- stdin, stdout = os.popen4('ps2txt "%s"' % self.full_text.path)
- for line in stdout:
- content.append(line)
- try:
- self.doi = p_doi.search(line).group(1)
- print self.doi
- except: pass
-
- self.extracted_text = ''.join(content)
- else:
- self.page_count = 0
- self.extracted_text = ''
- return self.extracted_text
-
class Admin:
list_display = ('id', 'doi', 'title')
=======================================
--- /importer/pdf_file.py Sun Mar 25 14:49:56 2012
+++ /importer/pdf_file.py Sat Mar 31 08:28:46 2012
@@ -1,9 +1,17 @@
from io import BytesIO
+import cStringIO
+import re
from pdfminer.pdfparser import PDFParser, PDFDocument, resolve1
+from pdfminer.pdfinterp import PDFResourceManager, process_pdf
+from pdfminer.layout import LAParams
+from pdfminer.converter import TextConverter
from logger import log_debug, log_info, log_error
+# A DOI consists of a numeric prefix starting with "10." followed by "/"
and
+# a more or less arbitrary suffix
+p_doi = re.compile('doi\s*:?\s*(10.[0-9]+/[^/].+)\s', re.IGNORECASE)
def get_paper_info_from_pdf(data):
fp = BytesIO(data)
@@ -39,12 +47,33 @@
paper_info['authors'] = author_list
title = info.get('Title')
if title:
- paper_info['title'] = title
+ # Some PDFs have the doi as a title
+ if title.lower().startswith('doi:'):
+ paper_info['doi'] = title[4:]
+ else:
+ paper_info['title'] = title
#TODO: Additional metadata?
#TODO: What about embedded BibTeX (as done by JabRef)?
- #TODO: Extract text
- #TODO: Find doi
+ #Extract text
+ rsrcmgr = PDFResourceManager()
+ content = cStringIO.StringIO()
+ device = TextConverter(rsrcmgr, content, codec='utf-8',
laparams=LAParams())
+ process_pdf(rsrcmgr, device, fp, check_extractable=True, caching=True)
+
+ paper_info['extracted_text'] = content.getvalue()
+
+ if not 'doi' in paper_info: # Try to find a DOI in the text
+ doi = p_doi.search(paper_info['extracted_text'])
+ if doi is not None:
+ doi = doi.group(1)
+ log_debug('Found a DOI: %s' % doi)
+ paper_info['doi'] = doi
+
+ log_debug('Extracted Content: %s' % paper_info['extracted_text'])
+ device.close()
+ content.close()
+
return paper_info
==============================================================================
Revision: 294f43e224e3
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:16:18 2012
Log: whitespace changes
http://code.google.com/p/gpapers/source/detail?r=294f43e224e3
Modified:
/importer/__init__.py
/importer/pdf_file.py
=======================================
--- /importer/__init__.py Sun Mar 25 14:49:56 2012
+++ /importer/__init__.py Sat Mar 31 13:16:18 2012
@@ -470,8 +470,7 @@
HTML page, it will be searched for potential links, then
:function:`import_from_urls` will be called with this link. Finally,
the
callback is called with the `paper_info` (a dictionary) and
`paper_data`
- (binary data)
- as an argument.
+ (binary data) as an argument.
'''
def data_received(session, message, user_data):
=======================================
--- /importer/pdf_file.py Sat Mar 31 08:28:46 2012
+++ /importer/pdf_file.py Sat Mar 31 13:16:18 2012
@@ -13,6 +13,7 @@
# a more or less arbitrary suffix
p_doi = re.compile('doi\s*:?\s*(10.[0-9]+/[^/].+)\s', re.IGNORECASE)
+
def get_paper_info_from_pdf(data):
fp = BytesIO(data)
# Create a PDF parser object associated with the file object.
==============================================================================
Revision: 04cf905ac0e9
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:16:37 2012
Log: Extract info from PDFs after downloading them
http://code.google.com/p/gpapers/source/detail?r=04cf905ac0e9
Modified:
/gPapers.py
=======================================
--- /gPapers.py Sat Mar 31 08:28:46 2012
+++ /gPapers.py Sat Mar 31 13:16:37 2012
@@ -365,10 +365,20 @@
# FIXME: This should be handled via an error callback
return
- log_debug('Calling paper_from_dictionary for %s' % str(paper_info))
- paper = paper_from_dictionary(paper_info)
- log_debug('paper_from_dictionary returned %s' % str(paper))
if paper_data is not None:
+ # Get some info from the PDF:
+ paper_info_pdf = pdf_file.get_paper_info_from_pdf(paper_data)
+
+ # Add everything that is not already known
+ if paper_info is None:
+ paper_info = paper_info_pdf
+ else:
+ for key in paper_info_pdf.keys():
+ if not key in paper_info:
+ paper_info[key] = paper_info_pdf[key]
+
+ paper = paper_from_dictionary(paper_info)
+
#TODO: What is a good filename? Make this configurable?
if paper.doi:
filename = 'doi_' + paper.doi
@@ -381,7 +391,13 @@
log_debug('Saving paper to "%s"' % filename)
paper.save_file(filename, paper_data)
log_debug('Paper saved')
-
+ else:
+ log_debug('Calling paper_from_dictionary for %s' %
str(paper_info))
+ paper = paper_from_dictionary(paper_info)
+
+ paper.save()
+
+ # TODO: Should be called automatically because of the save signal
self.refresh_middle_pane_from_my_library()
def import_url_dialog(self, o):
@@ -967,6 +983,7 @@
treeview.get_column(1).get_cells()[0].set_property('wrap-width',
width)
def handle_library_updates(self):
+ log_debug('handle_libray_updates called')
selection = self.ui.get_object('left_pane_selection')
liststore, row = selection.get_selected()
if liststore[row][4] == 'local':
==============================================================================
Revision: 2ec73542bbf3
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 13:38:25 2012
Log: fix PaperEditGUI.toolbutton_refresh_extracted_text_from_pdf
http://code.google.com/p/gpapers/source/detail?r=2ec73542bbf3
Modified:
/gPapers.py
=======================================
--- /gPapers.py Sat Mar 31 13:16:37 2012
+++ /gPapers.py Sat Mar 31 13:38:25 2012
@@ -2548,7 +2548,8 @@
self.ui.get_object('textview_abstract').get_buffer().set_text(self.paper.abstract)
self.ui.get_object('textview_bibtex').get_buffer().set_text(self.paper.bibtex)
self.ui.get_object('textview_extracted_text').get_buffer().set_text(self.paper.extracted_text)
- if self.paper.full_text:
self.ui.get_object('filechooserbutton').set_filename(self.paper.full_text.path)
+ if self.paper.full_text:
+
self.ui.get_object('filechooserbutton').set_filename(self.paper.full_text.path)
self.ui.get_object('rating').set_value(self.paper.rating)
self.ui.get_object('spinbutton_read_count').set_value(self.paper.read_count)
@@ -2580,12 +2581,13 @@
self.edit_dialog.show()
def toolbutton_refresh_extracted_text_from_pdf(self):
- self.paper.extract_document_information_from_pdf()
- self.authors_model.clear()
- for author in self.paper.get_authors_in_order():
- self.authors_model.append((author.id, author.name))
+ if self.paper.full_text:
+ fp = open(self.paper.full_text.path, 'rb')
+ paper_info = pdf_file.get_paper_info_from_pdf(fp.read())
+ fp.close()
+ self.paper.extracted_text = paper_info.get('extracted_text')
self.ui.get_object('textview_extracted_text').get_buffer().set_text(self.paper.extracted_text)
- self.ui.get_object('entry_title').set_text(self.paper.title)
+ self.paper.save()
def init_references_tab(self):
treeview_references = self.ui.get_object('treeview_references')
==============================================================================
Revision: 7c19695c8362
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 14:06:48 2012
Log: fix the search caching mechanism and simplify the use
of "user_data"
http://code.google.com/p/gpapers/source/detail?r=7c19695c8362
Modified:
/importer/__init__.py
/importer/pubmed.py
=======================================
--- /importer/__init__.py Sat Mar 31 13:16:18 2012
+++ /importer/__init__.py Sat Mar 31 14:06:48 2012
@@ -701,14 +701,35 @@
del self.search_cache[text]
def search(self, search_string, callback, error_callback):
+
+ # A tuple identifying the search, making it possible for the
callback
+ # function to deal with the results properly (otherwise results
arriving
+ # out of order could lead to wrongly displayed results)
+ user_data = (self.label, search_string)
+
if not search_string:
- return []
+ callback(user_data, [])
+ return
if search_string in self.search_cache:
- return self.search_cache[search_string]
+ log_debug('Result for "%s" already in cache.' % search_string)
+ callback(user_data, self.search_cache[search_string])
+ return
+
+ log_info('Search for "%s" is not cached by this provider, starting
new search' % search_string)
try:
- self.search_async(search_string, callback, error_callback)
+ def callback_wrapper(search_results):
+ '''
+ Before calling the actual callback, save the result in the
+ cache and add `user_data` (tuple identifying request and
search
+ provider) to the call.
+ '''
+ log_debug('Saving %s in cache for "%s"' % (search_results,
search_string))
+ self.search_cache[search_string] = search_results
+ callback(user_data, search_results)
+
+ self.search_async(search_string, callback_wrapper,
error_callback)
except Exception as ex:
error_callback(ex, None)
@@ -740,30 +761,23 @@
message = self.prepare_search_message(search_string)
def my_callback(session, message, user_data):
- self.response_received(message, callback, error_callback,
- user_data)
-
- # Provide a tuple of label and search string as `user_data` to
the
- # callback -- this way it is clear to what search a
result/error belongs
- soup_session.queue_message(message, my_callback, (self.label,
-
search_string))
+ self.response_received(message, callback, error_callback)
+
+ soup_session.queue_message(message, my_callback, None)
except Exception as ex:
error_callback(ex, search_string)
- def response_received(self, message, callback, error_callback,
- user_data):
+ def response_received(self, message, callback, error_callback):
'''
Will be called when the server returns a response.
'''
- log_info('Received a response for %s' % str(user_data))
if message.status_code == Soup.KnownStatusCode.OK:
#try:
- callback(user_data,
- self.parse_response(message.response_body.data))
+ callback(self.parse_response(message.response_body.data))
#except Exception as ex:
# error_callback(ex, user_data)
else:
- error_callback(message.status_code, user_data)
+ error_callback(message.status_code)
def import_paper_after_search(self, data, callback):
try:
=======================================
--- /importer/pubmed.py Sun Mar 25 15:07:36 2012
+++ /importer/pubmed.py Sat Mar 31 14:06:48 2012
@@ -24,17 +24,16 @@
def __init__(self):
WebSearchProvider.__init__(self)
- def _ids_received(self, message, callback, error_callback, user_data):
+ def _ids_received(self, message, callback, error_callback):
if not message.status_code == Soup.KnownStatusCode.OK:
- error_callback('Pubmed replied with error code %d.' %
message.status_code, user_data)
+ error_callback('Pubmed replied with error code %d.' %
message.status_code)
else:
response_data = message.response_body.flatten().get_data()
parsed_response =
BeautifulSoup.BeautifulStoneSoup(response_data)
- # Check wether there were any hits at all
+ # Check whether there were any hits at all
if int(parsed_response.esearchresult.count.string) == 0:
- log_info('No hits for search string "%s"' % user_data[1])
return # Nothing to do anymore
# Continue with a second request asking for the summaries
@@ -46,12 +45,11 @@
message = Soup.Message.new(method='GET', uri_string=query)
def mycallback(session, message, user_data):
- self._summaries_received(message, callback, error_callback,
- user_data)
-
- soup_session.queue_message(message, mycallback, user_data)
-
- def _summaries_received(self, message, callback, error_callback,
user_data):
+ self._summaries_received(message, callback, error_callback)
+
+ soup_session.queue_message(message, mycallback, None)
+
+ def _summaries_received(self, message, callback, error_callback):
if not message.status_code == Soup.KnownStatusCode.OK:
error_callback('Pubmed replied with error code %d.' %
message.status_code, user_data)
else:
@@ -89,7 +87,7 @@
papers.append(info)
- callback(user_data, papers)
+ callback(papers)
def search_async(self, search_text, callback, error_callback):
'''
@@ -103,10 +101,9 @@
message = Soup.Message.new(method='GET', uri_string=query)
def mycallback(session, message, user_data):
- self._ids_received(message, callback, error_callback,
user_data)
-
- soup_session.queue_message(message, mycallback, (self.label,
- search_text))
+ self._ids_received(message, callback, error_callback)
+
+ soup_session.queue_message(message, mycallback, None)
def _paper_info_received(self, message, callback, user_data):
if not message.status_code == Soup.KnownStatusCode.OK:
==============================================================================
Revision: 434244072e66
Author: Marcel Stimberg <marcel...@gmail.com>
Date: Sat Mar 31 14:20:15 2012
Log: list dependencies
http://code.google.com/p/gpapers/source/detail?r=434244072e66
Added:
/README
=======================================
--- /dev/null
+++ /README Sat Mar 31 14:20:15 2012
@@ -0,0 +1,14 @@
+Dependencies
+============
+
+gPapers is currently developed and tested under Ubuntu 12.04 and depends
on the
+following packages:
+
+gir1.2-glib-2.0: 1.32.0-1
+gir1.2-gtk-3.0: 3.4.0-0ubuntu2
+gir1.2-gdkpixbuf-2.0: 2.26.0-1
+gir1.2-pango-1.0: 1.30.0-0ubuntu1
+gir1.2-poppler-0.18: 0.18.4-1ubuntu2
+gir1.2-soup-2.4: 2.38.0-0ubuntu1
+python-django: 1.3.1-4ubuntu1
+python-pdfminer: 20110515+dfsg-1