Revision: 67643c06208d
Branch:   default
Author:   Andi Albrecht <albrecht.a...@gmail.com>
Date:     Sun Nov 18 00:01:30 2012
Log:      Handle incoming mail without encoding/charset correctly.

review: http://codereview.appspot.com/6852059/
http://code.google.com/p/rietveld/source/detail?r=67643c06208d

Modified:
  /codereview/views.py
  /tests/test_incomingmail.py

=======================================
--- /codereview/views.py        Thu Nov  8 11:48:03 2012
+++ /codereview/views.py        Sun Nov 18 00:01:30 2012
@@ -3946,6 +3946,13 @@
      # As a workaround we try to decode the payload ourselves.
      if payload.encoding == '8bit' and payload.charset:
        body = payload.payload.decode(payload.charset)
+    # If neither encoding not charset is set, but payload contains
+    # non-ASCII chars we can't use payload.decode() because it returns
+    # payload.payload unmodified. The later type cast to db.Text fails
+    # with a UnicodeDecodeError then.
+    elif payload.encoding is None and payload.charset is None:
+      # assume utf-8 but set replace flag to go for sure.
+      body = payload.payload.decode('utf-8', 'replace')
      else:
        body = payload.decode()
      break
=======================================
--- /tests/test_incomingmail.py Mon Nov 14 22:54:34 2011
+++ /tests/test_incomingmail.py Sun Nov 18 00:01:30 2012
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
  # Copyright 2011 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -185,3 +186,24 @@
      views._process_incoming_mail(msg.as_string(), 're...@example.com')
      imsg = models.Message.all().ancestor(self.issue).get()
      self.assertEqual(imsg.text.encode(jcode), jtxt)
+
+  def test_missing_encoding(self):
+    # make sure that incoming mails with missing encoding and
+    # charset are handled correctly.
+    body = 'Âfoo'
+    msg = ('From: sen...@example.com',
+           'Subject: subject (issue%s)' % self.issue.key().id(),
+           '',
+           body)
+    views._process_incoming_mail('\r\n'.join(msg), 're...@example.com')
+    imsg = models.Message.all().ancestor(self.issue).get()
+    self.assertEqual(imsg.text, u'Âfoo')
+    imsg.delete()
+    body = '\xf6'
+    msg = ('From: sen...@example.com',
+           'Subject: subject (issue%s)' % self.issue.key().id(),
+           '',
+           body)
+    views._process_incoming_mail('\r\n'.join(msg), 're...@example.com')
+    imsg = models.Message.all().ancestor(self.issue).get()
+    self.assertEqual(imsg.text, u'\ufffd')