[pubsubhubbub commit] r170 - Adds initial support for hub.secret and HMAC'ed payloads.
codesite...@google.com
Date: Tue Jul 14 00:27:16 2009
New Revision: 170
Modified:
trunk/hub/main.py
trunk/hub/main_test.py
trunk/hub/subscribe_debug.html
Log:
Adds initial support for hub.secret and HMAC'ed payloads.
Other bug fixes:
- Issue assigning RSS feeds the wrong content-type
- Accidentally re-submitting subscriptions twice after verification
Modified: trunk/hub/main.py
==============================================================================
--- trunk/hub/main.py (original)
+++ trunk/hub/main.py Tue Jul 14 00:27:16 2009
@@ -89,6 +89,7 @@
import datetime
import hashlib
+import hmac
import logging
import os
import random
@@ -212,6 +213,11 @@
return 'hash_' + sha1_hash(value)
+def sha1_hmac(secret, data):
+ """Returns the sha1 hmac for a chunk of data and a secret."""
+ return hmac.new(secret, data, hashlib.sha1).hexdigest()
+
+
def is_dev_env():
"""Returns True if we're running in the development environment."""
return 'Dev' in os.environ.get('SERVER_SOFTWARE', '')
@@ -402,6 +408,8 @@
eta = db.DateTimeProperty(auto_now_add=True)
confirm_failures = db.IntegerProperty(default=0)
verify_token = db.TextProperty()
+ secret = db.TextProperty()
+ hmac_algorithm = db.TextProperty()
subscription_state = db.StringProperty(default=STATE_NOT_VERIFIED,
choices=STATES)
@@ -419,7 +427,13 @@
return get_hash_key_name('%s\n%s' % (callback, topic))
@classmethod
- def insert(cls, callback, topic, lease_seconds=DEFAULT_LEASE_SECONDS,
+ def insert(cls,
+ callback,
+ topic,
+ verify_token,
+ secret,
+ hash_func='sha1',
+ lease_seconds=DEFAULT_LEASE_SECONDS,
now=datetime.datetime.now):
"""Marks a callback URL as being subscribed to a topic.
@@ -429,6 +443,10 @@
Args:
callback: URL that will receive callbacks.
topic: The topic to subscribe to.
+ verify_token: The verification token to use to confirm the
+ subscription request.
+ secret: Shared secret used for HMACs.
+ hash_func: String with the name of the hash function to use for
HMACs.
lease_seconds: Number of seconds the client would like the
subscription
to last before expiring. Must be a number.
now: Callable that returns the current time as a datetime instance.
Used
@@ -448,6 +466,9 @@
callback_hash=sha1_hash(callback),
topic=topic,
topic_hash=sha1_hash(topic),
+ verify_token=verify_token,
+ secret=secret,
+ hash_func=hash_func,
lease_seconds=lease_seconds,
expiration_time=(
now() + datetime.timedelta(seconds=lease_seconds)))
@@ -457,9 +478,14 @@
return db.run_in_transaction(txn)
@classmethod
- def request_insert(cls, callback, topic, verify_token,
- lease_seconds=DEFAULT_LEASE_SECONDS,
- now=datetime.datetime.now):
+ def request_insert(cls,
+ callback,
+ topic,
+ verify_token,
+ secret,
+ hash_func='sha1',
+ lease_seconds=DEFAULT_LEASE_SECONDS,
+ now=datetime.datetime.now):
"""Records that a callback URL needs verification before being
subscribed.
Creates a new subscription request (for asynchronous verification) if
None
@@ -472,6 +498,8 @@
topic: The topic to subscribe to.
verify_token: The verification token to use to confirm the
subscription request.
+ secret: Shared secret used for HMACs.
+ hash_func: String with the name of the hash function to use for
HMACs.
lease_seconds: Number of seconds the client would like the
subscription
to last before expiring. Must be a number.
now: Callable that returns the current time as a datetime instance.
Used
@@ -495,6 +523,8 @@
callback_hash=sha1_hash(callback),
topic=topic,
topic_hash=sha1_hash(topic),
+ secret=secret,
+ hash_func=hash_func,
verify_token=verify_token,
lease_seconds=lease_seconds,
expiration_time=(
@@ -969,6 +999,7 @@
retry_attempts = db.IntegerProperty(default=0)
last_modified = db.DateTimeProperty(required=True)
totally_failed = db.BooleanProperty(default=False)
+ content_type = db.TextProperty(default='')
@classmethod
def create_event_for_topic(cls, topic, format, header_footer,
entry_payloads,
@@ -990,8 +1021,10 @@
"""
if format == ATOM:
close_tag = '</feed>'
+ content_type = 'application/atom+xml'
elif format == RSS:
close_tag = '</channel>'
+ content_type = 'application/rss+xml'
else:
assert False, 'Invalid format "%s"' % format
@@ -1009,7 +1042,8 @@
topic=topic,
topic_hash=sha1_hash(topic),
payload=payload,
- last_modified=now())
+ last_modified=now(),
+ content_type=content_type)
def get_next_subscribers(self, chunk_size=None):
"""Retrieve the next set of subscribers to attempt delivery for this
event.
@@ -1247,7 +1281,8 @@
################################################################################
# Subscription handlers and workers
-def ConfirmSubscription(mode, topic, callback, verify_token,
lease_seconds):
+def ConfirmSubscription(mode, topic, callback, verify_token,
+ secret, lease_seconds):
"""Confirms a subscription request and updates a Subscription instance.
Args:
@@ -1255,6 +1290,7 @@
topic: URL of the topic being subscribed to.
callback: URL of the callback handler to confirm the subscription with.
verify_token: Opaque token passed to the callback.
+ secret: Shared secret used for HMACs.
lease_seconds: Number of seconds the client would like the subscription
to last before expiring. If more than max_lease_seconds, will be
capped
to that value. Should be an integer number.
@@ -1263,9 +1299,9 @@
True if the subscription was confirmed properly, False if the
subscription
request encountered an error or any other error has hit.
"""
- logging.info('Attempting to confirm %s for topic = %s, '
- 'callback = %s, verify_token = %s, lease_seconds = %s',
- mode, topic, callback, verify_token, lease_seconds)
+ logging.info('Attempting to confirm %s for topic = %s, callback = %s, '
+ 'verify_token = %s, secret = %s, lease_seconds = %s',
+ mode, topic, callback, verify_token, secret, lease_seconds)
parsed_url = list(urlparse.urlparse(callback))
challenge = get_random_challenge()
@@ -1290,7 +1326,8 @@
if 200 <= response.status_code < 300 and response.content == challenge:
if mode == 'subscribe':
- Subscription.insert(callback, topic, real_lease_seconds)
+ Subscription.insert(callback, topic, verify_token, secret,
+ lease_seconds=real_lease_seconds)
# Blindly put the feed's record so we have a record of all feeds.
db.put(KnownFeed.create(topic))
else:
@@ -1318,6 +1355,7 @@
topic = self.request.get('hub.topic', '')
verify_type_list = [s.lower() for s in
self.request.get_all('hub.verify')]
verify_token = self.request.get('hub.verify_token', '')
+ secret = self.request.get('hub.secret', None)
lease_seconds = self.request.get('hub.lease_seconds',
str(DEFAULT_LEASE_SECONDS))
mode = self.request.get('hub.mode', '').lower()
@@ -1368,16 +1406,16 @@
# Enqueue a background verification task, or immediately confirm.
# We prefer synchronous confirmation.
if verify_type == 'sync':
- if ConfirmSubscription(mode, topic, callback,
- verify_token, lease_seconds):
+ if ConfirmSubscription(mode, topic, callback, verify_token,
+ secret, lease_seconds):
return self.response.set_status(204)
else:
self.response.out.write('Error trying to confirm subscription')
return self.response.set_status(409)
else:
if mode == 'subscribe':
- Subscription.request_insert(callback, topic,
- verify_token, lease_seconds)
+ Subscription.request_insert(callback, topic, verify_token,
secret,
+ lease_seconds=lease_seconds)
else:
Subscription.request_remove(callback, topic, verify_token)
logging.info('Queued %s request for callback = %s, '
@@ -1409,13 +1447,8 @@
else:
mode = 'unsubscribe'
- if ConfirmSubscription(mode, sub.topic, sub.callback,
- sub.verify_token, sub.lease_seconds):
- if mode == 'subscribe':
- Subscription.insert(sub.callback, sub.topic)
- else:
- Subscription.remove(sub.callback, sub.topic)
- else:
+ if not ConfirmSubscription(mode, sub.topic, sub.callback,
+ sub.verify_token, sub.secret,
sub.lease_seconds):
sub.confirm_failed()
################################################################################
@@ -1766,11 +1799,18 @@
def create_callback(sub):
return lambda *args: callback(sub, *args)
+ payload_utf8 = work.payload.encode('utf-8')
for sub in subscription_list:
+ headers = {
+ # TODO(bslatkin): Remove the 'or' here once migration is done.
+ 'Content-Type': work.content_type or 'text/xml',
+ 'X-Hub-Signature':
+ 'sha1=%s' % sha1_hmac(sub.secret or sub.verify_token,
payload_utf8),
+ }
urlfetch_async.fetch(sub.callback,
method='POST',
-
headers={'content-type': 'application/atom+xml'},
- payload=work.payload.encode('utf-8'),
+ headers=headers,
+ payload=payload_utf8,
async_proxy=async_proxy,
callback=create_callback(sub))
Modified: trunk/hub/main_test.py
==============================================================================
--- trunk/hub/main_test.py (original)
+++ trunk/hub/main_test.py Tue Jul 14 00:27:16 2009
@@ -65,6 +65,10 @@
self.assertEquals('hash_54f6638eb67ad389b66bbc3fa65f7392b0c2d270',
get_hash_key_name('and now testing a key'))
+ def testSha1Hmac(self):
+ self.assertEquals('d95abcea4b2a8b0219da7cb04c261639a7bd8c94',
+ main.sha1_hmac('secrat', 'mydatahere'))
+
def testIsValidUrl(self):
self.assertTrue(main.is_valid_url(
'https://example.com:443/path/to?handler=1&b=2'))
@@ -297,6 +301,8 @@
self.callback3 = 'http://example.com/third-callback-url'
self.topic = 'http://example.com/my-topic-url'
self.topic2 = 'http://example.com/second-topic-url'
+ self.token = 'token'
+ self.secret = 'my secrat'
self.callback_key_map = dict(
(Subscription.create_key_name(cb, self.topic), cb)
for cb in (self.callback, self.callback2, self.callback3))
@@ -312,9 +318,11 @@
lease_seconds = 1234
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token', lease_seconds, now=now))
+ self.callback, self.topic, self.token,
+ self.secret, lease_seconds=lease_seconds, now=now))
self.assertFalse(Subscription.request_insert(
- self.callback, self.topic, 'token', lease_seconds, now=now))
+ self.callback, self.topic, self.token,
+ self.secret, lease_seconds=lease_seconds, now=now))
sub = self.get_subscription()
self.assertEquals(Subscription.STATE_NOT_VERIFIED,
sub.subscription_state)
@@ -322,6 +330,8 @@
self.assertEquals(sha1_hash(self.callback), sub.callback_hash)
self.assertEquals(self.topic, sub.topic)
self.assertEquals(sha1_hash(self.topic), sub.topic_hash)
+ self.assertEquals(self.token, sub.verify_token)
+ self.assertEquals(self.secret, sub.secret)
self.assertEquals(now_datetime +
datetime.timedelta(seconds=lease_seconds),
sub.expiration_time)
self.assertEquals(lease_seconds, sub.lease_seconds)
@@ -331,16 +341,20 @@
now = lambda: now_datetime
lease_seconds = 1234
- self.assertTrue(Subscription.insert(self.callback, self.topic,
- lease_seconds, now=now))
- self.assertFalse(Subscription.insert(self.callback, self.topic,
- lease_seconds, now=now))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret,
+ lease_seconds=lease_seconds, now=now))
+ self.assertFalse(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret,
+ lease_seconds=lease_seconds, now=now))
sub = self.get_subscription()
self.assertEquals(Subscription.STATE_VERIFIED, sub.subscription_state)
self.assertEquals(self.callback, sub.callback)
self.assertEquals(sha1_hash(self.callback), sub.callback_hash)
self.assertEquals(self.topic, sub.topic)
self.assertEquals(sha1_hash(self.topic), sub.topic_hash)
+ self.assertEquals(self.token, sub.verify_token)
+ self.assertEquals(self.secret, sub.secret)
self.assertEquals(now_datetime +
datetime.timedelta(seconds=lease_seconds),
sub.expiration_time)
self.assertEquals(lease_seconds, sub.lease_seconds)
@@ -348,41 +362,43 @@
def testInsert_override(self):
"""Tests that insert will override the existing subscription state."""
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
self.assertEquals(Subscription.STATE_NOT_VERIFIED,
self.get_subscription().subscription_state)
- self.assertFalse(Subscription.insert(self.callback, self.topic))
+ self.assertFalse(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
self.assertEquals(Subscription.STATE_VERIFIED,
self.get_subscription().subscription_state)
def testRemove(self):
self.assertFalse(Subscription.remove(self.callback, self.topic))
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
self.assertTrue(Subscription.remove(self.callback, self.topic))
self.assertFalse(Subscription.remove(self.callback, self.topic))
def testRequestRemove(self):
self.assertFalse(Subscription.request_remove(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token))
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
self.assertTrue(Subscription.request_remove(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token))
self.assertEquals(Subscription.STATE_TO_DELETE,
self.get_subscription().subscription_state)
self.assertFalse(Subscription.request_remove(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token))
def testHasSubscribers_unverified(self):
"""Tests that unverified subscribers do not make the subscription
active."""
self.assertFalse(Subscription.has_subscribers(self.topic))
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
self.assertFalse(Subscription.has_subscribers(self.topic))
def testHasSubscribers_verified(self):
- self.assertTrue(Subscription.insert(self.callback, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
self.assertTrue(Subscription.has_subscribers(self.topic))
self.assertTrue(Subscription.remove(self.callback, self.topic))
self.assertFalse(Subscription.has_subscribers(self.topic))
@@ -391,26 +407,32 @@
"""Tests that unverified subscribers will not be retrieved."""
self.assertEquals([], Subscription.get_subscribers(self.topic, 10))
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
self.assertTrue(Subscription.request_insert(
- self.callback2, self.topic, 'token'))
+ self.callback2, self.topic, self.token, self.secret))
self.assertTrue(Subscription.request_insert(
- self.callback3, self.topic, 'token'))
+ self.callback3, self.topic, self.token, self.secret))
self.assertEquals([], Subscription.get_subscribers(self.topic, 10))
def testGetSubscribers_verified(self):
self.assertEquals([], Subscription.get_subscribers(self.topic, 10))
- self.assertTrue(Subscription.insert(self.callback, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret))
sub_list = Subscription.get_subscribers(self.topic, 10)
found_keys = set(s.key().name() for s in sub_list)
self.assertEquals(set(self.callback_key_map.keys()), found_keys)
def testGetSubscribers_count(self):
- self.assertTrue(Subscription.insert(self.callback, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret))
sub_list = Subscription.get_subscribers(self.topic, 1)
self.assertEquals(1, len(sub_list))
@@ -425,9 +447,12 @@
all_keys = ['hash_' + h for h in all_hashes]
all_callbacks = [self.callback_key_map[k] for k in all_keys]
- self.assertTrue(Subscription.insert(self.callback, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret))
def key_list(starting_at_callback):
sub_list = Subscription.get_subscribers(
@@ -442,13 +467,18 @@
def testGetSubscribers_multipleTopics(self):
"""Tests that separate topics do not overlap in subscriber queries."""
self.assertEquals([], Subscription.get_subscribers(self.topic2, 10))
- self.assertTrue(Subscription.insert(self.callback, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret))
self.assertEquals([], Subscription.get_subscribers(self.topic2, 10))
- self.assertTrue(Subscription.insert(self.callback2, self.topic2))
- self.assertTrue(Subscription.insert(self.callback3, self.topic2))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic2, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic2, self.token, self.secret))
sub_list = Subscription.get_subscribers(self.topic2, 10)
found_keys = set(s.key().name() for s in sub_list)
self.assertEquals(
@@ -459,12 +489,14 @@
def testGetConfirmWork(self):
"""Verifies that we can retrieve subscription confirmation work."""
- self.assertTrue(Subscription.request_insert(self.callback, self.topic,
- 'token'))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
- self.assertTrue(Subscription.request_remove(self.callback3, self.topic,
- 'token'))
+ self.assertTrue(Subscription.request_insert(
+ self.callback, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret))
+ self.assertTrue(Subscription.request_remove(
+ self.callback3, self.topic, self.token))
key_name1 = Subscription.create_key_name(self.callback, self.topic)
key_name2 = Subscription.create_key_name(self.callback2, self.topic)
@@ -485,7 +517,7 @@
sub_key = Subscription.create_key_name(self.callback, self.topic)
self.assertTrue(Subscription.request_insert(
- self.callback, self.topic, 'token'))
+ self.callback, self.topic, self.token, self.secret))
sub_key = Subscription.create_key_name(self.callback, self.topic)
sub = Subscription.get_by_key_name(sub_key)
self.assertEquals(0, sub.confirm_failures)
@@ -608,6 +640,8 @@
self.callback3 = 'http://example.com/third-callback-123'
self.callback4 = 'http://example.com/fourth-callback-1205'
self.header_footer = '<feed>\n<stuff>blah</stuff>\n<xmldata/></feed>'
+ self.token = 'verify token'
+ self.secret = 'some secret'
self.test_payloads = [
'<entry>article1</entry>',
'<entry>article2</entry>',
@@ -630,10 +664,14 @@
event.put()
work_key = event.key()
- Subscription.insert(self.callback, self.topic)
- Subscription.insert(self.callback2, self.topic)
- Subscription.insert(self.callback3, self.topic)
- Subscription.insert(self.callback4, self.topic)
+ Subscription.insert(
+ self.callback, self.topic, self.token, self.secret)
+ Subscription.insert(
+ self.callback2, self.topic, self.token, self.secret)
+ Subscription.insert(
+ self.callback3, self.topic, self.token, self.secret)
+ Subscription.insert(
+ self.callback4, self.topic, self.token, self.secret)
sub_list = Subscription.get_subscribers(self.topic, 10)
sub_keys = [s.key() for s in sub_list]
self.assertEquals(4, len(sub_list))
@@ -1120,7 +1158,8 @@
testutil.HandlerTestBase.setUp(self)
self.callback = 'http://example.com/my-subscriber'
- self.assertTrue(Subscription.insert(self.callback, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback, self.topic, 'token', 'secret'))
def tearDown(self):
"""Tears down the test harness."""
@@ -1342,7 +1381,7 @@
"""Tests when the content doesn't parse correctly."""
topic = 'http://example.com/my-topic'
callback = 'http://example.com/my-subscriber'
- self.assertTrue(Subscription.insert(callback, topic))
+ self.assertTrue(Subscription.insert(callback,
topic, 'token', 'secret'))
FeedToFetch.insert([topic])
urlfetch_test_stub.instance.expect(
'get', topic, 200, 'this does not parse')
@@ -1356,7 +1395,7 @@
'<meep><entry>wooh</entry></meep>')
topic = 'http://example.com/my-topic'
callback = 'http://example.com/my-subscriber'
- self.assertTrue(Subscription.insert(callback, topic))
+ self.assertTrue(Subscription.insert(callback,
topic, 'token', 'secret'))
FeedToFetch.insert([topic])
urlfetch_test_stub.instance.expect('get', topic, 200, data)
self.handle('post', ('topic', topic))
@@ -1369,7 +1408,7 @@
'<entry><id>1</id><updated>123</updated>wooh</entry></feed>')
topic = 'http://example.com/my-topic'
callback = 'http://example.com/my-subscriber'
- self.assertTrue(Subscription.insert(callback, topic))
+ self.assertTrue(Subscription.insert(callback,
topic, 'token', 'secret'))
FeedToFetch.insert([topic])
urlfetch_test_stub.instance.expect('get', topic, 200, data)
self.handle('post', ('topic', topic))
@@ -1413,6 +1452,22 @@
'<entry>article3</entry>\n'
'</feed>'
)
+
+ self.header_footer_rss = '<rss><channel></channel></rss>'
+ self.test_payloads_rss = [
+ '<item>article1</item>',
+ '<item>article2</item>',
+ '<item>article3</item>',
+ ]
+ self.expected_payload_rss = (
+ '<?xml version="1.0" encoding="utf-8"?>\n'
+ '<rss><channel>\n'
+ '<item>article1</item>\n'
+ '<item>article2</item>\n'
+ '<item>article3</item>\n'
+ '</channel></rss>'
+ )
+
self.bad_key =
db.Key.from_path(EventToDeliver.kind(), 'does_not_exist')
def tearDown(self):
@@ -1425,9 +1480,12 @@
def testNoExtraSubscribers(self):
"""Tests when a single chunk of delivery is enough."""
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 3
urlfetch_test_stub.instance.expect(
'post', self.callback1, 204, '',
request_payload=self.expected_payload)
@@ -1442,11 +1500,67 @@
self.assertEquals([], list(EventToDeliver.all()))
testutil.get_tasks(main.EVENT_QUEUE, expected_count=0)
+ def testHmacData(self):
+ """Tests that the content is properly signed with an HMAC."""
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret3'))
+ # Secret is empty on purpose here, so the verify_token will be used
instead.
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'my-token', ''))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret-stuff'))
+ main.EVENT_SUBSCRIBER_CHUNK_SIZE = 3
+ urlfetch_test_stub.instance.expect(
+ 'post', self.callback1, 204, '',
+ request_payload=self.expected_payload,
+ request_headers={
+ 'Content-Type': 'application/atom+xml',
+ 'X-Hub-Signature': 'sha1=3e9caf971b0833d15393022f5f01a47adf597af5'})
+ urlfetch_test_stub.instance.expect(
+ 'post', self.callback2, 200, '',
+ request_payload=self.expected_payload,
+ request_headers={
+ 'Content-Type': 'application/atom+xml',
+ 'X-Hub-Signature': 'sha1=4847815aae8578eff55d351bc84a159b9bd8846e'})
+ urlfetch_test_stub.instance.expect(
+ 'post', self.callback3, 204, '',
+ request_payload=self.expected_payload,
+ request_headers={
+ 'Content-Type': 'application/atom+xml',
+ 'X-Hub-Signature': 'sha1=8b0a9da7204afa8ae04fc9439755c556b1e38d99'})
+ event = EventToDeliver.create_event_for_topic(
+ self.topic, main.ATOM, self.header_footer, self.test_payloads)
+ event.put()
+ self.handle('post', ('event_key', str(event.key())))
+ self.assertEquals([], list(EventToDeliver.all()))
+ testutil.get_tasks(main.EVENT_QUEUE, expected_count=0)
+
+ def testRssContentType(self):
+ """Tests that the content type of an RSS feed is properly supplied."""
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ main.EVENT_SUBSCRIBER_CHUNK_SIZE = 3
+ urlfetch_test_stub.instance.expect(
+ 'post', self.callback1, 204, '',
+ request_payload=self.expected_payload_rss,
+ request_headers={
+ 'Content-Type': 'application/rss+xml',
+ 'X-Hub-Signature': 'sha1=1607313b6195af74f29158421f0a31aa25d680da'})
+ event = EventToDeliver.create_event_for_topic(
+ self.topic, main.RSS, self.header_footer_rss,
self.test_payloads_rss)
+ event.put()
+ self.handle('post', ('event_key', str(event.key())))
+ self.assertEquals([], list(EventToDeliver.all()))
+ testutil.get_tasks(main.EVENT_QUEUE, expected_count=0)
+
def testExtraSubscribers(self):
"""Tests when there are more subscribers to contact after delivery."""
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 1
event = EventToDeliver.create_event_for_topic(
self.topic, main.ATOM, self.header_footer, self.test_payloads)
@@ -1475,9 +1589,12 @@
def testBrokenCallbacks(self):
"""Tests that when callbacks return errors and are saved for later."""
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 2
event = EventToDeliver.create_event_for_topic(
self.topic, main.ATOM, self.header_footer, self.test_payloads)
@@ -1513,9 +1630,12 @@
raise runtime.DeadlineExceededError()
main.async_proxy.wait = deadline
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 2
event = EventToDeliver.create_event_for_topic(
self.topic, main.ATOM, self.header_footer, self.test_payloads)
@@ -1541,10 +1661,14 @@
This is an end-to-end test for push delivery failures and retries.
We'll
simulate multiple times through the failure list.
"""
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
- self.assertTrue(Subscription.insert(self.callback3, self.topic))
- self.assertTrue(Subscription.insert(self.callback4, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback3, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback4, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 3
event = EventToDeliver.create_event_for_topic(
self.topic, main.ATOM, self.header_footer, self.test_payloads)
@@ -1597,8 +1721,10 @@
def testUrlFetchFailure(self):
"""Tests the UrlFetch API raising exceptions while sending
notifications."""
- self.assertTrue(Subscription.insert(self.callback1, self.topic))
- self.assertTrue(Subscription.insert(self.callback2, self.topic))
+ self.assertTrue(Subscription.insert(
+ self.callback1, self.topic, 'token', 'secret'))
+ self.assertTrue(Subscription.insert(
+ self.callback2, self.topic, 'token', 'secret'))
main.EVENT_SUBSCRIBER_CHUNK_SIZE = 3
event = EventToDeliver.create_event_for_topic(
self.topic, main.ATOM, self.header_footer, self.test_payloads)
@@ -1951,7 +2077,7 @@
self.assertEquals(409, self.response_code())
# Unsubscribe
- Subscription.insert(self.callback, self.topic)
+ Subscription.insert(self.callback, self.topic,
self.verify_token, 'secret')
urlfetch_test_stub.instance.expect('get',
self.verify_callback_querystring_template % 'unsubscribe', 500, '')
self.handle('post',
@@ -2072,6 +2198,7 @@
main.get_random_challenge = lambda: self.challenge
self.sub_key = Subscription.create_key_name(self.callback, self.topic)
self.verify_token = 'the_token'
+ self.secret = 'teh secrat'
self.verify_callback_querystring_template = (
self.callback +
'?hub.verify_token=the_token'
@@ -2099,7 +2226,8 @@
def testSubscribeSuccessful(self):
self.assertTrue(db.get(KnownFeed.create_key(self.topic)) is None)
self.assertTrue(Subscription.get_by_key_name(self.sub_key) is None)
- Subscription.request_insert(self.callback, self.topic,
self.verify_token)
+ Subscription.request_insert(
+ self.callback, self.topic, self.verify_token, self.secret)
urlfetch_test_stub.instance.expect(
'get', self.verify_callback_querystring_template % 'subscribe',
200,
self.challenge)
@@ -2109,7 +2237,8 @@
def testSubscribeFailed(self):
self.assertTrue(Subscription.get_by_key_name(self.sub_key) is None)
- Subscription.request_insert(self.callback, self.topic,
self.verify_token)
+ Subscription.request_insert(
+ self.callback, self.topic, self.verify_token, self.secret)
urlfetch_test_stub.instance.expect('get',
self.verify_callback_querystring_template % 'subscribe', 500, '')
self.handle('post', ('subscription_key_name', self.sub_key))
@@ -2124,7 +2253,8 @@
def testSubscribeBadChallengeResponse(self):
"""Tests when the subscriber responds with a bad challenge."""
self.assertTrue(Subscription.get_by_key_name(self.sub_key) is None)
- Subscription.request_insert(self.callback, self.topic,
self.verify_token)
+ Subscription.request_insert(
+ self.callback, self.topic, self.verify_token, self.secret)
urlfetch_test_stub.instance.expect('get',
self.verify_callback_querystring_template % 'subscribe',
200, 'bad')
self.handle('post', ('subscription_key_name', self.sub_key))
@@ -2138,7 +2268,8 @@
def testUnsubscribeSuccessful(self):
self.assertTrue(Subscription.get_by_key_name(self.sub_key) is None)
- Subscription.insert(self.callback, self.topic)
+ Subscription.insert(
+ self.callback, self.topic, self.verify_token, self.secret)
Subscription.request_remove(self.callback, self.topic,
self.verify_token)
urlfetch_test_stub.instance.expect(
'get', self.verify_callback_querystring_template % 'unsubscribe',
200,
@@ -2149,7 +2280,8 @@
def testUnsubscribeFailed(self):
self.assertTrue(Subscription.get_by_key_name(self.sub_key) is None)
- Subscription.insert(self.callback, self.topic)
+ Subscription.insert(
+ self.callback, self.topic, self.verify_token, self.secret)
Subscription.request_remove(self.callback, self.topic,
self.verify_token)
urlfetch_test_stub.instance.expect('get',
self.verify_callback_querystring_template % 'unsubscribe', 500, '')
@@ -2165,7 +2297,8 @@
def testConfirmError(self):
"""Tests when an exception is raised while confirming a
subscription."""
called = [False]
- Subscription.request_insert(self.callback, self.topic,
self.verify_token)
+ Subscription.request_insert(
+ self.callback, self.topic, self.verify_token, self.secret)
# All exceptions should just fall through.
old_confirm = main.ConfirmSubscription
try:
Modified: trunk/hub/subscribe_debug.html
==============================================================================
--- trunk/hub/subscribe_debug.html (original)
+++ trunk/hub/subscribe_debug.html Tue Jul 14 00:27:16 2009
@@ -35,6 +35,10 @@
<label for="verify_token">Verify token:</label>
<input type="text" name="hub.verify_token" id="verify_token" value="">
</p>
+ <p>
+ <label for="secret">HMAC secret: <em>(optional)</em></label>
+ <input type="text" name="hub.secret" id="secret" value="">
+ </p>
<p><input type="submit" value="Do it"></p>
</form>
<em>Note: submission will result in a HTTP 204 response to acknowledge; in
browsers this looks like a no-op</em>
Jeff Lindsay
I'd like to see several options for verification: none, challenge
echo, hmac signature, and request echo (paypal style).
--
Jeff Lindsay
http://webhooks.org -- Make the web more programmable
http://shdh.org -- A party for hackers and thinkers
http://tigdb.com -- Discover indie games
http://progrium.com -- More interesting things
Brett Slatkin
> Yay! HMAC is a popular method for ongoing hook verification. In fact,
> I'd like to see several options for verification: none, challenge
> echo, hmac signature, and request echo (paypal style).
Yay indeed. I believe we have hub verification (i.e., subscriber
verifies the hub it's using) with the verify_token. We have subscriber
verification with the hub.challenge. Now hub.secret adds one more
level of certainty to the actual callbacks, making them verifiable as
coming from the hub.
How do you think those other verification styles could factor in here?
My thinking is we may want to have other HMAC hash functions in the
future (like sha256) but I don't see much need for other signature
types (though the OAuth spec has a few).
Jeff Lindsay
you were doing Google Code post-commit hook style HMAC, which uses an
out of band shared key ... as far as my limited security mind knows,
that should satisfy all three verification (hub, subscriber, each
callback request). Does that sound right? If so, than maybe it can be
an alternative to this upfront verification stuff.
--
Brett Slatkin
Finally catching up on my backlog! Tracking this in this issue:
http://code.google.com/p/pubsubhubbub/issues/detail?id=32
On Tue, Jul 14, 2009 at 11:08 AM, Jeff Lindsay<prog...@gmail.com> wrote:
>
> Hmm, maybe I didn't look closely at the implementation. I was assuming
> you were doing Google Code post-commit hook style HMAC, which uses an
> out of band shared key ... as far as my limited security mind knows,
> that should satisfy all three verification (hub, subscriber, each
> callback request). Does that sound right? If so, than maybe it can be
> an alternative to this upfront verification stuff.
So there are three "keys" in the protocol, each with a different purpose.
1) verify_token
- Required
- Used by the subscriber to verify that the hub verifying a
subscription request is the correct hub and not an impostor.
- Used once and thrown away.
- Secret known by both the publisher and subscriber.
- Transmitted from the subscriber to the hub and back again to verify
the subscription.
- Could be stolen by a snooping party
2) challenge
- Required
- Used by the Hub to verify the subscriber callback is an active
handler speaking the hubbub protocol.
- Used once and thrown away.
- Generated by the hub, transmitted to the subscriber.
- Ensures that a random URL cannot be subscribed and DoSed by a Hub.
3) secret
- Optional
- Sent from the subscriber to the hub in the initial subscription
request as the "hub.secret" parameter.
- Never sent from the hub to anyone else!
- Used by the hub to HMAC all content bodies that are delivered to the
subscriber.
- No nonce needed because feeds themselves are 1) idempotent and 2)
have enough nonce already in the feed envelope.
Arguably we could combine verify_token and secret into a single token
somehow by using an HMAC. For example:
1) Subscriber gives hub a secret key
2) Hub contacts subscriber with challenge record and an HMAC header of
the challenge using the secret
3) Subscriber verifies HMAC and then responds with success/fail
Downsides of requiring HMAC:
- verify_token alone is enough for most subscribers' needs
- HMACs aren't simple for most people
- HMACs are more paranoid than necessary with obfuscated URLs,
especially if you're using HTTPS
- HMAC algorithms change over time; we'll need to support many of
them, but we don't want to bake any of them into the spec as permanent
So I guess hub.secret is one of our first optional "extensions" to the
protocol. I still haven't written up the formal proposal for how it's
going to work. The hardest thing is the standard HMAC algorithm
discovery thing, figuring out what algorithms the hub and subscriber
both support. I need to figure that part out before I put it in the
spec proper. Working with FriendFeed to demo this out has been very
informative, though!
What do you think?
-Brett
Jeff Lindsay
think HMAC should be optional and agree with all the downsides of
required (except that I think there are simple libraries in most
languages?). So figuring out how to do extensions will be helpful.
This reminds me ... do you really see any other way to do verification
other than sync and async? I've been curious what other options there
could be there since I first read the spec...
I like that nonces aren't required. The less state the better. Also, I
see how verify_token isn't necessary with the HMAC, but I think they
should be separate to avoid confusion, especially of hmac is optional.
-jeff
--
Brett Slatkin
Additional verify types could be used for bootstrapping protocol upgrades. For example, if a value is "xmpp:jid" that could indicate to the hub that the subscriber could receive XMPP messages on a particlar JID instead of using HTTP. We could also use the verify type to query existing subscriptions.
On Jul 20, 2009 1:29 AM, "Jeff Lindsay" <prog...@gmail.com> wrote:
Ah yes! The subscriber can provide the secret. That works fine. Yes, I
think HMAC should be optional and agree with all the downsides of
required (except that I think there are simple libraries in most
languages?). So figuring out how to do extensions will be helpful.
This reminds me ... do you really see any other way to do verification
other than sync and async? I've been curious what other options there
could be there since I first read the spec...
I like that nonces aren't required. The less state the better. Also, I
see how verify_token isn't necessary with the HMAC, but I think they
should be separate to avoid confusion, especially of hmac is optional.
-jeff
On Sun, Jul 19, 2009 at 11:25 PM, Brett Slatkin<bsla...@gmail.com> wrote: > > Hey Jeff, > > Finall...
-- Jeff Lindsay http://webhooks.org -- Make the web more programmable http://shdh.org -- A party fo...
Jeff Lindsay
"hub.options" or something?
Brett Slatkin
> But does that make sense semantically? Maybe it could be renamed to
> "hub.options" or something?
Yeah you're right that it doesn't make too much sense. I guess what
I'm saying is it's better to allow for variance here as opposed to
having strict validation baked into the spec, ya know? Who knows what
we'll use it for!
Jeff Lindsay
API that doesn't make sense! ;)
--
Pádraic Brady
do OAuth and some OpenID for Zend Framework).
The main risk I see is with anything like 'Hubbub is that of Man In
The Middle misuse, which is precisely why I would love to see HMAC
supported. It wouldn't surprise me to see it become standard practice,
and despite any reservations, it's extremely easy to implement in any
language. For example, PHP has the standard ext/hash extension
(enabled by default) which puts HMAC-SHA1|SHA256 support a single call
away. No problems there. ext/openssl also implements RSA support, so
RSA-SHA1 is easy - though I don't like any RSA based signing since if
the private key is compromised, so is everything else. HMAC is just
more fine grained so a single loss of a secret, only impacts on one
Subscriber or one Hub (not many).
For this reason, the verify_token sent by the Subscriber is perfectly
worthless. It verifies nothing except that someone on the other side
echoed it back correctly. If you assume no MITM, than all is well.
With HMAC, it's not even required since the HMAC secret and generated
digital signature replace it (I'd prefer to simply ignore verify_token
in those cases since it offer no additional benefit).
What I see as emerging here is the same thing as happened in OAuth -
two distinct modes: PLAINTEXT (uses no digital signing but includes
verify_token) and HMAC-SHA1|SHA256 (uses digital signing but excludes
verify_token). PLAINTEXT would be preferred mode where a) Subscribers
just don't implement it, or b) the requests occur over HTTPS making
MITM unlikely.
What remains is...how to discover the shared secret? Is it supplied by
the Subscriber or the Hubs? How is it transmitted without hitting the
MITM problem of being intercepted and misused. OAuth solves this
(usually) by having new applications register to a Provider to receive
a secret. OpenID assumes no centralised anything (it's authentication
anyway, not authorisation), so it implements the Diffie-Hellman
protocol to generate a shared secret between two parties in a way that
can operate over an unsecured HTTP connection without interception
working. If one were to select from those, Diffie-Hellman offers the
better options since the relationship between Hubs and Subscribers is
many-to-many (unlike OAuth where it's many-to-one, e.g. the only
Twitter API in existence is hosted by Twitter, which is like saying
there's only ever going to be one Pubsubhubbub Hub).
I'm not negating the downsides to HMAC - it's just an option for
anything not happening over HTTPS where a Subscriber or Hub would like
to be absolutely sure who constructed any given request. Even when
discussing HMAC, it's likely also a good idea not to specify a source
for shared secrets - leave that as another Extension (perhaps Diffie-
Hellman - I can help here if ever needed) and/or left optional for
Subscribers to send in the initial subscription request (I don't like
this option outside of secured HTTPS - just means a MITM can read it
in plain text and have some fun with it. In any case, DH support for
implementors like myself is as simple as HMAC - all languages either
have native support or ready to roll libraries. PHP has it natively
implemented in ext/openssl as standard, for example.
If discovering support for HMAC-XXX is a problem, I'd just agree on
one that MUST be supported, and perhaps another that SHOULD be
supported (mainly because I like HMAC-SHA256 but the OAuth team
haven't seen it as necessary to go beyond HMAC-SHA1). Keep the options
small.
The only other problem I see, since the topic of verifying request
sources (via signing) is raised, is the lack of a nonce. Digital
signing doesn't prevent the MITM from intercepting and storing
subscription or unsubscription requests that are pre-signed with the
shared secret outside of HTTPS. Once they have a pre-signed request,
they can continually ping either party with it and it will be
validated unchallenged. Perhaps going overboard ;), but if a
Subscriber unsubs a Topic from a Hub, a MITM could ping the Hub over
and over with a Subscribe request. The Subscriber is going to see the
Hub continuing (despite the unsub from their perspective) to ping
their callback URL with updates. You can come up with a story in the
reverse.
Using a time limited nonce ensures each request has a unique identity
(in fact maintaining a hub.verify_token is a bit like this but not the
same, since it's not associated with a specific point in time) and
timestamp. Either party can now track nonces, ignore/discard requests
coming in with the same nonce from the same party as replays from a
potential MITM, and use a timestamp as the basis for setting a
timestamp limit, where messages older than the limit are ignored (this
mitigates the problem of tracking all nonces, for all time - which is
unrealistic - and allows for nonce reuse).
Anyway, I ramble on...and then some ;). I just figured as an
implementor removed from the design process (and being the new guy!)
it might throw some light on it. If I had a conclusion, it was that
Diffie-Hellman use with HMAC would make another possibly great
optional Extension. OAuth can escape the need though registration, so
they ignore it, but OpenID is more similar to how 'Hubbub operates
with decentralised Hubs everywhere.
Best regards,
Paddy
On Jul 21, 5:09 am, Jeff Lindsay <progr...@gmail.com> wrote:
> This is what bothers me about writing specs ... getting married to an
> API that doesn't make sense! ;)
>
> > On Mon, Jul 20, 2009 at 9:00 PM, Jeff Lindsay<progr...@gmail.com> wrote:
> >> But does that make sense semantically? Maybe it could be renamed to
> >> "hub.options" or something?
>
> > Yeah you're right that it doesn't make too much sense. I guess what
> > I'm saying is it's better to allow for variance here as opposed to
> > having strict validation baked into the spec, ya know? Who knows what
> > we'll use it for!
>
> --
Brett Slatkin
Thanks for the detailed look into the Hubbub security model. We
definitely need to write a wiki doc on the security model so people
can quickly understand the tradeoffs. Responses follow in paragraph
form so this doesn't get too long.
In short, I'm against reimplementing SSL on top of a plain-text
channel like they've done for OAuth. This is too complicated for
subscribers to get right. If people want secure feeds and secure
subscriptions they should use SSL for all legs of the PubSubHubbub
protocol. CPUs are cheap these days so I don't buy that excuse. The
SSL overhead is worth the speed and size tradeoffs, especially when
combined with persistent HTTP connections. We're going to update the
spec to further emphasize that subscribers should use HTTPS for all
URLs when possible (callback URLs, feed URLs).
SSL addresses almost every concern you've mentioned. But a couple
other points/questions:
* The role of verify_token is to prevent any DoS attacks against a
subscriber URL by a third party. It ensures that the agent requesting
the subscription was the originator of the subscription request.
* The role of the challenge response from subscribers is to prevent
any DoS attacks where someone tries to set up a bunch of fake
subscriptions for URLs that don't speak the protocol.
* With the HMAC extension enabled, subscribers can use SSL to connect
to the Hub (for subscription and secret exchange), but then plaintext
for the rest of the protocol. I believe this is the approach that most
subscribers will use.
* Even in the plain-text version of the protocol, having a nonce with
the HMAC isn't super important because the feed itself already
contains enough Nonce. For example, the <updated> tag is always
changing to indicate the arrival of new feed items. And even if you
did achieve a replay attack, all of the feed items should have guids
and can be de-duped without any difficulty.
* Do you think HMACs are necessary in the case of SSL? The callback
URL should be an obfuscated URL that acts like a shared secret. It's
transmitted over SSL in both directions. So it should be secure enough
in itself for a trust relationship between the hub and the subscriber.
However, it's not as strong as the provable authentication that an
HMAC would provide. Thus right now I'm inclined to leave HMAC out of
the base spec, and make it an extension to the protocol that
implementors MAY implement if they see the need.
Let me know if that clears things up.
-Brett
Pádraic Brady
>
>Thanks for the detailed look into the Hubbub security model. We
>definitely need to write a wiki doc on the security model so people
>can quickly understand the tradeoffs. Responses follow in paragraph
>form so this doesn't get too long.
>In short, I'm against reimplementing SSL on top of a plain-text
>channel like they've done for OAuth. This is too complicated for
>subscribers to get right. If people want secure feeds and secure
>subscriptions they should use SSL for all legs of the PubSubHubbub
>protocol. CPUs are cheap these days so I don't buy that excuse. The
>SSL overhead is worth the speed and size tradeoffs, especially when
>combined with persistent HTTP connections. We're going to update the
>spec to further emphasize that subscribers should use HTTPS for all
>URLs when possible (callback URLs, feed URLs).
I think what propels the argument is what we perceive as the target audience for Pubsubhubbub, and what we perceive as the uses for Pubsubhubbub. It's a decentralised open protocol that allows anyone of any size or income stream setup Hubs, Publishers and Subscribers under their complete control. For example, say tomorrow I decide it would be a great idea to setup a feed aggregator. It's a small operation, volunteer based, with a few hundred subscribers, and perhaps a hundred or less sites being aggregated. We'd like to offer outselves as a Hub to feeds being aggregated (we don't trust theirs, or would prefer additional reliability). I now need to go use SSL? I'm aggregating as is, have no authenticated users, no service API other than RSS/Atom feeds, and no secure data to speak of other than my administrator password. But you still want me to use SSL?
You can also look at what service APIs have wrought in their fascination with escaping SSL. Twitter? No SSL. My blog? No SSL. At least we accompany each other down the road ;). Who else? Google? Well, took a while, but they made it opt-in for GMail. Otherwise? No. Flickr? Nope. I could keep going. But then this is why OAuth was created - too many alternatives and no clear standard - and very little working over HTTPS with the sole exception of getting users to login if their approval is needed.
The point is that the number of sites running web services over https is not particularly huge. OAuth, based on AuthSub and dozens more, is an emerging common replacement for SSL for web services and the backbone of that is its optional (now taken for granted almost everywhere) use of HMAC digital signing. It's no SSL, but it's creeping into its seat in certain uses.
That's why I think imposing HTTPS on all parties won't work. It hasn't worked so far. Stating all Subscribers and Hubs must use HTTPS isn't going to turn the tide back. HMAC signing at least offers an alternative that needs no additional server changes and comes closer to the security benefit of SSL (without the eavesdropping benefit, or the guarantee of identity - it only guarantees a known source).
On your point that "This is too complicated for subscribers to get right." I disagree. It's very easy to implement and very simple to understand. The main barrier is understanding the methodology of why it's done that way. The process itself isn't a challenge. You have parameters, you encode them, you order them, you concatenate them, you hmac them (usually with whatever a native language has built in - PHP has ext/hash by default), attach the signature, and send by request or Authorization header. OAuth makes it look hard because the smaller important pieces are scattered all over among the OAuth protocol pieces at times, and the method of signing differs depending on the request scheme for including the signature (Pubsubhubbub only uses GET/POST querystring/body so far - that's even simpler).
Sorry for the lengthy challange - I'm not great at summarising myself ;). I'm not suggesting this is added into the Core Specification, but as an Extension is would offer an immense benefit to anyone wishing to adopt the protocol. In the case of an Extension the language on SSL would need to be toned down (from that mentioned) to refer to an Extension being used as an alternative being acceptable.
>SSL addresses almost every concern you've mentioned. But a couple
>other points/questions:
>* The role of verify_token is to prevent any DoS attacks against a
>subscriber URL by a third party. It ensures that the agent requesting
>the subscription was the originator of the subscription request.
>
>* The role of the challenge response from subscribers is to prevent
>any DoS attacks where someone tries to set up a bunch of fake
>subscriptions for URLs that don't speak the protocol.
>* With the HMAC extension enabled, subscribers can use SSL to connect
>to the Hub (for subscription and secret exchange), but then plaintext
>for the rest of the protocol. I believe this is the approach that most
>subscribers will use.
>* Even in the plain-text version of the protocol, having a nonce with
>the HMAC isn't super important because the feed itself already
>contains enough Nonce. For example, the <updated> tag is always
>changing to indicate the arrival of new feed items. And even if you
>did achieve a replay attack, all of the feed items should have guids
>and can be de-duped without any difficulty.
None of which changes your point really ;). You can use the feed as the nonce, but what about subscriptions and unsubscriptions? Without a feed in the body, the digital signing would still need protection against replays. If it's needed there, it's a tiny effort to add it to feed update exchanges since the architecture in any implementation would be reusable.
>* Do you think HMACs are necessary in the case of SSL? The callback
>URL should be an obfuscated URL that acts like a shared secret. It's
>transmitted over SSL in both directions. So it should be secure enough
>in itself for a trust relationship between the hub and the subscriber.
>However, it's not as strong as the provable authentication that an
>HMAC would provide. Thus right now I'm inclined to leave HMAC out of
>the base spec, and make it an extension to the protocol that
>implementors MAY implement if they see the need.
On the topic of a HMAC digital signature Extension - if you need any assistance let me know. More than happy to volunteer the time and effort.
Paddy
http://blog.astrumfutura.com
http://www.survivethedeepend.com
OpenID Europe Foundation Irish Representative
From: Brett Slatkin <bsla...@gmail.com>
To: pubsub...@googlegroups.com
Sent: Monday, August 3, 2009 5:18:28 PM
Subject: [pubsubhubbub] Re: [pubsubhubbub commit] r170 - Adds initial support for hub.secret and HMAC'ed payloads.
Brett Slatkin
Took me a while to get through your long reply, so sorry if I missed something.
To be clear, there are two aspects of requiring SSL: the client-side
and the server-side. Your arguments are primarily against requiring
SSL on the server-side. I understand your concerns. However, every
language and HTTP fetching library out there supports SSL as a client;
this gives us an easy way to exchange secrets with the Hub during
initial subscription. The secret can then be used for calculating the
HMAC of the payload over an unencrypted channel. I think this is good
enough for the very common use-case of people who don't want to change
their server setup at all in order to use PubSubHubbub. Do you agree
with that?
Configuration issues aside, I don't agree with your summary of SSL in
the present time. SSL certs used to require lots of money and
paperwork, but now it takes less than 5 minutes and ~$20. The CPU cost
of SSL has also dropped to being insignificant. The tradeoff is
simple: If you don't want to use SSL, then you should be okay with the
reliability guarantees of HMAC (and use an SSL client for secret
exchange); if you want reliability and security, you should use SSL.
These options seem pretty natural to me.
I think we are in agreement, though, that we need to formalize this
HMAC extension to the spec, and seriously consider putting it in the
core spec for the benefit of all the users out there that won't take
the time to reconfigure their servers for SSL.
> None of which changes your point really ;). You can use the feed as the
> nonce, but what about subscriptions and unsubscriptions? Without a feed in
> the body, the digital signing would still need protection against replays.
> If it's needed there, it's a tiny effort to add it to feed update exchanges
> since the architecture in any implementation would be reusable.
I don't understand the risk you're highlighting here, could you
explain? I believe the subscription part of the protocol is reliable
as-is. I'd be curious to understand any attacks you can come up with
that don't require physical compromise of the subscriber's network.
Thanks,
-Brett
Pádraic Brady
>To be clear, there are two aspects of requiring SSL: the client-side
>and the server-side. Your arguments are primarily against requiring
>SSL on the server-side. I understand your concerns. However, every
>language and HTTP fetching library out there supports SSL as a client;
>this gives us an easy way to exchange secrets with the Hub during
>initial subscription. The secret can then be used for calculating the
>HMAC of the payload over an unencrypted channel. I think this is good
>enough for the very common use-case of people who don't want to change
>their server setup at all in order to use PubSubHubbub. Do you agree
>with that?
>Configuration issues aside, I don't agree with your summary of SSL in
>the present time. SSL certs used to require lots of money and
>paperwork, but now it takes less than 5 minutes and ~$20. The CPU cost
>of SSL has also dropped to being insignificant. The tradeoff is
>simple: If you don't want to use SSL, then you should be okay with the
>reliability guarantees of HMAC (and use an SSL client for secret
>exchange); if you want reliability and security, you should use SSL.
>These options seem pretty natural to me.
Like I said, I have no disagreement about SSL vs HMAC. SSL wins hands down.
>I think we are in agreement, though, that we need to formalize this
>HMAC extension to the spec, and seriously consider putting it in the
>core spec for the benefit of all the users out there that won't take
>the time to reconfigure their servers for SSL.
>> None of which changes your point really ;). You can use the feed as the
>> nonce, but what about subscriptions and unsubscriptions? Without a feed in
>> the body, the digital signing would still need protection against replays.
>> If it's needed there, it's a tiny effort to add it to feed update exchanges
>> since the architecture in any implementation would be reusable.
>
>I don't understand the risk you're highlighting here, could you
>explain? I believe the subscription part of the protocol is reliable
>as-is. I'd be curious to understand any attacks you can come up with
>that don't require physical compromise of the subscriber's network.
As a method to mitigate this, nonces are supposed to be stored forever to guarantee every single nonce generated is unique and reuse is avoided. Adding a timestamp removes "forever" and replaces it with "really long time" - any request older than the nonce horizon can be immediately discarded.
Although it's far-fetched, it's not hard to see how it can happen. All you need is someone using a poor method of generating random tokens, whether through ignorance, incompetence or an unintended bug in their implementation. The more Hubs/Subscribers using that implementation, the worse the impact. The spec also offers little guidance here. A subscriber is told a verify_token is OPTIONAL (the MITM can do half the work now) and no guidance is stated on whether it should be a static token, or a randomly generated one. The Hub's challenge is REQUIRED and is defined as a random string.
I think that made sense...:)