[PATCH 2/2] get: support "pulls" via --source-url URL

[Rob Browning]

Introduce a --source-url URL argument to allow pulling the data from a
remote as compared to "bup on REMOTE get" which actually has the
remote "push" the data to the current host.

Valid URLs are now file://PATH, ssh://[USER@]HOST[:PORT][/PATH], and
bup://HOST[:PORT][/PATH]. See the bup(1), bup-get(1), and release
notes for additional information.

The pull requires less trust in the remote, and avoids the need for a
remote index-cache.

Add client.rev_list(ref) support (single ref) for compatibility with
git.rev_list.

Signed-off-by: Rob Browning <r...@defaultvalue.org>
Tested-by: Rob Browning <r...@defaultvalue.org>
---
Documentation/bup-get.1.md | 5 +++--
Documentation/bup-on.1.md | 3 ++-
lib/bup/client.py | 10 +++++++---
lib/bup/cmd/get.py | 9 ++++++---
lib/bup/repo/__init__.py | 25 +++++++++++++++++++++----
note/main.md | 6 ++++++
test/ext/test_get.py | 10 +++++++---
7 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/Documentation/bup-get.1.md b/Documentation/bup-get.1.md
index 89c7e236..514bffdd 100644
--- a/Documentation/bup-get.1.md
+++ b/Documentation/bup-get.1.md
@@ -114,8 +114,9 @@ used to help test before/after results.)

# OPTIONS

--s, \--source=*path*
-: use *path* as the source repository, instead of the default.
+\-s *path*, \-S *url, \--source *path*, \--source-url *url*
+: use *path* or *url* as the source repository, instead of the
+ default. See `bup`(1) REPOSITORY URLS for further information.

-r, \--remote=[*user*@]*host*:[*path*], \--remote=URL
: write the *ref*s to the specified remote repository, by default
diff --git a/Documentation/bup-on.1.md b/Documentation/bup-on.1.md
index c8c4fae0..5db69b99 100644
--- a/Documentation/bup-on.1.md
+++ b/Documentation/bup-on.1.md
@@ -14,7 +14,8 @@ bup on [*user*@]*host*[:*port*] save ...

bup on [*user*@]*host*[:*port*] split ...

-bup on [*user*@]*host*[:*port*] get ...
+bup on [*user*@]*host*[:*port*] get ...
+(Prefer `bup get --source-url ssh://<hostname>...`)

# DESCRIPTION

diff --git a/lib/bup/client.py b/lib/bup/client.py
index 53f6e70b..2fb65b5e 100644
--- a/lib/bup/client.py
+++ b/lib/bup/client.py
@@ -577,7 +577,7 @@ class Client:
raise ClientError('Invalid reference name in %r' % line)
yield name, unhexlify(oidx)

- def rev_list(self, refs, parse=None, format=None):
+ def rev_list(self, ref_or_refs, parse=None, format=None):
"""See git.rev_list for the general semantics, but note that with the
current interface, the parse function must be able to handle
(consume) any blank lines produced by the format because the
@@ -585,12 +585,16 @@ class Client:
as a terminator for the entire rev-list result.

"""
+ if isinstance(ref_or_refs, bytes):
+ refs = (ref_or_refs,)
+ else:
+ refs = ref_or_refs
if format:
assert b'\n' not in format
assert parse
for ref in refs:
- assert ref
- assert b'\n' not in ref
+ assert ref, ref
+ assert b'\n' not in ref, ref
with self._line_based_call('rev-list') as call:
self.conn.write(b'\n')
if format:
diff --git a/lib/bup/cmd/get.py b/lib/bup/cmd/get.py
index d23e35b8..5c779ac4 100644
--- a/lib/bup/cmd/get.py
+++ b/lib/bup/cmd/get.py
@@ -29,7 +29,7 @@ from bup.helpers import \
from bup.io import path_msg
from bup.pwdgrp import userfullname, username
from bup.repair import valid_repair_id
-from bup.repo import LocalRepo
+from bup.repo import LocalRepo, parse_url_arg
from bup.rewrite import RepairInfo, Rewriter
import bup.repo as repo

@@ -51,8 +51,8 @@ argspec = (
('-v, --verbose',
'increase log output (can be specified more than once)'),
('-q, --quiet', "don't show progress meter"),
- ('-s SOURCE, --source SOURCE',
- 'path to the source repository (defaults to BUP_DIR)'),
+ ('-s PATH, -S URL --source PATH, --source-url URL',
+ 'the source repository (defaults to BUP_DIR)'),
('-r REMOTE, --remote REMOTE',
'hostname:/path/to/repo of remote destination repository'),
('-t --print-trees', 'output a tree id for each ref set'),
@@ -248,6 +248,9 @@ def parse_args(args):
opt.target_specs.append(make_spec(method=arg[2:-1].decode('ascii'),
src=ref, dest=dest))
pending_method_context = {}
+ elif arg in (b'-S', b'--source-url'):
+ (opt.source,), remaining = require_n_args_or_die(1, remaining)
+ opt.source_loc = parse_url_arg('--source-url', opt.source, misuse)
elif arg in (b'-s', b'--source'):
(opt.source,), remaining = require_n_args_or_die(1, remaining)
opt.source_loc = url.for_path(opt.source)
diff --git a/lib/bup/repo/__init__.py b/lib/bup/repo/__init__.py
index 93fa164b..55cc4690 100644
--- a/lib/bup/repo/__init__.py
+++ b/lib/bup/repo/__init__.py
@@ -4,14 +4,15 @@ from typing import Callable, NoReturn

from bup.compat import argv_bytes
from bup.config import url_for_remote_opt
+from bup.io import path_msg as pm
from bup.path import defaultrepo
-from bup.repo import local, remote
-from bup.url import URL, dot_decoded_url_path
+from bup.repo.local import LocalRepo
+from bup.repo.remote import RemoteRepo
+from bup.url import URL, dot_decoded_url_path, parse_bytes_path_url
import bup.client as client, bup.url as url


-LocalRepo = local.LocalRepo
-RemoteRepo = remote.RemoteRepo
+public_schemes = frozenset([b'file', b'ssh', b'bup'])


def location_url(location):
@@ -61,3 +62,19 @@ def for_location(location, **kwargs):
if isinstance(location, client.Config):
return RemoteRepo(location, **kwargs)
raise Exception(f'unexpected repository location {location}')
+
+
+def parse_url_arg(arg, val, misuse):
+ """Call misuse(err_msg) if val is not a valid URL, otherwise
+ return a corresponding URL instance."""
+ url = parse_bytes_path_url(val)
+ if not url:
+ misuse(f'invalid {arg} {pm(val)}')
+ if isinstance(url, str):
+ misuse(f'invalid {arg} {pm(val)} ({url})')
+ url = location_url(url)
+ if url.scheme not in public_schemes:
+ misuse(f'invalid {arg} schema in {pm(val)}')
+ if url.scheme == b'file' and url.auth:
+ misuse(f'{arg} {pm(val)} URL has extra leading slashes or an authority')
+ return url
diff --git a/note/main.md b/note/main.md
index c5682dc2..569fa373 100644
--- a/note/main.md
+++ b/note/main.md
@@ -130,6 +130,12 @@ General
example, replacing paths with missing contents with synthesized
"repair files". See `bup-get`(1) for additional information.

+* `bup get` has added a `-S, --source-url URL` option that specifies
+ the source repository and should generally be preferred to `bup on
+ HOST get` since it requires less trust in the remote, and avoids the
+ need for a remote index-cache. See `bup-get(1)` for additional
+ information.
+
* `bup` now verifies the hash of the incoming remote data when
requesting a specific hash (rather than an arbitrary "ref").
Combined with `--source-url`, this further decreases the trust
diff --git a/test/ext/test_get.py b/test/ext/test_get.py
index 99c30b2b..550e0cd9 100644
--- a/test/ext/test_get.py
+++ b/test/ext/test_get.py
@@ -262,8 +262,12 @@ def _run_get(disposition, method, what, rewrite=None):
get_cmd = (bup_cmd, b'-d', b'get-dest',
b'get', b'-vvct', b'--print-tags', b'-s', b'get-src',
b'-r', b'-:' + getcwd() + b'/get-dest')
+ elif disposition == 'get-from':
+ get_cmd = (bup_cmd, b'-d', b'get-dest',
+ b'get', b'-vvct', b'--print-tags',
+ b'--source-url', b'ssh://' + getcwd() + b'/get-src')
else:
- raise Exception('error: unexpected get disposition ' + repr(disposition))
+ raise Exception(f'error: unexpected get disposition {disposition!r}')

cmd = (*get_cmd, b'--rewrite' if rewrite else b'--copy')
if isinstance(what, bytes):
@@ -1084,10 +1088,10 @@ def create_get_src():
# FIXME: this fails in a strange way:
# WVPASS given nothing get --ff not-there

-dispositions_to_test = ('get',)
+dispositions_to_test = ('get-from',)

if int(environ.get(b'BUP_TEST_LEVEL', b'0')) >= 11:
- dispositions_to_test += ('get-on', 'get-to')
+ dispositions_to_test += ('get-on', 'get-to', 'get-from')

categories = ('replace', 'universal', 'ff', 'append', 'pick_force', 'pick_noforce', 'new_tag', 'unnamed')

--
2.47.3