[PATCH 1/2] Linux TPROXY support

[Roger]

This will enable the support for UDP and IPv6. See
Documentation/networking/tproxy.txt in linux kernel project.
---
client.py | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/client.py b/client.py
index 0ff5f2b..f0e4c2e 100644
--- a/client.py
+++ b/client.py
@@ -12,6 +12,8 @@ def got_signal(signum, frame):


_pidname = None
+IP_TRANSPARENT = 19
+
def check_daemon(pidfile):
global _pidname
_pidname = os.path.abspath(pidfile)
@@ -345,6 +347,10 @@ def main(listenip, ssh_cmd, remotename, python, latency_control, dns,
debug2(' %d' % port)
listener = socket.socket()
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ try:
+ listener.setsockopt(socket.SOL_IP, IP_TRANSPARENT, 1)
+ except socket.error, e:
+ last_e = e
dnslistener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
dnslistener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
--
1.7.4.1

[Roger]

The following 2 patches enables Linux TPROXY and then IPv6 sockets.

In Linux IPv6 there is no NAT table anymore, so TPROXY is necessary.
It also opens the possibility to forward UDP packets.

Thanks.

Roger (2):
Linux TPROXY support
Support for IPv6 mode based on TPROXY

client.py | 39 +++++++++++++++++++++--------
firewall.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++------
main.py | 56 +++++++++++++++++++++++++++++++++++------
server.py | 4 +-
ssnet.py | 7 +++-
5 files changed, 153 insertions(+), 32 deletions(-)

--
1.7.4.1

[Roger]

Added a new mode for IPv6 with switch '-6'. In this mode sshuttle
will listen on IPv6 address and handle the firewall with ip6tables.
---
client.py | 33 ++++++++++++++++--------

firewall.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++------
main.py | 56 +++++++++++++++++++++++++++++++++++------
server.py | 4 +-
ssnet.py | 7 +++-

5 files changed, 147 insertions(+), 32 deletions(-)

diff --git a/client.py b/client.py
index f0e4c2e..130e8aa 100644
--- a/client.py
+++ b/client.py
@@ -98,15 +98,16 @@ def original_dst(sock):


class FirewallClient:
- def __init__(self, port, subnets_include, subnets_exclude, dnsport):
+ def __init__(self, port, subnets_include, subnets_exclude, dnsport, ipv6):
self.port = port
self.auto_nets = []
self.subnets_include = subnets_include
self.subnets_exclude = subnets_exclude
self.dnsport = dnsport
+ self.ipv6 = ipv6
argvbase = ([sys.argv[1], sys.argv[0], sys.argv[1]] +
['-v'] * (helpers.verbose or 0) +
- ['--firewall', str(port), str(dnsport)])
+ ['--firewall', str(port), str(dnsport), str(ipv6 or '0')])
if ssyslog._p:
argvbase += ['--syslog']
argv_tries = [
@@ -178,7 +179,7 @@ class FirewallClient:


def _main(listener, fw, ssh_cmd, remotename, python, latency_control,
- dnslistener, seed_hosts, auto_nets,
+ dnslistener, ipv6, seed_hosts, auto_nets,
syslog, daemon):
handlers = []
if helpers.verbose >= 1:
@@ -190,7 +191,7 @@ def _main(listener, fw, ssh_cmd, remotename, python, latency_control,
try:
(serverproc, serversock) = ssh.connect(ssh_cmd, remotename, python,
stderr=ssyslog._p and ssyslog._p.stdin,
- options=dict(latency_control=latency_control))
+ options=dict(latency_control=latency_control, ipv6=ipv6))
except socket.error, e:
if e.args[0] == errno.EPIPE:
raise Fatal("failed to establish ssh session (1)")
@@ -267,7 +268,10 @@ def _main(listener, fw, ssh_cmd, remotename, python, latency_control,
return
else:
raise
- dstip = original_dst(sock)
+ if ipv6:
+ dstip = sock.getsockname();
+ else:
+ dstip = original_dst(sock)
debug1('Accept: %s:%r -> %s:%r.\n' % (srcip[0],srcip[1],
dstip[0],dstip[1]))
if dstip[1] == listener.getsockname()[1] and islocal(dstip[0]):
@@ -279,7 +283,7 @@ def _main(listener, fw, ssh_cmd, remotename, python, latency_control,
log('warning: too many open channels. Discarded connection.\n')
sock.close()
return
- mux.send(chan, ssnet.CMD_CONNECT, '%s,%s' % dstip)
+ mux.send(chan, ssnet.CMD_CONNECT, '%s,%r' % (dstip[0], dstip[1]))
outwrap = MuxWrapper(mux, chan)
handlers.append(Proxy(SockWrapper(sock, sock), outwrap))
handlers.append(Handler([listener], onaccept))
@@ -324,7 +328,7 @@ def _main(listener, fw, ssh_cmd, remotename, python, latency_control,

def main(listenip, ssh_cmd, remotename, python, latency_control, dns,

- seed_hosts, auto_nets,
+ ipv6, seed_hosts, auto_nets,
subnets_include, subnets_exclude, syslog, daemon, pidfile):
if syslog:
ssyslog.start_syslog()
@@ -345,13 +349,19 @@ def main(listenip, ssh_cmd, remotename, python, latency_control, dns,
debug2('Binding:')
for port in ports:
debug2(' %d' % port)
- listener = socket.socket()
+ if ipv6:
+ listener = socket.socket(socket.AF_INET6)
+ else:
+ listener = socket.socket(socket.AF_INET)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
listener.setsockopt(socket.SOL_IP, IP_TRANSPARENT, 1)
except socket.error, e:
last_e = e
- dnslistener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ if ipv6:
+ dnslistener = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+ else:
+ dnslistener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
dnslistener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
listener.bind((listenip[0], port))
@@ -376,12 +386,13 @@ def main(listenip, ssh_cmd, remotename, python, latency_control, dns,
dnsport = 0
dnslistener = None

- fw = FirewallClient(listenip[1], subnets_include, subnets_exclude, dnsport)
+ fw = FirewallClient(listenip[1], subnets_include, subnets_exclude, dnsport, ipv6)

try:
return _main(listener, fw, ssh_cmd, remotename,
python, latency_control, dnslistener,
- seed_hosts, auto_nets, syslog, daemon)
+ ipv6, seed_hosts, auto_nets, syslog,
+ daemon)
finally:
try:
if daemon:
diff --git a/firewall.py b/firewall.py
index 7767d43..417712b 100644
--- a/firewall.py
+++ b/firewall.py
@@ -7,8 +7,14 @@ from helpers import *
IPPROTO_DIVERT = 254


-def ipt_chain_exists(name):
- argv = ['iptables', '-t', 'nat', '-nL']
+def ipt_chain_exists(name, ipv6):
+ if ipv6:
+ table = 'mangle'
+ cmd = 'ip6tables'
+ else:
+ table = 'nat'
+ cmd = 'iptables'
+ argv = [cmd, '-t', table, '-nL']
p = ssubprocess.Popen(argv, stdout = ssubprocess.PIPE)
for line in p.stdout:
if line.startswith('Chain %s ' % name):
@@ -25,6 +31,13 @@ def ipt(*args):
if rv:
raise Fatal('%r returned %d' % (argv, rv))

+def ipt6(*args):
+ argv = ['ip6tables', '-t', 'mangle'] + list(args)
+ debug1('>> %s\n' % ' '.join(argv))
+ rv = ssubprocess.call(argv)
+ if rv:
+ raise Fatal('%r returned %d' % (argv, rv))
+

_no_ttl_module = False
def ipt_ttl(*args):
@@ -52,11 +65,11 @@ def ipt_ttl(*args):
# multiple copies shouldn't have overlapping subnets, or only the most-
# recently-started one will win (because we use "-I OUTPUT 1" instead of
# "-A OUTPUT").
-def do_iptables(port, dnsport, subnets):
+def do_iptables(port, dnsport, ipv6, subnets):
chain = 'sshuttle-%s' % port

# basic cleanup/setup of chains
- if ipt_chain_exists(chain):
+ if ipt_chain_exists(chain, ipv6):
ipt('-D', 'OUTPUT', '-j', chain)
ipt('-D', 'PREROUTING', '-j', chain)
ipt('-F', chain)
@@ -94,6 +107,51 @@ def do_iptables(port, dnsport, subnets):
'--dport', '53',
'--to-ports', str(dnsport))

+def do_ip6tables(port, dnsport, ipv6, subnets):
+ mark_chain = 'sshuttle-m-%s' % port
+ tproxy_chain = 'sshuttle-t-%s' % port
+
+ # basic cleanup/setup of chains
+ if ipt_chain_exists(mark_chain, ipv6):
+ ipt6('-D', 'OUTPUT', '-j', mark_chain)
+ ipt6('-F', mark_chain)
+ ipt6('-X', mark_chain)
+
+ if ipt_chain_exists(tproxy_chain, ipv6):
+ ipt6('-D', 'PREROUTING', '-j', tproxy_chain)
+ ipt6('-F', tproxy_chain)
+ ipt6('-X', tproxy_chain)
+
+ if subnets or dnsport:
+ ipt6('-N', mark_chain)
+ ipt6('-F', mark_chain)
+ ipt6('-N', tproxy_chain)
+ ipt6('-F', tproxy_chain)
+ ipt6('-I', 'OUTPUT', '1', '-j', mark_chain)
+ ipt6('-I', 'PREROUTING', '1', '-j', tproxy_chain)
+ ipt6('-A', tproxy_chain, '-m', 'socket', '-j', 'RETURN',
+ '-m', 'tcp', '-p', 'tcp')
+
+ #ipt6('-A', mark_chain, '-o', 'lo', '-j', 'RETURN')
+
+ if subnets:
+ for swidth,sexclude,snet in sorted(subnets, reverse=True):
+ if sexclude:
+ ipt6('-A', mark_chain, '-j', 'RETURN',
+ '--dest', '%s/%s' % (snet,swidth),
+ '-m', 'tcp', '-p', 'tcp')
+ ipt6('-A', tproxy_chain, '-j', 'RETURN',
+ '--dest', '%s/%s' % (snet,swidth),
+ '-m', 'tcp', '-p', 'tcp')
+ else:
+ ipt6('-A', mark_chain, '-j', 'MARK', '--set-mark', '1',
+ '--dest', '%s/%s' % (snet,swidth),
+ '-m', 'tcp', '-p', 'tcp')
+ ipt6('-A', tproxy_chain, '-j', 'TPROXY', '--tproxy-mark', '0x1/0x1',
+ '--dest', '%s/%s' % (snet,swidth),
+ '-m', 'tcp', '-p', 'tcp',
+ '--on-port', str(port))
+

def ipfw_rule_exists(n):
argv = ['ipfw', 'list']
@@ -199,7 +257,7 @@ def ipfw(*args):
raise Fatal('%r returned %d' % (argv, rv))


-def do_ipfw(port, dnsport, subnets):
+def do_ipfw(port, dnsport, ipv6, subnets):
sport = str(port)
xsport = str(port+1)

@@ -350,7 +408,7 @@ def restore_etc_hosts(port):
# exit. In case that fails, it's not the end of the world; future runs will
# supercede it in the transproxy list, at least, so the leftover rules
# are hopefully harmless.
-def main(port, dnsport, syslog):
+def main(port, dnsport, ipv6, syslog):
assert(port > 0)
assert(port <= 65535)
assert(dnsport >= 0)
@@ -362,7 +420,10 @@ def main(port, dnsport, syslog):
if program_exists('ipfw'):
do_it = do_ipfw
elif program_exists('iptables'):
- do_it = do_iptables
+ if ipv6:
+ do_it = do_ip6tables
+ else:
+ do_it = do_iptables
else:
raise Fatal("can't find either ipfw or iptables; check your PATH")

@@ -408,7 +469,7 @@ def main(port, dnsport, syslog):
try:
if line:
debug1('firewall manager: starting transproxy.\n')
- do_wait = do_it(port, dnsport, subnets)
+ do_wait = do_it(port, dnsport, ipv6, subnets)
sys.stdout.write('STARTED\n')

try:
@@ -437,5 +498,5 @@ def main(port, dnsport, syslog):
debug1('firewall manager: undoing changes.\n')
except:
pass
- do_it(port, 0, [])
+ do_it(port, 0, ipv6, [])
restore_etc_hosts(port)
diff --git a/main.py b/main.py
index 1cf00af..1963227 100644
--- a/main.py
+++ b/main.py
@@ -25,6 +25,23 @@ def parse_subnets(subnets_str):
subnets.append(('%d.%d.%d.%d' % (a,b,c,d), width))
return subnets

+# list of:
+# 1:2::3/64 or just 1:2::3
+def parse_subnets6(subnets_str):
+ subnets = []
+ for s in subnets_str:
+ m = re.match(r'(?:([a-fA-F\d:]+))?(?:/(\d+))?$', s)
+ if not m:
+ raise Fatal('%r is not a valid IP subnet format' % s)
+ (net,width) = m.groups()
+ if width == None:
+ width = 128
+ else:
+ width = int(width)
+ if width > 128:
+ raise Fatal('*/%d is greater than the maximum of 128' % width)
+ subnets.append((net, width))
+ return subnets

# 1.2.3.4:567 or just 1.2.3.4 or just 567
def parse_ipport(s):
@@ -43,6 +60,16 @@ def parse_ipport(s):
a = b = c = d = 0
return ('%d.%d.%d.%d' % (a,b,c,d), port)

+# [1:2::3]:456 or [1:2::3] or 456
+
+def parse_ipport6(s):
+ s = str(s)
+ m = re.match(r'(?:\[([^]]*)])?(?::)?(?:(\d+))?$', s)
+ if not m:
+ raise Fatal('%s is not a valid IP:port format' % s)
+ (ip,port) = m.groups()
+ (ip,port) = (ip or '::', int(port or 0))
+ return (ip, port)

optspec = """
sshuttle [-l [ip:]port] [-r [username@]sshserver[:port]] <subnets...>
@@ -50,10 +77,11 @@ sshuttle --server
sshuttle --firewall <port> <subnets...>
sshuttle --hostwatch
--
-l,listen= transproxy to this ip address and port number [127.0.0.1:0]
+l,listen= transproxy to this ip address and port number
H,auto-hosts scan for remote hostnames and update local /etc/hosts
N,auto-nets automatically determine subnets to route
dns capture local DNS requests and forward to the remote DNS server
+6,ipv6 mode for dealing with IPv6
python= path to python interpreter on the remote server
r,remote= ssh hostname (and optional username) of remote sshuttle server
x,exclude= exclude this subnet (can be used more than once)
@@ -84,18 +112,23 @@ try:
if len(extra) != 0:
o.fatal('no arguments expected')
server.latency_control = opt.latency_control
+ server.ipv6 = opt.ipv6
sys.exit(server.main())
elif opt.firewall:
- if len(extra) != 2:
- o.fatal('exactly two arguments expected')
- sys.exit(firewall.main(int(extra[0]), int(extra[1]), opt.syslog))
+ if len(extra) != 3:
+ o.fatal('exactly three arguments expected')
+ sys.exit(firewall.main(int(extra[0]), int(extra[1]),
+ int(extra[2]), opt.syslog))
elif opt.hostwatch:
sys.exit(hostwatch.hw_main(extra))
else:
if len(extra) < 1 and not opt.auto_nets:
o.fatal('at least one subnet (or -N) expected')
includes = extra
- excludes = ['127.0.0.0/8']
+ if not opt.ipv6:
+ excludes = ['127.0.0.0/8']
+ else:
+ excludes = [] #FIXME
for k,v in flags:
if k in ('-x','--exclude'):
excludes.append(v)
@@ -110,16 +143,23 @@ try:
sh = []
else:
sh = None
- sys.exit(client.main(parse_ipport(opt.listen or '0.0.0.0:0'),
+ if opt.ipv6:
+ ipport = parse_ipport6(opt.listen or '[::]:0')
+ do_parse_subnets = parse_subnets6
+ else:
+ ipport = parse_ipport(opt.listen or '127.0.0.1:0')
+ do_parse_subnets = parse_subnets
+ sys.exit(client.main(ipport,
opt.ssh_cmd,
remotename,
opt.python,
opt.latency_control,
opt.dns,
+ opt.ipv6,
sh,
opt.auto_nets,
- parse_subnets(includes),
- parse_subnets(excludes),
+ do_parse_subnets(includes),
+ do_parse_subnets(excludes),
opt.syslog, opt.daemon, opt.pidfile))
except Fatal, e:
log('fatal: %s\n' % e)
diff --git a/server.py b/server.py
index 45bc2fc..f04309f 100644
--- a/server.py
+++ b/server.py
@@ -131,7 +131,7 @@ def main():
else:
helpers.logprefix = 'server: '
debug1('latency control setting = %r\n' % latency_control)
-
+ debug1('IPv6 mode = %r\n' % ipv6)
routes = list(list_routes())
debug1('available routes:\n')
for r in routes:
@@ -180,7 +180,7 @@ def main():
def new_channel(channel, data):
(dstip,dstport) = data.split(',', 1)
dstport = int(dstport)
- outwrap = ssnet.connect_dst(dstip,dstport)
+ outwrap = ssnet.connect_dst(dstip,dstport,ipv6)
handlers.append(Proxy(MuxWrapper(mux, channel), outwrap))
mux.new_channel = new_channel

diff --git a/ssnet.py b/ssnet.py
index 2145431..1392cb2 100644
--- a/ssnet.py
+++ b/ssnet.py
@@ -493,9 +493,12 @@ class MuxWrapper(SockWrapper):
% (cmd, len(data)))


-def connect_dst(ip, port):
+def connect_dst(ip, port, ipv6):
debug2('Connecting to %s:%d\n' % (ip, port))
- outsock = socket.socket()
+ if ipv6:
+ outsock = socket.socket(socket.AF_INET6)
+ else:
+ outsock = socket.socket(socket.AF_INET)
outsock.setsockopt(socket.SOL_IP, socket.IP_TTL, 42)
return SockWrapper(outsock, outsock,
connect_to = (ip,port),
--
1.7.4.1

[Avery Pennarun]

On Sat, Feb 19, 2011 at 1:38 AM, Roger <wen...@gmail.com> wrote:
> Added a new mode for IPv6 with switch '-6'. In this mode sshuttle
> will listen on IPv6 address and handle the firewall with ip6tables.

Hmm, these patches themselves look okay, but I don't really like the
idea of having sshuttle have two "modes", one for ipv4 and one for
ipv6. Can't we just have it listen on both by default, and if one or
the other is unavailable on the local machine, just skip that one?

Secondly, is there anyone else on the list who can test Roger's
patches to confirm that they work correctly on their machine? I don't
have an ipv6 network to test with.

Thanks,

Avery

[Roger]

Avery Pennarun <apen...@gmail.com> writes:

> On Sat, Feb 19, 2011 at 1:38 AM, Roger <wen...@gmail.com> wrote:
>> Added a new mode for IPv6 with switch '-6'. In this mode sshuttle
>> will listen on IPv6 address and handle the firewall with ip6tables.
>
> Hmm, these patches themselves look okay, but I don't really like the
> idea of having sshuttle have two "modes", one for ipv4 and one for
> ipv6. Can't we just have it listen on both by default, and if one or
> the other is unavailable on the local machine, just skip that one?

I'll see that.

> Secondly, is there anyone else on the list who can test Roger's
> patches to confirm that they work correctly on their machine? I don't
> have an ipv6 network to test with.

To make it work, kernel 2.6.37 and iptables 1.4.11 (currently git ip) is
needed. And besides the ip6table rules which sshuttle will set for you,
the following route commands are needed to put the marked outgoing
packets back to 'lo' interface:

$ ip -6 route add local default dev lo table 100
$ ip -6 rule add fwmark 1 lookup 100

I've been using it for several days and it seems to be OK. BTW, I don't
have IPv6 network on my client, but the server is a dual stack host, so
I hacked sshuttle and add a fake IPv6 gateway here to access IPv6
network in a transparent way :)
--
Roger

[Avery Pennarun]

On Sat, Feb 19, 2011 at 3:52 PM, Roger <wen...@gmail.com> wrote:
> Avery Pennarun <apen...@gmail.com> writes:
>> On Sat, Feb 19, 2011 at 1:38 AM, Roger <wen...@gmail.com> wrote:
>>> Added a new mode for IPv6 with switch '-6'. In this mode sshuttle
>>> will listen on IPv6 address and handle the firewall with ip6tables.
>>
>> Hmm, these patches themselves look okay, but I don't really like the
>> idea of having sshuttle have two "modes", one for ipv4 and one for
>> ipv6.  Can't we just have it listen on both by default, and if one or
>> the other is unavailable on the local machine, just skip that one?
>
> I'll see that.

Thanks.

>> Secondly, is there anyone else on the list who can test Roger's
>> patches to confirm that they work correctly on their machine?  I don't
>> have an ipv6 network to test with.
>
> To make it work, kernel 2.6.37 and iptables 1.4.11 (currently git ip) is
> needed.

Wow, that's rather bleeding edge stuff. But oh well, so is anyone who
uses sshuttle, I suppose :)

> And besides the ip6table rules which sshuttle will set for you,
> the following route commands are needed to put the marked outgoing
> packets back to 'lo' interface:
>
> $ ip -6 route add local default dev lo table 100
> $ ip -6 rule add fwmark 1 lookup 100

Is this just because you don't have ipv6 on your client machine, or
are they always required?

Maybe we should have sshuttle add these entries automatically.

> I've been using it for several days and it seems to be OK. BTW, I don't
> have IPv6 network on my client, but the server is a dual stack host, so
> I hacked sshuttle and add a fake IPv6 gateway here to access IPv6
> network in a transparent way :)

That's interesting. I do have an ipv6-capable virtual host that at
linode that I can test against, I suppose.

What's the advantage here? Are there non-ipv4-capable hosts you need
to be able to access?

Have fun,

Avery

[Roger]

Avery Pennarun <apen...@gmail.com> writes:

> On Sat, Feb 19, 2011 at 3:52 PM, Roger <wen...@gmail.com> wrote:
>> And besides the ip6table rules which sshuttle will set for you,
>> the following route commands are needed to put the marked outgoing
>> packets back to 'lo' interface:
>>
>> $ ip -6 route add local default dev lo table 100
>> $ ip -6 rule add fwmark 1 lookup 100
>
> Is this just because you don't have ipv6 on your client machine, or
> are they always required?

They are always required. But when sshuttle quits they don't need to be
removed.

> Maybe we should have sshuttle add these entries automatically.

Yes. I'll see that.

>> I've been using it for several days and it seems to be OK. BTW, I don't
>> have IPv6 network on my client, but the server is a dual stack host, so
>> I hacked sshuttle and add a fake IPv6 gateway here to access IPv6
>> network in a transparent way :)
>
> That's interesting. I do have an ipv6-capable virtual host that at
> linode that I can test against, I suppose.
>
> What's the advantage here? Are there non-ipv4-capable hosts you need
> to be able to access?

We have a big IPv6 network which have lots of idle bandwidth. So people
are using it for P2P file exchange. btw, Here the 'transmission' BT
client seems to be confused with sshuttle when it's uploading -- the TCP
sockets are established immediately on the client side, but on the
server side it doesn't (still in SYN_SENT mode), and transmission
already starts uploading data then failed. The result is very low and
unstable uploading speed but downloading is very smooth.
--
Roger

[Avery Pennarun]

On Sat, Feb 19, 2011 at 4:19 PM, Roger <wen...@gmail.com> wrote:
> Avery Pennarun <apen...@gmail.com> writes:
>> On Sat, Feb 19, 2011 at 3:52 PM, Roger <wen...@gmail.com> wrote:
>>> And besides the ip6table rules which sshuttle will set for you,
>>> the following route commands are needed to put the marked outgoing
>>> packets back to 'lo' interface:
>>>
>>> $ ip -6 route add local default dev lo table 100
>>> $ ip -6 rule add fwmark 1 lookup 100
>>
>> Is this just because you don't have ipv6 on your client machine, or
>> are they always required?
>
> They are always required. But when sshuttle quits they don't need to be
> removed.

Well, it's generally rude for sshuttle to make changes that it doesn't
clear up afterwards. But if they're *always* required, then it would
make sense to both auto-add and auto-remove them in firewall.py.

> We have a big IPv6 network which have lots of idle bandwidth. So people
> are using it for P2P file exchange. btw, Here the 'transmission' BT
> client seems to be confused with sshuttle when it's uploading -- the TCP
> sockets are established immediately on the client side, but on the
> server side it doesn't (still in SYN_SENT mode), and transmission
> already starts uploading data then failed. The result is very low and
> unstable uploading speed but downloading is very smooth.

Yeah, because of the way transparent proxying works, sshuttle needs to
accept the connection before it can generate the outgoing connection
on the server side. So even if the remote connection is going to time
out (never goes beyond SYN_SENT), then client has no way of relaying
that back to the original session creator.

When I was implementing the --dns option for MacOS, I had to do some
weird stuff with "divert" sockets. A similar trick could be used to
catch TCP SYN packets and eat them until we connect to the remote end,
at which point we could either let through the SYN, or else send back
a "connection refused" error or something. It would be kind of hairy
to implement, though, and the advantages would be rather small in real
life. Although correctly reporting connection refused / timed out /
etc errors would be pretty great.

Have fun,

Avery

[Brian May]

On Feb 19, 7:38 pm, Roger <wen...@gmail.com> wrote:
> This will enable the support for UDP and IPv6. See
> Documentation/networking/tproxy.txt in linux kernel project.

Ok, just to ask some, maybe dumb, questions:

(a) does this allow UDP support for IPv4 too?

(b) Is tproxy IPv6 only or is it for IPv4 too?

(c) what is the current constraint preventing UDP support with IPv4
using NAT rules?

Thanks

[Roger]

On Thu, Apr 7, 2011 at 1:15 PM, Brian May
<br...@microcomaustralia.com.au> wrote:
> On Feb 19, 7:38 pm, Roger <wen...@gmail.com> wrote:
>> This will enable the support for UDP and IPv6. See
>> Documentation/networking/tproxy.txt in linux kernel project.
>
> Ok, just to ask some, maybe dumb, questions:
>
> (a) does this allow UDP support for IPv4 too?

yes. UDP support could be added with TPROXY.

> (b) Is tproxy IPv6 only or is it for IPv4 too?

for IPv4 too.

> (c) what is the current constraint preventing UDP support with IPv4
> using NAT rules?

The original destination is lost after NAT redirection.

HTH.

Roger

[Endre Szabo]

TProxy is a very good choice, we are using it for transparent proxying
since a very long time.

[Brian May]

On 7 April 2011 16:48, Roger <wen...@gmail.com> wrote:

yes. UDP support could be added with TPROXY.

Great!

Have you done any more for TPROXY support?

I tried applying the patches to the latest git version, but unfortunately, they don't apply cleanly any more.

(one hunk fails, will see if I can fix that)

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 7 April 2011 16:48, Roger <wen...@gmail.com> wrote:
>
>> yes. UDP support could be added with TPROXY.
>>
>>
> Great!
>
> Have you done any more for TPROXY support?

no. I got little time for it since then.

> I tried applying the patches to the latest git version, but unfortunately,
> they don't apply cleanly any more.
>
> (one hunk fails, will see if I can fix that)

Thanks for fixing it. I hope the work shouldn't be very much.
--
Roger

[Brian May]

On 12 May 2011 15:14, Roger <wen...@gmail.com> wrote:

Thanks for fixing it. I hope the work shouldn't be very much.

Done.

https://github.com/brianmay/sshuttle/tree/tproxy

What does the following commands do? Am still confused why you say these are required?

$ ip -6 route add local default dev lo table 100 

$ ip -6 rule add fwmark 1 lookup 100 

Also looks like iptables in Ubuntu natty isn't new enough (1.4.10-1ubuntu1), I get the error:

>> ip6tables -t mangle -N sshuttle-m-12300

>> ip6tables -t mangle -F sshuttle-m-12300

>> ip6tables -t mangle -N sshuttle-t-12300

>> ip6tables -t mangle -F sshuttle-t-12300

>> ip6tables -t mangle -I OUTPUT 1 -j sshuttle-m-12300

>> ip6tables -t mangle -I PREROUTING 1 -j sshuttle-t-12300

>> ip6tables -t mangle -A sshuttle-t-12300 -m socket -j RETURN -m tcp -p tcp

ip6tables v1.4.10: Couldn't load match `socket':/lib/xtables/libip6t_socket.so: cannot open shared object file: No such file or directory

Try `ip6tables -h' or 'ip6tables --help' for more information.

What is -m socket? It is really needed?

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 12 May 2011 15:14, Roger <wen...@gmail.com> wrote:
>
>> Thanks for fixing it. I hope the work shouldn't be very much.
>>
>
> Done.
>
> https://github.com/brianmay/sshuttle/tree/tproxy
>
> What does the following commands do? Am still confused why you say these are
> required?

I sent 2 patches. The first is TPROXY support, which is helpful to
enable UDP and IPv6. The 2nd patch is IPv6 support, all the following
(quoted by you) is required for IPv6.

> $ ip -6 route add local default dev lo table 100
> $ ip -6 rule add fwmark 1 lookup 100
>
> Also looks like iptables in Ubuntu natty isn't new enough (1.4.10-1ubuntu1),
> I get the error:

Yeah. For IPv6 you'll need iptables 1.4.11, which is currently git tip.

>>> ip6tables -t mangle -N sshuttle-m-12300
>>> ip6tables -t mangle -F sshuttle-m-12300
>>> ip6tables -t mangle -N sshuttle-t-12300
>>> ip6tables -t mangle -F sshuttle-t-12300
>>> ip6tables -t mangle -I OUTPUT 1 -j sshuttle-m-12300
>>> ip6tables -t mangle -I PREROUTING 1 -j sshuttle-t-12300
>>> ip6tables -t mangle -A sshuttle-t-12300 -m socket -j RETURN -m tcp -p tcp
> ip6tables v1.4.10: Couldn't load match
> `socket':/lib/xtables/libip6t_socket.so: cannot open shared object file: No
> such file or directory
>
> Try `ip6tables -h' or 'ip6tables --help' for more information.
>
>
> What is -m socket? It is really needed?

Yes. It's needed to match the packets which belongs to a local socket
(already established connection).

--
Roger

[Roger]

Roger <wen...@gmail.com> writes:

> Brian May <br...@microcomaustralia.com.au> writes:
>
>> On 12 May 2011 15:14, Roger <wen...@gmail.com> wrote:
>>
>>> Thanks for fixing it. I hope the work shouldn't be very much.
>>>
>>
>> Done.
>>
>> https://github.com/brianmay/sshuttle/tree/tproxy
>>
>> What does the following commands do? Am still confused why you say these are
>> required?
>
> I sent 2 patches. The first is TPROXY support, which is helpful to
> enable UDP and IPv6. The 2nd patch is IPv6 support, all the following
> (quoted by you) is required for IPv6.

btw, the ip6tables rules were written for the case that you need to
access v6 internet from local -- if your box acts as a router then the
rules are different.

--
Roger

[Brian May]

On 12 May 2011 17:20, Roger <wen...@gmail.com> wrote:

btw, the ip6tables rules were written for the case that you need to

access v6 internet from local -- if your box acts as a router then the
rules are different.

Still trying to work out what all these rules do. Only it doesn't quite make sense to me yet.

So, if my understanding is correct, when the system receives a packet that needs to be forwarded.

Reference figure 2 from

http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.filter_rules.html

1 PREROUTING table.

- executes tproxy_chain

- returns for local socket connections

- executes "TPROXY --tproxy-mark 0x1/0x1" if dest IP matches

(kind of confused what happens here. What does the 0x1/0x1 mean?)

2. Routing table.

- If packet marked 0x1 then redirect it to lo

- Packet is now considered "local"

3. OUTPUT table - only executed for "local" packets.

- executes mark_chain

- Sets the 0x1 mark if destination address matches.

(what is the point?)

How much have of this have I got correct?

--
Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 12 May 2011 19:14, Brian May <br...@microcomaustralia.com.au> wrote:

3. OUTPUT table - only executed for "local" packets.

- executes mark_chain

- Sets the 0x1 mark if destination address matches.

(what is the point?)

I guess some of my confusion is because all other tproxy examples I see mark the packets in the PREROUTING table (before routing), but your patch marks them to the OUTPUT table (after routing I think). 

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Yes. OUTPUT table is for packets _from_ your box. The examples you saw
are rules for the case that your box acts as a router, where OUTPUT
table is useless.

An iptables diagram is helpful to understand this:

http://jengelh.medozas.de/images/nf-packet-flow.png

In my case (rules for local box), packets flow in this way:

OUTPUT(eth0) -> routing -> PREROUTING (lo)

So it looks confusing.

HTH.

Roger

[Brian May]

On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:

An iptables diagram is helpful to understand this:

http://jengelh.medozas.de/images/nf-packet-flow.png

In my case (rules for local box), packets flow in this way:

OUTPUT(eth0) -> routing -> PREROUTING (lo)

How does this happen? Does the packet go out the right hand side of the diagram and back in on the left?

-- 

Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:

Yes. OUTPUT table is for packets _from_ your box. The examples you saw
are rules for the case that your box acts as a router, where OUTPUT
table is useless.

[...]

OUTPUT(eth0) -> routing -> PREROUTING (lo)

So why not do it in the PREROUTING table, where it looks like it should work for all packets?

It would appear PREROUTING gets invoked every time.

So checking my understanding:

TPROXY has two usage conditions:

1. It must only be used when the connection is initially established. For subsequent packets it remembers the previous tproxy condition.

2. It only works on packets that are routed locally. All packets that were initially going to be routed straight through have to be redirected locally, otherwise it won't have any affect.

-- 

Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:

So it looks confusing.

hmmm....

Everything looks fine. The packets are appearing on lo fine. The tproxy rule has a usage count that goes up every time I test it, etc.

However the SYN packets don't get redirected to the sshuttle process, they effectively go to a black hole. 

So how do I debug why the tproxy rule doesn't appear to work?
--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:
>
>> An iptables diagram is helpful to understand this:
>>
>> http://jengelh.medozas.de/images/nf-packet-flow.png
>>
>> In my case (rules for local box), packets flow in this way:
>
>
>> OUTPUT(eth0) -> routing -> PREROUTING (lo)
>>
>
>
> How does this happen? Does the packet go out the right hand side of the
> diagram and back in on the left?

Because it's two different interfaces (eth0 vs lo). The diagram is for
an single interface.
--
Roger

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:
>
>> Yes. OUTPUT table is for packets _from_ your box. The examples you saw
>> are rules for the case that your box acts as a router, where OUTPUT
>> table is useless.
>>
>> [...]
>
>>
>> OUTPUT(eth0) -> routing -> PREROUTING (lo)
>
>
> So why not do it in the PREROUTING table, where it looks like it should work
> for all packets?

Then you'll have no chance to catch the packets from local process --
they'll go through OUTPUT and leave the box directly.

> It would appear PREROUTING gets invoked every time.
>
> So checking my understanding:
>
> TPROXY has two usage conditions:
>
> 1. It must only be used when the connection is initially established. For
> subsequent packets it remembers the previous tproxy condition.
> 2. It only works on packets that are routed locally. All packets that were
> initially going to be routed straight through have to be redirected locally,
> otherwise it won't have any affect.

That's right.
--
Roger

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 12 May 2011 21:13, Roger <wen...@gmail.com> wrote:
>
>> So it looks confusing.
>>
>
> hmmm....
>
> Everything looks fine. The packets are appearing on lo fine. The tproxy rule
> has a usage count that goes up every time I test it, etc.
>
> However the SYN packets don't get redirected to the sshuttle process, they
> effectively go to a black hole.
>
> So how do I debug why the tproxy rule doesn't appear to work?

You can dump the packets and send it here (as well as the iptables
rules) so I'll see what's wrong.

--
Roger

[Brian May]

On 13 May 2011 12:02, Roger <wen...@gmail.com> wrote:

>> OUTPUT(eth0) -> routing -> PREROUTING (lo)
>
>
> So why not do it in the PREROUTING table, where it looks like it should work
> for all packets?

Then you'll have no chance to catch the packets from local process --
they'll go through OUTPUT and leave the box directly.

Oh, ok, my bad, if you don't intercept the packets in OUTPUT they will never reach the PREROUTING table. 

So I guess the ideal solution, to cover all cases, would be to have rules in both OUTPUT and PREROUTING to cover marking the packets.

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

>>
>> Chain sshuttle-m-12300 (1 references)
>> pkts bytes target prot opt in out source
>> destination
>> 0 0 MARK tcp * * ::/0
>> 2001:44b8:758e:21e4::/64 tcp MARK set 0x1
>>
>> Chain sshuttle-t-12300 (1 references)
>> pkts bytes target prot opt in out source
>> destination
>> 3363 1233K RETURN tcp * * ::/0 ::/0
>> socket tcp
>> 0 0 TPROXY tcp * * ::/0
>> 2001:44b8:758e:21e4::/64 tcp TPROXY redirect :::12300 mark 0x1/0x1
>>
>>
> Oops. Obviously I restarted sshuttle here, the counts are 0. After sending
> one SYN packet, the packet counts look like:

Did you run sshuttle with 'root' user? That should be required by
TPROXY, IIRC.

And what's your kernel version?
--
Roger

[Brian May]

On 16 May 2011 16:49, Roger <wen...@gmail.com> wrote:

Did you run sshuttle with 'root' user? That should be required by

TPROXY, IIRC.

No, I didn't know about that requirement. Is it documented anywhere?

Maybe I should try again as root.

 

And what's your kernel version?

2.6.38-8-generic, from Ubuntu natty.
--
Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 17 May 2011 09:46, Brian May <br...@microcomaustralia.com.au> wrote:

Maybe I should try again as root.

Yes, it does seem to need root, for more then just the firewall stuff. Don't understand why...

Now it intercepts packets fine:

 s:   Ready: 1 r=[4] w=[] x=[]

 s: <  channel=1 cmd=CONNECT len=25

 s: Connecting to 2001:44b8:758e:21e4::3:22

 s: creating new SockWrapper (1 now exist)

 s: SW#6:2001:44b8:758e:21e4::3:22: trying connect to ('2001:44b8:758e:21e4::3', 22)

Traceback (most recent call last):

  File "<string>", line 1, in <module>

  File "assembler.py", line 26, in <module>

  File "server.py", line 237, in main

  File "ssnet.py", line 547, in runonce

  File "ssnet.py", line 446, in callback

  File "ssnet.py", line 434, in handle

  File "ssnet.py", line 369, in got_packet

  File "server.py", line 218, in new_channel

  File "ssnet.py", line 523, in connect_dst

  File "ssnet.py", line 101, in __init__

  File "ssnet.py", line 131, in try_connect

socket.error: illegal IP address string passed to inet_aton

 s: SW#6:2001:44b8:758e:21e4::3:22: deleting (0 remain)

c :   Ready: 3 r=[8] w=[] x=[]

c : read: err was: [Errno 11] Resource temporarily unavailable

c : read: err was: [Errno 11] Resource temporarily unavailable

firewall manager: undoing changes.

>> ip6tables -t mangle -D OUTPUT -j sshuttle-m-12300

>> ip6tables -t mangle -F sshuttle-m-12300

>> ip6tables -t mangle -X sshuttle-m-12300

>> ip6tables -t mangle -D PREROUTING -j sshuttle-t-12300

>> ip6tables -t mangle -F sshuttle-t-12300

>> ip6tables -t mangle -X sshuttle-t-12300

c : fatal: server died with error code 1

c : SW#7:2001:388:60ac:104:226:18ff:feee:4d61:46369: deleting (1 remain)

 

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> On 16 May 2011 16:49, Roger <wen...@gmail.com> wrote:
>
>> Did you run sshuttle with 'root' user? That should be required by
>> TPROXY, IIRC.
>>
>
> No, I didn't know about that requirement. Is it documented anywhere?

It's required for the address spoofing to work -- sshuttle need this to
spoof the address of the destination host.

I've only seen it in squid's documentation.

--
Roger

[Brian May]

Ok, made some significant improvements in my tproxy branch of sshuttle. Now there is a --tproxy mode, default is NAT.

There are still some easy things that will be broken. Just need to get around to identifying them and fixing them. Perhaps the most serious is that things might break if run on a system doesn't doesn't have kernel ipv6 support (need to try and test this out). In particular there might be two different places it fails, one obvious, the other one more obscure.

Some challenges:

* Adding the nat rules doesn't affect existing connections. In contrast, adding tproxy rules kills existing connections, significant if the ssh connection goes to the subnet being redirected, it will kill the ssh connection. Solution is to add the address of this host to the excludes list.

* UDP support. How do I get the destination address of each packet via tproxy? I see some really old references, not sure what the current solution is.

-- 

Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

> * UDP support. How do I get the destination address of each packet via
> tproxy? I see some really old references, not sure what the current solution
> is.

http://stackoverflow.com/questions/5615579/how-to-get-original-destination-port-of-redirected-udp-message
--
Roger

[Brian May]

Hmmm... Looks like I need the recvmsg() function that is not yet available in Python :-(

http://bugs.python.org/issue6560

--
Brian May <br...@microcomaustralia.com.au>

[Roger]

Brian May <br...@microcomaustralia.com.au> writes:

Then another way besides tproxy should be the only choice -- look up the
conntrack table. It's described in the stackoverflow comments as well.

btw, I found another interesting program which is similar to
sshuttle. May it be helpful.

http://intrepidusgroup.com/insight/mallory/
--
Roger

[Brian May]

On 19 May 2011 15:50, Roger <wen...@gmail.com> wrote:

Then another way besides tproxy should be the only choice -- look up the

conntrack table. It's described in the stackoverflow comments as well.

I can't help but think that seems rather ugly :-(, had thought tproxy was going to improve this.

I found a library that supports recvmsg on python. http://www.pps.jussieu.fr/~ylg/PyXAPI/ - not sure yet if this is the best version, will come back to that later.

I note however that while I can get IPv4 UDP support working, it doesn't seem to provide anything when testing IPv6 UDP support.

So maybe IPv6 UDP support is broken in my kernel???

--
Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 20 May 2011 14:45, Brian May <br...@microcomaustralia.com.au> wrote:

I note however that while I can get IPv4 UDP support working, it doesn't seem to provide anything when testing IPv6 UDP support.

So maybe IPv6 UDP support is broken in my kernel???

I also note that the sockaddr format returned seems to be the older format. According to  :

http://uw714doc.sco.com/en/SDK_netapi/sockC.TheIPv6sockaddrstructure.html

The new format should start with two byte values, the old format has one integer. So maybe IPv6 simply isn't supported here???

How do I send UDP packets with the correct IP address? I would have assumed sendmsg, but so far that results in segmentation faults. Need to double check my code to ensure I am not doing something silly.
--
Brian May <br...@microcomaustralia.com.au>

[Tony Godshall]

Have you read Avery's essay re IPv6?

http://apenwarr.ca/log/?m=201103#28

--
Best Regards.

[Brian May]

On 20 May 2011 16:33, Brian May <br...@microcomaustralia.com.au> wrote:

How do I send UDP packets with the correct IP address? I would have assumed sendmsg, but so far that results in segmentation faults. Need to double check my code to ensure I am not doing something silly.

Oops. My bad. No need to use sendmsg, a simple bind(...) and sendto(...) is all that is needed.

So I now have IPv4 UDP working.

IPv6 still is a problem, the kernel doesn't seem to be giving the destination address - might have to look at the kernel source code to see if this is implemented in fact.

--
Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 21 May 2011 18:59, Brian May <br...@microcomaustralia.com.au> wrote:

IPv6 still is a problem, the kernel doesn't seem to be giving the destination address - might have to look at the kernel source code to see if this is implemented in fact.

Now got IPv6 UDP going. When it doubt read the source :-).

Needed to the SOL_IPV6,  IPV6_RECVORIGDSTADDR and IPV6_ORIGDSTADDR for IPv6; SOL_IP, IP_RECVORIGDSTADDR, and IP_ORIGDSTADDR are IPv4 only.

--
Brian May <br...@microcomaustralia.com.au>

[Brian May]

On 22 May 2011 10:13, Brian May <br...@microcomaustralia.com.au> wrote:

On 21 May 2011 18:59, Brian May <br...@microcomaustralia.com.au> wrote:

IPv6 still is a problem, the kernel doesn't seem to be giving the destination address - might have to look at the kernel source code to see if this is implemented in fact.

Now got IPv6 UDP going. When it doubt read the source :-).

Also got UDP support working with the proposed patches to Python 3.3.

However my port to Python 3.x, while it basically works for Python 3.x, is a bit of a mess at the moment,  so no plans to contribute it just yet. I also believe it might be difficult/impossible to get the same code to work with Python 2.6 and Python 3.x. Python 2.7 I think should work but doesn't.

--
Brian May <br...@microcomaustralia.com.au>