Update templates in parallel

[fiftyfour...@gmail.com]

I have a ton of templates and standalones (>10), so updating them one by
one serially is a pain. I found a convenient dom0 script so I thought I'd
share.

Basically, take this and paste it into dom0 then make it executable.
There'a also a handy test function. All credit goes to Andrea Micheloni.
Anyone have similarly handy modifications/scripts?

https://m7i.org/tips/qubes-update-all-templates/

#!/usr/bin/python2
#
# Copyright (c) Andrea Micheloni 2018
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import sys, os, time, subprocess, unittest

from Queue import Queue
from threading import Thread, current_thread
class QubesVm:
def __init__(self, name, template, is_running):
self.name = name
self.template = template
self._is_running = is_running
self._is_updated = False

def is_running(self):
return self._is_running

def is_template(self):
return (self.template is None)

def is_updated(self):
return self._is_updated

def run(self, command, autostart = False, verbose = True, user = None,
notify_function = None,
passio = False, localcmd = None, gui = True, filter_esc = True
):
qvm_command = ['qvm-run']
if autostart:
qvm_command += ['--autostart']
else:
qvm_command += ['--no-autostart']
if not verbose:
qvm_command += ['--quiet']
if user:
qvm_command += ['--user=%s' % user]
if gui:
qvm_command += ['--gui']
else:
qvm_command += ['--no-gui']
qvm_command += [self.name, command]
subprocess.call(qvm_command)
class QubesHandler:
def __init__(self):
raw_result = self._get_raw_qubes_string()
self._qubes = {}
for qube_string in raw_result.split("\n"):
if len(qube_string)>0:
qube_properties = qube_string.split("|")
qube_class = qube_properties[0]
qube_name = qube_properties[1]
qube_template = None;
if qube_properties[2] != '-':
qube_template = qube_properties[2]
qube_running = True
if qube_properties[3] == 'Halted':
qube_running = False
if qube_class != "AdminVM":
new_qube = self._create_qube(qube_name, qube_template,
qube_running)
self._qubes[qube_name] = new_qube
def _create_qube(self, qube_name, qube_template, qube_running):
return QubesVm(qube_name, qube_template, qube_running)
def _get_raw_qubes_string(self):
return subprocess.check_output(['qvm-ls', '--fields',
'CLASS,NAME,TEMPLATE,STATE', '--raw-data'])
def _get_template_dictionary(self, template_vm, rank):
return {'name': template_vm.name,
'qvm': template_vm,
'rank': rank}
def _get_ranked_templates(self):
templates_ranked = {}

for vm in self._qubes.values():
if vm.is_template():
if not vm.name in templates_ranked:
templates_ranked[vm.name] = self._get_template_dictionary(vm,
0)
else:
if vm.template:
if vm.is_running():
rank = 3
else:
rank = 1

if not vm.template in templates_ranked:
templates_ranked[vm.template] = \
self._get_template_dictionary(self._qubes[vm.template],
rank)
else:
templates_ranked[vm.template]['rank'] += rank

return templates_ranked

def get_templates(self):
templates_ranked = self._get_ranked_templates()

return [templates_ranked[template] for template in sorted(templates_ranked,

key=lambda name: templates_ranked[name][
'rank'],
reverse = True)]

def update_template(self, template, print_function):
print_function('Updating template...')

try:
template['qvm'].run('touch /tmp/update-all; chmod go+r
/tmp/update-all; { if [ -f /usr/bin/snap ' +
']; then /usr/bin/snap refresh || exit; fi;
if [ -f /usr/bin/apt-get ' +
']; then /usr/bin/apt-get update &&
/usr/bin/apt-get -y upgrade </dev/null ' +
'&& /usr/bin/apt-get -y autoremove &&
/usr/bin/apt-get autoclean && ' +
'/sbin/poweroff; else /usr/bin/dnf -y
upgrade </dev/null && /usr/bin/dnf ' +
'clean all && /usr/sbin/poweroff; fi; } 2>&1
| tee -a /tmp/update-all',
autostart = True, verbose = False, user =
'root', notify_function = None,
passio = False, localcmd = None, gui = False,
filter_esc = True )
print_function('Update complete.')
except:
print_function('Update failed.', error=True)

raise

class UpdateWorker(Thread):
def __init__(self, name, queue, qubes_handler):
Thread.__init__(self)
self.daemon = True
self.name = name
self._queue = queue
self._qubes_handler = qubes_handler
self._template = None

def _print_function(self, text, error=False):
if error:
prefix = 'ERR'
else:
prefix = 'INF'

prefix += ' ['+self.name+' '+self._template['name']+'] '

print prefix + text

def run(self):
while True:
self._template = {'name':'None'}
self._template = self._queue.get()
try:
self._qubes_handler.update_template(self._template, self
._print_function)
self._template = {'name':'None'}
finally:
self._template = {'name':'None'}
self._queue.task_done()

self._template = {'name':'None'}

class UpdateHandler:
def __init__(self):
self._queue = Queue()

def update_templates(self, thread_number):
qubes_handler = self._get_qubes_handler()

print 'INF Updating templates: ', [template['name'] for template in
qubes_handler.get_templates()]

workers = [self._get_worker('w'+str(number), qubes_handler) for number
in range(thread_number)]

for worker in workers:
worker.start()

for template in qubes_handler.get_templates():
self._queue.put(template, block = True)

self._queue.join()

def _get_worker(self, name, qubes_handler):
return UpdateWorker(name, self._queue, qubes_handler)

def _get_qubes_handler(self):
return QubesHandler()

class MockQubesVm(QubesVm):
def __init__(self, name, template, is_running):
QubesVm.__init__(self, name, template, is_running)
self.name = name
self.template = template
self._is_running = is_running
self._is_updated = False

def is_running(self):
return self._is_running

def is_template(self):
return (self.template is None)

def is_updated(self):
return self._is_updated

def __repr__(self):
return "MockQubesVm(%s,%s,%s)" % (self.name, self.template, self
._is_running)
def __eq__(self, other):
return repr(self) == repr(other)
def run(self, command, autostart = False, verbose = True, user = None,
notify_function = None,
passio = False, localcmd = None, gui = True, filter_esc = True
):
if (autostart):
self._is_running = True

if (user == 'root' and self._is_running):
self._is_updated = True
self._is_running = False

class MockQubesHandler(QubesHandler, unittest.TestCase):
def __init__(self, *args, **kwargs):
if (args):
unittest.TestCase.__init__(self, *args, **kwargs)
self.maxDiff=None
self._template_f = MockQubesVm('fedora-24', None, False)
self._template_fm = MockQubesVm('fedora-24-minimal', None, True)
self._template_d = MockQubesVm('debian-8', None, False)
self._template_ws = MockQubesVm('whonix-ws', None, True)
self._template_wg = MockQubesVm('whonix-wg', None, True)

self._ranked_f = {'name': 'fedora-24', 'qvm': self._template_f,
'rank': 9}
self._ranked_fm = {'name': 'fedora-24-minimal', 'qvm': self._template_fm,
'rank': 4}
self._ranked_d = {'name': 'debian-8', 'qvm': self._template_d,
'rank': 5}
self._ranked_ws = {'name': 'whonix-ws', 'qvm': self._template_ws,
'rank': 1}
self._ranked_wg = {'name': 'whonix-wg', 'qvm': self._template_wg,
'rank': 3}

self._expected_ranked = {
'fedora-24': self._ranked_f,
'fedora-24-minimal': self._ranked_fm,
'debian-8': self._ranked_d,
'whonix-ws': self._ranked_ws,
'whonix-wg': self._ranked_wg }

self._expected = [ self._ranked_f, self._ranked_d, self._ranked_fm,
self._ranked_wg, self._ranked_ws]
QubesHandler.__init__(self)
def _create_qube(self, qube_name, qube_template, qube_running):
return MockQubesVm(qube_name, qube_template, qube_running)

def _get_raw_qubes_string(self):
return 'AppVM|sys-net|fedora-24|Running\n' + \
'AppVM|sys-whonix|whonix-wg|Running\n' + \
'AppVM|personal|debian-8|Halted\n' + \
'AppVM|sys-firewall|fedora-24-minimal|Running\n' + \
'TemplateVM|debian-8|-|Halted\n' + \
'AdminVM|dom0|-|Running\n' + \
'AppVM|gnupg|fedora-24-minimal|Halted\n' + \
'AppVM|disp5|fedora-24|Suspended\n' + \
'AppVM|anonymous|whonix-ws|Halted\n' + \
'TemplateVM|fedora-24-minimal|-|True\n' + \
'TemplateVM|whonix-ws|-|Running\n' + \
'TemplateVM|whonix-wg|-|Running\n' + \
'AppVM|work|debian-8|Suspended\n' + \
'AppVM|development|debian-8|Halted\n' + \
'AppVM|disp2|fedora-24|Running\n' + \
'TemplateVM|fedora-24|-|Halted\n'

def get_updated_status(self):
return [self._qubes[self._template_f.name].is_updated(),
self._qubes[self._template_fm.name].is_updated(),
self._qubes[self._template_d.name].is_updated(),
self._qubes[self._template_ws.name].is_updated(),
self._qubes[self._template_wg.name].is_updated()]

def test_get_template_dictionary(self):
self.assert_get_template_dictionary(self._template_f)
self.assert_get_template_dictionary(self._template_fm)
self.assert_get_template_dictionary(self._template_d)
self.assert_get_template_dictionary(self._template_ws)
self.assert_get_template_dictionary(self._template_wg)

def assert_get_template_dictionary(self, template):
found = self._get_template_dictionary(template, len(template.name))

self.assertEquals(3, len(found.keys()))
self.assertEquals(template, found['qvm'])
self.assertEquals(len(template.name), found['rank'])
self.assertEquals(template.name, found['name'])

def test_get_template_dictionary_custom(self):
template = MockQubesVm('test-template', None, False)
found = self._get_template_dictionary(template, 0)

self.assertEquals(3, len(found.keys()))
self.assertEquals(template, found['qvm'])
self.assertEquals(0, found['rank'])
self.assertEquals('test-template', found['name'])

def test_get_templates(self):
found = self.get_templates()

self.assertEquals(self._expected, found)

def test_get_ranked_templates(self):
found = self._get_ranked_templates()

self.assertEquals(self._expected_ranked, found)

class MockUpdateWorker(UpdateWorker):
def _print_function(self, text, error=False):
pass

class MockUpdateHandler(UpdateHandler, unittest.TestCase):
def __init__(self, *args, **kwargs):
UpdateHandler.__init__(self)
unittest.TestCase.__init__(self, *args, **kwargs)
self.generate_qubes_handler()

def _get_worker(self, name, qubes_handler):
return MockUpdateWorker(name, self._queue, qubes_handler)

def _get_qubes_handler(self):
return self._qubes_handler

def generate_qubes_handler(self):
self._qubes_handler = MockQubesHandler()

def test_update_all(self):
for number in range(1,10):
self.assert_update_all(number)
self.generate_qubes_handler()

def assert_update_all(self, thread_number):
self.assertEquals([False]*5 , self
._qubes_handler.get_updated_status())
self.update_templates(thread_number)
self.assertEquals([True]*5 , self
._qubes_handler.get_updated_status())

if __name__ == '__main__':
if len(sys.argv) == 1:
UpdateHandler().update_templates(3)
elif len(sys.argv) >2 or sys.argv[1] != 'test':
print "ERR Usage: dom0-update-all [test]"
else:
unittest.main(argv=sys.argv[:1])

[Chris Laprise]

On 8/2/20 8:32 AM, fiftyfour...@gmail.com wrote:
> I have a ton of templates and standalones (>10), so updating them one by
> one serially is a pain. I found a convenient dom0 script so I thought
> I'd share.
>
> Basically, take this and paste it into dom0 then make it executable.
> There'a also a handy test function. All credit goes to Andrea Micheloni.
> Anyone have similarly handy modifications/scripts?
>
> https://m7i.org/tips/qubes-update-all-templates/

IIRC there is an option somewhere in 'qubesctl' for parallel template
updates.

You can check out my github for some interesting stuff. The
'Qubes-scripts' project has a (serial) template updater that lets you
select by certain criteria. It could be parallelized pretty easily.

Since you have a lot of templates+standalones, the 'findpref' tool might
also be of interest. It can bulk search/replace VM prefs, such as
changing all the VMs that are using 'sys-vpn1' to use 'sys-vpn3'
instead, or change VMs to use a different template.

I also wrote 'wyng-backup', a fast LVM incremental backup tool that only
scans where LVM reports new activity.

Finally, there is a VPN tool and one to enhance VM internal security.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB 4AB3 1DC4 D106 F07F 1886

[fiftyfour...@gmail.com]

On Sunday, 2 August 2020 22:42:31 UTC+8, Chris Laprise wrote:

IIRC there is an option somewhere in 'qubesctl' for parallel template
updates.

You can check out my github for some interesting stuff. The
'Qubes-scripts' project has a (serial) template updater that lets you
select by certain criteria. It could be parallelized pretty easily.

Since you have a lot of templates+standalones, the 'findpref' tool might
also be of interest. It can bulk search/replace VM prefs, such as
changing all the VMs that are using 'sys-vpn1' to use 'sys-vpn3'
instead, or change VMs to use a different template. 

I also wrote 'wyng-backup', a fast LVM incremental backup tool that only
scans where LVM reports new activity.

Finally, there is a VPN tool and one to enhance VM internal security.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886

Hi Chris,

After testing, the m7i solution works, but isn't ideal--it sometimes fails due to pipe errors and attempts to update all my appVMs as well, which might be problematic from a security standpoint. I looked in man qubesctl and didn't see anything for parallelizing updates, but since I'm hesitatnt to install salt management in my minimal templates to begin with, that's beside the point.

I noticed that you have a qubes4-multi-update tool which I'm likely going to switch to, since parallelizing isn't my true concern, but clearing 10+ dom0 prompts for sudo authentication every time I update. Since qubes4-multi-update looks cleaner and also uses -u root, this seems to be promising. I'll report back.

halt-vm-by-window and system-stats-xen seem incredibly promising since I constantly have xentop running and I'm looking to simplify my system to one app per VM. 

On that topic, I'm having huge difficulties making it so starting a qube, say app-firefox, automatically starts a program, like firefox. I've tried rc.local but that's pre-boot, before X11. Others have suggested something related to /etc/config or something, but that involves fiddling with templates and implies an ungodly amount of templates. Would you happen to have any suggestions?

[Steve Coleman]

On Sun, Aug 2, 2020, 10:26 PM <fiftyfour...@gmail.com> wrote:

On that topic, I'm having huge difficulties making it so starting a qube, say app-firefox, automatically starts a program, like firefox. I've tried rc.local but that's pre-boot, before X11. Others have suggested something related to /etc/config or something, but that involves fiddling with templates and implies an ungodly amount of templates. Would you happen to have any suggestions?

To start an app automatically in a appvm when the vm starts you can copy the app.desktop file into (from my vague memory) your .config/autostart (?) directory. Each time that vm starts that application will be launched just as if you started it manually from the menu. 

But if that vm starts on system startup, and before you login, this still may not work any better than the rc.local method. You might then create a .desktop to call a script that sleeps until the x11 is available, and have that script launch the app when the system is ready.

[fiftyfour...@gmail.com]

On Sunday, 2 August 2020 22:42:31 UTC+8, Chris Laprise wrote:

You can check out my github for some interesting stuff. The
'Qubes-scripts' project has a (serial) template updater that lets you
select by certain criteria. It could be parallelized pretty easily.

[...]

Finally, there is a VPN tool and one to enhance VM internal security.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886

I tested your halt-vm-by-window and system-stats-xen and found them very useful. I also tried your qubes4-multi-update but ran into three issues: one is that it relies on curl, which my Fedora minimal wasn't happy about; another is that it [Y/n] prompts me for upgrades, which it shouldn't do, according to the script; the last is that it attempts to update mirage firewall standalones and when it fails, the whole process stops.

Your Qubes-VM-Hardening tool was one of the first things installed into my first Qubes, but I'm still not very familiar with how it works. I think vm-boot-protect might be blocking me from adding a .desktop file into ~/.config/autostart, as Steve suggested (Steve: does this need to be done in templates? If done in an appVM, wouldn't it get purged upon restart?).

Anyways, your tools are very convnient and I think they should be more widely known, if not integrated into Qubes proper. Thank you

[fiftyfour...@gmail.com]

On Monday, 3 August 2020 16:11:40 UTC+8, 54th Parallel wrote:

I tested your halt-vm-by-window and system-stats-xen and found them very useful. I also tried your qubes4-multi-update but ran into three issues: one is that it relies on curl, which my Fedora minimal wasn't happy about; another is that it [Y/n] prompts me for upgrades, which it shouldn't do, according to the script; the last is that it attempts to update mirage firewall standalones and when it fails, the whole process stops.

 

Now that I've actually read the documentation, my problems can be solved by using the -atu option--a classic case of RTFM.

[Chris Laprise]

On 8/3/20 4:11 AM, fiftyfour...@gmail.com wrote:
>
>
> On Sunday, 2 August 2020 22:42:31 UTC+8, Chris Laprise wrote:
>
> You can check out my github for some interesting stuff. The
> 'Qubes-scripts' project has a (serial) template updater that lets you
> select by certain criteria. It could be parallelized pretty easily.
>
> [...]
>
> Finally, there is a VPN tool and one to enhance VM internal security.
>
> --
> Chris Laprise, tas...@posteo.net
> https://github.com/tasket
> https://twitter.com/ttaskett
> PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886
>
>
> I tested your halt-vm-by-window and system-stats-xen and found them very
> useful. I also tried your qubes4-multi-update but ran into three issues:
> one is that it relies on curl, which my Fedora minimal wasn't happy
> about; another is that it [Y/n] prompts me for upgrades, which it
> shouldn't do, according to the script; the last is that it attempts to
> update mirage firewall standalones and when it fails, the whole process
> stops.

'curl' would only be used in a Whonix template. This is to signal Qubes'
proxy to start the Tor-based updateVM as soon as possible. It should not
try to run curl in a Fedora or regular Debian template.

To suppress interactive prompts, you need to run the script with '-u' or
'--unattended'.

>
> Your Qubes-VM-Hardening tool was one of the first things installed into
> my first Qubes, but I'm still not very familiar with how it works. I
> think vm-boot-protect might be blocking me from adding a .desktop file
> into ~/.config/autostart, as Steve suggested (Steve: does this need to
> be done in templates? If done in an appVM, wouldn't it get purged upon
> restart?).

Yes, vm-boot-protect does lock down that dir, along with other startup
files and dirs in /home. The way it does this is with the 'immutable'
flag. To change it (re)start the VM and do:

sudo chattr -i -R .config/autostart

Then change what you need to in that path and restart the VM. During the
startup process the dir and its contents will be automatically made
immutable again.

>
> Anyways, your tools are very convnient and I think they should be more
> widely known, if not integrated into Qubes proper. Thank you

[Chris Laprise]

> On 8/3/20 4:11 AM, fiftyfour...@gmail.com wrote:

>> Your Qubes-VM-Hardening tool was one of the first things installed
>> into my first Qubes, but I'm still not very familiar with how it
>> works. I think vm-boot-protect might be blocking me from adding a
>> .desktop file into ~/.config/autostart, as Steve suggested (Steve:
>> does this need to be done in templates? If done in an appVM, wouldn't
>> it get purged upon restart?).

BTW, I think the appVM is the right place to make the .config/autostart
change if the custom .desktop file is being applied on a per-VM basis.

If you want it for _all_ VMs based on that template, that's a little
harder. Putting the .desktop file in /etc/skel would only make the
change when an appVM is first created, so existing VMs using that
template would not benefit. However, vm-boot-protect-root has the
ability to copy or "deploy" files into /home on each boot; you would
have to save the .desktop file under
/etc/default/vms/vms.all/rw/home/user/.config/autostart in the template.

Another idea is to use rc.local to launch the app via 'systemd-run'
using its "timer" features or some other way to delay execution. Or you
could even try adding the .desktop file to /home using rc.local.

[fiftyfour...@gmail.com]

On Monday, 3 August 2020 18:36:28 UTC+8, Chris Laprise wrote:

'curl' would only be used in a Whonix template. This is to signal Qubes' 
proxy to start the Tor-based updateVM as soon as possible. It should not 
try to run curl in a Fedora or regular Debian template. 

To suppress interactive prompts, you need to run the script with '-u' or 
'--unattended'. 

Thanks--I managed to figure that out and respond right before you posted, so now I'm using the -atu option.

Not sure if it's a bug, but it seems like your script attempted to run curl in Fedora. I can't copy the output, but the VM basically goes, "Errors during downloading metadata for repository 'updates': Curl error (28); Curl Error (23)" several times before throwing up its arms and giving up. Then the script tells me that fedora-32-minimal update returned non-zero status.

 

 

Yes, vm-boot-protect does lock down that dir, along with other startup 
files and dirs in /home. The way it does this is with the 'immutable' 
flag. To change it (re)start the VM and do: 

sudo chattr -i -R .config/autostart 

Then change what you need to in that path and restart the VM. During the 
startup process the dir and its contents will be automatically made 
immutable again. 

Tested and confirmed working. When combined with your halt-vm-by-window script, my Qube Manager is now basically my start menu. Who said security doesn't mix well with useability? Now, if only there were a modification that allowed you to start VMs by double-clicking on them in the Qube Manager...

On Monday, 3 August 2020 19:05:29 UTC+8, Chris Laprise wrote:

BTW, I think the appVM is the right place to make the .config/autostart
change if the custom .desktop file is being applied on a per-VM basis.

Tested and confirmed to persist across reboots. Thanks a bunch! 

 

[fiftyfour...@gmail.com]

Oh, and while I have you here, Chris, I thought I'd let you know that your Wireguard guide in Qubes-VPN-Support doesn't work--I followed it step-by-step but was left frustrated, so I took another route.

I just came across this Reddit post where the poster seems to have gone through the same experience, so that reminded me to let you know: 

https://old.reddit.com/r/Qubes/comments/i2cza9/cant_for_the_life_of_me_get_wireguard_to_work/

[Chris Laprise]

On 8/3/20 10:18 AM, fiftyfour...@gmail.com wrote:
> On Monday, 3 August 2020 18:36:28 UTC+8, Chris Laprise wrote:
>
> 'curl' would only be used in a Whonix template. This is to signal
> Qubes'
> proxy to start the Tor-based updateVM as soon as possible. It should
> not
> try to run curl in a Fedora or regular Debian template.
>
> To suppress interactive prompts, you need to run the script with
> '-u' or
> '--unattended'.
>
>
> Thanks--I managed to figure that out and respondright before you posted

> <https://groups.google.com/d/msg/qubes-users/SbLGJ1CWAWw/Bk1SyD-JAQAJ>,

> so now I'm using the -atu option.
>
> Not sure if it's a bug, but it seems like your script attempted to run
> curl in Fedora. I can't copy the output, but the VM basically goes,
> "Errors during downloading metadata for repository 'updates': Curl error
> (28); Curl Error (23)" several times before throwing up its arms and
> giving up. Then the script tells me that fedora-32-minimal update
> returned non-zero status.

Seems like dnf itself uses curl. Searching for 'dnf curl' shows a lot of
hits. dnf is saying it couldn't use curl to retrieve metadata for the
updates repo. Maybe there is an issue with the way the updatevm is setup
on your system?

To test manually, here is the command the script uses:

dnf update -y --best

I also just tested the script with a fresh fedora-32-minimal and it
works (interesting to note that "curl-minimal" was one of the updated
packages).

[Chris Laprise]

Yes, the requirements to get it running keep changing. Right now the
easiest way is to install 'kernel-latest-qubes-vm' from dom0 to get a
5.x kernel for VMs (the 5.x kernels have wg module included), then
install the wireguard-tools package without dependencies in your template.

I'll be switching to wireguard in the next few weeks so I'll be updating
the wiki then.

[fiftyfour...@gmail.com]

On Tuesday, 4 August 2020 00:03:08 UTC+8, Chris Laprise wrote:

Yes, the requirements to get it running keep changing. Right now the
easiest way is to install 'kernel-latest-qubes-vm' from dom0 to get a
5.x kernel for VMs (the 5.x kernels have wg module included), then
install the wireguard-tools package without dependencies in your template.

I'll be switching to wireguard in the next few weeks so I'll be updating
the wiki then.

I forgot to mention that I was following your latest instructions via this thread and still wasn't able to get it working: https://groups.google.com/d/msg/qubes-users/f974-MsbZyM/xh93RU7NAQAJ

1. Install the 'kernel-latest-qubes-vm' package in dom0. This will
provide a 5.x kernel with wireguard module built-in. Set your VPN VM to
use this kernel.

 

2. Install only the 'wireguard-tools' package (from testing) in Debian
10. Otherwise, there may be a conflict between the built-in and DKMS
modules.

 

3. Given the above, it may now be possible to skip using HVM mode
altogether. 

[Ulrich Windl]

On 8/2/20 4:42 PM, Chris Laprise wrote:
> On 8/2/20 8:32 AM, fiftyfour...@gmail.com wrote:
>> I have a ton of templates and standalones (>10), so updating them one
>> by one serially is a pain. I found a convenient dom0 script so I
>> thought I'd share.
>>
>> Basically, take this and paste it into dom0 then make it executable.
>> There'a also a handy test function. All credit goes to Andrea
>> Micheloni. Anyone have similarly handy modifications/scripts?
>>
>> https://m7i.org/tips/qubes-update-all-templates/
>
> IIRC there is an option somewhere in 'qubesctl' for parallel template
> updates.

Actually instead of parallel updates (assuming limited bandwidth) I'd
vote for a more verbose progress indicator (in the graphical update app):
Currently the VMs start, update starts, and then ...long time
nothing..., then the list of packages updated.

Regards,
Ulrich

[fiftyfour...@gmail.com]

On Wednesday, 5 August 2020 07:08:04 UTC+8, Ulrich Windl wrote:

Actually instead of parallel updates (assuming limited bandwidth) I'd
vote for a more verbose progress indicator (in the graphical update app):
Currently the VMs start, update starts, and then ...long time
nothing..., then the list of packages updated.

Regards,
Ulrich

Check out Chris' qubes4-multi-update script--it gives you exactly what you want, alone with some more options.

[Qubes]

I vote for the local proxy to cache updates. Is that not possible?

Would it not make sense if a Fedora-32 or Fedora-32-xx template
downloads for example the latest Firefox that it should be cached
locally. When the next Fedora-32 template checks for updates the system
can check what the latest version is on the repository, if it is the
same as the cached version use the cached copy. This will dramatically
reduce update bandwidth requirements and make updating quick and
painless no matter the number of templateVMs.

There must be a fancy way of making this secure. When an update is
downloaded for the first time it is verified so we must ensure the
cache's integrity. Maybe the downloaded files can be signed by the users
own key kept in the vault AppVM.

[unman]

Yes, it is. A simple mechanism is to install apt-cacher-ng, and
configure it to listen on port 8082, while removing or masking
tinyproxy.
This acts as a simple drop in replacement with benefit of caching.
To use https in the repositories you need to rewrite them as
"http://HTTPS///", and the caching proxy will then connect using https.
To use for Fedora you need to keep an eye on the multiplicity of Fedora
repos, and update the repo definitions.
Debian and debian based archives work out of the box.

>
> Would it not make sense if a Fedora-32 or Fedora-32-xx template downloads
> for example the latest Firefox that it should be cached locally. When the
> next Fedora-32 template checks for updates the system can check what the
> latest version is on the repository, if it is the same as the cached version
> use the cached copy. This will dramatically reduce update bandwidth
> requirements and make updating quick and painless no matter the number of
> templateVMs.
>
> There must be a fancy way of making this secure. When an update is
> downloaded for the first time it is verified so we must ensure the cache's
> integrity. Maybe the downloaded files can be signed by the users own key
> kept in the vault AppVM.
>

The security isnt to be found at the proxy level, but at the package
management level. It's there that verification is (and should be) done.

[fiftyfour...@gmail.com]

On Thursday, 6 August 2020 09:03:31 UTC+8, unman wrote:

The security isnt to be found at the proxy level, but at the package
management level. It's there that verification is (and should be) done.

Unman, speaking of verification at the package management level, would you happen to know the algorithm that's used to verify dom0 and domu packages? I've been looking for this info since I'm worried that it might be the now-deprecated SHA1 (like Github) but I haven't found anything yet. 

[sysad.andes]

-- I'm not unman, but I just checked the repo data and it appears they use sha256

[fiftyfour...@gmail.com]

On Thursday, 6 August 2020 12:31:44 UTC+8, Emily wrote:

-- I'm not unman, but I just checked the repo data and it appears they use sha256

This is reassuring. Thanks, Emily

[Chris Laprise]

I hate to break that feeling, but Fedora is unique in that it doesn't
sign its repo metadata, and sadly that is what matters. They put a
bandaid on it by fetching more hashes via https... so the update
security in Fedora is based on the strength of https. That is bad, as
https can be subverted by resourceful attackers.

https://bugzilla.redhat.com/show_bug.cgi?id=1130491

What this potentially allows is an attacker to blind Fedora systems to
specific package updates, where the systems appear to retrieve updates
normally without the users being aware that particular packages with
known vulnerabilities have been held back.

Note that RHEL and Centos _do_ sign their repomd.xml. So we're looking
at some kind of decision made either by Red Hat's marketing department
(keep Fedora off RHEL's expensive turf) or by some idea that Fedora is
not for serious mission critical environments, or both.

So this is a sizable hole in Qubes security. The best advice I can give
is to avoid using Fedora templates and pay attention to Qubes Security
Bulletins when they mention which dom0 components will be updated (and
pay close attention when running qubes-dom0-update to look for the
mentioned components).

[fiftyfour...@gmail.com]

On Thursday, 6 August 2020 18:05:25 UTC+8, Chris Laprise wrote:

I hate to break that feeling, but Fedora is unique in that it doesn't
sign its repo metadata, and sadly that is what matters. They put a
bandaid on it by fetching more hashes via https... so the update
security in Fedora is based on the strength of https. That is bad, as
https can be subverted by resourceful attackers.

https://bugzilla.redhat.com/show_bug.cgi?id=1130491

What this potentially allows is an attacker to blind Fedora systems to
specific package updates, where the systems appear to retrieve updates
normally without the users being aware that particular packages with
known vulnerabilities have been held back.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886

That's highly concerning and might put me off from using Qubes for sensitive work, which defeats the entire purpose of installing Qubes. This is a massive gaping whole that, to me, invalidates all the other security strengths of Qubes, since dom0 is the key to the kingdom.

The reason why I'm anxious about the security of packages is because my dom0 has exhibited strange behavior not present before my dom0 update (and I know because I spent a lot of time with my OS before connecting it for the first time). My dom0 update itself has been behaving strangely and I made a post about it earlier, where I also asked about package verification, but received no response.

Hi all,
 
Every time I use qubes-dom0-update in a fresh installation (which I've done around ten times now), I get strange outputs where the repositories aren't shown being queried but the update proceeds. It looks something like this: 

error:could not delete old database at /var/lib/qubes/dom0-updates/home/user/.rpmdbold.965
https://mirrors.phx.ms/qubes/repo/yum/r4.0/current/dom0/fc25/repodata/repomd.xml:[Errno 14]curl#6-"Could not resolve host:mirror.phx.ms"
Trying other mirror.
https://mirror.linux.pizza/qubes-os.org/repo/yum/r4.0/current/dom0/fc25/repodata/repomd.xml:[Errno14]HTTPS Error 404 -Not Found
Trying other mirror.
https://mirror.linux.pizza/qubes-os.org/repo/yum/r4.0/templates-til/repodata/repomd.xml:[Errno 14] HTTPS Error 404 - Not Found
Trying other mirror.
No Match for argument
No Match for argument
No Match for argument
No Match for argument
No Match for argument
No Match for argument
No Match for argument
No Match for argument
-->Running transaction check
--->Package kernel[...] will be installed


[...]
--->Finished Dependency Resolution
[Starts downloading]
This is consistent even when updating over tor, and has been bugging me. Does anyone else see this when they first update dom0? 

 Also, it dom0 update consistently gives me two [Y/n] prompts in a row before installation, which seems very strange.

[fiftyfour...@gmail.com]

I hate to break that feeling, but Fedora is unique in that it doesn't
sign its repo metadata, and sadly that is what matters. They put a
bandaid on it by fetching more hashes via https... so the update
security in Fedora is based on the strength of https. That is bad, as
https can be subverted by resourceful attackers.

On the other hand, following the instructions on these sites shows me that /etc/yum.conf and the repos in /etc/yum.repos.d/  all have gpgcheck=1. I'm not sure what this means.

https://www.qubes-os.org/doc/security-guidelines/

https://docs.fedoraproject.org/en-US/Fedora/12/html/Deployment_Guide/sec-Configuring_Yum_and_Yum_Repositories.html 

[Chris Laprise]

IIRC that setting refers to checking packages, not the repomd.xml files.
That's why an attacker can't replace packages with their own versions;
they have to manipulate the metadata to hold back packages from
receiving updates.

[fiftyfour...@gmail.com]

On Friday, 7 August 2020 00:13:52 UTC+8, Chris Laprise wrote:

IIRC that setting refers to checking packages, not the repomd.xml files.
That's why an attacker can't replace packages with their own versions;
they have to manipulate the metadata to hold back packages from
receiving updates.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886

So as long as I verify that the version numbers of packages in dom0 match those of the actual repo website, I can assume that my dom0 updates have not been tampered with by adversaries? 

[Chris Laprise]

Yes. Note that Qubes Security Bulletins are issued for vulns that affect
dom0 and they reference the package versions that contain the patches.
For example:

https://groups.google.com/d/msgid/qubes-users/34eddc9a-300c-743c-cb12-acc677f5784f%40qubes-os.org

However, most vulns that affect templates are not addressed by QSBs
because they're not Qubes-specific. That's one reason to avoid Fedora
templates in general.

[fiftyfour...@gmail.com]

On Friday, 7 August 2020 02:40:58 UTC+8, Chris Laprise wrote:

Yes. Note that Qubes Security Bulletins are issued for vulns that affect
dom0 and they reference the package versions that contain the patches.
For example:

https://groups.google.com/d/msgid/qubes-users/34eddc9a-300c-743c-cb12-acc677f5784f%40qubes-os.org

However, most vulns that affect templates are not addressed by QSBs
because they're not Qubes-specific. That's one reason to avoid Fedora
templates in general.

--
Chris Laprise, tas...@posteo.net
https://github.com/tasket
https://twitter.com/ttaskett
PGP: BEE2 20C5 356E 764A 73EB  4AB3 1DC4 D106 F07F 1886

That's great to know. I was worried my efforts at secure computing have been in vain. I suppose those who need to use Fedora should stick with CentOS. Thanks a bunch, Chris! 

[Qubes]

Why does the Qubes project continue using Fedora as the base for a
default install. Even dom0 is Fedora. I assume they are well aware of
this issue.

Do the Qubes core team not regard this as a problem or what is the
rationale?

[Qubes]

On 8/3/20 1:05 PM, Chris Laprise wrote:
>> On 8/3/20 4:11 AM, fiftyfour...@gmail.com wrote:
>
>>> Your Qubes-VM-Hardening tool was one of the first things installed
>>> into my first Qubes, but I'm still not very familiar with how it
>>> works. I think vm-boot-protect might be blocking me from adding a
>>> .desktop file into ~/.config/autostart, as Steve suggested (Steve:
>>> does this need to be done in templates? If done in an appVM, wouldn't
>>> it get purged upon restart?).
>
> BTW, I think the appVM is the right place to make the .config/autostart
> change if the custom .desktop file is being applied on a per-VM basis.
>
> If you want it for _all_ VMs based on that template, that's a little
> harder. Putting the .desktop file in /etc/skel would only make the
> change when an appVM is first created, so existing VMs using that
> template would not benefit. However, vm-boot-protect-root has the
> ability to copy or "deploy" files into /home on each boot; you would
> have to save the .desktop file under
> /etc/default/vms/vms.all/rw/home/user/.config/autostart in the template.
>

Chris, do I understand you correct. The
/etc/default/vms/vms.all/rw/home/user/.config/autostart directory
(structure) needs to be created? My debian-10 templateVM only has
/etc/default/grub.d, there are no other directories.