[PATCH 1/3] find_live_objects: correct progress ref count and verbosity

[Rob Browning]

Only print the final progress message when verbose (matching the other
progress output), and fix the ref count.

Signed-off-by: Rob Browning <r...@defaultvalue.org>
Tested-by: Rob Browning <r...@defaultvalue.org>
---
lib/bup/gc.py | 4 ++--
test/ext/test-get-repair-bupm | 5 ++---
2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/bup/gc.py b/lib/bup/gc.py
index fa554b79..ce76438c 100644
--- a/lib/bup/gc.py
+++ b/lib/bup/gc.py
@@ -103,7 +103,7 @@ def find_live_objects(existing_count, cat_pipe, refs=None, *,
ref_n = len(scan_refs)
def progress_msg(ref_i):
return 'scanned %s of %s ref%s (%02.2f%% of all objects)' \
- % (ref_i, ref_n, 's' if ref_n > 1 else '',
+ % (ref_i + 1, ref_n, 's' if ref_n > 1 else '',
approx_live_count * 100.0 / existing_count)
for ref_i, (ref_name, ref_id) in enumerate(scan_refs):
for item_path in walk_object(cat_pipe.get, hexlify(ref_id),
@@ -131,7 +131,7 @@ def find_live_objects(existing_count, cat_pipe, refs=None, *,
approx_live_count += 1
qprogress(progress_msg(ref_i) + '\r')
live_blobs.add(item.oid)
- if scan_refs:
+ if verbosity and scan_refs:
log(progress_msg(ref_i) + '\n')
maybe_close_bloom.pop_all()
return live_blobs, live_trees
diff --git a/test/ext/test-get-repair-bupm b/test/ext/test-get-repair-bupm
index 5efc90f0..5eeeb125 100755
--- a/test/ext/test-get-repair-bupm
+++ b/test/ext/test-get-repair-bupm
@@ -60,9 +60,8 @@ unset saves
# May as well double-check, though test-validate-refs already does this
WVEXPRC 1 eval 'bup validate-refs --bupm 2>&1 | tee validate.log'
btl-display-file validate.log
-wv-match-rx "$(< validate.log)" \
-"abridged-bupm refs/heads/src $broken_tree_oid:\.bupm/?
-scanned 0 of 1 ref \(0\.00% of all objects\)"
+wv-match-rx "$(< validate.log)" \
+ "abridged-bupm refs/heads/src $broken_tree_oid:\.bupm/?"

# Ensure a normal rewrite rejects the abridged bupm
WVEXPRC 2 eval 'bup get --rewrite --append: src dst 2>&1 | tee rewrite.log'
--
2.47.3

[Rob Browning]

Pushed to main.

[Rob Browning]

Unfortunately, with current python (3.13), __setattr__ is too
expensive, taking a sample index run down from 25k paths/s to ~16k.

Signed-off-by: Rob Browning <r...@defaultvalue.org>
Tested-by: Rob Browning <r...@defaultvalue.org>
---

lib/bup/metadata.py | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py
index 9ee5fe65..c6b8d83d 100644
--- a/lib/bup/metadata.py
+++ b/lib/bup/metadata.py
@@ -8,6 +8,7 @@
from copy import deepcopy
from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
from io import BytesIO
+from os import environb as environ
from time import gmtime, strftime
import copy, errno, os, sys, stat, socket, struct

@@ -803,13 +804,16 @@ class Metadata:

def freeze(self): self._frozen = True; return self
def thaw(self): self._frozen = False; return self
- def __setattr__(self, k, v):
- if k == '_frozen':
+ if int(environ.get(b'BUP_TEST_LEVEL', b'0')) >= 11:
+ # Unfortunately expensive (took an "index ~" from 25k paths/s
+ # to 16k with 3.13).
+ def __setattr__(self, k, v):
+ if k == '_frozen':
+ return super().__setattr__(k, v)
+ if getattr(self, '_frozen', False):
+ raise AttributeError(f'Cannot change frozen instance attribute {k}',
+ name=k, obj=self)
return super().__setattr__(k, v)
- if getattr(self, '_frozen', False):
- raise AttributeError(f'Cannot change frozen instance attribute {k}',
- name=k, obj=self)
- return super().__setattr__(k, v)
def __copy__(self):
result = self.__new__(self.__class__)
for k in [x for x in self.__slots__ if x != '_frozen']:
--
2.47.3

[Rob Browning]

Python added support for nanosecond timestamps in 3.3, so we shouldn't
need our C *stat helpers anymore.

Signed-off-by: Rob Browning <r...@defaultvalue.org>
Tested-by: Rob Browning <r...@defaultvalue.org>
---

configure | 25 ------------
lib/bup/_helpers.c | 100 ---------------------------------------------
lib/bup/xstat.py | 36 +++++++---------
3 files changed, 15 insertions(+), 146 deletions(-)

diff --git a/configure b/configure
index 7e78b217..a8efbcaf 100755
--- a/configure
+++ b/configure
@@ -360,31 +360,6 @@ if test "${c_define[BUP_HAVE_READLINE]:-}"; then
fi
fi

-info -n 'checking for ns resolution stat times'
-stat_code='
-#include <sys/stat.h>
-
-int main(int argc, char **argv)
-{
- struct stat st;
- stat(argv[0], &st);
- return (int) st.BUP_TIME_FIELD;
-}
-'
-if try-c-code -DBUP_TIME_FIELD=st_atim.tv_nsec -- "$stat_code"; then
- info ' (found, tim)'
- c_define[BUP_STAT_NS_FLAVOR_TIM]=1
-elif try-c-code -DBUP_TIME_FIELD=st_atimensec.tv_nsec -- "$stat_code"; then
- info ' (found, timensec)'
- c_define[BUP_STAT_NS_FLAVOR_TIMENSEC]=1
-elif try-c-code -DBUP_TIME_FIELD=st_atimespec.tv_nsec -- "$stat_code"; then
- info ' (found, timespec)'
- c_define[BUP_STAT_NS_FLAVOR_TIMESPEC]=1
-else
- info ' (not found)'
- c_define[BUP_STAT_NS_FLAVOR_NONE]=1
-fi
-
info -n 'checking for tm tm_gmtoff field'
if try-c-code \
-- \
diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index d192001a..cfa9af0b 100644
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -1093,100 +1093,6 @@ static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args)
#endif /* def BUP_HAVE_FILE_ATTRS */


-#ifdef BUP_STAT_NS_FLAVOR_TIM
-# define BUP_STAT_ATIME_NS(st) (st)->st_atim.tv_nsec
-# define BUP_STAT_MTIME_NS(st) (st)->st_mtim.tv_nsec
-# define BUP_STAT_CTIME_NS(st) (st)->st_ctim.tv_nsec
-#elif defined BUP_STAT_NS_FLAVOR_TIMENSEC
-# define BUP_STAT_ATIME_NS(st) (st)->st_atimensec.tv_nsec
-# define BUP_STAT_MTIME_NS(st) (st)->st_mtimensec.tv_nsec
-# define BUP_STAT_CTIME_NS(st) (st)->st_ctimensec.tv_nsec
-#elif defined BUP_STAT_NS_FLAVOR_TIMESPEC
-# define BUP_STAT_ATIME_NS(st) (st)->st_atimespec.tv_nsec
-# define BUP_STAT_MTIME_NS(st) (st)->st_mtimespec.tv_nsec
-# define BUP_STAT_CTIME_NS(st) (st)->st_ctimespec.tv_nsec
-#elif defined BUP_STAT_NS_FLAVOR_NONE
-# define BUP_STAT_ATIME_NS(st) 0
-# define BUP_STAT_MTIME_NS(st) 0
-# define BUP_STAT_CTIME_NS(st) 0
-#else
-# error "./configure did not define a BUP_STAT_NS_FLAVOR"
-#endif
-
-
-static PyObject *stat_struct_to_py(const struct stat *st,
- const char *filename,
- int fd)
-{
- // We can check the known (via POSIX) signed and unsigned types at
- // compile time, but not (easily) the unspecified types, so handle
- // those via BUP_LONGISH_TO_PY(). Assumes ns values will fit in a
- // long.
- return Py_BuildValue("NKNNNNNL(Nl)(Nl)(Nl)",
- BUP_LONGISH_TO_PY(st->st_mode),
- (unsigned PY_LONG_LONG) st->st_ino,
- BUP_LONGISH_TO_PY(st->st_dev),
- BUP_LONGISH_TO_PY(st->st_nlink),
- BUP_LONGISH_TO_PY(st->st_uid),
- BUP_LONGISH_TO_PY(st->st_gid),
- BUP_LONGISH_TO_PY(st->st_rdev),
- (PY_LONG_LONG) st->st_size,
- BUP_LONGISH_TO_PY(st->st_atime),
- (long) BUP_STAT_ATIME_NS(st),
- BUP_LONGISH_TO_PY(st->st_mtime),
- (long) BUP_STAT_MTIME_NS(st),
- BUP_LONGISH_TO_PY(st->st_ctime),
- (long) BUP_STAT_CTIME_NS(st));
-}
-
-
-static PyObject *bup_stat(PyObject *self, PyObject *args)
-{
- int rc;
- char *filename;
-
- if (!PyArg_ParseTuple(args, cstr_argf, &filename))
- return NULL;
-
- struct stat st;
- rc = stat(filename, &st);
- if (rc != 0)
- return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
- return stat_struct_to_py(&st, filename, 0);
-}
-
-
-static PyObject *bup_lstat(PyObject *self, PyObject *args)
-{
- int rc;
- char *filename;
-
- if (!PyArg_ParseTuple(args, cstr_argf, &filename))
- return NULL;
-
- struct stat st;
- rc = lstat(filename, &st);
- if (rc != 0)
- return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
- return stat_struct_to_py(&st, filename, 0);
-}
-
-
-static PyObject *bup_fstat(PyObject *self, PyObject *args)
-{
- int rc, fd;
-
- if (!PyArg_ParseTuple(args, "i", &fd))
- return NULL;
-
- struct stat st;
- rc = fstat(fd, &st);
- if (rc != 0)
- return PyErr_SetFromErrno(PyExc_OSError);
- return stat_struct_to_py(&st, NULL, fd);
-}
-
-
#ifdef HAVE_TM_TM_GMTOFF
static PyObject *bup_localtime(PyObject *self, PyObject *args)
{
@@ -1913,12 +1819,6 @@ static PyMethodDef helper_methods[] = {
{ "set_linux_file_attr", bup_set_linux_file_attr, METH_VARARGS,
"Set the Linux attributes for the given file." },
#endif
- { "stat", bup_stat, METH_VARARGS,
- "Extended version of stat." },
- { "lstat", bup_lstat, METH_VARARGS,
- "Extended version of lstat." },
- { "fstat", bup_fstat, METH_VARARGS,
- "Extended version of fstat." },
#ifdef HAVE_TM_TM_GMTOFF
{ "localtime", bup_localtime, METH_VARARGS,
"Return struct_time elements plus the timezone offset and name." },
diff --git a/lib/bup/xstat.py b/lib/bup/xstat.py
index 90f80a79..382a7b76 100644
--- a/lib/bup/xstat.py
+++ b/lib/bup/xstat.py
@@ -4,8 +4,6 @@ from time import strftime
import os, sys, time
import stat as pystat

-from bup import _helpers
-

def timespec_to_nsecs(ts):
ts_s, ts_ns = ts
@@ -64,24 +62,20 @@ class stat_result:
__slots__ = ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid', 'st_gid',
'st_rdev', 'st_size', 'st_atime', 'st_mtime', 'st_ctime')
@staticmethod
- def from_xstat_rep(st):
- global _cygwin_sys
+ def from_py_stat(st):
result = stat_result()
- (result.st_mode,
- result.st_ino,
- result.st_dev,
- result.st_nlink,
- result.st_uid,
- result.st_gid,
- result.st_rdev,
- result.st_size,
- result.st_atime,
- result.st_mtime,
- result.st_ctime) = st
+ result.st_mode = st.st_mode
+ result.st_ino = st.st_ino
+ result.st_dev = st.st_dev
+ result.st_nlink = st.st_nlink
+ result.st_uid = st.st_uid
+ result.st_gid = st.st_gid
+ result.st_rdev = st.st_rdev
+ result.st_size = st.st_size
# Inlined timespec_to_nsecs after profiling
- result.st_atime = result.st_atime[0] * 10**9 + result.st_atime[1]
- result.st_mtime = result.st_mtime[0] * 10**9 + result.st_mtime[1]
- result.st_ctime = result.st_ctime[0] * 10**9 + result.st_ctime[1]
+ result.st_atime = st.st_atime_ns
+ result.st_mtime = st.st_mtime_ns
+ result.st_ctime = st.st_ctime_ns
if _cygwin_sys:
result.st_uid = _fix_cygwin_id(result.st_uid)
result.st_gid = _fix_cygwin_id(result.st_gid)
@@ -89,15 +83,15 @@ class stat_result:


def stat(path):
- return stat_result.from_xstat_rep(_helpers.stat(path))
+ return stat_result.from_py_stat(os.stat(path))


def fstat(path):
- return stat_result.from_xstat_rep(_helpers.fstat(path))
+ return stat_result.from_py_stat(os.fstat(path))


def lstat(path):
- return stat_result.from_xstat_rep(_helpers.lstat(path))
+ return stat_result.from_py_stat(os.lstat(path))


def mode_str(mode):
--
2.47.3