[PATCH] loader.py: use with statement when writing trace file

[Waldemar Kozaczuk]

The 'osv trace2file' does not work with python3 so let us fix it by
replacing the code to open and write to a file with more portable
and succinct "with" construct.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
scripts/loader.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/loader.py b/scripts/loader.py
index 0ce782d0..97c831e9 100755
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -1508,9 +1508,8 @@ class osv_trace_file(gdb.Command):
def __init__(self):
gdb.Command.__init__(self, 'osv trace2file', gdb.COMMAND_USER, gdb.COMPLETE_NONE)
def invoke(self, arg, from_tty):
- fout = file("trace.txt", "wt")
- dump_trace(fout.write)
- fout.close()
+ with open("trace.txt", 'wt') as fout:
+ dump_trace(fout.write)

class osv_leak(gdb.Command):
def __init__(self):
--
2.35.1

[Waldemar Kozaczuk]

Eliminate any tracepoint code specific frames from the profiler backtraces.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

scripts/osv/prof.py | 2 ++
1 file changed, 2 insertions(+)

diff --git a/scripts/osv/prof.py b/scripts/osv/prof.py
index 2de0d4ac..edb8a5c1 100644
--- a/scripts/osv/prof.py
+++ b/scripts/osv/prof.py
@@ -103,6 +103,8 @@ def strip_garbage(backtrace):
def is_good(src_addr):
if not src_addr.name:
return True
+ if src_addr.filename and src_addr.filename.endswith("trace.hh"):
+ return False
return not src_addr.name in unimportant_functions

for chain in unimportant_prefixes:
--
2.35.1

[Waldemar Kozaczuk]

Some collected tracepoints may have missing the filename field. So before
trying to filter the frames by filename test if it is present.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

scripts/osv/trace.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/osv/trace.py b/scripts/osv/trace.py
index fb14c4fe..1b979215 100644
--- a/scripts/osv/trace.py
+++ b/scripts/osv/trace.py
@@ -33,7 +33,7 @@ class BacktraceFormatter:

frames = list(debug.resolve_all(self.resolver, (x - 1 for x in backtrace if x)))

- while frames[0].name and (frames[0].name.startswith("tracepoint") or frames[0].filename.endswith("trace.hh")):
+ while frames[0].name and (frames[0].name.startswith("tracepoint") or (frames[0].filename and frames[0].filename.endswith("trace.hh"))):
frames.pop(0)

if self.multiline:
--
2.35.1

[Waldemar Kozaczuk]

Enhance tracing/profiling tool by adding a new -S option that instructs
trace.py to use tracefile.symbols file for symbol resolution instead of addr2line utility:

./scripts/trace.py prof -FLS

In essence, this new patch enhances loader.py to generate new file tracefile.symbols
along the trace file with an information like this:

0x100000119277 /usr/lib/golang/src/runtime/mheap.go 1950 runtime.newMarkBits
0x100000114a04 /usr/lib/golang/src/runtime/mgcsweep.go 471 runtime.(*mspan).sweep
0x10000010824a /usr/lib/golang/src/runtime/mcentral.go 214 runtime.(*mcentral).uncacheSpan
0x100000107c3b /usr/lib/golang/src/runtime/mcache.go 276 runtime.(*mcache).releaseAll
0x40351825 core/mempool.cc 356 memory::pool::free(void*)
0x402403b6 ./bsd/sys/sys/mbuf.h 609 m_freem
0x40388884 /usr/include/c++/11/bits/std_mutex.h 228 epoll_file::wait(epoll_event*, int, int)

Each line above contains white-space separated address, filename,
file number (if any) and name of all referenced symbols of both kernel and application.

Please note the 'trace.py extract' calls 'osv syms' to load all application objects.
But this only helps if the objects are still loaded by OSv. For example, if one runs some
server app to profile under load and uses 'trace.py extract', then it will work because
the app would be still running as OSv has not entered into a shutdown phase.

However, if one tests it with any app that simply completes like native-example, then
it will not work because objects are already unloaded. In this case, one has to invoke run.py
with -w option, set a breakpoint in OSv code after it loads objects but before it starts
the app (for example at core/app.cc:220) and then manually run 'osv syms' to force resolution
of application symbols.

The extra benefit of this new approach is that 'trace.py prof' works
much faster as it simply reads new file .symbols instead of calling
addr2line against loader.elf for each symbol which can take half a
minute or longer.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

scripts/loader.py | 21 +++++++++++++++++++++
scripts/osv/debug.py | 36 ++++++++++++++++++++++++++++++++++++
scripts/trace.py | 6 ++++++
3 files changed, 63 insertions(+)

diff --git a/scripts/loader.py b/scripts/loader.py
index 97c831e9..a7f82e6c 100755
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -101,6 +101,15 @@ class syminfo_resolver(object):
def clear_cache(clazz):
clazz.cache.clear()

+ @classmethod
+ def output_cache(clazz, output_func):
+ for source_addr in clazz.cache.values():
+ addr = source_addr[0]
+ if addr.line:
+ output_func("0x%x %s %d %s\n" % (addr.addr, addr.filename, addr.line, addr.name))
+ else:
+ output_func("0x%x %s <?> %s\n" % (addr.addr, addr.filename, addr.name))
+
symbol_resolver = syminfo_resolver()

def symbol_formatter(src_addr):
@@ -1304,6 +1313,17 @@ def all_traces():
def save_traces_to_file(filename):
trace.write_to_file(filename, list(all_traces()))

+def save_backtrace_symbols_to_file(filename):
+ # Iterate over all traces and force resolution of symbols in
+ # included backtrace if any
+ for trace in all_traces():
+ if trace.backtrace:
+ for address in list(x - 1 for x in trace.backtrace if x):
+ symbol_resolver(address)
+ # Save resolved symbol information from cache into a file
+ with open(filename, 'wt') as sout:
+ syminfo_resolver.output_cache(sout.write)
+
def make_symbolic(addr):
return str(syminfo(addr))

@@ -1503,6 +1523,7 @@ class osv_trace_save(gdb.Command):

gdb.write('Saving traces to %s ...\n' % arg)
save_traces_to_file(arg)
+ save_backtrace_symbols_to_file("%s.symbols" % arg)

class osv_trace_file(gdb.Command):
def __init__(self):
diff --git a/scripts/osv/debug.py b/scripts/osv/debug.py
index 83372ada..d7d34637 100644
--- a/scripts/osv/debug.py
+++ b/scripts/osv/debug.py
@@ -99,6 +99,42 @@ class SymbolResolver(object):
self.addr2line.stdin.close()
self.addr2line.wait()

+class SymbolsFileResolver(object):
+ def __init__(self, symbols_file, fallback_resolver=DummyResolver()):
+ if not os.path.exists(symbols_file):
+ raise Exception('File not found: ' + object_path)
+ self.fallback_resolver = fallback_resolver
+ self.cache = dict()
+
+ try:
+ symbol_lines = open(symbols_file).read().split('\n')
+ except IOError:
+ symbol_lines = []
+
+ for symbol_line in symbol_lines:
+ tokens = symbol_line.split(maxsplit=3)
+ if len(tokens) > 0:
+ addr = int(tokens[0], 16)
+ filename = tokens[1]
+ if tokens[2] == '<?>':
+ line = None
+ else:
+ line = int(tokens[2])
+ name = tokens[3]
+ self.cache[addr] = [SourceAddress(addr, name=name, filename=filename, line=line)]
+
+ def __call__(self, addr):
+ """
+ Returns an iterable of SourceAddress objects for given addr.
+
+ """
+
+ result = self.cache.get(addr, None)
+ if result:
+ return result
+ else:
+ return self.fallback_resolver(addr)
+
def resolve_all(resolver, raw_addresses):
"""
Returns iterable of SourceAddress objects for given list of raw addresses
diff --git a/scripts/trace.py b/scripts/trace.py
index 3d3d0dce..17a6faeb 100755
--- a/scripts/trace.py
+++ b/scripts/trace.py
@@ -63,6 +63,7 @@ def add_symbol_resolution_options(parser):
group.add_argument("-L", "--show-line-number", action='store_true', help="show line numbers")
group.add_argument("-A", "--show-address", action='store_true', help="show raw addresses")
group.add_argument("-F", "--show-file-name", action='store_true', help="show file names")
+ group.add_argument("-S", "--use-symbols-file", action='store_true', help="use <tracefile>.symbols to resolve symbols")

class BeautifyingResolver(object):
def __init__(self, delegate):
@@ -82,6 +83,10 @@ def symbol_resolver(args):
if args.no_resolve:
return debug.DummyResolver()

+ if args.use_symbols_file:
+ symbols_file = "%s.symbols" % args.tracefile
+ return BeautifyingResolver(debug.SymbolsFileResolver(symbols_file))
+
if args.exe:
elf_path = args.exe
elif args.debug:
@@ -281,6 +286,7 @@ def extract(args):
cmdline.extend(['-ex', 'target remote ' + args.remote])
else:
cmdline.extend(['-ex', 'conn'])
+ cmdline.extend(['-ex', 'osv syms'])
cmdline.extend(['-ex', 'osv trace save ' + args.tracefile])
proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
--
2.35.1

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

loader.py: use with statement when writing trace file

The 'osv trace2file' does not work with python3 so let us fix it by
replacing the code to open and write to a file with more portable
and succinct "with" construct.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---

diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

prof.py: remove extra noise from profiler stack traces

Eliminate any tracepoint code specific frames from the profiler backtraces.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---

diff --git a/scripts/osv/prof.py b/scripts/osv/prof.py

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

trace.py: inspect backtrace filename in a safe manner

Some collected tracepoints may have missing the filename field. So before
trying to filter the frames by filename test if it is present.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---

diff --git a/scripts/osv/trace.py b/scripts/osv/trace.py
--- a/scripts/osv/trace.py
+++ b/scripts/osv/trace.py
@@ -33,7 +33,7 @@ def __call__(self, backtrace):

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

trace: add mechanism to resolve application symbols

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---

diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py

@@ -101,6 +101,15 @@ def __call__(self, addr):

def clear_cache(clazz):
clazz.cache.clear()

+ @classmethod
+ def output_cache(clazz, output_func):
+ for source_addr in clazz.cache.values():
+ addr = source_addr[0]
+ if addr.line:
+ output_func("0x%x %s %d %s\n" % (addr.addr, addr.filename, addr.line, addr.name))
+ else:
+ output_func("0x%x %s <?> %s\n" % (addr.addr, addr.filename, addr.name))
+
symbol_resolver = syminfo_resolver()

def symbol_formatter(src_addr):

@@ -1304,6 +1313,17 @@ def one_cpu_trace(cpu):

def save_traces_to_file(filename):
trace.write_to_file(filename, list(all_traces()))

+def save_backtrace_symbols_to_file(filename):
+ # Iterate over all traces and force resolution of symbols in
+ # included backtrace if any
+ for trace in all_traces():
+ if trace.backtrace:
+ for address in list(x - 1 for x in trace.backtrace if x):
+ symbol_resolver(address)
+ # Save resolved symbol information from cache into a file
+ with open(filename, 'wt') as sout:
+ syminfo_resolver.output_cache(sout.write)
+
def make_symbolic(addr):
return str(syminfo(addr))

@@ -1503,6 +1523,7 @@ def invoke(self, arg, from_tty):

gdb.write('Saving traces to %s ...\n' % arg)
save_traces_to_file(arg)
+ save_backtrace_symbols_to_file("%s.symbols" % arg)

class osv_trace_file(gdb.Command):
def __init__(self):
diff --git a/scripts/osv/debug.py b/scripts/osv/debug.py

--- a/scripts/osv/debug.py
+++ b/scripts/osv/debug.py
@@ -99,6 +99,42 @@ def close():