[PATCH] scripts: upgrade to python 3 - part 2

11 views

Skip to first unread message

Waldemar Kozaczuk

unread,

Feb 17, 2020, 6:22:38 PM2/17/20

to osv...@googlegroups.com, Waldemar Kozaczuk

This is the second batch of the python 3 upgrade changes.
This one focuces on httpserver API unit tests and most changes
revolve around changes to urllib.* modules.

The scripts have been tested by runnin following tests in
the root of the modules/httpserver-api directory:

- make check-http
- make check-ssl

Ref #1056

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
modules/httpserver-api/json2code.py | 2 +-
modules/httpserver-api/tests/api/testenv.py | 2 +-
modules/httpserver-api/tests/api/testfile.py | 8 ++++----
modules/httpserver-api/tests/api/testfs.py | 2 +-
modules/httpserver-api/tests/api/testjolokia.py | 2 +-
modules/httpserver-api/tests/api/testjvm.py | 8 ++++----
modules/httpserver-api/tests/api/testnetwork.py | 2 +-
modules/httpserver-api/tests/api/testos.py | 8 ++++----
modules/httpserver-api/tests/api/testtrace.py | 4 +---
modules/httpserver-api/tests/basetest.py | 6 +++---
modules/httpserver-api/tests/testhttpserver-api.py | 2 +-
11 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/modules/httpserver-api/json2code.py b/modules/httpserver-api/json2code.py
index 1e42ddb3..0b947e70 100755
--- a/modules/httpserver-api/json2code.py
+++ b/modules/httpserver-api/json2code.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import json
import sys
import re
diff --git a/modules/httpserver-api/tests/api/testenv.py b/modules/httpserver-api/tests/api/testenv.py
index 06a00e5e..edf605c5 100755
--- a/modules/httpserver-api/tests/api/testenv.py
+++ b/modules/httpserver-api/tests/api/testenv.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import basetest

class testenv(basetest.Basetest):
diff --git a/modules/httpserver-api/tests/api/testfile.py b/modules/httpserver-api/tests/api/testfile.py
index e2e08aa0..567ae171 100755
--- a/modules/httpserver-api/tests/api/testfile.py
+++ b/modules/httpserver-api/tests/api/testfile.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
import basetest
import subprocess

@@ -45,10 +45,10 @@ class testfile(basetest.Basetest):

def test_put_file_cmd(self):
path = "/file"
- self.curl(path + "/etc/hosts?op=COPY&destination="+urllib.quote("/etc/hosts1"), method='PUT')
+ self.curl(path + "/etc/hosts?op=COPY&destination="+urllib.parse.quote("/etc/hosts1"), method='PUT')
hosts = self.curl(path + "/etc/hosts1?op=GETFILESTATUS")
self.assertEqual(hosts["type"], "FILE")
- self.curl(path + "/etc/hosts1?op=RENAME&destination="+urllib.quote("/etc/hosts2"), method='PUT')
+ self.curl(path + "/etc/hosts1?op=RENAME&destination="+urllib.parse.quote("/etc/hosts2"), method='PUT')
hosts = self.curl(path + "/etc/hosts2?op=GETFILESTATUS")
self.assertEqual(hosts["type"], "FILE")
self.assertHttpError(path + "/etc/hosts1?op=GETFILESTATUS")
diff --git a/modules/httpserver-api/tests/api/testfs.py b/modules/httpserver-api/tests/api/testfs.py
index 08fdbe5a..b0924747 100755
--- a/modules/httpserver-api/tests/api/testfs.py
+++ b/modules/httpserver-api/tests/api/testfs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import basetest

class testfs(basetest.Basetest):
diff --git a/modules/httpserver-api/tests/api/testjolokia.py b/modules/httpserver-api/tests/api/testjolokia.py
index 87e1e515..e0c57cf9 100644
--- a/modules/httpserver-api/tests/api/testjolokia.py
+++ b/modules/httpserver-api/tests/api/testjolokia.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import basetest
import json

diff --git a/modules/httpserver-api/tests/api/testjvm.py b/modules/httpserver-api/tests/api/testjvm.py
index a3b9ac1b..a9cbc69b 100755
--- a/modules/httpserver-api/tests/api/testjvm.py
+++ b/modules/httpserver-api/tests/api/testjvm.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import time
import basetest
-import urllib
+import urllib.request, urllib.parse, urllib.error

class testjvm(basetest.Basetest):
def test_jvm_version(self):
@@ -31,14 +31,14 @@ class testjvm(basetest.Basetest):

def test_get_mbean(self):
mbean = self.curl(self.path_by_nick(self.jvm_api, "getMbeanList") +
- urllib.quote("java.lang:name=PS Old Gen,type=MemoryPool"))
+ urllib.parse.quote("java.lang:name=PS Old Gen,type=MemoryPool"))
self.assertGreaterEqual(len(mbean), 15)
self.assert_key_in("type", mbean[0])
self.assert_key_in("name", mbean[0])
self.assert_key_in("value", mbean[0])

def test_set_mbean(self):
- path = self.path_by_nick(self.jvm_api, "getMbeanList") + urllib.quote("java.lang:name=PS Old Gen,type=MemoryPool")
+ path = self.path_by_nick(self.jvm_api, "getMbeanList") + urllib.parse.quote("java.lang:name=PS Old Gen,type=MemoryPool")
mbean = self.curl(path)
usage = next((item for item in mbean if item["name"] == "UsageThreshold"), None)
self.assertTrue(usage != None)
diff --git a/modules/httpserver-api/tests/api/testnetwork.py b/modules/httpserver-api/tests/api/testnetwork.py
index d8fa95f5..62b4fc83 100755
--- a/modules/httpserver-api/tests/api/testnetwork.py
+++ b/modules/httpserver-api/tests/api/testnetwork.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import time
import basetest

diff --git a/modules/httpserver-api/tests/api/testos.py b/modules/httpserver-api/tests/api/testos.py
index 0208de71..73ef6423 100755
--- a/modules/httpserver-api/tests/api/testos.py
+++ b/modules/httpserver-api/tests/api/testos.py
@@ -1,11 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import time
import basetest

class testos(basetest.Basetest):
def test_os_version(self):
path = self.path_by_nick(self.os_api, "os_version")
- self.assertRegexpMatches(self.curl(path), r"v0\.\d+(-rc\d+)?(-\d+-[0-9a-z]+)?" , path)
+ self.assertRegex(self.curl(path), r"v0\.\d+(-rc\d+)?(-\d+-[0-9a-z]+)?" , path)

def test_vendor(self):
self.validate_path(self.os_api, "os_vendor", "Cloudius Systems")
@@ -18,8 +18,8 @@ class testos(basetest.Basetest):

def test_os_date(self):
path = self.path_by_nick(self.os_api, "os_date")
- val = self.curl(path).encode('ascii', 'ignore')
- self.assertRegexpMatches(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+UTC\\s+20..", path)
+ val = self.curl(path)
+ self.assertRegex(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+UTC\\s+20..", path)

def test_os_total_memory(self):
path = self.path_by_nick(self.os_api, "os_memory_total")
diff --git a/modules/httpserver-api/tests/api/testtrace.py b/modules/httpserver-api/tests/api/testtrace.py
index 0916593c..f4b302c4 100644
--- a/modules/httpserver-api/tests/api/testtrace.py
+++ b/modules/httpserver-api/tests/api/testtrace.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import basetest

class testtrace(basetest.Basetest):
@@ -17,14 +17,12 @@ class testtrace(basetest.Basetest):

def test_get_status(self):
status = self.curl(self.path + "/status")
- self.assertGreaterEqual(status, 0)
self.check_status_list(status)

def test_set_status(self):
for bt in [True, False]:
for en in [True, False]:
status = self.curl(self.path + '/status?enabled=' + str(en) + '&backtrace=' + str(bt), method='POST')
- self.assertGreaterEqual(status, 0)
self.check_status_list(status)
status = self.curl(self.path + "/status")
for s in status:
diff --git a/modules/httpserver-api/tests/basetest.py b/modules/httpserver-api/tests/basetest.py
index f3941711..40ca64ce 100755
--- a/modules/httpserver-api/tests/basetest.py
+++ b/modules/httpserver-api/tests/basetest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import json
import os
import subprocess
@@ -33,7 +33,7 @@ class Basetest(unittest.TestCase):

@classmethod
def get_json_api_from_directory(cls, directory, name):
- json_data = open(os.path.join(directory, name))
+ json_data = open(os.path.join(directory, name))
data = json.load(json_data)
json_data.close()
return data
@@ -83,7 +83,7 @@ class Basetest(unittest.TestCase):

def validate_path_regex(self, api_definition, nickname, expr):
path = self.path_by_nick(api_definition, nickname)
- self.assertRegexpMatches(self.curl(path), expr)
+ self.assertRegex(self.curl(path), expr)

def assertHttpError(self, url, code=404):
try:
diff --git a/modules/httpserver-api/tests/testhttpserver-api.py b/modules/httpserver-api/tests/testhttpserver-api.py
index 49da740d..25600c25 100755
--- a/modules/httpserver-api/tests/testhttpserver-api.py
+++ b/modules/httpserver-api/tests/testhttpserver-api.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import sys
import argparse
import os
--
2.20.1

Waldemar Kozaczuk

unread,

Feb 17, 2020, 6:22:40 PM2/17/20

to osv...@googlegroups.com, Waldemar Kozaczuk

This is the third and possibly last batch of the python 3 upgrade changes.

Following scripts have either been adapted to version 3 or
verified that they need to be changed by all 3 patches:

./modules/httpserver-api/json2code.py
./modules/httpserver-api/tests/api/testenv.py
./modules/httpserver-api/tests/api/testfile.py
./modules/httpserver-api/tests/api/testfs.py
./modules/httpserver-api/tests/api/testjolokia.py
./modules/httpserver-api/tests/api/testjvm.py
./modules/httpserver-api/tests/api/testnetwork.py
./modules/httpserver-api/tests/api/testos.py
./modules/httpserver-api/tests/api/testtrace.py
./modules/httpserver-api/tests/basetest.py
./modules/httpserver-api/tests/testhttpserver-api.py
./modules/httpserver-html5-gui/tests/static/teststatic.py
./modules/httpserver-html5-gui/tests/testhttpserver.py
./modules/openjdk8-from-host/module.py
./scripts/export_manifest.py
./scripts/firecracker.py
./scripts/freq.py
./scripts/gen-rofs-img.py
./scripts/imgedit.py
./scripts/libosv.py
./scripts/loader.py
./scripts/manifest_common.py
./scripts/memory_analyzer.py
./scripts/metadata.py
./scripts/mkbootfs.py
./scripts/nbd_client.py
./scripts/osv/client.py
./scripts/osv/debug.py
./scripts/osv/modules/api.py
./scripts/osv/modules/filemap.py
./scripts/osv/modules/resolve.py
./scripts/osv/prof.py
./scripts/osv/trace.py
./scripts/osv/tree.py
./scripts/prof.py
./scripts/run.py
./scripts/setup.py
./scripts/tester.py
./scripts/test.py
./scripts/tests/test_app.py
./scripts/tests/test_app_with_test_script.py
./scripts/tests/test_http_app_with_curl_and_ab.py
./scripts/tests/testing.py
./scripts/tests/test_net.py
./scripts/tests/test_tracing.py
./scripts/top.py
./scripts/trace.py
./scripts/upload_manifest.py
./tests/tst-tcp-hash-cli.py

Following relevant python files have been left out from the
upgrade and possibly never will be upgraded:

./bsd/scripts/init-order.py
./bsd/sys/xen/interface/foreign/mkchecker.py
./bsd/sys/xen/interface/foreign/mkheader.py
./bsd/sys/xen/interface/foreign/structs.py
./scripts/benchs/zfs_mount.py
./scripts/ec2-gc.py
./scripts/ec2-make-ami.py
./scripts/ec2-simulator.py
./scripts/post-processing/scheduler/histo.py
./scripts/post-processing/scheduler/trace_sched_timings.py
./scripts/silentant.py
./scripts/tester.py
./scripts/test-ruby.py

Fixes #1056

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

.../tests/testhttpserver.py | 4 +++-
scripts/osv/modules/api.py | 6 +-----
scripts/osv/modules/resolve.py | 6 +++---
tests/tst-tcp-hash-cli.py | 18 +++++++++---------
4 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/modules/httpserver-html5-gui/tests/testhttpserver.py b/modules/httpserver-html5-gui/tests/testhttpserver.py
index b48daeb0..4f5c8324 100755
--- a/modules/httpserver-html5-gui/tests/testhttpserver.py
+++ b/modules/httpserver-html5-gui/tests/testhttpserver.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import sys
import argparse
import os

@@ -17,6 +17,8 @@ parser.add_argument('--run_script', help='path to the run image script', default
parser.add_argument('--cmd', help='the command to execute')
parser.add_argument('--use_sudo', help='Use sudo with -n option instead of port forwarding', action='store_true')
parser.add_argument('--jsondir', help='location of the json files', default=os.path.join(module_base, '../httpserver-api/api-doc/listings/'))
+parser.add_argument('--test_image', help='the path to the test image')
+parser.add_argument('--hypervisor', action="store", default="qemu", help="choose hypervisor to run: qemu, firecracker")
client.Client.add_arguments(parser)

if __name__ == '__main__':
diff --git a/scripts/osv/modules/api.py b/scripts/osv/modules/api.py
index 9a91fdf9..03b2b5f7 100644
--- a/scripts/osv/modules/api.py
+++ b/scripts/osv/modules/api.py
@@ -47,11 +47,7 @@ class java_app(object):
return []

def get_string_object():
- import sys
- if sys.version < '3':
- return basestring
- else:
- return str
+ return str

def _to_args_list(text_or_list):
if not text_or_list:
diff --git a/scripts/osv/modules/resolve.py b/scripts/osv/modules/resolve.py
index 17871d59..5d5c82f7 100644
--- a/scripts/osv/modules/resolve.py
+++ b/scripts/osv/modules/resolve.py
@@ -57,7 +57,7 @@ def get_required_modules():
according to dependency graph

"""
- return list(unique(_modules.values()))
+ return list(unique(list(_modules.values())))

def _is_direct(module_config):
return module_config["type"] == "direct-dir"
@@ -87,7 +87,7 @@ def find_module_config(module_name):
def all_module_directories():
config = read_config()

- for module_name, module_config in config["modules"].items():
+ for module_name, module_config in list(config["modules"].items()):
if module_name == "repositories":
for repo in config["modules"]["repositories"]:
repo_path = os.path.expandvars(repo)
@@ -127,7 +127,7 @@ def require_if_other_module_present(module_name,other_module_name):

def resolve_required_modules_if_other_is_present():
required_module_names = set()
- for module_name in _modules_to_be_added_if_other_module_present.keys():
+ for module_name in list(_modules_to_be_added_if_other_module_present.keys()):
# If module is present then add modules that should be required implictly
if( _modules.get(module_name)):
modules_to_be_added = _modules_to_be_added_if_other_module_present[module_name]
diff --git a/tests/tst-tcp-hash-cli.py b/tests/tst-tcp-hash-cli.py
index 270c0942..cf81184f 100755
--- a/tests/tst-tcp-hash-cli.py
+++ b/tests/tst-tcp-hash-cli.py
@@ -1,6 +1,6 @@
-#!/bin/env python2
+#!/usr/bin/env python3
import socket
-from Queue import Queue
+from queue import Queue
from threading import Thread
import sys

@@ -16,7 +16,7 @@ class Worker(Thread):
while True:
func, args, kargs = self.tasks.get()
try: func(*args, **kargs)
- except Exception, e: print e
+ except Exception as e: print(e)
self.tasks.task_done()

class ThreadPool:
@@ -67,16 +67,16 @@ if __name__ == "__main__":
nthreads = int(sys.argv[1])
connections = int(sys.argv[2])
except:
- print "Usage: ./tst-tcp-hash-cli.py <nthreads> <connections>"
+ print("Usage: ./tst-tcp-hash-cli.py <nthreads> <connections>")
sys.exit()

#data = range(0, (4096**2)*2, 11)
- data = range(0,4096, 11)
- data = map(lambda x: chr(x % 256), data)
+ data = list(range(0,4096, 11))
+ data = [chr(x % 256) for x in data]
expected = hash_function(data)

- print "Sending %d bytes requests, expected hash: %d" % (len(data), expected)
- print "Creating %d threads and making %d connections, please wait..." % (nthreads, connections)
+ print("Sending %d bytes requests, expected hash: %d" % (len(data), expected))
+ print("Creating %d threads and making %d connections, please wait..." % (nthreads, connections))

drops = 0
hash_errors = 0
@@ -88,6 +88,6 @@ if __name__ == "__main__":
pool.wait_completion()

# FIXME: these metrics may not be accurate as I didn't use locks and interfere with the test
- print "Test completed with %d drops and %d hash errors" % (drops, hash_errors)
+ print("Test completed with %d drops and %d hash errors" % (drops, hash_errors))

--
2.20.1

Commit Bot

unread,

Feb 21, 2020, 1:07:06 PM2/21/20

to osv...@googlegroups.com, Waldemar Kozaczuk

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

scripts: upgrade to python 3 - part 2

Ref #1056

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---

diff --git a/modules/httpserver-api/json2code.py b/modules/httpserver-api/json2code.py
--- a/modules/httpserver-api/json2code.py
+++ b/modules/httpserver-api/json2code.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import json
import sys
import re
diff --git a/modules/httpserver-api/tests/api/testenv.py b/modules/httpserver-api/tests/api/testenv.py

--- a/modules/httpserver-api/tests/api/testenv.py
+++ b/modules/httpserver-api/tests/api/testenv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import basetest

class testenv(basetest.Basetest):
diff --git a/modules/httpserver-api/tests/api/testfile.py b/modules/httpserver-api/tests/api/testfile.py

--- a/modules/httpserver-api/tests/api/testfile.py
+++ b/modules/httpserver-api/tests/api/testfile.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
import basetest
import subprocess

@@ -45,10 +45,10 @@ def test_file_status_cmd(self):

def test_put_file_cmd(self):
path = "/file"
- self.curl(path + "/etc/hosts?op=COPY&destination="+urllib.quote("/etc/hosts1"), method='PUT')
+ self.curl(path + "/etc/hosts?op=COPY&destination="+urllib.parse.quote("/etc/hosts1"), method='PUT')
hosts = self.curl(path + "/etc/hosts1?op=GETFILESTATUS")
self.assertEqual(hosts["type"], "FILE")
- self.curl(path + "/etc/hosts1?op=RENAME&destination="+urllib.quote("/etc/hosts2"), method='PUT')
+ self.curl(path + "/etc/hosts1?op=RENAME&destination="+urllib.parse.quote("/etc/hosts2"), method='PUT')
hosts = self.curl(path + "/etc/hosts2?op=GETFILESTATUS")
self.assertEqual(hosts["type"], "FILE")
self.assertHttpError(path + "/etc/hosts1?op=GETFILESTATUS")
diff --git a/modules/httpserver-api/tests/api/testfs.py b/modules/httpserver-api/tests/api/testfs.py

--- a/modules/httpserver-api/tests/api/testfs.py
+++ b/modules/httpserver-api/tests/api/testfs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import basetest

class testfs(basetest.Basetest):
diff --git a/modules/httpserver-api/tests/api/testjolokia.py b/modules/httpserver-api/tests/api/testjolokia.py

--- a/modules/httpserver-api/tests/api/testjolokia.py
+++ b/modules/httpserver-api/tests/api/testjolokia.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import basetest
import json

diff --git a/modules/httpserver-api/tests/api/testjvm.py b/modules/httpserver-api/tests/api/testjvm.py

--- a/modules/httpserver-api/tests/api/testjvm.py
+++ b/modules/httpserver-api/tests/api/testjvm.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import time
import basetest
-import urllib
+import urllib.request, urllib.parse, urllib.error

class testjvm(basetest.Basetest):
def test_jvm_version(self):

@@ -31,14 +31,14 @@ def test_get_mbeans(self):

def test_get_mbean(self):
mbean = self.curl(self.path_by_nick(self.jvm_api, "getMbeanList") +
- urllib.quote("java.lang:name=PS Old Gen,type=MemoryPool"))
+ urllib.parse.quote("java.lang:name=PS Old Gen,type=MemoryPool"))
self.assertGreaterEqual(len(mbean), 15)
self.assert_key_in("type", mbean[0])
self.assert_key_in("name", mbean[0])
self.assert_key_in("value", mbean[0])

def test_set_mbean(self):
- path = self.path_by_nick(self.jvm_api, "getMbeanList") + urllib.quote("java.lang:name=PS Old Gen,type=MemoryPool")
+ path = self.path_by_nick(self.jvm_api, "getMbeanList") + urllib.parse.quote("java.lang:name=PS Old Gen,type=MemoryPool")
mbean = self.curl(path)
usage = next((item for item in mbean if item["name"] == "UsageThreshold"), None)
self.assertTrue(usage != None)
diff --git a/modules/httpserver-api/tests/api/testnetwork.py b/modules/httpserver-api/tests/api/testnetwork.py

--- a/modules/httpserver-api/tests/api/testnetwork.py
+++ b/modules/httpserver-api/tests/api/testnetwork.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import time
import basetest

diff --git a/modules/httpserver-api/tests/api/testos.py b/modules/httpserver-api/tests/api/testos.py

--- a/modules/httpserver-api/tests/api/testos.py
+++ b/modules/httpserver-api/tests/api/testos.py
@@ -1,11 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import time
import basetest

class testos(basetest.Basetest):
def test_os_version(self):
path = self.path_by_nick(self.os_api, "os_version")
- self.assertRegexpMatches(self.curl(path), r"v0\.\d+(-rc\d+)?(-\d+-[0-9a-z]+)?" , path)
+ self.assertRegex(self.curl(path), r"v0\.\d+(-rc\d+)?(-\d+-[0-9a-z]+)?" , path)

def test_vendor(self):
self.validate_path(self.os_api, "os_vendor", "Cloudius Systems")

@@ -18,8 +18,8 @@ def test_os_uptime(self):

def test_os_date(self):
path = self.path_by_nick(self.os_api, "os_date")
- val = self.curl(path).encode('ascii', 'ignore')
- self.assertRegexpMatches(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+UTC\\s+20..", path)
+ val = self.curl(path)
+ self.assertRegex(val, "...\\s+...\\s+\\d+\\s+\\d\\d:\\d\\d:\\d\\d\\s+UTC\\s+20..", path)

def test_os_total_memory(self):
path = self.path_by_nick(self.os_api, "os_memory_total")
diff --git a/modules/httpserver-api/tests/api/testtrace.py b/modules/httpserver-api/tests/api/testtrace.py

--- a/modules/httpserver-api/tests/api/testtrace.py
+++ b/modules/httpserver-api/tests/api/testtrace.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import basetest

class testtrace(basetest.Basetest):
@@ -17,14 +17,12 @@ def check_status_list(self, list):

def test_get_status(self):
status = self.curl(self.path + "/status")
- self.assertGreaterEqual(status, 0)
self.check_status_list(status)

def test_set_status(self):
for bt in [True, False]:
for en in [True, False]:
status = self.curl(self.path + '/status?enabled=' + str(en) + '&backtrace=' + str(bt), method='POST')
- self.assertGreaterEqual(status, 0)
self.check_status_list(status)
status = self.curl(self.path + "/status")
for s in status:
diff --git a/modules/httpserver-api/tests/basetest.py b/modules/httpserver-api/tests/basetest.py

--- a/modules/httpserver-api/tests/basetest.py
+++ b/modules/httpserver-api/tests/basetest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3

import json
import os
import subprocess

@@ -33,7 +33,7 @@ def get_json_api(cls, name):

@classmethod
def get_json_api_from_directory(cls, directory, name):
- json_data = open(os.path.join(directory, name))
+ json_data = open(os.path.join(directory, name))
data = json.load(json_data)
json_data.close()
return data

@@ -83,7 +83,7 @@ def validate_path(self, api_definition, nickname, value):

def validate_path_regex(self, api_definition, nickname, expr):
path = self.path_by_nick(api_definition, nickname)
- self.assertRegexpMatches(self.curl(path), expr)
+ self.assertRegex(self.curl(path), expr)

def assertHttpError(self, url, code=404):
try:
diff --git a/modules/httpserver-api/tests/testhttpserver-api.py b/modules/httpserver-api/tests/testhttpserver-api.py

--- a/modules/httpserver-api/tests/testhttpserver-api.py
+++ b/modules/httpserver-api/tests/testhttpserver-api.py

Waldemar Kozaczuk

unread,

Feb 22, 2020, 9:20:13 AM2/22/20

to osv...@googlegroups.com, Waldemar Kozaczuk

This is the third and the last batch of the python 3 upgrade changes and
focuses on debug (loader.py) and trace related scripts.

Following scripts have either been adapted to version 3 or

verified that they do NOT need to be changed by all patches the series:

Fixes #1056

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

scripts/loader.py | 5 +--
scripts/osv/debug.py | 3 +-
scripts/osv/prof.py | 16 ++++++---
scripts/osv/trace.py | 45 ++++++++++++++----------
scripts/osv/tree.py | 4 +--
scripts/trace.py | 84 ++++++++++++++++++++++++--------------------
6 files changed, 90 insertions(+), 67 deletions(-)

diff --git a/scripts/loader.py b/scripts/loader.py
index 500d864a..4b82fd4a 100644
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python

import gdb
import re
@@ -1034,7 +1034,7 @@ class osv_info_callouts(gdb.Command):
fname = callout['c_fn']

# time
- t = int(callout['c_to_ns'])
+ t = int(callout['c_to_ns']['__d']['__r'])

# flags
CALLOUT_ACTIVE = 0x0002
@@ -1176,6 +1176,7 @@ def all_traces():
max_trace = ulong(trace_buffer['_size'])

if not trace_log_base:
+ print('!!! Could not find any trace data! Make sure "--trace" option matches some tracepoints.')
raise StopIteration

trace_log = inf.read_memory(trace_log_base, max_trace)
diff --git a/scripts/osv/debug.py b/scripts/osv/debug.py
index fe42be60..83372ada 100644
--- a/scripts/osv/debug.py
+++ b/scripts/osv/debug.py
@@ -38,7 +38,7 @@ class SymbolResolver(object):
if show_inline:
flags += 'i'
self.addr2line = subprocess.Popen(['addr2line', '-e', object_path, flags],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
self.cache = {}

def next_line(self):
@@ -82,6 +82,7 @@ class SymbolResolver(object):
if self.show_inline:
self.addr2line.stdin.write('0\n')

+ self.addr2line.stdin.flush()
result = self.parse_line(addr, self.next_line())

if self.show_inline:
diff --git a/scripts/osv/prof.py b/scripts/osv/prof.py
index 95db15b8..3a000013 100644
--- a/scripts/osv/prof.py
+++ b/scripts/osv/prof.py
@@ -51,7 +51,7 @@ time_units = [
]

def parse_time_as_nanos(text, default_unit='ns'):
- for level, name in sorted(time_units, key=lambda (level, name): -len(name)):
+ for level, name in sorted(time_units, key=lambda level_name: -len(level_name[1])):
if text.endswith(name):
return float(text.rstrip(name)) * level
for level, name in time_units:
@@ -60,7 +60,7 @@ def parse_time_as_nanos(text, default_unit='ns'):
raise Exception('Unknown unit: ' + default_unit)

def format_time(time, format="%.2f %s"):
- for level, name in sorted(time_units, key=lambda (level, name): -level):
+ for level, name in sorted(time_units, key=lambda level_name1: -level_name1[0]):
if time >= level:
return format % (float(time) / level, name)
return str(time)
@@ -207,10 +207,16 @@ class timed_trace_producer(object):
self.last_time = None

def __call__(self, sample):
+ if not sample.time:
+ return
+
if not sample.cpu in self.earliest_trace_per_cpu:
self.earliest_trace_per_cpu[sample.cpu] = sample

- self.last_time = max(self.last_time, sample.time)
+ if not self.last_time:
+ self.last_time = sample.time
+ else:
+ self.last_time = max(self.last_time, sample.time)

matcher = self.matcher_by_name.get(sample.name, None)
if not matcher:
@@ -239,7 +245,7 @@ class timed_trace_producer(object):
return trace.TimedTrace(entry_trace, duration)

def finish(self):
- for sample in self.open_samples.itervalues():
+ for sample in self.open_samples.values():
duration = self.last_time - sample.time
yield trace.TimedTrace(sample, duration)

@@ -402,7 +408,7 @@ def print_profile(samples, symbol_resolver, caller_oriented=False,
if not order:
order = lambda node: (-node.resident_time, -node.hit_count)

- for group, tree_root in sorted(groups.iteritems(), key=lambda (thread, node): order(node)):
+ for group, tree_root in sorted(iter(groups.items()), key=lambda thread_node: order(thread_node[1])):
collapse_similar(tree_root)

if max_levels:
diff --git a/scripts/osv/trace.py b/scripts/osv/trace.py
index 2c28582b..636ea0a4 100644
--- a/scripts/osv/trace.py
+++ b/scripts/osv/trace.py
@@ -65,7 +65,12 @@ class TimeRange(object):
return self.end - self.begin

def intersection(self, other):
- begin = max(self.begin, other.begin)
+ if not self.begin:
+ begin = other.begin
+ elif not other.begin:
+ begin = self.begin
+ else:
+ begin = max(self.begin, other.begin)

if self.end is None:
end = other.end
@@ -143,11 +148,11 @@ class Trace:
class TimedTrace:
def __init__(self, trace, duration=None):
self.trace = trace
- self.duration = duration
+ self.duration_ = duration

@property
def duration(self):
- return self.duration
+ return self.duration_

@property
def time(self):
@@ -183,6 +188,8 @@ def do_split_format(format_str):

_split_cache = {}
def split_format(format_str):
+ if not format_str:
+ return []
result = _split_cache.get(format_str, None)
if not result:
result = list(do_split_format(format_str))
@@ -190,7 +197,7 @@ def split_format(format_str):
return result

formatters = {
- '*': lambda bytes: '{' + ' '.join('%02x' % ord(b) for b in bytes) + '}'
+ '*': lambda bytes: '{' + ' '.join('%02x' % b for b in bytes) + '}'
}

def get_alignment_of(fmt):
@@ -238,16 +245,15 @@ class SlidingUnpacker:
size = struct.calcsize(fmt)
val, = struct.unpack_from(fmt, self.buffer[self.offset:self.offset+size])
self.offset += size
- values.append(val)
+ if fmt.startswith('50p'):
+ values.append(val.decode('utf-8'))
+ else:
+ values.append(val)

return tuple(values)

- def __nonzero__(self):
- return self.offset < len(self.buffer)
-
- # Python3
def __bool__(self):
- return self.__nonzero__()
+ return self.offset < len(self.buffer)

class WritingPacker:
def __init__(self, writer):
@@ -270,7 +276,10 @@ class WritingPacker:
if fmt == '*':
self.pack_blob(arg)
else:
- self.writer(struct.pack(fmt, arg))
+ if fmt == '50p':
+ self.writer(struct.pack(fmt, arg.encode('utf-8')))
+ else:
+ self.writer(struct.pack(fmt, arg))
self.offset += struct.calcsize(fmt)

def pack_blob(self, arg):
@@ -298,7 +307,7 @@ class TraceDumpReaderBase :
self.endian = '<'
self.file = open(filename, 'rb')
try:
- tag = self.file.read(4)
+ tag = self.file.read(4).decode()
if tag == "OSVT":
endian = '>'
elif tag != "TVSO":
@@ -347,7 +356,7 @@ class TraceDumpReaderBase :

def readString(self):
len = self.read('H')
- return self.file.read(len)
+ return self.file.read(len).decode()

class TraceDumpReader(TraceDumpReaderBase) :
def __init__(self, filename):
@@ -378,7 +387,7 @@ class TraceDumpReader(TraceDumpReaderBase) :
sig = ""
for j in range(0, n_args):
arg_name = self.readString()
- arg_sig = self.file.read(1)
+ arg_sig = self.file.read(1).decode()
if arg_sig == 'p':
arg_sig = '50p'
sig += arg_sig
@@ -405,7 +414,7 @@ class TraceDumpReader(TraceDumpReaderBase) :

backtrace = None
if flags & 1:
- backtrace = filter(None, unpacker.unpack('Q' * self.backtrace_len))
+ backtrace = [_f for _f in unpacker.unpack('Q' * self.backtrace_len) if _f]

data = unpacker.unpack(tp.signature)
unpacker.align_up(8)
@@ -414,7 +423,7 @@ class TraceDumpReader(TraceDumpReaderBase) :
yield last_trace

def traces(self):
- iters = map(lambda data: self.oneTrace(data), self.trace_buffers)
+ iters = [self.oneTrace(data) for data in self.trace_buffers]
return heapq.merge(*iters)

@@ -523,7 +532,7 @@ def read(buffer_view):

while unpacker:
tp_key, thread_ptr, thread_name, time, cpu = unpacker.unpack('QQ16sQI')
- thread_name = thread_name.rstrip('\0')
+ thread_name = thread_name.rstrip(b'\0').decode('utf-8')
tp = tracepoints[tp_key]

backtrace = []
@@ -551,7 +560,7 @@ def write(traces, writer):
trace.time, trace.cpu)

if trace.backtrace:
- for frame in filter(None, trace.backtrace):
+ for frame in [_f for _f in trace.backtrace if _f]:
packer.pack('Q', frame)
packer.pack('Q', 0)

diff --git a/scripts/osv/tree.py b/scripts/osv/tree.py
index 594b00e2..86345157 100644
--- a/scripts/osv/tree.py
+++ b/scripts/osv/tree.py
@@ -18,11 +18,11 @@ class TreeNode(object):

def squash_child(self):
assert self.has_only_one_child()
- self.children_by_key = next(self.children_by_key.itervalues()).children_by_key
+ self.children_by_key = next(iter(self.children_by_key.values())).children_by_key

@property
def children(self):
- return self.children_by_key.itervalues()
+ return iter(self.children_by_key.values())

def has_only_one_child(self):
return len(self.children_by_key) == 1
diff --git a/scripts/trace.py b/scripts/trace.py
index 34cfb2ab..1b35e568 100755
--- a/scripts/trace.py
+++ b/scripts/trace.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
import sys
import errno
import argparse
@@ -13,6 +13,7 @@ from collections import defaultdict
from osv import trace, debug, prof
from osv.client import Client
import memory_analyzer
+from functools import reduce

class InvalidArgumentsException(Exception):
def __init__(self, message):
@@ -114,7 +115,7 @@ def list_trace(args):
with get_trace_reader(args) as reader:
for t in reader.get_traces():
if t.time in time_range:
- print t.format(backtrace_formatter, data_formatter=data_formatter)
+ print(t.format(backtrace_formatter, data_formatter=data_formatter))

def mem_analys(args):
mallocs = {}
@@ -276,7 +277,7 @@ def extract(args):
stderr=subprocess.STDOUT)
_stdout, _ = proc.communicate()
if proc.returncode or not os.path.exists(args.tracefile):
- print(_stdout)
+ print(_stdout.decode())
sys.exit(1)
else:
print("error: %s not found" % (elf_path))
@@ -332,8 +333,10 @@ def write_sample_to_pcap(sample, pcap_writer):
}

pkt = dpkt.ethernet.Ethernet()
- pkt.data = sample.data[1]
pkt.type = eth_types[proto]
+ pkt.src = b''
+ pkt.dst = b''
+ pkt.data = sample.data[1]
pcap_writer.writepkt(pkt, ts=ts)

def format_packet_sample(sample):
@@ -343,7 +346,7 @@ def format_packet_sample(sample):
pcap = dpkt.pcap.Writer(proc.stdin)
write_sample_to_pcap(sample, pcap)
pcap.close()
- assert(proc.stdout.readline() == "reading from file -, link-type EN10MB (Ethernet)\n")
+ assert(proc.stdout.readline().decode() == "reading from file -, link-type EN10MB (Ethernet)\n")
packet_line = proc.stdout.readline().rstrip()
proc.wait()
return packet_line
@@ -361,7 +364,7 @@ def pcap_dump(args, target=None):
needs_dpkt()

if not target:
- target = sys.stdout
+ target = sys.stdout.buffer

pcap_file = dpkt.pcap.Writer(target)
try:
@@ -439,7 +442,10 @@ def print_summary(args, printer=sys.stdout.write):
else:
min_time = min(min_time, t.time)

- max_time = max(max_time, t.time)
+ if not max_time:
+ max_time = t.time
+ else:
+ max_time = max(max_time, t.time)

if args.timed:
timed = timed_producer(t)
@@ -450,42 +456,42 @@ def print_summary(args, printer=sys.stdout.write):
timed_samples.extend((timed_producer.finish()))

if count == 0:
- print "No samples"
+ print("No samples")
return

- print "Collected %d samples spanning %s" % (count, prof.format_time(max_time - min_time))
+ print("Collected %d samples spanning %s" % (count, prof.format_time(max_time - min_time)))

- print "\nTime ranges:\n"
- for cpu, r in sorted(cpu_time_ranges.items(), key=lambda (c, r): r.min):
- print " CPU 0x%02d: %s - %s = %10s" % (cpu,
+ print("\nTime ranges:\n")
+ for cpu, r in sorted(list(cpu_time_ranges.items()), key=lambda c_r: c_r[1].min):
+ print(" CPU 0x%02d: %s - %s = %10s" % (cpu,
trace.format_time(r.min),
trace.format_time(r.max),
- prof.format_time(r.max - r.min))
+ prof.format_time(r.max - r.min)))

- max_name_len = reduce(max, map(lambda tp: len(tp.name), count_per_tp.iterkeys()))
+ max_name_len = reduce(max, [len(tp.name) for tp in iter(count_per_tp.keys())])
format = " %%-%ds %%8s" % (max_name_len)
- print "\nTracepoint statistics:\n"
- print format % ("name", "count")
- print format % ("----", "-----")
+ print("\nTracepoint statistics:\n")
+ print(format % ("name", "count"))
+ print(format % ("----", "-----"))

- for tp, count in sorted(count_per_tp.iteritems(), key=lambda (tp, count): tp.name):
- print format % (tp.name, count)
+ for tp, count in sorted(iter(count_per_tp.items()), key=lambda tp_count: tp_count[0].name):
+ print(format % (tp.name, count))

if args.timed:
format = " %-20s %8s %8s %8s %8s %8s %8s %8s %15s"
- print "\nTimed tracepoints [ms]:\n"
+ print("\nTimed tracepoints [ms]:\n")

- timed_samples = filter(lambda t: t.time_range.intersection(time_range), timed_samples)
+ timed_samples = [t for t in timed_samples if t.time_range.intersection(time_range)]

if not timed_samples:
- print " None"
+ print(" None")
else:
- print format % ("name", "count", "min", "50%", "90%", "99%", "99.9%", "max", "total")
- print format % ("----", "-----", "---", "---", "---", "---", "-----", "---", "-----")
+ print(format % ("name", "count", "min", "50%", "90%", "99%", "99.9%", "max", "total"))
+ print(format % ("----", "-----", "---", "---", "---", "---", "-----", "---", "-----"))

- for name, traces in get_timed_traces_per_function(timed_samples).iteritems():
+ for name, traces in get_timed_traces_per_function(timed_samples).items():
samples = sorted(list((t.time_range.intersection(time_range).length() for t in traces)))
- print format % (
+ print(format % (
name,
len(samples),
format_duration(get_percentile(samples, 0)),
@@ -494,9 +500,9 @@ def print_summary(args, printer=sys.stdout.write):
format_duration(get_percentile(samples, 0.99)),
format_duration(get_percentile(samples, 0.999)),
format_duration(get_percentile(samples, 1)),
- format_duration(sum(samples)))
+ format_duration(sum(samples))))

- print
+ print()

def list_cpu_load(args):
load_per_cpu = {}
@@ -550,7 +556,7 @@ def list_timed(args):

for timed in timed_traces:
t = timed.trace
- print '0x%016x %-15s %2d %20s %7s %-20s %s%s' % (
+ print('0x%016x %-15s %2d %20s %7s %-20s %s%s' % (
t.thread.ptr,
t.thread.name,
t.cpu,
@@ -558,7 +564,7 @@ def list_timed(args):
trace.format_duration(timed.duration),
t.name,
trace.Trace.format_data(t),
- bt_formatter(t.backtrace))
+ bt_formatter(t.backtrace)))

def list_wakeup_latency(args):
bt_formatter = get_backtrace_formatter(args)
@@ -575,9 +581,9 @@ def list_wakeup_latency(args):
return "%4.6f" % (float(nanos) / 1e6)

if not args.no_header:
- print '%-18s %-15s %3s %20s %13s %9s %s' % (
+ print('%-18s %-15s %3s %20s %13s %9s %s' % (
"THREAD", "THREAD-NAME", "CPU", "TIMESTAMP[s]", "WAKEUP[ms]", "WAIT[ms]", "BACKTRACE"
- )
+ ))

with get_trace_reader(args) as reader:
for t in reader.get_traces():
@@ -594,14 +600,14 @@ def list_wakeup_latency(args):
if t.cpu == waiting_thread.wait.cpu:
wakeup_delay = t.time - waiting_thread.wake.time
wait_time = t.time - waiting_thread.wait.time
- print '0x%016x %-15s %3d %20s %13s %9s %s' % (
+ print('0x%016x %-15s %3d %20s %13s %9s %s' % (
t.thread.ptr,
t.thread.name,
t.cpu,
trace.format_time(t.time),
format_wakeup_latency(wakeup_delay),
trace.format_duration(wait_time),
- bt_formatter(t.backtrace))
+ bt_formatter(t.backtrace)))

def add_trace_listing_options(parser):
add_time_slicing_options(parser)
@@ -615,7 +621,7 @@ def convert_dump(args):
if os.path.exists(args.tracefile):
os.remove(args.tracefile)
assert(not os.path.exists(args.tracefile))
- print "Converting dump %s -> %s" % (args.dumpfile, args.tracefile)
+ print("Converting dump %s -> %s" % (args.dumpfile, args.tracefile))
td = trace.TraceDumpReader(args.dumpfile)
trace.write_to_file(args.tracefile, list(td.traces()))
else:
@@ -631,7 +637,7 @@ def download_dump(args):
client = Client(args)
url = client.get_url() + "/trace/buffers"

- print "Downloading %s -> %s" % (url, file)
+ print("Downloading %s -> %s" % (url, file))

r = requests.get(url, stream=True, **client.get_request_kwargs())
size = int(r.headers['content-length'])
@@ -641,7 +647,7 @@ def download_dump(args):
for chunk in r.iter_content(8192):
out_file.write(chunk)
current += len(chunk)
- sys.stdout.write("[{0:8d} / {1:8d} k] {3} {2:.2f}%\r".format(current/1024, size/1024, 100.0*current/size, ('='*32*(current/size)) + '>'))
+ sys.stdout.write("[{0:8d} / {1:8d} k] {3} {2:.2f}%\r".format(current//1024, size//1024, 100.0*current//size, ('='*32*(current//size)) + '>'))
if current >= size:
sys.stdout.write("\n")
sys.stdout.flush()
@@ -789,7 +795,7 @@ if __name__ == "__main__":
args = parser.parse_args()

if getattr(args, 'paginate', False):
- less_process = subprocess.Popen(['less', '-FX'], stdin=subprocess.PIPE)
+ less_process = subprocess.Popen(['less', '-FX'], stdin=subprocess.PIPE, text=True)
sys.stdout = less_process.stdin
else:
less_process = None
@@ -797,7 +803,7 @@ if __name__ == "__main__":
try:
args.func(args)
except InvalidArgumentsException as e:
- print "Invalid arguments:", e.message
+ print("Invalid arguments:", e.message)
except IOError as e:
if e.errno != errno.EPIPE:
raise
--
2.20.1

Nadav Har'El

unread,

Feb 23, 2020, 3:20:10 AM2/23/20

to Waldemar Kozaczuk, Osv Dev

On Sat, Feb 22, 2020 at 4:20 PM Waldemar Kozaczuk <jwkoz...@gmail.com> wrote:

This is the third and the last batch of the python 3 upgrade changes and
focuses on debug (loader.py) and trace related scripts.

Following scripts have either been adapted to version 3 or
verified that they do NOT need to be changed by all patches the series:

./scripts/export_manifest.py

Please note that most or not all of the scripts that have "#!/usr/bin/python" on top

(not python2 or python3) have been tested for years by different people who had either

python2 or python3 as their default "python", and should ideally be working for both.

That's especially true in the scripts used in regular scripts/build, which people would

have noticed if they were broken.

So it's indeed worth testing that this is in fact true, but the last thing I want to see is

rushing patches to "convert" a script that already works on Python3 to Python3.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
scripts/loader.py | 5 +--
scripts/osv/debug.py | 3 +-
scripts/osv/prof.py | 16 ++++++---
scripts/osv/trace.py | 45 ++++++++++++++----------
scripts/osv/tree.py | 4 +--
scripts/trace.py | 84 ++++++++++++++++++++++++--------------------
6 files changed, 90 insertions(+), 67 deletions(-)

diff --git a/scripts/loader.py b/scripts/loader.py
index 500d864a..4b82fd4a 100644
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python

import gdb
import re
@@ -1034,7 +1034,7 @@ class osv_info_callouts(gdb.Command):
fname = callout['c_fn']

# time
- t = int(callout['c_to_ns'])
+ t = int(callout['c_to_ns']['__d']['__r'])

Thanks, this was an already-needed fix (not related to Python 3). It was probably casued by changing C++

ABI, and while this change is good for us, it may not work for people using older version of the compiler.

Traditionally we handled this with code like:

try:
return self.map_header['_M_bbegin']
except gdb.error:
return self.map_header['_M_before_begin']['_M_nxt']

Would be good to do this here too.

But I'll not block this commit. I'll commit it as-is, and for your consideration if you want to submit a followup patch to make this change backward-compatible with older compilers.

--
You received this message because you are subscribed to the Google Groups "OSv Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+u...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/20200222142004.9909-1-jwkozaczuk%40gmail.com.

Commit Bot

unread,

Feb 23, 2020, 3:24:54 AM2/23/20

to osv...@googlegroups.com, Waldemar Kozaczuk

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Nadav Har'El <n...@scylladb.com>
Branch: master

scripts: upgrade to python 3 - part 3

Message-Id: <20200222142004.9...@gmail.com>

---
diff --git a/scripts/loader.py b/scripts/loader.py

--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python

import gdb
import re

@@ -1034,7 +1034,7 @@ def invoke(self, arg, for_tty):

fname = callout['c_fn']

# time
- t = int(callout['c_to_ns'])
+ t = int(callout['c_to_ns']['__d']['__r'])

# flags
CALLOUT_ACTIVE = 0x0002

@@ -1176,6 +1176,7 @@ def one_cpu_trace(cpu):

max_trace = ulong(trace_buffer['_size'])

if not trace_log_base:
+ print('!!! Could not find any trace data! Make sure "--trace" option matches some tracepoints.')
raise StopIteration

trace_log = inf.read_memory(trace_log_base, max_trace)
diff --git a/scripts/osv/debug.py b/scripts/osv/debug.py

--- a/scripts/osv/debug.py
+++ b/scripts/osv/debug.py
@@ -38,7 +38,7 @@ def __init__(self, object_path, fallback_resolver=DummyResolver(), show_inline=T

if show_inline:
flags += 'i'
self.addr2line = subprocess.Popen(['addr2line', '-e', object_path, flags],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
self.cache = {}

def next_line(self):

@@ -82,6 +82,7 @@ def __call__(self, addr):

if self.show_inline:
self.addr2line.stdin.write('0\n')

+ self.addr2line.stdin.flush()
result = self.parse_line(addr, self.next_line())

if self.show_inline:
diff --git a/scripts/osv/prof.py b/scripts/osv/prof.py

--- a/scripts/osv/prof.py
+++ b/scripts/osv/prof.py
@@ -51,7 +51,7 @@ def intersection(self, time_range):

]

def parse_time_as_nanos(text, default_unit='ns'):
- for level, name in sorted(time_units, key=lambda (level, name): -len(name)):
+ for level, name in sorted(time_units, key=lambda level_name: -len(level_name[1])):
if text.endswith(name):
return float(text.rstrip(name)) * level
for level, name in time_units:
@@ -60,7 +60,7 @@ def parse_time_as_nanos(text, default_unit='ns'):
raise Exception('Unknown unit: ' + default_unit)

def format_time(time, format="%.2f %s"):
- for level, name in sorted(time_units, key=lambda (level, name): -level):
+ for level, name in sorted(time_units, key=lambda level_name1: -level_name1[0]):
if time >= level:
return format % (float(time) / level, name)
return str(time)

@@ -207,10 +207,16 @@ def __init__(self):

self.last_time = None

def __call__(self, sample):
+ if not sample.time:
+ return
+
if not sample.cpu in self.earliest_trace_per_cpu:
self.earliest_trace_per_cpu[sample.cpu] = sample

- self.last_time = max(self.last_time, sample.time)
+ if not self.last_time:
+ self.last_time = sample.time
+ else:
+ self.last_time = max(self.last_time, sample.time)

matcher = self.matcher_by_name.get(sample.name, None)
if not matcher:

@@ -239,7 +245,7 @@ def __call__(self, sample):

return trace.TimedTrace(entry_trace, duration)

def finish(self):
- for sample in self.open_samples.itervalues():
+ for sample in self.open_samples.values():
duration = self.last_time - sample.time
yield trace.TimedTrace(sample, duration)

@@ -402,7 +408,7 @@ def format_node(node, root):

if not order:
order = lambda node: (-node.resident_time, -node.hit_count)

- for group, tree_root in sorted(groups.iteritems(), key=lambda (thread, node): order(node)):
+ for group, tree_root in sorted(iter(groups.items()), key=lambda thread_node: order(thread_node[1])):
collapse_similar(tree_root)

if max_levels:
diff --git a/scripts/osv/trace.py b/scripts/osv/trace.py

--- a/scripts/osv/trace.py
+++ b/scripts/osv/trace.py
@@ -65,7 +65,12 @@ def length(self):

return self.end - self.begin

def intersection(self, other):
- begin = max(self.begin, other.begin)
+ if not self.begin:
+ begin = other.begin
+ elif not other.begin:
+ begin = self.begin
+ else:
+ begin = max(self.begin, other.begin)

if self.end is None:
end = other.end

@@ -143,11 +148,11 @@ def __lt__(self, b):

class TimedTrace:
def __init__(self, trace, duration=None):
self.trace = trace
- self.duration = duration
+ self.duration_ = duration

@property
def duration(self):
- return self.duration
+ return self.duration_

@property
def time(self):

@@ -183,14 +188,16 @@ def do_split_format(format_str):

_split_cache = {}
def split_format(format_str):
+ if not format_str:
+ return []
result = _split_cache.get(format_str, None)
if not result:
result = list(do_split_format(format_str))

_split_cache[format_str] = result

return result

formatters = {
- '*': lambda bytes: '{' + ' '.join('%02x' % ord(b) for b in bytes) + '}'
+ '*': lambda bytes: '{' + ' '.join('%02x' % b for b in bytes) + '}'
}

def get_alignment_of(fmt):

@@ -238,16 +245,15 @@ def unpack(self, format):

size = struct.calcsize(fmt)
val, = struct.unpack_from(fmt, self.buffer[self.offset:self.offset+size])
self.offset += size
- values.append(val)
+ if fmt.startswith('50p'):
+ values.append(val.decode('utf-8'))
+ else:
+ values.append(val)

return tuple(values)

- def __nonzero__(self):
- return self.offset < len(self.buffer)
-
- # Python3
def __bool__(self):
- return self.__nonzero__()
+ return self.offset < len(self.buffer)

class WritingPacker:
def __init__(self, writer):

@@ -270,7 +276,10 @@ def pack(self, format, *data):

if fmt == '*':
self.pack_blob(arg)
else:
- self.writer(struct.pack(fmt, arg))
+ if fmt == '50p':
+ self.writer(struct.pack(fmt, arg.encode('utf-8')))
+ else:
+ self.writer(struct.pack(fmt, arg))
self.offset += struct.calcsize(fmt)

def pack_blob(self, arg):

@@ -298,7 +307,7 @@ def __init__(self, filename):

self.endian = '<'
self.file = open(filename, 'rb')
try:
- tag = self.file.read(4)
+ tag = self.file.read(4).decode()
if tag == "OSVT":
endian = '>'
elif tag != "TVSO":

@@ -347,7 +356,7 @@ def readStruct0(self):

def readString(self):
len = self.read('H')
- return self.file.read(len)
+ return self.file.read(len).decode()

class TraceDumpReader(TraceDumpReaderBase) :
def __init__(self, filename):

@@ -378,7 +387,7 @@ def readTraceDict(self, size):

sig = ""
for j in range(0, n_args):
arg_name = self.readString()
- arg_sig = self.file.read(1)
+ arg_sig = self.file.read(1).decode()
if arg_sig == 'p':
arg_sig = '50p'
sig += arg_sig

@@ -405,7 +414,7 @@ def oneTrace(self, trace_log):

backtrace = None
if flags & 1:
- backtrace = filter(None, unpacker.unpack('Q' * self.backtrace_len))
+ backtrace = [_f for _f in unpacker.unpack('Q' * self.backtrace_len) if _f]

data = unpacker.unpack(tp.signature)
unpacker.align_up(8)

@@ -414,7 +423,7 @@ def oneTrace(self, trace_log):

yield last_trace

def traces(self):
- iters = map(lambda data: self.oneTrace(data), self.trace_buffers)
+ iters = [self.oneTrace(data) for data in self.trace_buffers]
return heapq.merge(*iters)

@@ -523,7 +532,7 @@ def read(buffer_view):

while unpacker:
tp_key, thread_ptr, thread_name, time, cpu = unpacker.unpack('QQ16sQI')
- thread_name = thread_name.rstrip('\0')
+ thread_name = thread_name.rstrip(b'\0').decode('utf-8')
tp = tracepoints[tp_key]

backtrace = []
@@ -551,7 +560,7 @@ def write(traces, writer):
trace.time, trace.cpu)

if trace.backtrace:
- for frame in filter(None, trace.backtrace):
+ for frame in [_f for _f in trace.backtrace if _f]:
packer.pack('Q', frame)
packer.pack('Q', 0)

diff --git a/scripts/osv/tree.py b/scripts/osv/tree.py

--- a/scripts/osv/tree.py
+++ b/scripts/osv/tree.py
@@ -18,11 +18,11 @@ def add(self, node):

def squash_child(self):
assert self.has_only_one_child()
- self.children_by_key = next(self.children_by_key.itervalues()).children_by_key
+ self.children_by_key = next(iter(self.children_by_key.values())).children_by_key

@property
def children(self):
- return self.children_by_key.itervalues()
+ return iter(self.children_by_key.values())

def has_only_one_child(self):
return len(self.children_by_key) == 1
diff --git a/scripts/trace.py b/scripts/trace.py

--- a/scripts/trace.py
+++ b/scripts/trace.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
import sys
import errno
import argparse
@@ -13,6 +13,7 @@

from osv import trace, debug, prof
from osv.client import Client
import memory_analyzer
+from functools import reduce

class InvalidArgumentsException(Exception):
def __init__(self, message):

@@ -114,7 +115,7 @@ def data_formatter(sample):

@@ -439,7 +442,10 @@ def update(self, time):

else:
min_time = min(min_time, t.time)

- max_time = max(max_time, t.time)
+ if not max_time:
+ max_time = t.time
+ else:
+ max_time = max(max_time, t.time)

if args.timed:
timed = timed_producer(t)

@@ -450,42 +456,42 @@ def update(self, time):

@@ -494,9 +500,9 @@ def update(self, time):

format_duration(get_percentile(samples, 0.99)),
format_duration(get_percentile(samples, 0.999)),
format_duration(get_percentile(samples, 1)),
- format_duration(sum(samples)))
+ format_duration(sum(samples))))

- print
+ print()

def list_cpu_load(args):
load_per_cpu = {}

@@ -550,15 +556,15 @@ def list_timed(args):

for timed in timed_traces:
t = timed.trace
- print '0x%016x %-15s %2d %20s %7s %-20s %s%s' % (
+ print('0x%016x %-15s %2d %20s %7s %-20s %s%s' % (
t.thread.ptr,
t.thread.name,
t.cpu,

trace.format_time(t.time),

trace.format_duration(timed.duration),
t.name,
trace.Trace.format_data(t),
- bt_formatter(t.backtrace))
+ bt_formatter(t.backtrace)))

def list_wakeup_latency(args):
bt_formatter = get_backtrace_formatter(args)

@@ -575,9 +581,9 @@ def format_wakeup_latency(nanos):

return "%4.6f" % (float(nanos) / 1e6)

if not args.no_header:
- print '%-18s %-15s %3s %20s %13s %9s %s' % (
+ print('%-18s %-15s %3s %20s %13s %9s %s' % (
"THREAD", "THREAD-NAME", "CPU", "TIMESTAMP[s]", "WAKEUP[ms]", "WAIT[ms]", "BACKTRACE"
- )
+ ))

with get_trace_reader(args) as reader:
for t in reader.get_traces():

@@ -594,14 +600,14 @@ def format_wakeup_latency(nanos):

@@ -789,15 +795,15 @@ def download_dump(args):

args = parser.parse_args()

if getattr(args, 'paginate', False):
- less_process = subprocess.Popen(['less', '-FX'], stdin=subprocess.PIPE)
+ less_process = subprocess.Popen(['less', '-FX'], stdin=subprocess.PIPE, text=True)
sys.stdout = less_process.stdin
else:
less_process = None

Waldek Kozaczuk

unread,

Feb 24, 2020, 5:15:03 PM2/24/20

to OSv Development

I am about to send a patch accommodating your request.

I think we have a similar issue with this code (probably line in bold):

def runqueue(cpuid, node=None):

if node == None:

cpus = gdb.lookup_global_symbol('sched::cpus').value()

cpu = cpus['_M_impl']['_M_start'][cpuid]

rq = cpu['runqueue']

p = rq['data_']['node_plus_pred_']

node = p['header_plus_size_']['header_']['parent_']

if node:

offset = gdb.parse_and_eval('(int)&((sched::thread *)0)->_runqueue_link')

thread = node.cast(gdb.lookup_type('void').pointer()) - offset

thread = thread.cast(gdb.lookup_type('sched::thread').pointer())

for x in runqueue(cpuid, node['left_']):

yield x

yield thread

for x in runqueue(cpuid, node['right_']):

yield x

Trying to run 'osv runqueue' gives this error:

osv runqueue

CPU 0:

Python Exception <class 'gdb.error'> There is no member or method named data_.:

Error occurred in Python: There is no member or method named data_.

I could not quite figure out how exactly the 'runqueue' has changed - it does not seem to have '_data' field anymore. Do you have any idea how we should fix it?

To unsubscribe from this group and stop receiving emails from it, send an email to osv...@googlegroups.com.

Reply all

Reply to author

Forward

0 new messages