[COMMIT osv master] scripts: upgrade to python 3 - part 1.1

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

scripts: upgrade to python 3 - part 1.1

This patch incorporates changes from original patch
submitted by Matt Bass as well as extra changes made
by Waldemar Kozaczuk to upgrade python scripts from version
2 to version 3.

Some trivial changes are the result of automated upgrade tools like
"Future" module or 2to3. Others involving proper handling of binary and string
data especially when using subprocess module have been manually made. Same goes
for any code that tried to compare non-uniform data which python 3 is more
strict.

Most of the scripts have been tested but there maybe be scenarios/flows
which have been missed.

Ref #1056

Signed-off-by: Matt Bass <bassma...@gmail.com>
Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

---
diff --git a/docker/Dockerfile.Fedora b/docker/Dockerfile.Fedora
--- a/docker/Dockerfile.Fedora
+++ b/docker/Dockerfile.Fedora
@@ -8,10 +8,9 @@
# This Docker file defines a container intended to build, test and publish
# OSv kernel as well as many applications ...
#
-#FROM fedora:27 #On Jenkins build machine
FROM fedora:29

-RUN yum install -y git python2 file which
+RUN yum install -y git python2 python3 file which

#
# PREPARE ENVIRONMENT
diff --git a/docker/Dockerfile.Ubuntu b/docker/Dockerfile.Ubuntu
--- a/docker/Dockerfile.Ubuntu
+++ b/docker/Dockerfile.Ubuntu
@@ -16,7 +16,7 @@ ENV TERM=linux
COPY ./etc/keyboard /etc/default/keyboard
COPY ./etc/console-setup /etc/default/console-setup

-RUN apt-get update -y && apt-get install -y git python2
+RUN apt-get update -y && apt-get install -y git python2 python3

#
# PREPARE ENVIRONMENT
diff --git a/modules/openjdk8-from-host/module.py b/modules/openjdk8-from-host/module.py
--- a/modules/openjdk8-from-host/module.py
+++ b/modules/openjdk8-from-host/module.py
@@ -11,12 +11,12 @@
print('Could not find any jdk on the host. Please install openjdk8!')
os.exit(-1)

-java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
+java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT).decode('utf-8')
if not 'openjdk version "1.8.0' in java_version:
print('Could not find openjdk version 8 on the host. Please install openjdk8!')
os.exit(-1)

-javac_path = subprocess.check_output(['which', 'javac']).split('\n')[0]
+javac_path = subprocess.check_output(['which', 'javac']).decode('utf-8').split('\n')[0]
javac_real_path = os.path.realpath(javac_path)
jdk_path = os.path.dirname(os.path.dirname(javac_real_path))

diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py
--- a/scripts/export_manifest.py
+++ b/scripts/export_manifest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import optparse, os, shutil
from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file
@@ -8,7 +8,7 @@
# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts.
def export_package(manifest, dest):
abs_dest = os.path.abspath(dest)
- print "[INFO] exporting into directory %s" % abs_dest
+ print("[INFO] exporting into directory %s" % abs_dest)

# Remove and create the base directory where we are going to put all package files.
if os.path.exists(abs_dest):
@@ -39,7 +39,7 @@ def export_package(manifest, dest):
os.makedirs(target_dir)

os.symlink(link_source, name)
- print "[INFO] added link %s -> %s" % (name, link_source)
+ print("[INFO] added link %s -> %s" % (name, link_source))

else:
# If it is a symlink, then resolve it add to the list of host symlinks to be created later
@@ -58,23 +58,23 @@ def export_package(manifest, dest):
hostname = strip_file(hostname)

shutil.copy(hostname, name)
- print "[INFO] exported %s" % name
+ print("[INFO] exported %s" % name)
elif os.path.isdir(hostname):
# If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy.
if not os.path.exists(name):
os.makedirs(name)
- print "[INFO] created dir %s" % name
+ print("[INFO] created dir %s" % name)

else:
# Inform the user that the rule cannot be applied. For example, this happens for links in OSv.
- print "[ERR] unable to export %s" % hostname
+ print("[ERR] unable to export %s" % hostname)

for link_source, name in host_symlinks:
target_dir = os.path.dirname(name)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
os.symlink(link_source, name)
- print "[INFO] added link %s -> %s" % (name, link_source)
+ print("[INFO] added link %s -> %s" % (name, link_source))

def main():
make_option = optparse.make_option
diff --git a/scripts/firecracker.py b/scripts/firecracker.py
--- a/scripts/firecracker.py
+++ b/scripts/firecracker.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# pip install requests-unixsocket
import sys
@@ -152,14 +152,14 @@ def print_time(msg):
def setup_tap_interface(mode, tap_interface_name, tap_ip=None, physical_nic=None, bridge_name=None):
# Setup tun tap interface if does not exist
# sudo ip link delete fc_tap0 - this deletes the tap device
- tuntap_interfaces = subprocess.check_output(['ip', 'tuntap'])
+ tuntap_interfaces = subprocess.check_output(['ip', 'tuntap']).decode('utf-8')
if tuntap_interfaces.find(tap_interface_name) < 0:
print("The tap interface %s not found -> needs to set it up!" % tap_interface_name)
dirname = os.path.dirname(os.path.abspath(__file__))
setup_networking_script = os.path.join(dirname, 'setup_fc_networking.sh')
# Check if the bridge exists if user specified it
if mode == 'bridged' and bridge_name:
- bridges = subprocess.check_output(['brctl', 'show'])
+ bridges = subprocess.check_output(['brctl', 'show']).decode('utf-8')
if bridges.find(bridge_name) < 0:
print("The bridge %s does not exist per brctl. Please create one!" % bridge_name)
exit(-1)
@@ -182,7 +182,7 @@ def find_firecracker(dirname):
if not os.path.exists(firecracker_path):
url_base = 'https://github.com/firecracker-microvm/firecracker/releases/download'
download_url = '%s/%s/firecracker-%s' % (url_base, firecracker_version, firecracker_version)
- answer = raw_input("Firecracker executable has not been found under %s. "
+ answer = input("Firecracker executable has not been found under %s. "
"Would you like to download it from %s and place it under %s? [y|n]" %
(firecracker_path, download_url, firecracker_path))
if answer.capitalize() != 'Y':
@@ -233,7 +233,7 @@ def start_firecracker(firecracker_path, socket_path):
def start_firecracker_with_no_api(firecracker_path, firecracker_config_json):
# Start firecracker process and pass configuration JSON as a file
api_file = tempfile.NamedTemporaryFile(delete=False)
- api_file.write(firecracker_config_json)
+ api_file.write(bytes(firecracker_config_json, 'utf-8'))
stty_save()
return subprocess.Popen([firecracker_path, "--no-api", "--config-file", api_file.name],
stdout=sys.stdout, stderr=subprocess.STDOUT), api_file.name
diff --git a/scripts/gen-rofs-img.py b/scripts/gen-rofs-img.py
--- a/scripts/gen-rofs-img.py
+++ b/scripts/gen-rofs-img.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

#
# Copyright (c) 2015 Carnegie Mellon University.
@@ -93,9 +93,7 @@ def write(self,fp):
pos = fp.tell()
fp.write(c_ulonglong(self.inode_no))
fp.write(c_ushort(len(self.filename)))
- for c in self.filename:
- fp.write(c_char(c))
-
+ fp.write(bytes(self.filename,'utf-8'))
return fp.tell() - pos

class SymbolicLink(object):
@@ -105,9 +103,7 @@ def __init__(self,path):
def write(self,fp):
pos = fp.tell()
fp.write(c_ushort(len(self.path)))
- for c in self.path:
- fp.write(c_char(c))
-
+ fp.write(bytes(self.path,'utf-8'))
return fp.tell() - pos

directory_entries = []
@@ -151,7 +147,7 @@ def next_inode():
return inode

def pad(fp, size):
- fp.write('\0' * size)
+ fp.write(b'\0' * size)
return size

def write_initial_superblock(fp):
@@ -218,13 +214,13 @@ def write_dir(fp, manifest, dirpath, parent_dir):
inode.data_offset = symlinks_count
inode.count = 1
next_symlink(val[2:],manifest)
- print 'Link %s to %s' % (dirpath + '/' + entry, val[2:])
+ print('Link %s to %s' % (dirpath + '/' + entry, val[2:]))
else: #file
inode.mode = REG_MODE
global block
inode.data_offset = block
inode.count = write_file(fp, val)
- print 'Adding %s' % (dirpath + '/' + entry)
+ print('Adding %s' % (dirpath + '/' + entry))

# This needs to be added so that later we can walk the tree
# when fining symlinks
@@ -264,15 +260,15 @@ def write_fs(fp, manifest):
return (block_no, bytes_written)

def gen_image(out, manifest):
- print 'Writing image'
+ print('Writing image')
fp = open(out, 'wb')

# write the initial superblock
write_initial_superblock(fp)

system_structure_block, bytes_written = write_fs(fp, manifest)
structure_info_last_block_bytes = bytes_written % OSV_BLOCK_SIZE
- structure_info_blocks_count = bytes_written / OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0)
+ structure_info_blocks_count = bytes_written // OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0)

pad(fp,OSV_BLOCK_SIZE - structure_info_last_block_bytes)

@@ -290,10 +286,10 @@ def gen_image(out, manifest):
sb.symlinks_count = len(symlinks)
sb.inodes_count = len(inodes)

- print 'First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count)
- print 'Directory entries count %d' % sb.directory_entries_count
- print 'Symlinks count %d' % sb.symlinks_count
- print 'Inodes count %d' % sb.inodes_count
+ print('First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count))
+ print('Directory entries count %d' % sb.directory_entries_count)
+ print('Symlinks count %d' % sb.symlinks_count)
+ print('Inodes count %d' % sb.inodes_count)

fp.seek(0)
fp.write(sb)
diff --git a/scripts/imgedit.py b/scripts/imgedit.py
--- a/scripts/imgedit.py
+++ b/scripts/imgedit.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import sys, struct
import subprocess
@@ -31,14 +31,15 @@ def chs(x):
return c, h, s

def read_chars_up_to_null(file):
- while True:
+ keep_reading = True
+ while keep_reading:
try:
- c = file.read(1)
+ c = file.read(1).decode()
if c == '\0':
- raise StopIteration
+ keep_reading = False
yield c
except ValueError:
- raise StopIteration
+ keep_reading = False

def read_cstr(file):
return ''.join(read_chars_up_to_null(file))
diff --git a/scripts/libosv.py b/scripts/libosv.py
--- a/scripts/libosv.py
+++ b/scripts/libosv.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
import sys
import re

@@ -30,7 +30,7 @@
try:
value, tp, bnd, sym = elf.match(line).groups()
out.write("%s = 0x%s;\n"%(sym, value))
- print (".global %s\n.type %s,@%s"%(sym, sym, asm_type[tp]))
+ print(".global %s\n.type %s,@%s"%(sym, sym, asm_type[tp]))
except AttributeError:
pass

diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py
--- a/scripts/manifest_common.py
+++ b/scripts/manifest_common.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import os, io, re, subprocess

diff --git a/scripts/metadata.py b/scripts/metadata.py
--- a/scripts/metadata.py
+++ b/scripts/metadata.py
@@ -1,7 +1,7 @@
-import SimpleHTTPServer as http
-import SocketServer
import subprocess
import os
+import http.server as http
+import socketserver

METADATA_IP = '169.254.169.254'
port = 80
@@ -32,7 +32,7 @@ def start_server(path):
try:
os.chdir(path)
handler = http.SimpleHTTPRequestHandler
- server = SocketServer.TCPServer(("", port), handler, False)
+ server = socketserver.TCPServer(("", port), handler, False)
server.allow_reuse_address = True
server.server_bind()
server.server_activate()
diff --git a/scripts/mkbootfs.py b/scripts/mkbootfs.py
--- a/scripts/mkbootfs.py
+++ b/scripts/mkbootfs.py
@@ -1,10 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import os, struct, optparse, io
-try:
- import configparser
-except ImportError:
- import ConfigParser as configparser
+import configparser
from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file

def main():
@@ -80,7 +77,7 @@ def main():
if hostname.startswith("->"):
link = hostname[2:]
out.write(link.encode())
- out.write('\0')
+ out.write(b'\0')
else:
out.write(open(hostname, 'rb').read())

diff --git a/scripts/module.py b/scripts/module.py
--- a/scripts/module.py
+++ b/scripts/module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import re
import os
diff --git a/scripts/nbd_client.py b/scripts/nbd_client.py
--- a/scripts/nbd_client.py
+++ b/scripts/nbd_client.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 Nodalink, SARL.
diff --git a/scripts/osv/modules/filemap.py b/scripts/osv/modules/filemap.py
--- a/scripts/osv/modules/filemap.py
+++ b/scripts/osv/modules/filemap.py
@@ -19,15 +19,26 @@ def _pattern_to_regex(path):
% bad_token.group(1))

path = '^' + re.escape(path) + '$'
+ # Python 3.7 and above does NOT escape '/' so we need to detect it and handle accordingly
+ slash_escaped = '\\/' in path

# Normalize path
- path = re.sub(r'(\\/)+', '\/', path)
+ if slash_escaped:
+ path = re.sub(r'(\\/)+', '\/', path)
+ else:
+ path = re.sub(r'(/)+', '/', path)

# Merge consecutive **/ components
- path = _reduce_path(path, r'\\\*\\\*\\/\\\*\\\*', '\*\*')
+ if slash_escaped:
+ path = _reduce_path(path, r'\\\*\\\*\\/\\\*\\\*', '\*\*')
+ else:
+ path = _reduce_path(path, r'\\\*\\\*/\\\*\\\*', '\*\*')

# Transform ** component
- path = re.sub(r'(\^|\\/)\\\*\\\*(\\/|\$)', '((^|\/).*($|\/|^))', path)
+ if slash_escaped:
+ path = re.sub(r'(\^|\\/)\\\*\\\*(\\/|\$)', '((^|\/).*($|\/|^))', path)
+ else:
+ path = re.sub(r'(\^|/)\\\*\\\*(/|\$)', '((^|/).*($|\|^))', path)

path = path.replace('\\*', '[^/]*')
path = path.replace('\\?', '.')
diff --git a/scripts/run.py b/scripts/run.py
--- a/scripts/run.py
+++ b/scripts/run.py
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-from __future__ import print_function
+#!/usr/bin/env python3
+
import subprocess
import sys
import argparse
diff --git a/scripts/test.py b/scripts/test.py
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import atexit
import subprocess
import argparse
@@ -81,7 +81,7 @@ def is_not_skipped(test):
return test.name not in blacklist

def run_tests_in_single_instance():
- run(filter(lambda test: not isinstance(test, TestRunnerTest), tests))
+ run([test for test in tests if not isinstance(test, TestRunnerTest)])

blacklist_tests = ' '.join(blacklist)
args = run_py_args + ["-s", "-e", "/testrunner.so -b %s" % (blacklist_tests)]
@@ -103,7 +103,7 @@ def pluralize(word, count):

def make_export_and_conf():
export_dir = tempfile.mkdtemp(prefix='share')
- os.chmod(export_dir, 0777)
+ os.chmod(export_dir, 0o777)
(conf_fd, conf_path) = tempfile.mkstemp(prefix='export')
conf = os.fdopen(conf_fd, "w")
conf.write("%s 127.0.0.1(insecure,rw)\n" % export_dir)
@@ -155,12 +155,12 @@ def run_tests():
"/tst-nfs.so --server 192.168.122.1 --share %s" %
export_dir) ]

- line = proc.stdout.readline()
+ line = proc.stdout.readline().decode()
while line:
print(line)
if "/tmp" in line:
break
- line = proc.stdout.readline()
+ line = proc.stdout.readline().decode()


run(tests_to_run)
diff --git a/scripts/tests/test_app.py b/scripts/tests/test_app.py
--- a/scripts/tests/test_app.py
+++ b/scripts/tests/test_app.py
@@ -1,7 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
from testing import *
import argparse
-import subprocess
from time import sleep

def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, host_port=None, input_lines=[], kill_app=False):
diff --git a/scripts/tests/test_app_with_test_script.py b/scripts/tests/test_app_with_test_script.py
--- a/scripts/tests/test_app_with_test_script.py
+++ b/scripts/tests/test_app_with_test_script.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
from testing import *
import argparse
import runpy
diff --git a/scripts/tests/test_http_app_with_curl_and_ab.py b/scripts/tests/test_http_app_with_curl_and_ab.py
--- a/scripts/tests/test_http_app_with_curl_and_ab.py
+++ b/scripts/tests/test_http_app_with_curl_and_ab.py
@@ -1,11 +1,11 @@
-#!/usr/bin/python
+#!/usr/bin/python3
from testing import *
import argparse
import subprocess
from time import sleep

def check_with_curl(url, expected_http_line):
- output = subprocess.check_output(["curl", "-s", url])
+ output = subprocess.check_output(["curl", "-s", url]).decode('utf-8')
print(output)
if expected_http_line not in output:
print("FAILED curl: wrong output")
@@ -39,9 +39,9 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt
check_with_curl(app_url, expected_http_line)

if no_keep_alive:
- output = subprocess.check_output(["ab", "-l", "-c", str(concurrency), "-n", str(count), app_url]).split('\n')
+ output = subprocess.check_output(["ab", "-l", "-c", str(concurrency), "-n", str(count), app_url]).decode('utf-8').split('\n')
else:
- output = subprocess.check_output(["ab", "-l", "-k", "-c", str(concurrency), "-n", str(count), app_url]).split('\n')
+ output = subprocess.check_output(["ab", "-l", "-k", "-c", str(concurrency), "-n", str(count), app_url]).decode('utf-8').split('\n')

failed_requests = 1
complete_requests = 0
@@ -74,11 +74,11 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt
success = False

if failed_requests > 0:
- print("FAILED ab - encountered failed requests: %d" % failed_requests)
+ print("FAILED ab - encountered failed requests: %d" % failed_requests)
success = False

if complete_requests < count:
- print("FAILED ab - too few complete requests : %d ? %d" % (complete_requests, count))
+ print("FAILED ab - too few complete requests : %d ? %d" % (complete_requests, count))
success = False

if success:
diff --git a/scripts/tests/testing.py b/scripts/tests/testing.py
--- a/scripts/tests/testing.py
+++ b/scripts/tests/testing.py
@@ -132,7 +132,7 @@ def append_line(line):
self.cv.release()

line = ''
- ch_bytes = ''
+ ch_bytes = bytes()
while True:
ch_bytes = ch_bytes + self.process.stdout.read(1)
try:
@@ -143,7 +143,7 @@ def append_line(line):
if ch == '\n':
append_line(line)
line = ''
- ch_bytes = ''
+ ch_bytes = bytes()
except UnicodeError:
continue

diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py
--- a/scripts/upload_manifest.py
+++ b/scripts/upload_manifest.py
@@ -1,17 +1,11 @@
-#!/usr/bin/python
+#!/usr/bin/python3

import optparse, os, subprocess, socket, threading, stat, sys
from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file
from contextlib import closing

-try:
- import StringIO
- # This works on Python 2
- StringIO = StringIO.StringIO
-except ImportError:
- import io
- # This works on Python 3
- StringIO = io.StringIO
+import io
+StringIO = io.StringIO

def find_free_port():
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
@@ -53,8 +47,7 @@ def cpio_send(data):
def cpio_field(number, length):
return ("%.*x" % (length, number)).encode()
def cpio_header(filename, mode, filesize):
- if sys.version_info >= (3, 0, 0):
- filename = filename.encode("utf-8")
+ filename = filename.encode("utf-8")
return (b"070701" # magic
+ cpio_field(0, 8) # inode
+ cpio_field(mode, 8) # mode