[PATCH 3 of 6] packaging: build Mercurial with py2exe under python3

[Matt Harbison]

# HG changeset patch
# User Matt Harbison <matt_h...@yahoo.com>
# Date 1650479168 14400
# Wed Apr 20 14:26:08 2022 -0400
# Node ID fab9c0c5275c6df84ff216489f10f7df9c235371
# Parent 83a3e09fce3708e3a6b128733d947bb75f8288aa
# EXP-Topic windows-py3-packaging
packaging: build Mercurial with py2exe under python3

This is a reproduction of the process used by the winbuild repo's setup.py to
build Mercurial. The one major difference here is that the html man pages are
built using Mercurial's `build_doc` target instead of a lot of manual work.
That in turn normalizes the html files to LF endings, but if that's OK upstream,
it's OK here.

The fact that the hg.1.html file is missing the zstd entry under "Available
Compression Engines" and is missing the "purge" and "strip" entries in the
disabled extensions list is a bit odd. On a new system, neither of these are
problems. I suspect the latter is due to PATH and/or PYTHONPATH hijink with the
current build process, picking up an old Mercurial somewhere- these extensions
were deprecated in 5.7 and mostly hidden. The former, I have no explanation
for.

diff --git a/contrib/packaging/thgpackaging/cli.py b/contrib/packaging/thgpackaging/cli.py
--- a/contrib/packaging/thgpackaging/cli.py
+++ b/contrib/packaging/thgpackaging/cli.py
@@ -96,7 +96,7 @@
raise Exception("--python arg must be an absolute path")

kwargs = {
- "source_dir": SOURCE_DIR,
+ "source_dirs": SOURCE_DIRS,
"version": version,
}

diff --git a/contrib/packaging/thgpackaging/py2exe.py b/contrib/packaging/thgpackaging/py2exe.py
--- a/contrib/packaging/thgpackaging/py2exe.py
+++ b/contrib/packaging/thgpackaging/py2exe.py
@@ -10,7 +10,9 @@

import os
import pathlib
+import shutil
import subprocess
+import sys

from .downloads import download_entry
from .util import (
@@ -18,6 +20,7 @@
extract_zip_to_directory,
process_install_rules,
python_exe_info,
+ SourceDirs,
)


@@ -57,7 +60,7 @@


def build_py2exe(
- source_dir: pathlib.Path,
+ source_dirs: SourceDirs,
build_dir: pathlib.Path,
python_exe: pathlib.Path,
build_name: str,
@@ -73,6 +76,10 @@
Build files will be placed in ``build_dir``.
"""

+ env = dict(os.environ)
+ env["HGPLAIN"] = "1"
+ env["HGRCPATH"] = ""
+
py_info = python_exe_info(python_exe)

vc_x64 = py_info['arch'] == '64bit'
@@ -105,8 +112,6 @@
[str(venv_pip), 'install', '-r', str(venv_requirements_txt)], check=True
)

- env = dict(os.environ)
-
if extra_packages_script:
more_packages = set(
subprocess.check_output(extra_packages_script, cwd=build_dir)
@@ -145,9 +150,53 @@
)

print('building Mercurial')
+ hg_dir = source_dirs.hg
+
+ # Clear files added by previous installer runs
subprocess.run(
- [str(venv_python), 'setup.py', 'py2exe', 'build_doc', '--html'],
- cwd=str(source_dir),
+ ["hg.exe", "--config", "extensions.purge=", "purge", "--all"],
+ cwd=str(hg_dir),
+ env=env,
+ check=True,
+ )
+
+ oldpath = env.get('PYTHONPATH', '').split(os.pathsep)
+ path = [str(hg_dir), str(source_dirs.evolve)]
+ path.extend(oldpath)
+ env['PYTHONPATH'] = os.pathsep.join(path)
+
+ subprocess.run(
+ [str(venv_python), 'setup.py', '--version'],
+ cwd=str(hg_dir),
+ env=env,
+ check=True,
+ )
+ subprocess.run(
+ [str(venv_python), 'setup.py', 'build_py', '-c', '-d', '.', 'build_mo'],
+ cwd=str(hg_dir),
+ env=env,
+ check=True,
+ )
+ subprocess.run(
+ [str(venv_python), 'setup.py', 'build_ext', '-i'],
+ cwd=str(hg_dir),
+ env=env,
+ check=True,
+ )
+ subprocess.run(
+ [str(venv_python), 'setup.py', 'build_hgextindex'],
+ cwd=str(hg_dir),
+ env=env,
+ check=True,
+ )
+ # TODO: the zstd entry isn't being added to doc/hg.1.html for some reason on
+ # some systems. It would appear that cext isn't loading. (shell=True
+ # doesn't help)
+ subprocess.run(
+ # Use python_exe instead of venv_python so it has access to the docutils
+ # module in the bootstrap venv.
+ [str(python_exe), 'setup.py', 'build_doc', '--html'],
+ cwd=str(hg_dir),
env=env,
check=True,
)
diff --git a/contrib/packaging/thgpackaging/wix.py b/contrib/packaging/thgpackaging/wix.py
--- a/contrib/packaging/thgpackaging/wix.py
+++ b/contrib/packaging/thgpackaging/wix.py
@@ -31,6 +31,7 @@
python_exe_info,
read_version_py,
sign_with_signtool,
+ SourceDirs,
)


@@ -43,7 +44,7 @@
}

EXTRA_INCLUDES = {
- '_curses',
+ #'_curses', # TODO: figure out why py3's py2exe doesn't see this
'_curses_panel',
}

@@ -255,7 +256,7 @@


def build_installer_py2exe(
- source_dir: pathlib.Path,
+ source_dirs: SourceDirs,
python_exe: pathlib.Path,
msi_name='tortoisehg',
version=None,
@@ -266,7 +267,8 @@
):
"""Build a WiX MSI installer using py2exe.

- ``source_dir`` is the path to the TortoiseHg source tree to use.
+ ``source_dirs`` is the collection of paths to the TortoiseHg source tree and
+ other included dependencies to use.
``arch`` is the target architecture. either ``x86`` or ``x64``.
``python_exe`` is the path to the Python executable to use/bundle.
``version`` is the TortoiseHg version string. If not defined,
@@ -283,15 +285,18 @@

arch = 'x64' if py_info['arch'] == '64bit' else 'x86'

- hg_build_dir = source_dir / 'build'
+ thg_build_dir = source_dirs.original / 'build'

requirements_txt = (
- source_dir / 'contrib' / 'packaging' / 'requirements-windows-py2.txt'
+ source_dirs.original
+ / 'contrib'
+ / 'packaging'
+ / 'requirements-windows-pyqt5-installer.txt'
)

build_py2exe(
- source_dir,
- hg_build_dir,
+ source_dirs,
+ thg_build_dir,
python_exe,
'wix',
requirements_txt,
@@ -300,27 +305,31 @@
extra_includes=EXTRA_INCLUDES,
)

- build_dir = hg_build_dir / ('wix-%s' % arch)
+ build_dir = thg_build_dir / ('wix-%s' % arch)
staging_dir = build_dir / 'stage'

build_dir.mkdir(exist_ok=True)

+ # TODO: stage the build once the automatic *.wxs file generation is sorted
+ # out.
# Purge the staging directory for every build so packaging is pristine.
- if staging_dir.exists():
- print('purging %s' % staging_dir)
- shutil.rmtree(staging_dir)
+# if staging_dir.exists():
+# print('purging %s' % staging_dir)
+# shutil.rmtree(staging_dir)

- stage_install(source_dir, staging_dir, lower_case=True)
+# stage_install(source_dirs.original, staging_dir, lower_case=True)

# We also install some extra files.
- process_install_rules(EXTRA_INSTALL_RULES, source_dir, staging_dir)
+# process_install_rules(
+# EXTRA_INSTALL_RULES, source_dirs.original, staging_dir
+# )

# And remove some files we don't want.
- for f in STAGING_REMOVE_FILES:
- p = staging_dir / f
- if p.exists():
- print('removing %s' % p)
- p.unlink()
+# for f in STAGING_REMOVE_FILES:
+# p = staging_dir / f
+# if p.exists():
+# print('removing %s' % p)
+# p.unlink()

return run_wix_packaging(
source_dir,

[Yuya Nishihara]

On Mon, 02 May 2022 19:04:44 -0400, Matt Harbison wrote:
> # HG changeset patch
> # User Matt Harbison <matt_h...@yahoo.com>
> # Date 1650479168 14400
> # Wed Apr 20 14:26:08 2022 -0400
> # Node ID fab9c0c5275c6df84ff216489f10f7df9c235371
> # Parent 83a3e09fce3708e3a6b128733d947bb75f8288aa
> # EXP-Topic windows-py3-packaging
> packaging: build Mercurial with py2exe under python3

> build_py2exe(
> - source_dir,
> - hg_build_dir,
> + source_dirs,
> + thg_build_dir,
> python_exe,
> 'wix',
> requirements_txt,

> return run_wix_packaging(
> source_dir,
^^^^^^^^^^
Queued, but this run_wix_packaging would be broken at this revision.

[Matt Harbison]

Yep, thanks.  I've been trying to chunk this up into smaller related/reviewable pieces, but it won't be fully functional until I post the last patch(es?) to assemble the MSI.