[gentoo-dev] [PATCH 7/8] Move run_in_build_dir() to multibuild.eclass.

[Michał Górny]

---
gx86/eclass/multibuild.eclass | 19 +++++++++++++++++++
gx86/eclass/python-r1.eclass | 19 -------------------
2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
index a4d5d11..e15b74e 100644
--- a/gx86/eclass/multibuild.eclass
+++ b/gx86/eclass/multibuild.eclass
@@ -173,5 +173,24 @@ multibuild_parallel_foreach() {
return ${ret}
}

+# @FUNCTION: run_in_build_dir
+# @USAGE: <argv>...
+# @DESCRIPTION:
+# Run the given command in the directory pointed by BUILD_DIR.
+run_in_build_dir() {
+ debug-print-function ${FUNCNAME} "${@}"
+ local ret
+
+ [[ ${#} -ne 0 ]] || die "${FUNCNAME}: no command specified."
+ [[ ${BUILD_DIR} ]] || die "${FUNCNAME}: BUILD_DIR not set."
+
+ pushd "${BUILD_DIR}" >/dev/null || die
+ "${@}"
+ ret=${?}
+ popd >/dev/null || die
+
+ return ${ret}
+}
+
_MULTIBUILD=1
fi
diff --git a/gx86/eclass/python-r1.eclass b/gx86/eclass/python-r1.eclass
index fb9032e..6c1dc3e 100644
--- a/gx86/eclass/python-r1.eclass
+++ b/gx86/eclass/python-r1.eclass
@@ -735,24 +735,5 @@ python_replicate_script() {
done
}

-# @FUNCTION: run_in_build_dir
-# @USAGE: <argv>...
-# @DESCRIPTION:
-# Run the given command in the directory pointed by BUILD_DIR.
-run_in_build_dir() {
- debug-print-function ${FUNCNAME} "${@}"
- local ret
-
- [[ ${#} -ne 0 ]] || die "${FUNCNAME}: no command specified."
- [[ ${BUILD_DIR} ]] || die "${FUNCNAME}: BUILD_DIR not set."
-
- pushd "${BUILD_DIR}" >/dev/null || die
- "${@}"
- ret=${?}
- popd >/dev/null || die
-
- return ${ret}
-}
-
_PYTHON_R1=1
fi
--
1.8.1.4

[Michał Górny]

This allows us to spawn 'tee' as separate process while keeping
the function code executed in the main shell.
---
gx86/eclass/multibuild.eclass | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
index 716d34f..d42b8a7 100644
--- a/gx86/eclass/multibuild.eclass
+++ b/gx86/eclass/multibuild.eclass
@@ -108,18 +108,20 @@ multibuild_foreach() {
local MULTIBUILD_VARIANT=${v}
local MULTIBUILD_ID=${prev_id}${v}
local BUILD_DIR=${bdir%%/}-${v}
+ local log_fd

- einfo "${v}: running ${@}" \
- | tee -a "${T}/build-${MULTIBUILD_ID}.log"
+ # redirect_alloc_fd accepts files only. so we need to open
+ # a random file and then reuse the fd for logger process.
+ redirect_alloc_fd log_fd /dev/null
+ eval "exec ${log_fd}> "'>(exec tee -a "${T}/build-${MULTIBUILD_ID}.log")'
+
+ eval 'einfo "${v}: running ${@}" >&'${log_fd}' 2>&1'

# _multibuild_parallel() does redirection internally.
# this is a hidden API to avoid writing multilib_foreach twice.
- if [[ ${1} == _multibuild_parallel ]]; then
- "${@}"
- else
- "${@}" 2>&1 | tee -a "${T}/build-${MULTIBUILD_ID}.log"
- fi
+ eval '"${@}" >&'${log_fd}' 2>&1'
lret=${?}
+ eval "exec ${log_fd}>&-"
done
[[ ${ret} -eq 0 && ${lret} -ne 0 ]] && ret=${lret}

@@ -145,8 +147,7 @@ multibuild_parallel_foreach() {
_multibuild_parallel() {
(
multijob_child_init
- "${@}" 2>&1 | tee -a "${T}/build-${MULTIBUILD_ID}.log"
- exit ${PIPESTATUS[0]}
+ "${@}"
) &
multijob_post_fork
}
--
1.8.1.4

[Michał Górny]

---
gx86/eclass/multilib-build.eclass | 45 ++++++++++++++++++---------------------
1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/gx86/eclass/multilib-build.eclass b/gx86/eclass/multilib-build.eclass
index b1afd85..4321e45 100644
--- a/gx86/eclass/multilib-build.eclass
+++ b/gx86/eclass/multilib-build.eclass
@@ -23,7 +23,7 @@ case ${EAPI:-0} in
*) die "EAPI=${EAPI} is not supported" ;;
esac

-inherit multilib multiprocessing
+inherit multibuild multilib

# @ECLASS-VARIABLE: _MULTILIB_FLAGS
# @INTERNAL
@@ -85,6 +85,19 @@ multilib_get_enabled_abis() {
fi
}

+# @FUNCTION: _multilib_multibuild_wrapper
+# @USAGE: <argv>...
+# @INTERNAL
+# @DESCRIPTION:
+# Initialize the environment for ABI selected for multibuild.
+_multilib_multibuild_wrapper() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ local ABI=${MULTIBUILD_VARIANT}
+ multilib_toolchain_setup "${ABI}"
+ "${@}"
+}
+
# @FUNCTION: multilib_foreach_abi
# @USAGE: <argv>...
# @DESCRIPTION:
@@ -95,14 +108,11 @@ multilib_get_enabled_abis() {
# If multilib support is disabled, it just runs the commands. No setup
# is done.
multilib_foreach_abi() {
- local initial_dir=${BUILD_DIR:-${S}}

+ debug-print-function ${FUNCNAME} "${@}"

- local ABI
- for ABI in $(multilib_get_enabled_abis); do
- multilib_toolchain_setup "${ABI}"
- local BUILD_DIR=${initial_dir%%/}-${ABI}
- "${@}" | tee -a "${T}/build-${ABI}.log"
- done
+ local MULTIBUILD_VARIANTS=( $(multilib_get_enabled_abis) )
+
+ multibuild_foreach _multilib_multibuild_wrapper "${@}"
}

# @FUNCTION: multilib_parallel_foreach_abi
@@ -118,24 +128,11 @@ multilib_foreach_abi() {
#
# Useful for running configure scripts.
multilib_parallel_foreach_abi() {
- local initial_dir=${BUILD_DIR:-${S}}
-
- multijob_init
-
- local ABI
- for ABI in $(multilib_get_enabled_abis); do
- (
- multijob_child_init
-
- multilib_toolchain_setup "${ABI}"
- local BUILD_DIR=${initial_dir%%/}-${ABI}
- "${@}" 2>&1 | tee -a "${T}/build-${ABI}.log"
- ) &

+ debug-print-function ${FUNCNAME} "${@}"

- multijob_post_fork
- done
+ local MULTIBUILD_VARIANTS=( $(multilib_get_enabled_abis) )

- multijob_finish
+ multibuild_parallel_foreach _multilib_multibuild_wrapper "${@}"
}

# @FUNCTION: multilib_check_headers
--
1.8.1.4

[Michał Górny]

---
gx86/eclass/multibuild.eclass | 4 ++++
1 file changed, 4 insertions(+)

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
index d42b8a7..a4d5d11 100644
--- a/gx86/eclass/multibuild.eclass
+++ b/gx86/eclass/multibuild.eclass
@@ -99,6 +99,10 @@ multibuild_foreach() {
|| die "MULTIBUILD_VARIANTS need to be set"

local bdir=${BUILD_DIR:-${S}}
+
+ # Avoid writing outside WORKDIR if S=${WORKDIR}.
+ [[ ${bdir%%/} == ${WORKDIR%%/} ]] && bdir=${WORKDIR}/build
+
local prev_id=${MULTIBUILD_ID:+${MULTIBUILD_ID}-}
local ret=0 lret=0 v

--
1.8.1.4

[Michał Górny]

---
gx86/eclass/python-r1.eclass | 74 +++++++++++++-------------------------------
1 file changed, 21 insertions(+), 53 deletions(-)

diff --git a/gx86/eclass/python-r1.eclass b/gx86/eclass/python-r1.eclass
index a1d9228..fb9032e 100644
--- a/gx86/eclass/python-r1.eclass
+++ b/gx86/eclass/python-r1.eclass
@@ -48,7 +48,7 @@ elif [[ ${_PYTHON_ANY_R1} ]]; then
die 'python-r1.eclass can not be used with python-any-r1.eclass.'
fi

-inherit python-utils-r1
+inherit multibuild python-utils-r1

# @ECLASS-VARIABLE: PYTHON_COMPAT
# @REQUIRED
@@ -602,6 +602,21 @@ _python_obtain_impls() {
done
}

+# @FUNCTION: _python_multibuild_wrapper
+# @USAGE: <command> [<args>...]
+# @INTERNAL
+# @DESCRIPTION:
+# Initialize the environment for Python implementation selected
+# for multibuild.
+_python_multibuild_wrapper() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ local -x EPYTHON PYTHON
+ python_export "${MULTIBUILD_VARIANT}" EPYTHON PYTHON
+

+ "${@}"
+}
+

# @FUNCTION: python_foreach_impl
# @USAGE: <command> [<args>...]
# @DESCRIPTION:
@@ -621,36 +636,10 @@ python_foreach_impl() {
_python_validate_useflags
_python_check_USE_PYTHON

- local impl
- local bdir=${BUILD_DIR:-${S}}
- local ret=0 lret=0
local MULTIBUILD_VARIANTS
-
- debug-print "${FUNCNAME}: bdir = ${bdir}"
_python_obtain_impls
- for impl in "${MULTIBUILD_VARIANTS[@]}"; do
- local EPYTHON PYTHON
- python_export "${impl}" EPYTHON PYTHON
- local BUILD_DIR=${bdir%%/}-${impl}
- export EPYTHON PYTHON
-
- einfo "${EPYTHON}: running ${@}" \
- | tee -a "${T}/build-${EPYTHON}.log"
-
- # _python_parallel() does redirection internally.
- # note: this is a hidden API to avoid writing python_foreach_impl
- # twice. do *not* even think of using it anywhere else.
- if [[ ${1} == _python_parallel ]]; then

- "${@}"
- else

- "${@}" 2>&1 | tee -a "${T}/build-${EPYTHON}.log"
- fi
- lret=${?}

- [[ ${ret} -eq 0 && ${lret} -ne 0 ]] && ret=${lret}
- done

-
- return ${ret}

+ multibuild_foreach _python_multibuild_wrapper "${@}"
}

# @FUNCTION: python_parallel_foreach_impl
@@ -672,32 +661,11 @@ python_foreach_impl() {
python_parallel_foreach_impl() {

debug-print-function ${FUNCNAME} "${@}"

- local ret lret
-
- _python_parallel() {
- (
- multijob_child_init
- "${@}" 2>&1 | tee -a "${T}/build-${EPYTHON}.log"

- exit ${PIPESTATUS[0]}

- ) &
- multijob_post_fork
- }
-
- local opts
- if [[ ${DISTUTILS_JOBS} ]]; then
- opts=-j${DISTUTILS_JOBS}
- else
- opts=${MAKEOPTS}
- fi
-
- multijob_init "${opts}"
- python_foreach_impl _python_parallel "${@}"
- ret=${?}
- multijob_finish
- lret=${?}
+ local MULTIBUILD_JOBS=${MULTIBUILD_JOBS:-${DISTUTILS_JOBS}}
+ local MULTIBUILD_VARIANTS
+ _python_obtain_impls

- [[ ${ret} -eq 0 ]] && ret=${lret}
- return ${ret}
+ multibuild_parallel_foreach _python_multibuild_wrapper "${@}"
}

# @FUNCTION: python_export_best
--
1.8.1.4

[Michał Górny]

---
gx86/eclass/python-r1.eclass | 61 +++++++++++++++++++++++++++-----------------
1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/gx86/eclass/python-r1.eclass b/gx86/eclass/python-r1.eclass
index 310859e..a1d9228 100644
--- a/gx86/eclass/python-r1.eclass
+++ b/gx86/eclass/python-r1.eclass
@@ -586,6 +586,22 @@ _python_check_USE_PYTHON() {
fi
}

+# @FUNCTION: _python_obtain_impls
+# @INTERNAL
+# @DESCRIPTION:
+# Set up the enabled implementation list.
+_python_obtain_impls() {
+ MULTIBUILD_VARIANTS=()
+
+ for impl in "${_PYTHON_ALL_IMPLS[@]}"; do
+ if has "${impl}" "${PYTHON_COMPAT[@]}" \
+ && use "python_targets_${impl}"
+ then
+ MULTIBUILD_VARIANTS+=( "${impl}" )
+ fi
+ done

+}
+
# @FUNCTION: python_foreach_impl
# @USAGE: <command> [<args>...]
# @DESCRIPTION:

@@ -608,33 +624,30 @@ python_foreach_impl() {
local impl

local bdir=${BUILD_DIR:-${S}}

local ret=0 lret=0
+ local MULTIBUILD_VARIANTS

debug-print "${FUNCNAME}: bdir = ${bdir}"

- for impl in "${_PYTHON_ALL_IMPLS[@]}"; do
- if has "${impl}" "${PYTHON_COMPAT[@]}" \
- && _python_impl_supported "${impl}" \
- && use "python_targets_${impl}"
- then

- local EPYTHON PYTHON
- python_export "${impl}" EPYTHON PYTHON
- local BUILD_DIR=${bdir%%/}-${impl}
- export EPYTHON PYTHON
-
- einfo "${EPYTHON}: running ${@}" \
- | tee -a "${T}/build-${EPYTHON}.log"
-
- # _python_parallel() does redirection internally.
- # note: this is a hidden API to avoid writing python_foreach_impl
- # twice. do *not* even think of using it anywhere else.
- if [[ ${1} == _python_parallel ]]; then
- "${@}"
- else
- "${@}" 2>&1 | tee -a "${T}/build-${EPYTHON}.log"
- fi
- lret=${?}
-
- [[ ${ret} -eq 0 && ${lret} -ne 0 ]] && ret=${lret}

+ _python_obtain_impls
+ for impl in "${MULTIBUILD_VARIANTS[@]}"; do
+ local EPYTHON PYTHON
+ python_export "${impl}" EPYTHON PYTHON
+ local BUILD_DIR=${bdir%%/}-${impl}
+ export EPYTHON PYTHON
+
+ einfo "${EPYTHON}: running ${@}" \
+ | tee -a "${T}/build-${EPYTHON}.log"
+
+ # _python_parallel() does redirection internally.
+ # note: this is a hidden API to avoid writing python_foreach_impl
+ # twice. do *not* even think of using it anywhere else.
+ if [[ ${1} == _python_parallel ]]; then
+ "${@}"
+ else
+ "${@}" 2>&1 | tee -a "${T}/build-${EPYTHON}.log"
fi
+ lret=${?}
+
+ [[ ${ret} -eq 0 && ${lret} -ne 0 ]] && ret=${lret}
done

return ${ret}
--
1.8.1.4

[Michał Górny]

Based on the code from python-r1.
---
gx86/eclass/multibuild.eclass | 172 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 172 insertions(+)
create mode 100644 gx86/eclass/multibuild.eclass

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
new file mode 100644
index 0000000..716d34f
--- /dev/null
+++ b/gx86/eclass/multibuild.eclass
@@ -0,0 +1,172 @@
+# Copyright 1999-2013 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: $
+
+# @ECLASS: multibuild
+# @MAINTAINER:
+# Michał Górny <mgo...@gentoo.org>
+# @AUTHOR:
+# Author: Michał Górny <mgo...@gentoo.org>
+# @BLURB: A generic eclass for building multiple variants of packages.
+# @DESCRIPTION:
+# The multibuild eclass aims to provide a generic framework for building
+# multiple 'variants' of a package (e.g. multilib, Python
+# implementations).
+
+case "${EAPI:-0}" in
+ 0|1|2|3|4)
+ die "Unsupported EAPI=${EAPI:-0} (too old) for ${ECLASS}"
+ ;;
+ 5)
+ ;;
+ *)
+ die "Unsupported EAPI=${EAPI} (unknown) for ${ECLASS}"
+ ;;
+esac
+
+if [[ ! ${_MULTIBUILD} ]]; then
+
+inherit multiprocessing
+
+# @ECLASS-VARIABLE: MULTIBUILD_VARIANTS
+# @DESCRIPTION:
+# An array specifying all enabled variants which multilib_foreach* will
+# execute the process for.
+#
+# In ebuild, it can be set in global scope. Eclasses should set it
+# locally in function scope to support nesting properly.
+#
+# Example:
+# @CODE
+# python_foreach_impl() {
+# local MULTIBUILD_VARIANTS=( python{2_5,2_6,2_7} ... )
+# multibuild_foreach_impl python_compile
+# }
+# @CODE
+
+# @ECLASS-VARIABLE: MULTIBUILD_VARIANT
+# @DESCRIPTION:
+# The current variant which the function was executed for.
+#
+# Example value:
+# @CODE
+# python2_6
+# @CODE
+
+# @ECLASS-VARIABLE: MULTIBUILD_ID
+# @DESCRIPTION:
+# The unique identifier for a multibuild run. In a simple run, it is
+# equal to MULTIBUILD_VARIANT. In a nested multibuild environment, it
+# contains the complete selection tree.
+#
+# It can be used to create variant-unique directories and files.
+#
+# Example value:
+# @CODE
+# amd64-double
+# @CODE
+
+# @ECLASS-VARIABLE: BUILD_DIR
+# @DESCRIPTION:
+# The current build directory. In global scope, it is supposed to
+# contain an 'initial' build directory; if unset, it defaults to ${S}.
+#
+# multibuild_foreach() sets BUILD_DIR locally to variant-specific build
+# directories based on the initial value of BUILD_DIR.
+#
+# Example value:
+# @CODE
+# ${WORKDIR}/foo-1.3-python2_6
+# @CODE
+
+# @FUNCTION: multibuild_foreach
+# @USAGE: [<argv>...]
+# @DESCRIPTION:
+# Run the passed command repeatedly for each of the enabled package
+# variants.
+#
+# Each of the runs will have variant-specific BUILD_DIR set, and output
+# teed to a separate log in ${T}.
+#
+# The function returns 0 if all commands return 0, or the first non-zero
+# exit status otherwise. However, it performs all the invocations
+# nevertheless. It is preferred to call 'die' inside of the passed
+# function.
+multibuild_foreach() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ [[ ${MULTIBUILD_VARIANTS} ]] \
+ || die "MULTIBUILD_VARIANTS need to be set"
+
+ local bdir=${BUILD_DIR:-${S}}

+ local prev_id=${MULTIBUILD_ID:+${MULTIBUILD_ID}-}

+ local ret=0 lret=0 v
+
+ debug-print "${FUNCNAME}: initial build_dir = ${bdir}"
+
+ for v in "${MULTIBUILD_VARIANTS[@]}"; do
+ local MULTIBUILD_VARIANT=${v}
+ local MULTIBUILD_ID=${prev_id}${v}
+ local BUILD_DIR=${bdir%%/}-${v}
+
+ einfo "${v}: running ${@}" \
+ | tee -a "${T}/build-${MULTIBUILD_ID}.log"
+
+ # _multibuild_parallel() does redirection internally.
+ # this is a hidden API to avoid writing multilib_foreach twice.
+ if [[ ${1} == _multibuild_parallel ]]; then

+ "${@}"
+ else

+ "${@}" 2>&1 | tee -a "${T}/build-${MULTIBUILD_ID}.log"
+ fi
+ lret=${?}
+ done

+ [[ ${ret} -eq 0 && ${lret} -ne 0 ]] && ret=${lret}

+
+ return ${ret}
+}
+

+# @FUNCTION: multibuild_parallel_foreach
+# @USAGE: [<argv>...]
+# @DESCRIPTION:
+# Run the passed command repeatedly for each of the enabled package
+# variants alike multibuild_foreach. Multiple invocations of the command
+# will be performed in parallel, up to MULTIBUILD_JOBS tasks.
+#
+# The function returns 0 if all commands return 0, or the first non-zero
+# exit status otherwise. However, it performs all the invocations
+# nevertheless. It is preferred to call 'die' inside of the passed
+# function.
+multibuild_parallel_foreach() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ local ret lret
+
+ _multibuild_parallel() {
+ (
+ multijob_child_init
+ "${@}" 2>&1 | tee -a "${T}/build-${MULTIBUILD_ID}.log"
+ exit ${PIPESTATUS[0]}
+ ) &
+ multijob_post_fork
+ }
+
+ local opts
+ if [[ ${MULTIBUILD_JOBS} ]]; then
+ opts=-j${MULTIBUILD_JOBS}
+ else
+ opts=${MAKEOPTS}
+ fi
+
+ multijob_init "${opts}"
+ multibuild_foreach _multibuild_parallel "${@}"
+ ret=${?}
+ multijob_finish
+ lret=${?}
+
+ [[ ${ret} -eq 0 ]] && ret=${lret}

+ return ${ret}
+}
+

+_MULTIBUILD=1
+fi
--
1.8.1.4

[Michał Górny]

Just a quick, dirty example. Not even tested thoroughly ;).
---
gx86/sci-libs/fftw/fftw-3.3.3-r1.ebuild | 38 +++++++++++++--------------------
1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/gx86/sci-libs/fftw/fftw-3.3.3-r1.ebuild b/gx86/sci-libs/fftw/fftw-3.3.3-r1.ebuild
index 18554f0..ddca8e4 100644
--- a/gx86/sci-libs/fftw/fftw-3.3.3-r1.ebuild
+++ b/gx86/sci-libs/fftw/fftw-3.3.3-r1.ebuild
@@ -7,7 +7,7 @@ EAPI=5
#AUTOTOOLS_AUTORECONF=1
FORTRAN_NEEDED=fortran

-inherit autotools-multilib eutils flag-o-matic fortran-2 toolchain-funcs versionator
+inherit autotools-multilib eutils flag-o-matic fortran-2 multibuild toolchain-funcs versionator

DESCRIPTION="Fast C library for the Discrete Fourier Transform"
HOMEPAGE="http://www.fftw.org/"
@@ -24,6 +24,8 @@ DEPEND="${RDEPEND}
test? ( dev-lang/perl )"

pkg_setup() {
+ # XXX: this looks like it should be used with BUILD_TYPE!=binary
+
if use openmp; then
if [[ $(tc-getCC) == *gcc ]] && ! tc-has-openmp; then
ewarn "OpenMP is not available in your current selected gcc"
@@ -32,13 +34,13 @@ pkg_setup() {
FORTRAN_NEED_OPENMP=1
fi
fortran-2_pkg_setup
- FFTW_DIRS="single double longdouble"
+ MULTIBUILD_VARIANTS=( single double longdouble )
if use quad; then
if [[ $(tc-getCC) == *gcc ]] && ! version_is_at_least 4.6 $(gcc-version); then
ewarn "quad precision only available for gcc >= 4.6"
die "need quad precision capable gcc"
fi
- FFTW_DIRS+=" quad"
+ MULTIBUILD_VARIANTS+=( quad )
fi
}

@@ -57,7 +59,9 @@ src_configure() {
# filter -Os according to docs
replace-flags -Os -O2

- for x in ${FFTW_DIRS}; do
+ my_configure() {
+ local x=${MULTIBUILD_VARIANT}
+
myeconfargs=(
$(use_enable fma)
$(use_enable fortran)
@@ -93,42 +97,30 @@ src_configure() {
die "${x} precision not implemented in this ebuild"
fi

- einfo "Configuring for ${x} precision"
- BUILD_DIR="${S}-${x}" \
- autotools-multilib_src_configure
- done
+ autotools-multilib_src_configure
+ }
+
+ multibuild_foreach my_configure
}

src_compile() {
- for x in ${FFTW_DIRS}; do
- einfo "Compiling for ${x} precision"
- BUILD_DIR="${S}-${x}" \
- autotools-multilib_src_compile
- done
+ multibuild_foreach autotools-multilib_src_compile
}

src_test () {
- do_smalltest() { cd "${BUILD_DIR}" && emake -C tests smallcheck; }
# We want this to be a reasonably quick test, but that is still hard...
ewarn "This test series will take 30 minutes on a modern 2.5Ghz machine"
# Do not increase the number of threads, it will not help your performance
#local testbase="perl check.pl --nthreads=1 --estimate"
# ${testbase} -${p}d || die "Failure: $n"
- for x in ${FFTW_DIRS}; do
- einfo "Testing ${x} precision"
- BUILD_DIR="${S}-${x}" \
- multilib_foreach_abi do_smalltest
- done
+ multibuild_foreach autotools-multilib_src_compile -C tests smallcheck
}

src_install () {
local u x
DOCS=( AUTHORS ChangeLog NEWS README TODO COPYRIGHT CONVENTIONS )
HTML_DOCS=( doc/html/ )
- for x in ${FFTW_DIRS}; do
- BUILD_DIR="${S}-${x}" \
- autotools-multilib_src_install
- done
+ multibuild_foreach autotools-multilib_src_install

if use doc; then
dodoc doc/*.pdf
--
1.8.1.4

[Michał Górny]

Hello,

Recently python-r1 and multilib-build started to share a few bits
of code related to running the build process for multiple 'variants'
of the same package. Over time, the code extended and it is a bit
cumbersome to maintain the two copies and keep them in sync.

Therefore, I'd like to propose a new multibuild.eclass which will
provide a generic framework for eclasses and ebuilds to perform
multi-variant builds.


The eclass currently is intended to provide two functions:
- multibuild_foreach,
- multibuild_parallel_foreach.

These functions are supposed to execute the child process for each
of the enabled variants with support for the following:

- setting BUILD_DIR to an unique location which can be used to build
the sources,

- teeing logs to a variant-specific log file to make reading them
easier,

- nesting multiple uses of multibuild.eclass -- e.g. fftw build 3-4
'float type' variants + multilib.

It also fixes some of the issues that were noticed within
distutils-r1 / python-r1.


I will send patches in the reply to this message. The patches:

1) introduce the initial version of multibuild based on the code
from python-r1 and distutils-r1,

2) improve running 'tee' to a method not requiring subshelling
the called function,

3) fix an issue of trying to write ${WORKDIR%/}-${impl} if S=${WORKDIR},

4) convert multilib-build to use the new eclass,

5) provide python-r1 with a function to obtain enabled implementation
list to make the migration possible,

6) convert python-r1 and distutils-r1 to use the new eclass,

7) move run_in_build_dir() common function to the new eclass,

8) convert sci-libs/fftw to the new eclass as an example of nested use.


That's just the initial approach. If it works fine, the todo lists:

1) supporting nested parallel builds,

2) adding a function to run a snippet with just one ('default') variant,

and possibly more.


What are your thoughts?

--
Best regards,
Michał Górny

[Alec Warner]

On Wed, Feb 27, 2013 at 1:43 PM, Michał Górny <mgo...@gentoo.org> wrote:
> This allows us to spawn 'tee' as separate process while keeping
> the function code executed in the main shell.

Can you explain why this is 'better'? I'd prefer way more
documentation in the code itself. You are using a number of what I'd
consider 'banned' constructs and you don't go very far out of your way
to explain why you cannot live without them.

> ---
> gx86/eclass/multibuild.eclass | 19 ++++++++++---------
> 1 file changed, 10 insertions(+), 9 deletions(-)
>
> diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
> index 716d34f..d42b8a7 100644
> --- a/gx86/eclass/multibuild.eclass
> +++ b/gx86/eclass/multibuild.eclass
> @@ -108,18 +108,20 @@ multibuild_foreach() {
> local MULTIBUILD_VARIANT=${v}
> local MULTIBUILD_ID=${prev_id}${v}
> local BUILD_DIR=${bdir%%/}-${v}
> + local log_fd
>
> - einfo "${v}: running ${@}" \
> - | tee -a "${T}/build-${MULTIBUILD_ID}.log"
> + # redirect_alloc_fd accepts files only. so we need to open
> + # a random file and then reuse the fd for logger process.
> + redirect_alloc_fd log_fd /dev/null
> + eval "exec ${log_fd}> "'>(exec tee -a "${T}/build-${MULTIBUILD_ID}.log")'

The quoting here is a nightmare.
Are the single quotes required?
Do the leading single quotes have to be *right next to* the double quotes?
Why is there a space between the first > and the "?
vim hi-lighting makes this more readable, but still weird looking.
Also please document clearly why 'eval' is required (it is not clear
to me..or I suspect most readers ;p)
Are we not just making an fd and pointing the 'read end' at a tee subprocess?

> +
> + eval 'einfo "${v}: running ${@}" >&'${log_fd}' 2>&1'

And again here, why can't we just redirect directly to log_fd?

>
> # _multibuild_parallel() does redirection internally.
> # this is a hidden API to avoid writing multilib_foreach twice.

We do not use the _multibuild_parellel stuff anymore...is this comment relevant?

[Michał Górny]

On Thu, 28 Feb 2013 05:09:15 -0800
Alec Warner <ant...@gentoo.org> wrote:

> On Wed, Feb 27, 2013 at 1:43 PM, Michał Górny <mgo...@gentoo.org> wrote:
> > This allows us to spawn 'tee' as separate process while keeping
> > the function code executed in the main shell.
>
> Can you explain why this is 'better'? I'd prefer way more
> documentation in the code itself. You are using a number of what I'd
> consider 'banned' constructs and you don't go very far out of your way
> to explain why you cannot live without them.

Well, if we used simple pipes, like:

foo | tee ...

Then any variable exported by foo would be discarded. This broke a few
ebuilds already.

By banned constructs, do you mean 'eval'? I hate it too but wasn't able
to solve it anyhow better.

> > gx86/eclass/multibuild.eclass | 19 ++++++++++---------
> > 1 file changed, 10 insertions(+), 9 deletions(-)
> >
> > diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
> > index 716d34f..d42b8a7 100644
> > --- a/gx86/eclass/multibuild.eclass
> > +++ b/gx86/eclass/multibuild.eclass
> > @@ -108,18 +108,20 @@ multibuild_foreach() {
> > local MULTIBUILD_VARIANT=${v}
> > local MULTIBUILD_ID=${prev_id}${v}
> > local BUILD_DIR=${bdir%%/}-${v}
> > + local log_fd
> >
> > - einfo "${v}: running ${@}" \
> > - | tee -a "${T}/build-${MULTIBUILD_ID}.log"
> > + # redirect_alloc_fd accepts files only. so we need to open
> > + # a random file and then reuse the fd for logger process.
> > + redirect_alloc_fd log_fd /dev/null
> > + eval "exec ${log_fd}> "'>(exec tee -a "${T}/build-${MULTIBUILD_ID}.log")'
> The quoting here is a nightmare.

> Do the leading single quotes have to be *right next to* the double quotes?
> Why is there a space between the first > and the "?
> vim hi-lighting makes this more readable, but still weird looking.

Yes, I know. I'm thinking of trying to put it all into a single eval,
and just use \$ rather than mixing single and double quotes.

> Also please document clearly why 'eval' is required (it is not clear
> to me..or I suspect most readers ;p)

Well, for the very simple reason that 'fd=2; echo 11 >&${fd}' doesn't
work. The eval simply changes that into 'echo 11>&2'.

> Are we not just making an fd and pointing the 'read end' at a tee subprocess?

Yes, exactly that.

> > +
> > + eval 'einfo "${v}: running ${@}" >&'${log_fd}' 2>&1'
> And again here, why can't we just redirect directly to log_fd?
>
> >
> > # _multibuild_parallel() does redirection internally.
> > # this is a hidden API to avoid writing multilib_foreach twice.
> We do not use the _multibuild_parellel stuff anymore...is this comment relevant?

You are correct, my mistake.

[Michał Górny]

Shift the unnecessary 'private' commands from the printed commands when
executing.

That is:

python_parallel_foreach_impl foo

will print:

* pythonX.Y: foo

rather than:

* pythonX.Y: _multibuild_parallel _python_multibuild_wrapper ...
---
gx86/eclass/multibuild.eclass | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
index 1c9058a..1cc33a9 100644
--- a/gx86/eclass/multibuild.eclass
+++ b/gx86/eclass/multibuild.eclass
@@ -118,12 +118,23 @@ multibuild_foreach_variant() {

# redirect_alloc_fd accepts files only. so we need to open

# a random file and then reuse the fd for logger process.

redirect_alloc_fd log_fd /dev/null
+
+ _multibuild_run() {
+ # find the first non-private command
+ local i=1
+ while [[ ${!i} == _* ]]; do
+ (( i += 1 ))
+ done
+
+ einfo "${v}: running ${@:${i}}"

+ "${@}"
+ }
+

# bash can't handle ${log_fd} in redirections,
# we need to use eval to pass fd numbers directly.
eval "
exec ${log_fd}> >(exec tee -a \"\${T}/build-\${MULTIBUILD_ID}.log\")
- einfo \"\${v}: running \${@}\" >&${log_fd} 2>&1
- \"\${@}\" >&${log_fd} 2>&1
+ _multibuild_run \"\${@}\" >&${log_fd} 2>&1
lret=\${?}
exec ${log_fd}>&-
"
--
1.8.1.4

[Michał Górny]

---
gx86/eclass/distutils-r1.eclass | 25 ++++---------------------
1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/gx86/eclass/distutils-r1.eclass b/gx86/eclass/distutils-r1.eclass
index 40dae36..ba48a35 100644
--- a/gx86/eclass/distutils-r1.eclass
+++ b/gx86/eclass/distutils-r1.eclass
@@ -549,11 +549,6 @@ distutils-r1_run_phase() {
then
popd >/dev/null || die
fi
-
- # Store them for reuse.
- _DISTUTILS_BEST_IMPL=(
- "${EPYTHON}" "${PYTHON}" "${BUILD_DIR}" "${PYTHONPATH}"
- )
}

# @FUNCTION: _distutils-r1_run_common_phase
@@ -567,23 +562,11 @@ distutils-r1_run_phase() {
_distutils-r1_run_common_phase() {
local DISTUTILS_ORIG_BUILD_DIR=${BUILD_DIR}

- local EPYTHON=${_DISTUTILS_BEST_IMPL[0]}
- local PYTHON=${_DISTUTILS_BEST_IMPL[1]}
- local BUILD_DIR=${_DISTUTILS_BEST_IMPL[2]}
- local PYTHONPATH=${_DISTUTILS_BEST_IMPL[3]}
-
- export EPYTHON PYTHON PYTHONPATH
-
- if [[ ${DISTUTILS_IN_SOURCE_BUILD} ]]; then
- pushd "${BUILD_DIR}"/.. >/dev/null || die
- fi

+ local MULTIBUILD_VARIANTS
+ _python_obtain_impls

- einfo "common: running ${1}"
- "${@}"
-
- if [[ ${DISTUTILS_IN_SOURCE_BUILD} ]]; then
- popd >/dev/null || die
- fi
+ multibuild_for_best_variant _python_multibuild_wrapper \
+ distutils-r1_run_phase "${@}"
}

# @FUNCTION: _distutils-r1_run_foreach_impl
--
1.8.1.4

[Michał Górny]

Four more patches so far:

1) print only 'public' part of command-line

That is, shift over all the arguments starting with an underscore
for einfo.

multibuild_foreach_variant _python_multibuild_wrapper foo

will be printed as:

pythonX.Y: foo


2) add a function to run the command for the 'best' (default, preferred)
variant.


3) use it to implement multilib_for_best_abi().

A few ebuilds will happily use it instead of the current hacks.


4) use it to run *_all() phases in distutils-r1.

This fixes bug 460016 (python_configure_all() doesn't have best-impl if
there's no python_configure()).

[Michał Górny]

---
gx86/eclass/multilib-build.eclass | 12 ++++++++++++
1 file changed, 12 insertions(+)

diff --git a/gx86/eclass/multilib-build.eclass b/gx86/eclass/multilib-build.eclass
index b6409d8..0fe46a0 100644
--- a/gx86/eclass/multilib-build.eclass
+++ b/gx86/eclass/multilib-build.eclass
@@ -139,6 +139,18 @@ multilib_parallel_foreach_abi() {
multibuild_parallel_foreach_variant _multilib_multibuild_wrapper "${@}"
}

+# @FUNCTION: multilib_for_best_abi

+# @USAGE: <argv>...
+# @DESCRIPTION:

+# Runs the given command with setup for the 'best' (usually native) ABI.
+multilib_for_best_abi() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ local MULTIBUILD_VARIANTS=( $(multilib_get_enabled_abis) )
+

+ multibuild_for_best_variant _multilib_multibuild_wrapper "${@}"
+}
+
# @FUNCTION: multilib_check_headers
# @DESCRIPTION:
# Check whether the header files are consistent between ABIs.
--
1.8.1.4

[Michał Górny]

---
gx86/eclass/multibuild.eclass | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)

diff --git a/gx86/eclass/multibuild.eclass b/gx86/eclass/multibuild.eclass
index 1cc33a9..3ffe9e8 100644
--- a/gx86/eclass/multibuild.eclass
+++ b/gx86/eclass/multibuild.eclass
@@ -185,6 +185,26 @@ multibuild_parallel_foreach_variant() {
return ${ret}
}

+# @FUNCTION: multibuild_for_best_variant

+# @USAGE: [<argv>...]
+# @DESCRIPTION:

+# Run the passed command once, for the best of the enabled package
+# variants.
+#
+# The run will have a proper, variant-specificBUILD_DIR set, and output

+# teed to a separate log in ${T}.
+#

+# The function returns command exit status.
+multibuild_for_best_variant() {

+ debug-print-function ${FUNCNAME} "${@}"
+

+ [[ ${MULTIBUILD_VARIANTS} ]] \
+ || die "MULTIBUILD_VARIANTS need to be set"
+

+ local MULTIBUILD_VARIANTS=( "${MULTIBUILD_VARIANTS[-1]}" )
+ multibuild_foreach_variant "${@}"
+}
+
# @FUNCTION: run_in_build_dir

# @USAGE: <argv>...
# @DESCRIPTION:

--
1.8.1.4

[Alec Warner]

So this is an einfo with an assignment side-effect? Can we perhaps
make the assignment explicit?

-A

[Michał Górny]

Assignment side-effect? I don't understand.

Maybe I'm missing something but by design it was supposed to print ${@}
starting from ${i}-th item.

[Christoph Junghans]

+1, feel free to commit, when multibuild.eclass was added.

2013/2/27 Michał Górny <mgo...@gentoo.org>:

--
Christoph Junghans
http://dev.gentoo.org/~ottxor/