CustomTool installation issue with large Matrix job (race condition)

[Geoffroy J]

Hello

I have a fairly large matrix job (30 environments).

For one matrix environment only, I need to install a dedicated tool (Sonarqube build-wrapper).

CustomTool works perfectly fine for this purpose.

However, as CustomTool installation is defined as a build wrapper step, it get activated for all matrix environments, resulting in the tool being installed 30 times.

As you can imagine, because several jobs are executed in // on the same machine, we regularly face some installation race conditions (mostly on windows) as one job is installing the tool and another one also tries to install it (and the first step of installation seems to be uninstalling)...

Any recommendations on how to workaround this behavior?

Maybe there could be an option not to fail in case of tool installation failure. But as the plugin seems "abandoned" it might not be a great idea to 

Elsewhere i'll switch to a manual install...

[CustomTools] - scm-sonarqube-build-wrapper: Starting installation
FATAL: Failed to install https://sonarqubeserver/sonarqube-prod/static/cpp/build-wrapper-win-x86.zip to E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper
Also:   Also:   java.nio.file.AccessDeniedException: E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86\build-wrapper-win-x86-64.exe
at sun.nio.fs.WindowsException.translateToIOException(WindowsException.java:83)
at sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:97)
at sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:102)
at sun.nio.fs.WindowsFileSystemProvider.implDelete(WindowsFileSystemProvider.java:269)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
Also:   java.nio.file.AccessDeniedException: E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86\build-wrapper-win-x86-64.exe
at sun.nio.fs.WindowsException.translateToIOException(WindowsException.java:83)
at sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:97)
at sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:102)
at sun.nio.fs.WindowsFileSystemProvider.implDelete(WindowsFileSystemProvider.java:269)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
jenkins.util.io.CompositeIOException: Unable to remove file E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86\build-wrapper-win-x86-64.exe
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:252)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.forceRemoveDirectoryContents(PathRemover.java:87)
Also:   Also:   java.nio.file.DirectoryNotEmptyException: E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86
at sun.nio.fs.WindowsFileSystemProvider.implDelete(WindowsFileSystemProvider.java:266)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
Also:   java.nio.file.DirectoryNotEmptyException: E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86
at sun.nio.fs.WindowsFileSystemProvider.implDelete(WindowsFileSystemProvider.java:266)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
jenkins.util.io.CompositeIOException: Unable to remove directory E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86 with directory contents: [E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper\build-wrapper-win-x86\build-wrapper-win-x86-64.exe]
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:250)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.forceRemoveDirectoryContents(PathRemover.java:87)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to JNLP4-connect connection from buildhost.company.com/10.0.60.52:61450
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1788)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:356)
at hudson.remoting.Channel.call(Channel.java:998)
at hudson.FilePath.act(FilePath.java:1069)
at hudson.FilePath.act(FilePath.java:1058)
at hudson.FilePath.deleteContents(FilePath.java:1280)
at hudson.FilePath.installIfNecessaryFrom(FilePath.java:904)
at hudson.FilePath.installIfNecessaryFrom(FilePath.java:850)
at hudson.tools.ZipExtractionInstaller.performInstallation(ZipExtractionInstaller.java:83)
at hudson.tools.InstallerTranslator.getToolHome(InstallerTranslator.java:69)
at hudson.tools.ToolLocationNodeProperty.getToolHome(ToolLocationNodeProperty.java:109)
at hudson.tools.ToolInstallation.translateFor(ToolInstallation.java:206)
at com.cloudbees.jenkins.plugins.customtools.CustomTool.forNode(CustomTool.java:163)
at com.cloudbees.jenkins.plugins.customtools.CustomToolInstallWrapper.decorateLauncher(CustomToolInstallWrapper.java:183)
at hudson.model.AbstractBuild$AbstractBuildExecution.createLauncher(AbstractBuild.java:536)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:462)
at hudson.model.Run.execute(Run.java:1880)
at hudson.matrix.MatrixRun.run(MatrixRun.java:153)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:428)
jenkins.util.io.CompositeIOException: Unable to delete 'E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper'. Tried 3 times (of a maximum of 3) waiting 0.1 sec between attempts.
at jenkins.util.io.PathRemover.forceRemoveDirectoryContents(PathRemover.java:90)
at hudson.Util.deleteContentsRecursive(Util.java:261)
at hudson.FilePath$DeleteContents.invoke(FilePath.java:1286)
at hudson.FilePath$DeleteContents.invoke(FilePath.java:1282)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3073)
at hudson.remoting.UserRequest.perform(UserRequest.java:211)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at hudson.remoting.Engine$1.lambda$newThread$0(Engine.java:117)
at java.lang.Thread.run(Thread.java:748)
Caused: java.io.IOException: Failed to install https://sonarqubeserver/sonarqube-prod/static/cpp/build-wrapper-win-x86.zip to E:\j\tools\com.cloudbees.jenkins.plugins.customtools.CustomTool\scm-sonarqube-build-wrapper
at hudson.FilePath.installIfNecessaryFrom(FilePath.java:937)
at hudson.FilePath.installIfNecessaryFrom(FilePath.java:850)
at hudson.tools.ZipExtractionInstaller.performInstallation(ZipExtractionInstaller.java:83)
at hudson.tools.InstallerTranslator.getToolHome(InstallerTranslator.java:69)
at hudson.tools.ToolLocationNodeProperty.getToolHome(ToolLocationNodeProperty.java:109)
at hudson.tools.ToolInstallation.translateFor(ToolInstallation.java:206)
at com.cloudbees.jenkins.plugins.customtools.CustomTool.forNode(CustomTool.java:163)
at com.cloudbees.jenkins.plugins.customtools.CustomToolInstallWrapper.decorateLauncher(CustomToolInstallWrapper.java:183)
at hudson.model.AbstractBuild$AbstractBuildExecution.createLauncher(AbstractBuild.java:536)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:462)
at hudson.model.Run.execute(Run.java:1880)
at hudson.matrix.MatrixRun.run(MatrixRun.java:153)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:428)

[geoffroy...@gmail.com]

Hello

i was not able to fix this issue using CustomTool plugin, so the solution was to get rid of it, and create a dedicated jenkins job to install/update the SonarQube build wrapper tool.

In case it can help, find below an excerpt of the pipeline code, which relies on File Operations && Pipeline Utility Steps plugins so that it is OS agnostic (windows & unix).

This should be wrapped within a Pipeline job taking as parameter a string "InstallOnLabel". Each node has a property "SONAR_BUILDWRAPPER_HOME" defining installation folder.

properties(

    [parameters([string(defaultValue: 'sonar', name: 'InstallOnLabel')])]  // will execute on all nodes having label 

)

def nodes = nodesByLabel label: "${InstallOnLabel}"

nodes = nodes.sort()

Map tasks = [:]

for (int i = 0; i < nodes.size(); i++) {

    def label = nodes[i]

    tasks[label] = {

        node(label) {

            stage(label) {

                stage("Install") {

                    withSonarQubeEnv('sonarqube-prod') {

                        _hostflavor = isUnix() ? "linux" : "win"

                        _bwDlUrl = "${env.SONAR_HOST_URL}/static/cpp/build-wrapper-${_hostflavor}-x86.zip"

                        echo "[INFO] Installing SonarQube ${_hostflavor} build-wrapper into ${env.SONAR_BUILDWRAPPER_HOME}/bin"

                        echo "[INFO] Downloading from ${_bwDlUrl}"

                        fileOperations([

                            folderDeleteOperation(folderPath: "${env.SONAR_BUILDWRAPPER_HOME}"),

                            folderCreateOperation(folderPath: "${env.SONAR_BUILDWRAPPER_HOME}"),

                            fileDownloadOperation(url: "${_bwDlUrl}", targetFileName: 'build-wrapper-x86.zip', targetLocation: ".", userName: "", password: ""),

                            fileUnZipOperation(filePath: 'build-wrapper-x86.zip', targetLocation: "${env.SONAR_BUILDWRAPPER_HOME}"),

                            folderRenameOperation(source: "${env.SONAR_BUILDWRAPPER_HOME}/build-wrapper-${_hostflavor}-x86", destination: "${env.SONAR_BUILDWRAPPER_HOME}/bin"),

                        ])

                    }

                }

                stage("Validate") {

                    if (isUnix()) {

                        sh '''

                        set +e

                        ls -Rl ${SONAR_BUILDWRAPPER_HOME}

                        build-wrapper-linux-x86-64

                        (( $? == 10 )) && exit 0

                        '''

                    } else {

                        bat '''

                        dir /s "%SONAR_BUILDWRAPPER_HOME%"

                        build-wrapper-win-x86-64

                        if ERRORLEVEL 1 ( exit /b 0 )

                        exit /b 1

                        '''

                    }

                }

            }

        }

    }

}

timeout(time: 5, unit: 'MINUTES') {

    parallel(tasks)

}