diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index d4fa09f8b5c..77dfd5d43eb 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -26,11 +26,10 @@ EasyBuild support for bundles of Julia packages, implemented as an easyblock @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Davide Grassano (CECAM, EPFL) """ -import os - from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.juliapackage import EXTS_FILTER_JULIA_PACKAGES, JuliaPackage +from easybuild.easyblocks.generic.juliapackage import JuliaPackage class JuliaBundle(Bundle, JuliaPackage): @@ -53,7 +52,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.cfg['exts_defaultclass'] = 'JuliaPackage' - self.cfg['exts_filter'] = EXTS_FILTER_JULIA_PACKAGES + # Ensure that Julia packages such as LLVM do not try to use the EB_LLVM easyblock + # many packages have names that overlap with EB easyblocks, and would end up using them as default class for + # extensions, which is not what we want here. We force to always use JuliaPackage unless explicitly specified + # otherwise in the easyconfig file. + self.cfg['exts_default_options'] = { + 'easyblock': 'JuliaPackage', + } # need to disable templating to ensure that actual value for exts_default_options is updated... with self.cfg.disable_templating(): @@ -74,25 +79,7 @@ def __init__(self, *args, **kwargs): } ] - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) - - def prepare_step(self, *args, **kwargs): - """Prepare for installing bundle of Julia packages.""" - super().prepare_step(*args, **kwargs) + # The name of the bundle can be arbitrary and not necessarily a Julia package + self.cfg['exts_filter'] = None - def install_step(self): - """Prepare installation environment and dd all dependencies to project environment.""" - self.prepare_julia_env() - self.include_pkg_dependencies() - - def sanity_check_step(self, *args, **kwargs): - """Custom sanity check for bundle of Julia packages""" - custom_paths = { - 'files': [], - 'dirs': [os.path.join('packages', self.name)], - } - super().sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self, *args, **kwargs): - """Custom module environment from JuliaPackage""" - return super().make_module_extra(*args, **kwargs) + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index a3e641b2178..f063d3ba1f2 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -26,11 +26,14 @@ EasyBuild support for Julia Packages, implemented as an easyblock @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Davide Grassano (CECAM, EPFL) """ import ast -import glob import os import re +import tempfile + +from typing import List, Dict, Tuple, Union from easybuild.tools import LooseVersion @@ -38,28 +41,24 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.filetools import copy_dir, mkdir +from easybuild.tools.filetools import move_file from easybuild.tools.run import run_shell_cmd -from easybuild.tools.utilities import trace_msg +from easybuild.tools.utilities import trace_msg, print_msg +from easybuild.tools.hooks import BUILD_STEP, TEST_STEP -EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") -USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") +from easybuild.easyblocks.j.julia import EB_JULIA_DEPOT_PATH_VAR, EB_JULIA_LOAD_PATH_VAR + +_COMPILECACHE_CHECK = ' | '.join([ + # "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"), Base.get_world_counter())'", + # compilecache_path is not part of the stable API and changes arguments across versions + # allow internal cache resolution and use debug statements to get the path of the loaded cache + "JULIA_DEBUG=loading julia -e 'using %(ext_name)s' 2>&1 1>/dev/null", + "grep -E 'Loading (object )?cache file .*%(grep_loc)s'" +]) -JULIA_PATHS_SOFT_INIT = { - "Lua": """ -if ( mode() == "load" ) then - if ( os.getenv("JULIA_DEPOT_PATH") == nil ) then setenv("JULIA_DEPOT_PATH", ":") end - if ( os.getenv("JULIA_LOAD_PATH") == nil ) then setenv("JULIA_LOAD_PATH", ":") end -end -""", - "Tcl": """ -if { [ module-info mode load ] } { - if {![info exists env(JULIA_DEPOT_PATH)]} { setenv JULIA_DEPOT_PATH : } - if {![info exists env(JULIA_LOAD_PATH)]} { setenv JULIA_LOAD_PATH : } -} -""", -} +USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") class JuliaPackage(ExtensionEasyBlock): @@ -71,17 +70,22 @@ class JuliaPackage(ExtensionEasyBlock): - remove paths in user depot '~/.julia' from DEPOT_PATH and LOAD_PATH - put installation directory as top DEPOT_PATH, the target depot for installations with Pkg - put installation environment as top LOAD_PATH, needed to precompile installed packages - - add Julia packages found in dependencies of the easyconfig to installation environment, needed - for Pkg to be aware of those packages and not install them again - - add newly installed Julia packages to installation environment (automatically done by Pkg) + - merge the environment of julia dependencies of this installation to the environment of this installation. + - add newly installed Julia packages to installation environment + - install all packages (main and deps) in parallel using Pkg.instantiate() command Julia environment setup on module load: User depot and its shared environment for this version of Julia are kept as top paths of DEPOT_PATH and LOAD_PATH respectively. This ensures that the user can keep using its own environment after loading JuliaPackage modules, installing additional software on its personal depot while still using packages provided by the module. Effectively, this translates to: - - append installation directory to list of DEPOT_PATH, only really needed to load artifacts (JLL packages) - - append installation Project.toml file to list of LOAD_PATH, needed to load packages with `using` command + - prepend installation directory to list of EB_JULIA_DEPOT_PATH, only really needed to load JLL artifacts + - prepend installation Project.toml file to list of EB_JULIA_LOAD_PATH, needed to load packages with `using` + - Generate JULIA_DEPOT_PATH by append the default DEPOT_PATH (:) to EB_JULIA_DEPOT_PATH + - Generate JULIA_LOAD_PATH by prepending the default LOAD_PATH (:) to EB_JULIA_LOAD_PATH + The prepending here is necessary as the default LOAD_PATH [@, @v#.#, @stdlib] can cause stdlib's packages + that have been made upgradable (https://github.com/JuliaLang/julia/issues/50697) to fail matching the cache. + See https://github.com/easybuilders/easybuild-easyblocks/issues/4123#issuecomment-4386990190 for more details. """ @staticmethod @@ -92,6 +96,26 @@ def extra_options(extra_vars=None): 'download_pkg_deps': [ False, "Let Julia download and bundle all needed dependencies for this installation", CUSTOM ], + 'is_test_dependency': [ + False, + "Whether this package is only needed for testing and should not be added to installation environment", + CUSTOM + ], + 'julia_debug': [ + False, + "Whether to set JULIA_DEBUG=all during installation and testing, to get more verbose output.", + CUSTOM + ], + # Arguably this should be set to True by default. In many cases the test environment in Julia packages + # will attempt to re-install the package being tested. If not all required packages to be brought in the + # test environment were specified in test/Project.toml, running Pkg.test() in offline mode will fail. + # Some test tools like Aqua will also re-create their own tmp environment which can be broken even if + # test/Project.toml is properly set up/patched. + 'test_online': [ + False, + "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", + CUSTOM + ], }) return extra_vars @@ -106,11 +130,13 @@ def get_julia_env(env_var): "LOAD_PATH": "julia -E 'Base.load_path()'", } - try: - res = run_shell_cmd(julia_read_cmd[env_var], hidden=True) - except KeyError: + if env_var not in julia_read_cmd: raise EasyBuildError("Unknown Julia environment variable requested: %s", env_var) + cmd = julia_read_cmd[env_var] + + res = run_shell_cmd(cmd, hidden=True) + try: parsed_var = ast.literal_eval(res.output) except SyntaxError: @@ -118,39 +144,153 @@ def get_julia_env(env_var): return parsed_var - def julia_env_path(self, absolute=True, base=True): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if self.cfg['is_test_dependency'] and not self.is_extension: + raise EasyBuildError("Test dependencies can only be defined as an extension, not as a bundle itself") + + self._conflicts = [] + self._julia_deps = {} + self._julia_deps_test = {} + self._julia_version = None + self._pkg_deps = [] + self._pkgs_to_test_online = [] + self._pkgs_to_test_offline = [] + self._tmp_test_dir = None + self._installdir_created = False + + @property + def julia_version(self) -> str: + """Needs to get Julia version from dependencies to allow --sanity-check-only to generate the fake module""" + if self._julia_version is None: + deps = self.cfg.dependencies(runtime_only=True) + for dep in deps: + if dep['name'] == 'Julia': + self._julia_version = dep['version'] + break + else: + raise EasyBuildError( + "Julia not included as dependency, cannot determine Julia version for installation of: %s", + self.name + ) + return self._julia_version + + @property + def conflicts(self) -> List[Tuple[str, str, str, str]]: + """List of 4-tuple conflicting software (same package included from multiple locations) including: + - extension name where conflict is found + - package name with conflict + - previous source of the package that caused the conflict + - new source of the package that caused the conflict + """ + if self.is_extension: + return self.master.conflicts + return self._conflicts + + @property + def is_last_extension(self) -> bool: + """Whether this extension is the last one to be installed.""" + if not self.is_extension: + return True + + return self.master.ext_instances and self.master.ext_instances[-1] is self + + @property + def julia_deps(self) -> Dict[str, str]: + """List of Julia dependencies found in this installation excluding test dependencies + - key: package name + - value: package source + """ + if self.is_extension: + return self.master.julia_deps + return self._julia_deps + + @property + def julia_deps_test(self) -> Dict[str, str]: + """List of Julia dependencies found in this installation including test dependencies + - key: package name + - value: package source + """ + if self.is_extension: + return self.master.julia_deps_test + return self._julia_deps_test + + @property + def pkg_deps(self) -> List[Tuple[str, str]]: + """List of easybuild runtime dependencies that will need to be sanity-checked. 2-tuple including: + - dependency name + - dependency root (equivalent to get_software_root) + """ + if self.is_extension: + return self.master.pkg_deps + return self._pkg_deps + + @property + def pkgs_to_test_online(self) -> List[str]: + """List extension names with `runtest` set to True that should be tested in online mode.""" + if self.is_extension: + return self.master.pkgs_to_test_online + return self._pkgs_to_test_online + + @property + def pkgs_to_test_offline(self) -> List[str]: + """List extension names with `runtest` set to True that should be tested in offline mode.""" + if self.is_extension: + return self.master.pkgs_to_test_offline + return self._pkgs_to_test_offline + + @property + def tmp_test_dir(self) -> str: + """Temporary path used for running the test step.""" + if self.is_extension: + return self.master.tmp_test_dir + + if not self._tmp_test_dir: + try: + self._tmp_test_dir = tempfile.mkdtemp(suffix='-julia_tests') + except Exception as e: + raise EasyBuildError("Failed to create temporary directory for Julia package testing: %s", str(e)) + return self._tmp_test_dir + + def julia_env_path(self, absolute: bool = True, base: bool = True, basedir: Union[str, None] = None) -> str: """ Return path to installation environment file. + + :param absolute: whether to return absolute path to environment file or relative path to `basedir` + :param base: whether to return path to environment file or its parent directory + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. """ - julia_version = get_software_version('Julia').split('.') + basedir = basedir or self.installdir + julia_version = self.julia_version.split('.') env_dir = "v{}.{}".format(*julia_version[:2]) project_env = os.path.join("environments", env_dir, "Project.toml") if absolute: - project_env = os.path.join(self.installdir, project_env) + project_env = os.path.join(basedir, project_env) if base: project_env = os.path.dirname(project_env) return project_env - def set_pkg_offline(self): - """Enable offline mode of Julia Pkg""" - - if not self.cfg['download_pkg_deps']: - julia_version = get_software_version('Julia') - if LooseVersion(julia_version) >= LooseVersion('1.5'): - # Enable offline mode of Julia Pkg - # https://pkgdocs.julialang.org/v1/api/#Pkg.offline - env.setvar('JULIA_PKG_OFFLINE', 'true') - else: - errmsg = ( - "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " - "Enable easyconfig option 'download_pkg_deps' to allow installation " - "with any extra downloaded dependencies." - ) - raise EasyBuildError(errmsg, julia_version) + def set_pkg_remote(self, online: bool = False): + """Enable online/offline mode of Julia Pkg""" - def prepare_julia_env(self): + julia_version = get_software_version('Julia') + if LooseVersion(julia_version) >= LooseVersion('1.5'): + # Enable offline mode of Julia Pkg + # https://pkgdocs.julialang.org/v1/api/#Pkg.offline + env.setvar('JULIA_PKG_OFFLINE', 'false' if online else 'true') + else: + errmsg = ( + "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " + "Enable easyconfig option 'download_pkg_deps' to allow installation " + "with any extra downloaded dependencies." + ) + raise EasyBuildError(errmsg, julia_version) + + def prepare_julia_env(self, basedir: Union[str, None] = None, online: bool = False): """ 1. Remove user depot and prepend installation directory to DEPOT_PATH. Top directory in Julia DEPOT_PATH is the target installation directory. @@ -165,7 +305,14 @@ def prepare_julia_env(self): 3. Enable offline mode in Julia to avoid automatic downloads of packages. 4. Enable automatic precompilation of packages after each build. + + 5. Enable/disable debug mode in Julia to get more verbose output during installation and testing. + + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. + :param online: whether to allow online operations of Julia Pkg """ + basedir = basedir or self.installdir # Grab both DEPOT_PATH and LOAD_PATH before any changes are made # given that Julia might automatically update LOAD_PATH from a change on DEPOT_PATH dirty_depot = self.get_julia_env("DEPOT_PATH") @@ -175,51 +322,94 @@ def prepare_julia_env(self): # First set DEPOT_PATH and then LOAD_PATH to avoid any automatic changes made by Julia clean_depot = [path for path in dirty_depot if not USER_DEPOT_PATTERN.search(path) and path != self.installdir] - install_depot = os.pathsep.join([self.installdir] + clean_depot) + install_depot = os.pathsep.join([basedir] + clean_depot) self.log.debug("Preparing Julia 'DEPOT_PATH' for installation: %s", install_depot) env.setvar("JULIA_DEPOT_PATH", install_depot) - project_toml = self.julia_env_path(base=False) + project_toml = self.julia_env_path(base=False, basedir=basedir) clean_load = [path for path in dirty_load if not USER_DEPOT_PATTERN.search(path) and path != project_toml] install_load = os.pathsep.join([project_toml] + clean_load) self.log.debug("Preparing Julia 'LOAD_PATH' for installation: %s", install_load) env.setvar("JULIA_LOAD_PATH", install_load) - if self.julia_env_path(base=False) not in self.get_julia_env("LOAD_PATH"): + # Disable EB controlled PATHS to enforce the specific configurations + env.setvar(EB_JULIA_DEPOT_PATH_VAR, "") + env.setvar(EB_JULIA_LOAD_PATH_VAR, "") + + if project_toml not in self.get_julia_env("LOAD_PATH"): errmsg = "Failed to prepare Julia environment for installation of: %s" raise EasyBuildError(errmsg, self.name) - # Enable offline mode - self.set_pkg_offline() + # Enable/disable offline mode + self.set_pkg_remote(online=online) + env.setvar('JULIA_DEBUG', 'all' if self.cfg['julia_debug'] else '') + + # Set the maximum number of concurrent package builds + env.setvar('JULIA_NUM_PRECOMPILE_TASKS', str(self.cfg.parallel)) # Enable automatic precompilation env.setvar('JULIA_PKG_PRECOMPILE_AUTO', 'true') - def install_pkg_source(self, pkg_source, environment, trace=True): - """Execute Julia.Pkg command to install package from its sources""" + def add_package(self, pkg_source, test_only=False): + """Add a Julia package to the list of dependencies to be installed. - julia_pkg_cmd = [ - 'using Pkg', - 'Pkg.activate("%s")' % environment, - ] + :param pkg_source: path to the package source to be added as dependency + :param test_only: whether this package is only needed for testing and should not be added to installation + environment + """ + pkg_name = os.path.basename(pkg_source) + if pkg_name in self.julia_deps: + prev_source = self.julia_deps[pkg_name] + if prev_source != pkg_source: + self.conflicts.append((self.name, pkg_name, prev_source, pkg_source)) + + self.julia_deps_test[pkg_name] = pkg_source + if not test_only: + self.julia_deps[pkg_name] = pkg_source + + def _check_conflicts(self): + """Check for conflicts in Julia dependencies and raise an error if any are found.""" + if self.conflicts: + conflict_msgs = [] + for ext_name, pkg_name, prev_source, new_source in self.conflicts: + conflict_msgs.append( + f"Conflict detected for package '{pkg_name}' in extension '{ext_name}': " + f"already added from source '{prev_source}', cannot add from source '{new_source}'" + ) + raise EasyBuildError("Conflicts detected in Julia dependencies:\n" + "\n".join(conflict_msgs)) - if os.path.isdir(os.path.join(pkg_source, '.git')): - # sources from git repos can be installed as any remote package - self.log.debug('Installing Julia package in normal mode (Pkg.add)') + def install_pkg(self, basedir: Union[str, None] = None): + """Execute Julia.Pkg command to install all packages in `self.julia_deps` in the install environment. - julia_pkg_cmd.extend([ - # install package from local path preserving existing dependencies - 'Pkg.add(url="%s"; preserve=PRESERVE_ALL)' % pkg_source, - ]) - else: - # plain sources have to be installed in develop mode - self.log.debug('Installing Julia package in develop mode (Pkg.develop)') + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified + """ + self._check_conflicts() + basedir = basedir or self.installdir + env = self.julia_env_path(basedir=basedir) + + if len(self.julia_deps) == 0: + if len(self.julia_deps_test) > 0: + raise EasyBuildError( + "Only test dependencies found for Julia package %s, cannot proceed with installation. " + "Please check that all needed dependencies are properly specified in the easyconfig file.", + self.name + ) + raise EasyBuildError("No Julia packages to install for: %s", self.name) - julia_pkg_cmd.extend([ - # install package from local path preserving existing dependencies - 'Pkg.develop(PackageSpec(path="%s"); preserve=PRESERVE_ALL)' % pkg_source, - 'Pkg.build("%s")' % os.path.basename(pkg_source), - ]) + package_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps.values()) + + julia_pkg_cmd = [ + 'using Pkg', + f'Pkg.activate("{env}")', + + f'Pkg.develop([{package_specs}]; preserve=PRESERVE_ALL)', + # Usage `Pkg.instantiate()` after all sources are in place to let Pkg handle all dependencies. + # This has the advantage of letting Pkg deal with the order and parallel installation. + 'Pkg.instantiate()', + # Ensure operations done only by build.jl are done for all packages if needed + 'Pkg.build()', + ] julia_pkg_cmd = '; '.join(julia_pkg_cmd) cmd = ' '.join([ @@ -231,96 +421,269 @@ def install_pkg_source(self, pkg_source, environment, trace=True): return res.output + def _deps_from_project(self, environment: str) -> List[str]: + """Get list of dependencies from a Julia environment + + :param environment: path to Julia environment to query for dependencies + """ + julia_root = get_software_root('Julia') + cmd = "; ".join([ + 'using Pkg', + f'Pkg.activate("{environment}")', + 'for (_, pkg) in Pkg.dependencies()', + 'println("$(pkg.source)")', + 'end' + ]) + + res = run_shell_cmd(f"julia -E '{cmd}'", split_stderr=True, hidden=True) + deps = res.output.strip().splitlines() + + # Filter out Julia's stdlib dependencies + deps = filter(lambda d: not d.startswith(julia_root), deps) + deps = filter(lambda d: d != 'nothing', deps) + + return list(deps) + def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" - # Location of project environment files in install dir - mkdir(self.julia_env_path(), parents=True) - + build_only_deps = self.cfg.dependencies(build_only=True) # add packages found in dependencies to this installation environment for dep in self.cfg.dependencies(): - dep_root = get_software_root(dep['name']) - for pkg in glob.glob(os.path.join(dep_root, 'packages/*')): - trace_msg("incorporating Julia package from dependencies: %s" % os.path.basename(pkg)) - self.install_pkg_source(pkg, self.julia_env_path(), trace=False) - - def install_pkg(self): - """Install Julia package""" + dep_name = dep['name'] + dep_root = get_software_root(dep_name) + dep_env = self.julia_env_path(absolute=True, base=True, basedir=dep_root) + if not os.path.isdir(dep_env): + self.log.warning("No environment directory found in dependency %s, skipping: %s", dep_name, dep_env) + continue + + test_only = dep in build_only_deps + if test_only: + trace_msg( + f"Incorporating Julia package in TEST environment {dep_name}: {dep_env}" + ) + else: + self.pkg_deps.append((dep_name, dep_root)) + trace_msg( + f"Incorporating Julia package in INSTALL environment {dep_name}: {dep_env}" + ) - # determine source type of current installation - if os.path.isdir(os.path.join(self.start_dir, '.git')): - pkg_source = self.start_dir - else: - # copy non-git sources to install directory - pkg_source = os.path.join(self.installdir, 'packages', self.name) - copy_dir(self.start_dir, pkg_source) + for pkg_path in self._deps_from_project(dep_env): + self.add_package(pkg_path, test_only=test_only) - return self.install_pkg_source(pkg_source, self.julia_env_path()) + def make_installdir(self): + """Create new installation directory and ensure this is done only once""" + # Stop installdir from being re-created at the install step + if not self._installdir_created: + super().make_installdir() + self._installdir_created = True def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" super().prepare_step(*args, **kwargs) + self.make_installdir() if get_software_root('Julia') is None: raise EasyBuildError("Julia not included as dependency!") + self.include_pkg_dependencies() + def configure_step(self): - """No separate configuration for JuliaPackage.""" + """Configure step for Julia packages is a no-op, as there is no configuration needed before installation.""" pass + def install_source(self): + """Add the Julia package source files in the installation directory.""" + if self.cfg['is_test_dependency']: + self.log.debug( + f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" + ) + trace_msg("Installing as a test dependency...") + package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) + else: + package_dir = os.path.join(self.installdir, 'packages', self.name) + + move_file(self.start_dir, package_dir) + self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) + + def _build_install_step(self, basedir: Union[str, None] = None): + """Install step commons between single-package and extensions based installs. + + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. + """ + self.install_source() + if self.is_last_extension: + self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) + return self.install_pkg(basedir=basedir) + else: + trace_msg("Delegating build+install to last extension") + def build_step(self): - """No separate build procedure for JuliaPackage.""" - pass + """Build step for Julia packages is a no-op, as there is no build needed before installation.""" + return self._build_install_step() def test_step(self): - """No separate (standard) test procedure for JuliaPackage.""" - pass + """ + Test the built Julia package. - def install_step(self): - """Prepare installation environment and install Julia package.""" + :param return_output: return output and exit code of test command + """ + if self.cfg['runtest']: + if self.cfg['test_online']: + self.pkgs_to_test_online.append(self.name) + else: + self.pkgs_to_test_offline.append(self.name) + if not self.is_last_extension: + trace_msg("Delegating testing to last extension") + return + + env = self.julia_env_path(basedir=self.tmp_test_dir) + testcmd = [ + 'using Pkg', + f'Pkg.activate("{env}")', + # f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)', + # 'Pkg.test([{pkg_test}])', + ] + if self.julia_deps_test: + pkg_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) + testcmd.append(f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)') + testcmd.append('Pkg.test([{pkg_test}])') + testcmd = '; '.join(testcmd) + testcmd = f"julia -e '{testcmd}'" + + # Also add normal DEPOT/LOAD paths to environment to try and re-use pre-compiled caches as much as possible + # But also ensure that artifacts for packages that do not need recompilation are still found in the tests self.prepare_julia_env() - self.include_pkg_dependencies() + self.prepare_julia_env(basedir=self.tmp_test_dir) + + # Run offline tests first just in case the online ones could modify the environment + if self.pkgs_to_test_offline: + trace_msg("Running offline tests:") + self.set_pkg_remote(online=False) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test_offline) + + cmd = ' '.join([ + self.cfg['pretestopts'], + testcmd.format(pkg_test=pkg_test), + self.cfg['testopts'], + ]) - return self.install_pkg() + res = run_shell_cmd(cmd, fail_on_error=False) - def install_extension(self): - """Install Julia package as an extension.""" + if res.exit_code != 0: + self.report_test_failure(f"Offline tests failed for Julia package {self.name}:\n{res.output}") + + if self.pkgs_to_test_online: + trace_msg("Running online tests:") + self.set_pkg_remote(online=True) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test_online) + + cmd = ' '.join([ + self.cfg['pretestopts'], + testcmd.format(pkg_test=pkg_test), + self.cfg['testopts'], + ]) + res = run_shell_cmd(cmd, fail_on_error=False) + + if res.exit_code != 0: + self.report_test_failure(f"Tests failed for Julia package {self.name}:\n{res.output}") + + def install_step(self): + """Install step for Julia packages is a no-op, as there is no build needed before installation.""" + pass + + def install_extension(self, *args, **kwargs): + """Install Julia package as an extension.""" if not self.src: errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) - ExtensionEasyBlock.install_extension(self, unpack_src=True) - self.prepare_julia_env() - self.install_pkg() + super().install_extension(*args, unpack_src=True) + + steps = [ + (BUILD_STEP, 'building', [lambda x: x._build_install_step], True), + (TEST_STEP, 'testing', [lambda x: x._test_step], True), + ] + self.skip = False # --skip does not apply here + self.silent = build_option('silent') + # See EasyBlock.run_all_steps + for (step_name, descr, step_methods, skippable) in steps: + if self.skip_step(step_name, skippable): + print_msg("\t%s [skipped]" % descr, log=self.log, silent=self.silent) + else: + if self.dry_run: + self.dry_run_msg("\t%s... [DRY RUN]\n", descr) + else: + print_msg("\t%s..." % descr, log=self.log, silent=self.silent) + for step_method in step_methods: + step_method(self)() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" + if self.cfg['is_test_dependency']: + self.log.debug(f"Package {self.name} is only used for testing, skipping sanity check") + return (True, "Test dependency, skipping sanity check") + + exts_filter = ["julia -e 'using %(ext_name)s'"] + cc_check = False + if LooseVersion(self.julia_version) >= LooseVersion('1.8'): + cc_check = True + exts_filter.insert(0, _COMPILECACHE_CHECK % {'ext_name': self.name, 'grep_loc': self.name}) + + if self.is_extension: + pkg_dir = os.path.join('packages', self.name) + + custom_commands = [] + # Check that the compile cache of the dependencies can still be loaded + if not self.is_extension and cc_check: + for dep, _ in self.pkg_deps: + # root_comp = os.path.join(root, 'compiled') + custom_commands.append( + _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} + + ) + kwargs.setdefault('custom_commands', []).extend(custom_commands) + + custom_paths = { + 'files': [], + 'dirs': [pkg_dir], + } + + exts_filter = " && ".join(exts_filter) + exts_filter = (exts_filter, "") + + self.cfg['exts_filter'] = exts_filter + else: + custom_paths = { + 'files': [], + 'dirs': ['packages', 'environments'], + } - pkg_dir = os.path.join('packages', self.name) - - custom_paths = { - 'files': [], - 'dirs': [pkg_dir], - } kwargs.update({'custom_paths': custom_paths}) - return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs) + return super().sanity_check_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """ - Module load initializes JULIA_DEPOT_PATH and JULIA_LOAD_PATH with default values if they are not set. - - Path to installation directory is appended to JULIA_DEPOT_PATH. - Path to the environment file of this installation is prepended to JULIA_LOAD_PATH. - This configuration fulfils the rule that user depot has to be the first path in JULIA_DEPOT_PATH, - allowing user to add custom Julia packages while having packages in this installation available. - See issue easybuilders/easybuild-easyconfigs#17455 + Set EB-specific PATH like variables `EB_JULIA_DEPOT_PATH` and `EB_JULIA_LOAD_PATH`. + These are then used by Easybuild's custom Julia startup script (etc/julia/startup.jl) to set JULIA_DEPOT_PATH + and JULIA_LOAD_PATH correctly at runtime. """ mod = super().make_module_extra() - if self.module_generator.SYNTAX: - mod += JULIA_PATHS_SOFT_INIT[self.module_generator.SYNTAX] - mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) - mod += self.module_generator.append_paths('JULIA_LOAD_PATH', [self.julia_env_path(absolute=False, base=False)]) + + mod += self.module_generator.prepend_paths(EB_JULIA_DEPOT_PATH_VAR, ['']) + mod += self.module_generator.prepend_paths( + EB_JULIA_LOAD_PATH_VAR, + [self.julia_env_path(absolute=False, base=True)] + ) return mod + + def make_extension_list(self): + """Generate list of extensions while filtering out test dependencies.""" + exts = super().make_extension_list() + test_dep_names = self.julia_deps_test.keys() - self.julia_deps.keys() + + return list(filter(lambda ext: ext[0] not in test_dep_names, exts)) diff --git a/easybuild/easyblocks/j/julia.py b/easybuild/easyblocks/j/julia.py new file mode 100644 index 00000000000..d0c7ebfd2b4 --- /dev/null +++ b/easybuild/easyblocks/j/julia.py @@ -0,0 +1,103 @@ +## +# Copyright 2022-2026 Vrije Universiteit Brussel +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Custom EasyBlock for installing Julia binaries with an extra startup script to give special treatment to. + +@author: Davide Grassano (CECAM) +""" +import os +import stat + +from easybuild.tools.filetools import write_file, move_file, adjust_permissions +from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.easyblocks.generic.tarball import Tarball + +EB_JULIA_DEPOT_PATH_VAR = "EBJULIA_DEPOT_PATH" +EB_JULIA_LOAD_PATH_VAR = "EBJULIA_LOAD_PATH" + +STARTUP_CONTENT = rf""" +# Read EB environment variables +function get_env_list(name) + haskey(ENV, name) ? split(ENV[name], ':') : [] +end + +EB_LOAD_PATH = get_env_list("{EB_JULIA_LOAD_PATH_VAR}") +EB_DEPOT_PATH = get_env_list("{EB_JULIA_DEPOT_PATH_VAR}") + +# Append EB_LOAD_PATH to the default LOAD_PATH +if !isempty(EB_LOAD_PATH) + pushfirst!(LOAD_PATH, EB_LOAD_PATH...) +end +# Prepend EB_DEPOT_PATH to the default DEPOT_PATH +if !isempty(EB_DEPOT_PATH) + push!(DEPOT_PATH, EB_DEPOT_PATH...) +end +""" + +WRAPPER_CONTENT = r"""#!/bin/sh +LD_LIBRARY_PATH="$EBROOTJULIA/lib:$EBROOTJULIA/lib/julia:$LD_LIBRARY_PATH" julia.bin "$@" +""" + + +class EB_Julia(Tarball): + """Add custom startup script and sanity checks to Julia installations.""" + + def sanity_check_step(self): + """Custom sanity check for Julia.""" + + shlib_ext = get_shared_lib_ext() + + custom_files = [ + 'bin/julia', 'bin/julia.bin', + 'include/julia/julia.h', f'lib/libjulia.{shlib_ext}', 'etc/julia/startup.jl', + ] + custom_dirs = ['bin', 'etc', 'include', 'lib', 'share'] + custom_commands = [ + "julia --help", + "julia --version", + "julia -E '1+2 == 3' | grep true", + ] + custom_paths = { + 'files': custom_files, + 'dirs': custom_dirs, + } + + super().sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def install_step(self, *args, **kwargs): + """Install procedure for Julia.""" + super().install_step(*args, **kwargs) + + startup_script = os.path.join(self.installdir, 'etc', 'julia', 'startup.jl') + os.makedirs(os.path.dirname(startup_script), exist_ok=True) + + write_file(startup_script, STARTUP_CONTENT) + julia_bin = os.path.join(self.installdir, 'bin', 'julia') + julia_bin_new = os.path.join(self.installdir, 'bin', 'julia.bin') + + # install wrapper with linking safeguards for Julia libraries + move_file(julia_bin, julia_bin_new) + write_file(julia_bin, WRAPPER_CONTENT) + adjust_permissions(julia_bin, stat.S_IRUSR | stat.S_IXUSR) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 846c2c3ef33..28f2d11b23f 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -451,7 +451,7 @@ def innertest(self): write_file(os.path.join(TMPDIR, 'modules', 'all', prgenv, '1.2.3'), "#%Module") # add empty module files for dependencies that are required for testing easyblocks - for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1'): + for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1', "Julia/1.6.7"): write_file(os.path.join(TMPDIR, 'modules', 'all', dep_mod_name), "#%Module") for easyblock in easyblocks: @@ -505,6 +505,10 @@ def innertest(self): elif eb_fn in ['python.py', 'tkinter.py']: # custom easyblock for Python (ensurepip) requires version >= 3.4.0 innertest = make_inner_test(easyblock, name=eb_fn.replace('_', '-')[:-3], version='3.4.0') + elif eb_fn in ['juliapackage.py', 'juliabundle.py']: + # Building a Julia package/bundle requires Julia as a dependency + extra_txt = 'dependencies = [("Julia", "1.6.7")]' + innertest = make_inner_test(easyblock, name='julia-stuff', extra_txt=extra_txt) elif eb_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency extra_txt = "dependencies = [('PyTorch', '1.12.1')]"