From 95ee3ecb6425000bb8341f8c3954b81645f4c627 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 1 May 2026 17:18:25 +0200 Subject: [PATCH 01/32] Allow running julia package tests in a dedicated test_step --- easybuild/easyblocks/generic/juliapackage.py | 45 +++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index a3e641b2178..bf8b5d5c2f4 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -31,6 +31,7 @@ import glob import os import re +import tempfile from easybuild.tools import LooseVersion @@ -272,8 +273,47 @@ def build_step(self): pass def test_step(self): - """No separate (standard) test procedure for JuliaPackage.""" - pass + """ + Test the built Julia package. + + :param return_output: return output and exit code of test command + """ + testcmd = None + if isinstance(self.cfg['runtest'], str): + testcmd = self.cfg['runtest'] + + if self.cfg['runtest']: + if testcmd is None: + testcmd = f"julia -e 'using Pkg; Pkg.test(\"{self.name}\")'" + + try: + tmpdir = tempfile.mkdtemp() + except OSError as err: + raise EasyBuildError("Failed to create test install dir: %s", err) + + depot_path = self.get_julia_env("DEPOT_PATH") + load_path = self.get_julia_env("LOAD_PATH") + self.log.info("Original DEPOT_PATH for testing: %s", os.pathsep.join(depot_path)) + self.log.info("Original LOAD_PATH for testing: %s", os.pathsep.join(load_path)) + + depot_path = os.pathsep.join([tmpdir] + depot_path) + + extrapath = " && ".join([ + f"export JULIA_DEPOT_PATH=\"{depot_path}\"", + # Ensure TEST dependencies can be downloaded + # The alternative is to install them as normal dependencies of the package + "export JULIA_PKG_OFFLINE=false", + "" + ]) + + cmd = ' '.join([ + extrapath, + self.cfg['pretestopts'], + testcmd, + self.cfg['testopts'], + ]) + + run_shell_cmd(cmd) def install_step(self): """Prepare installation environment and install Julia package.""" @@ -293,6 +333,7 @@ def install_extension(self): self.prepare_julia_env() self.install_pkg() + self._test_step() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" From 986870ac795ee386f49446f38f9aadc6e76a4da4 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 5 May 2026 16:23:14 +0200 Subject: [PATCH 02/32] WIP --- easybuild/easyblocks/generic/juliapackage.py | 159 +++++++++++++++++-- 1 file changed, 149 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index bf8b5d5c2f4..1ac0d7763dd 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -28,7 +28,6 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ import ast -import glob import os import re import tempfile @@ -40,9 +39,10 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.filetools import copy_dir, mkdir +from easybuild.tools.filetools import copy_dir, mkdir, change_dir, move_file from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import trace_msg +from easybuild.tools import tomllib EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") @@ -93,6 +93,17 @@ def extra_options(extra_vars=None): 'download_pkg_deps': [ False, "Let Julia download and bundle all needed dependencies for this installation", CUSTOM ], + 'is_test_dependency': [ + False, + "Whether this package is only needed for testing and should not be added to installation environment", + CUSTOM + ], + 'is_main_package': [ + False, + "Whether this package is the main package of the installation. Only this will be installed and Julia " + "will take care of installing dependencies.", + CUSTOM + ], }) return extra_vars @@ -119,6 +130,17 @@ def get_julia_env(env_var): return parsed_var + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._env_toml = {'deps': {}, 'sources': {}} + + @property + def env_toml(self): + """Property to store content of installation environment Project.toml file.""" + if self.is_extension: + return self.master.env_toml + return self._env_toml + def julia_env_path(self, absolute=True, base=True): """ Return path to installation environment file. @@ -232,17 +254,91 @@ def install_pkg_source(self, pkg_source, environment, trace=True): return res.output + @staticmethod + def write_project_toml(file_path, project_toml): + """Write a Project.toml file with the given content to the given path""" + + with open(file_path, 'w') as env_proj: + for section, items in project_toml.items(): + env_proj.write(f'[{section}]\n') + for key, value in items.items(): + value = repr(value) + value = value.replace("'", '"') + value = value.replace(': ', ' = ') + env_proj.write(f'{key} = {value}\n') + env_proj.write('\n') + + @staticmethod + def read_project_toml(file_path): + """Read a Project.toml file and return its content as a dict""" + + with open(file_path, 'rb') as env_proj: + project_toml = tomllib.load(env_proj) + + return project_toml + + def write_env_toml(self, toml_data): + """Write environment Project.toml file""" + + self.write_project_toml(self.julia_env_path(base=False), toml_data) + + + # def include_pkg_dependencies(self): + # """Add to installation environment all Julia packages already present in its dependencies""" + # # Location of project environment files in install dir + # mkdir(self.julia_env_path(), parents=True) + + # # add packages found in dependencies to this installation environment + # for dep in self.cfg.dependencies(): + # dep_root = get_software_root(dep['name']) + # for pkg in glob.glob(os.path.join(dep_root, 'packages/*')): + # trace_msg("incorporating Julia package from dependencies: %s" % os.path.basename(pkg)) + # self.install_pkg_source(pkg, self.julia_env_path(), trace=False) + def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" # Location of project environment files in install dir mkdir(self.julia_env_path(), parents=True) + sections = ['deps', 'sources'] + # add packages found in dependencies to this installation environment + to_remove = set() for dep in self.cfg.dependencies(): - dep_root = get_software_root(dep['name']) - for pkg in glob.glob(os.path.join(dep_root, 'packages/*')): - trace_msg("incorporating Julia package from dependencies: %s" % os.path.basename(pkg)) - self.install_pkg_source(pkg, self.julia_env_path(), trace=False) + dep_name = dep['name'] + dep_root = get_software_root(dep_name) + dep_env = os.path.join(dep_root, self.julia_env_path(absolute=False, base=False)) + if not os.path.isfile(dep_env): + self.log.warning("No environment file found in dependency %s, skipping: %s", dep_name, dep_env) + continue + + to_remove.add(dep_name) + trace_msg + + dep_toml = self.read_project_toml(dep_env) + for section in sections: + self.env_toml.setdefault(section, {}).update(dep_toml.get(section, {})) + + if to_remove: + trace_msg( + "Removing dependencies from installation environment. " + "These dependencies will not appear in the resulting module:" + ) + for dep_type in ['dependencies', 'builddependencies']: + to_remove_idx = [] + ref = self.cfg.get_ref(dep_type) + for idx, dep in enumerate(ref): + if dep['name'] in to_remove: + trace_msg(f'- {dep}') + to_remove_idx.insert(0, idx) + for idx in to_remove_idx: + del ref[idx] + + # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) + # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) + + self.write_env_toml(self.env_toml) + def install_pkg(self): """Install Julia package""" @@ -295,6 +391,8 @@ def test_step(self): load_path = self.get_julia_env("LOAD_PATH") self.log.info("Original DEPOT_PATH for testing: %s", os.pathsep.join(depot_path)) self.log.info("Original LOAD_PATH for testing: %s", os.pathsep.join(load_path)) + # print("Testing with DEPOT_PATH: %s", os.pathsep.join(depot_path)) + # print("Testing with LOAD_PATH: %s", os.pathsep.join(load_path)) depot_path = os.pathsep.join([tmpdir] + depot_path) @@ -302,7 +400,7 @@ def test_step(self): f"export JULIA_DEPOT_PATH=\"{depot_path}\"", # Ensure TEST dependencies can be downloaded # The alternative is to install them as normal dependencies of the package - "export JULIA_PKG_OFFLINE=false", + # "export JULIA_PKG_OFFLINE=false", "" ]) @@ -329,11 +427,52 @@ def install_extension(self): if not self.src: errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) + + if self.cfg['is_test_dependency']: + self.log.info("Package %s is only needed for testing, skipping installation", self.name) + return + ExtensionEasyBlock.install_extension(self, unpack_src=True) - self.prepare_julia_env() - self.install_pkg() - self._test_step() + if not self.cfg['is_main_package']: + self.log.info("Package %s is not the main package of the installation, skipping installation", self.name) + + change_dir(self.master.start_dir) + package_dir = os.path.join(self.installdir, 'packages', self.name) + move_file(self.ext_dir, package_dir) + + self.add_to_env_toml() + else: + self.write_project_toml(self.julia_env_path(base=False), self.env_toml) + + self.prepare_julia_env() + self.install_pkg() + self._test_step() + + @classmethod + def update_toml_data(cls, toml_data, pkg_name, pkg_path): + """Update given toml data with new package and return updated data""" + if os.path.isdir(pkg_path): + pkg_dir = pkg_path + pkg_path = os.path.join(pkg_path, 'Project.toml') + else: + pkg_dir = os.path.dirname(pkg_path) + + if not os.path.isfile(pkg_path): + raise EasyBuildError("Project.toml file not found for package %s in path: %s", pkg_name, pkg_dir) + + project_toml = cls.read_project_toml(pkg_path) + pkg_uuid = project_toml['uuid'] + + toml_data.setdefault('deps', {})[pkg_name] = pkg_uuid + toml_data.setdefault('sources', {})[pkg_name] = {'path': pkg_dir} + + return toml_data + + def add_to_env_toml(self): + """Add package to environment toml file of the installation""" + package_dir = os.path.join(self.installdir, 'packages', self.name) + self.update_toml_data(self.env_toml, self.name, package_dir) def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" From a9dfae3677b9fce257d7ab56c2fcc6d8dd3a3c83 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 6 May 2026 17:31:07 +0200 Subject: [PATCH 03/32] WIP --- easybuild/easyblocks/generic/juliapackage.py | 136 +++++++++++++++---- 1 file changed, 107 insertions(+), 29 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 1ac0d7763dd..6ca39edbb5e 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -44,7 +44,18 @@ from easybuild.tools.utilities import trace_msg from easybuild.tools import tomllib -EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") +_COMPILECACHE_CHECK = ' | '.join([ + "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"))'", + "grep '%(ext_name)s'" +]) + +EXTS_FILTER_JULIA_PACKAGES = ( + " && ".join([ + _COMPILECACHE_CHECK, + "julia -e 'using %(ext_name)s'", + ]), + "" +) USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") JULIA_PATHS_SOFT_INIT = { @@ -58,7 +69,35 @@ if { [ module-info mode load ] } { if {![info exists env(JULIA_DEPOT_PATH)]} { setenv JULIA_DEPOT_PATH : } if {![info exists env(JULIA_LOAD_PATH)]} { setenv JULIA_LOAD_PATH : } +""", } + +JULIA_MODULE_FOOTER = { + "Lua": """ +setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) +setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") +""", +# "Lua": """ +# if ( mode() == "load" ) then +# setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) +# setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") +# end +# if (mode() == "unload") then +# if (os.getenv("JULIA_DEPOT_PATH") == nil) then +# setenv("JULIA_DEPOT_PATH", "") +# else +# setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) +# setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") +# end +# end +# """, + "Lua": """ +execute{cmd="export JULIA_DEPOT_PATH=:${EBJULIA_DEPOT_PATH}", modeA={"load", "unload"}} +execute{cmd="export JULIA_LOAD_PATH=${EBJULIA_LOAD_PATH}:", modeA={"load", "unload"}} +""", + "Tcl": """ +setenv JULIA_DEPOT_PATH [file join $env(:) $env(EBJULIA_DEPOT_PATH)] +setenv JULIA_LOAD_PATH [file join $env(EBJULIA_LOAD_PATH) :] """, } @@ -104,6 +143,10 @@ def extra_options(extra_vars=None): "will take care of installing dependencies.", CUSTOM ], + 'test_online': [ + False, "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", + CUSTOM + ], }) return extra_vars @@ -133,6 +176,7 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._env_toml = {'deps': {}, 'sources': {}} + self.julia_deps = [] @property def env_toml(self): @@ -303,7 +347,7 @@ def include_pkg_dependencies(self): sections = ['deps', 'sources'] # add packages found in dependencies to this installation environment - to_remove = set() + # to_remove = set() for dep in self.cfg.dependencies(): dep_name = dep['name'] dep_root = get_software_root(dep_name) @@ -312,27 +356,32 @@ def include_pkg_dependencies(self): self.log.warning("No environment file found in dependency %s, skipping: %s", dep_name, dep_env) continue - to_remove.add(dep_name) - trace_msg + self.julia_deps.append(dep_name) + trace_msg( + f"Incorporating Julia package dependencies from {dep_name} in installation environment: {dep_env}" + ) + + # to_remove.add(dep_name) + # trace_msg dep_toml = self.read_project_toml(dep_env) for section in sections: self.env_toml.setdefault(section, {}).update(dep_toml.get(section, {})) - if to_remove: - trace_msg( - "Removing dependencies from installation environment. " - "These dependencies will not appear in the resulting module:" - ) - for dep_type in ['dependencies', 'builddependencies']: - to_remove_idx = [] - ref = self.cfg.get_ref(dep_type) - for idx, dep in enumerate(ref): - if dep['name'] in to_remove: - trace_msg(f'- {dep}') - to_remove_idx.insert(0, idx) - for idx in to_remove_idx: - del ref[idx] + # if to_remove: + # trace_msg( + # "Removing dependencies from installation environment. " + # "These dependencies will not appear in the resulting module:" + # ) + # for dep_type in ['dependencies', 'builddependencies']: + # to_remove_idx = [] + # ref = self.cfg.get_ref(dep_type) + # for idx, dep in enumerate(ref): + # if dep['name'] in to_remove: + # trace_msg(f'- {dep}') + # to_remove_idx.insert(0, idx) + # for idx in to_remove_idx: + # del ref[idx] # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) @@ -396,13 +445,18 @@ def test_step(self): depot_path = os.pathsep.join([tmpdir] + depot_path) - extrapath = " && ".join([ - f"export JULIA_DEPOT_PATH=\"{depot_path}\"", - # Ensure TEST dependencies can be downloaded - # The alternative is to install them as normal dependencies of the package - # "export JULIA_PKG_OFFLINE=false", - "" - ]) + # extrapath = " && ".join([ + # f"export JULIA_DEPOT_PATH=\"{depot_path}\"", + # # Ensure TEST dependencies can be downloaded + # # The alternative is to install them as normal dependencies of the package + # # "export JULIA_PKG_OFFLINE=false", + # # "" + # ]) + + extrapath = f"export JULIA_DEPOT_PATH=\"{depot_path}\"" + if self.cfg['test_online']: + extrapath += " && export JULIA_PKG_OFFLINE=false" + extrapath += " && " cmd = ' '.join([ extrapath, @@ -479,6 +533,16 @@ def sanity_check_step(self, *args, **kwargs): pkg_dir = os.path.join('packages', self.name) + custom_commands = [] + # Check that the compile cache of the dependencies can still be loaded. The check is only w.r.t the package + # name as rebuilding using PkgA and PkgB as dependenices can retrigger a compilation of PkgB in case e.g. it + # was a `weakdep` of PkgA with an associated extension + for julia_dep in self.julia_deps: + custom_commands.append( + _COMPILECACHE_CHECK % {'ext_name': julia_dep} + ) + kwargs.setdefault('custom_commands', []).extend(custom_commands) + custom_paths = { 'files': [], 'dirs': [pkg_dir], @@ -498,9 +562,23 @@ def make_module_extra(self, *args, **kwargs): See issue easybuilders/easybuild-easyconfigs#17455 """ mod = super().make_module_extra() - if self.module_generator.SYNTAX: - mod += JULIA_PATHS_SOFT_INIT[self.module_generator.SYNTAX] - mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) - mod += self.module_generator.append_paths('JULIA_LOAD_PATH', [self.julia_env_path(absolute=False, base=False)]) + # if self.module_generator.SYNTAX: + # mod += JULIA_PATHS_SOFT_INIT[self.module_generator.SYNTAX] + # mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) + + mod += self.module_generator.prepend_paths('EBJULIA_DEPOT_PATH', ['']) + mod += self.module_generator.prepend_paths('EBJULIA_LOAD_PATH', [self.julia_env_path(absolute=False, base=False)]) return mod + + def make_module_footer(self): + """ + Extend module footer with statements to set up shell completion for Click-based Python tools. + """ + footer = super().make_module_footer() + + if self.module_generator.SYNTAX: + extra_footer = JULIA_MODULE_FOOTER[self.module_generator.SYNTAX] + footer += '\n' + extra_footer + '\n' + + return footer From 17f13410e5b9d4eb2b8c308a0928180ebe984547 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 7 May 2026 16:57:28 +0200 Subject: [PATCH 04/32] WIP - Use `Pkg.instantiate()` which removes the need to define a `main_package` as all packages in the environment will be installed --- easybuild/easyblocks/generic/juliapackage.py | 338 ++++++++----------- 1 file changed, 135 insertions(+), 203 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 6ca39edbb5e..a97d215a285 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -39,7 +39,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.filetools import copy_dir, mkdir, change_dir, move_file +from easybuild.tools.filetools import mkdir, move_file from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import trace_msg from easybuild.tools import tomllib @@ -58,46 +58,28 @@ ) USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") -JULIA_PATHS_SOFT_INIT = { - "Lua": """ -if ( mode() == "load" ) then - if ( os.getenv("JULIA_DEPOT_PATH") == nil ) then setenv("JULIA_DEPOT_PATH", ":") end - if ( os.getenv("JULIA_LOAD_PATH") == nil ) then setenv("JULIA_LOAD_PATH", ":") end -end -""", - "Tcl": """ -if { [ module-info mode load ] } { - if {![info exists env(JULIA_DEPOT_PATH)]} { setenv JULIA_DEPOT_PATH : } - if {![info exists env(JULIA_LOAD_PATH)]} { setenv JULIA_LOAD_PATH : } -""", -} - -JULIA_MODULE_FOOTER = { - "Lua": """ -setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) -setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") -""", +# JULIA_PATHS_SOFT_INIT = { # "Lua": """ # if ( mode() == "load" ) then -# setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) -# setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") -# end -# if (mode() == "unload") then -# if (os.getenv("JULIA_DEPOT_PATH") == nil) then -# setenv("JULIA_DEPOT_PATH", "") -# else -# setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) -# setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") -# end +# if ( os.getenv("JULIA_DEPOT_PATH") == nil ) then setenv("JULIA_DEPOT_PATH", ":") end +# if ( os.getenv("JULIA_LOAD_PATH") == nil ) then setenv("JULIA_LOAD_PATH", ":") end # end # """, +# "Tcl": """ +# if { [ module-info mode load ] } { +# if {![info exists env(JULIA_DEPOT_PATH)]} { setenv JULIA_DEPOT_PATH : } +# if {![info exists env(JULIA_LOAD_PATH)]} { setenv JULIA_LOAD_PATH : } +# """, +# } + +JULIA_MODULE_FOOTER = { "Lua": """ -execute{cmd="export JULIA_DEPOT_PATH=:${EBJULIA_DEPOT_PATH}", modeA={"load", "unload"}} -execute{cmd="export JULIA_LOAD_PATH=${EBJULIA_LOAD_PATH}:", modeA={"load", "unload"}} +setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) +setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") """, "Tcl": """ -setenv JULIA_DEPOT_PATH [file join $env(:) $env(EBJULIA_DEPOT_PATH)] -setenv JULIA_LOAD_PATH [file join $env(EBJULIA_LOAD_PATH) :] +setenv JULIA_DEPOT_PATH ":$::env(EBJULIA_DEPOT_PATH)" +setenv JULIA_LOAD_PATH "$::env(EBJULIA_LOAD_PATH):" """, } @@ -111,17 +93,22 @@ class JuliaPackage(ExtensionEasyBlock): - remove paths in user depot '~/.julia' from DEPOT_PATH and LOAD_PATH - put installation directory as top DEPOT_PATH, the target depot for installations with Pkg - put installation environment as top LOAD_PATH, needed to precompile installed packages - - add Julia packages found in dependencies of the easyconfig to installation environment, needed - for Pkg to be aware of those packages and not install them again - - add newly installed Julia packages to installation environment (automatically done by Pkg) + - merge the environment of julia dependencies of this installation to the environment of this installation. + - add newly installed Julia packages to installation environment + - install all packages (main and deps) in parallel using Pkg.instantiate() command Julia environment setup on module load: User depot and its shared environment for this version of Julia are kept as top paths of DEPOT_PATH and LOAD_PATH respectively. This ensures that the user can keep using its own environment after loading JuliaPackage modules, installing additional software on its personal depot while still using packages provided by the module. Effectively, this translates to: - - append installation directory to list of DEPOT_PATH, only really needed to load artifacts (JLL packages) - - append installation Project.toml file to list of LOAD_PATH, needed to load packages with `using` command + - prepend installation directory to list of EB_JULIA_DEPOT_PATH, only really needed to load JLL artifacts + - prepend installation Project.toml file to list of EB_JULIA_LOAD_PATH, needed to load packages with `using` + - Generate JULIA_DEPOT_PATH by append the default DEPOT_PATH (:) to EB_JULIA_DEPOT_PATH + - Generate JULIA_LOAD_PATH by prepending the default LOAD_PATH (:) to EB_JULIA_LOAD_PATH + The prepending here is necessary as the default LOAD_PATH [@, @v#.#, @stdlib] can cause stdlib's packages + that have been made upgradable (https://github.com/JuliaLang/julia/issues/50697) to fail matching the cache. + See https://github.com/easybuilders/easybuild-easyblocks/issues/4123#issuecomment-4386990190 for more details. """ @staticmethod @@ -137,12 +124,6 @@ def extra_options(extra_vars=None): "Whether this package is only needed for testing and should not be added to installation environment", CUSTOM ], - 'is_main_package': [ - False, - "Whether this package is the main package of the installation. Only this will be installed and Julia " - "will take care of installing dependencies.", - CUSTOM - ], 'test_online': [ False, "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", CUSTOM @@ -177,6 +158,23 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._env_toml = {'deps': {}, 'sources': {}} self.julia_deps = [] + self._julia_version = None + + @property + def julia_version(self) -> str: + """Needs to get Julia version from dependencies to allow --sanity-check-only to generate the fake module""" + if self._julia_version is None: + deps = self.cfg.dependencies(runtime_only=True) + for dep in deps: + if dep['name'] == 'Julia': + self._julia_version = dep['version'] + break + else: + raise EasyBuildError( + "Julia not included as dependency, cannot determine Julia version for installation of: %s", + self.name + ) + return self._julia_version @property def env_toml(self): @@ -185,6 +183,65 @@ def env_toml(self): return self.master.env_toml return self._env_toml + @property + def is_last_extension(self): + """Whether this extension is the last one to be installed in the installation.""" + if not self.is_extension: + return False + + return self.master.ext_instances and self.master.ext_instances[-1] is self + + @staticmethod + def write_project_toml(file_path, project_toml): + """Write a Project.toml file with the given content to the given path""" + with open(file_path, 'w') as env_proj: + for section, items in project_toml.items(): + env_proj.write(f'[{section}]\n') + for key, value in items.items(): + value = repr(value) + value = value.replace("'", '"') + value = value.replace(': ', ' = ') + env_proj.write(f'{key} = {value}\n') + env_proj.write('\n') + + @staticmethod + def read_project_toml(file_path): + """Read a Project.toml file and return its content as a dict""" + with open(file_path, 'rb') as env_proj: + project_toml = tomllib.load(env_proj) + return project_toml + + @classmethod + def update_toml_data(cls, toml_data, pkg_name, pkg_path): + """Update given toml data with new package and return updated data""" + if os.path.isdir(pkg_path): + pkg_dir = pkg_path + pkg_path = os.path.join(pkg_path, 'Project.toml') + else: + pkg_dir = os.path.dirname(pkg_path) + + if not os.path.isfile(pkg_path): + raise EasyBuildError("Project.toml file not found for package %s in path: %s", pkg_name, pkg_dir) + + project_toml = cls.read_project_toml(pkg_path) + pkg_uuid = project_toml['uuid'] + + toml_data.setdefault('deps', {})[pkg_name] = pkg_uuid + toml_data.setdefault('sources', {})[pkg_name] = {'path': pkg_dir} + + return toml_data + + def write_env_toml(self, toml_data): + """Write environment Project.toml file""" + dir_path = self.julia_env_path() + mkdir(dir_path, parents=True) + self.write_project_toml(self.julia_env_path(base=False), toml_data) + + def add_to_env_toml(self): + """Add package to environment toml file of the installation""" + package_dir = os.path.join(self.installdir, 'packages', self.name) + self.update_toml_data(self.env_toml, self.name, package_dir) + def julia_env_path(self, absolute=True, base=True): """ Return path to installation environment file. @@ -259,35 +316,21 @@ def prepare_julia_env(self): # Enable offline mode self.set_pkg_offline() + # Set the maximum number of concurrent package builds + env.setvar('JULIA_NUM_PRECOMPILE_TASKS', str(self.cfg.parallel)) # Enable automatic precompilation env.setvar('JULIA_PKG_PRECOMPILE_AUTO', 'true') - def install_pkg_source(self, pkg_source, environment, trace=True): + def install_pkg(self): """Execute Julia.Pkg command to install package from its sources""" - julia_pkg_cmd = [ 'using Pkg', - 'Pkg.activate("%s")' % environment, + 'Pkg.activate("%s")' % self.julia_env_path(), + # Usage `Pkg.instantiate()` after all sources are in place to let Pkg handle all dependencies. + # This has the advantage of letting Pkg deal with the order and parallel installation. + 'Pkg.instantiate()', ] - if os.path.isdir(os.path.join(pkg_source, '.git')): - # sources from git repos can be installed as any remote package - self.log.debug('Installing Julia package in normal mode (Pkg.add)') - - julia_pkg_cmd.extend([ - # install package from local path preserving existing dependencies - 'Pkg.add(url="%s"; preserve=PRESERVE_ALL)' % pkg_source, - ]) - else: - # plain sources have to be installed in develop mode - self.log.debug('Installing Julia package in develop mode (Pkg.develop)') - - julia_pkg_cmd.extend([ - # install package from local path preserving existing dependencies - 'Pkg.develop(PackageSpec(path="%s"); preserve=PRESERVE_ALL)' % pkg_source, - 'Pkg.build("%s")' % os.path.basename(pkg_source), - ]) - julia_pkg_cmd = '; '.join(julia_pkg_cmd) cmd = ' '.join([ self.cfg['preinstallopts'], @@ -298,56 +341,11 @@ def install_pkg_source(self, pkg_source, environment, trace=True): return res.output - @staticmethod - def write_project_toml(file_path, project_toml): - """Write a Project.toml file with the given content to the given path""" - - with open(file_path, 'w') as env_proj: - for section, items in project_toml.items(): - env_proj.write(f'[{section}]\n') - for key, value in items.items(): - value = repr(value) - value = value.replace("'", '"') - value = value.replace(': ', ' = ') - env_proj.write(f'{key} = {value}\n') - env_proj.write('\n') - - @staticmethod - def read_project_toml(file_path): - """Read a Project.toml file and return its content as a dict""" - - with open(file_path, 'rb') as env_proj: - project_toml = tomllib.load(env_proj) - - return project_toml - - def write_env_toml(self, toml_data): - """Write environment Project.toml file""" - - self.write_project_toml(self.julia_env_path(base=False), toml_data) - - - # def include_pkg_dependencies(self): - # """Add to installation environment all Julia packages already present in its dependencies""" - # # Location of project environment files in install dir - # mkdir(self.julia_env_path(), parents=True) - - # # add packages found in dependencies to this installation environment - # for dep in self.cfg.dependencies(): - # dep_root = get_software_root(dep['name']) - # for pkg in glob.glob(os.path.join(dep_root, 'packages/*')): - # trace_msg("incorporating Julia package from dependencies: %s" % os.path.basename(pkg)) - # self.install_pkg_source(pkg, self.julia_env_path(), trace=False) - def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" - # Location of project environment files in install dir - mkdir(self.julia_env_path(), parents=True) - sections = ['deps', 'sources'] # add packages found in dependencies to this installation environment - # to_remove = set() for dep in self.cfg.dependencies(): dep_name = dep['name'] dep_root = get_software_root(dep_name) @@ -361,47 +359,12 @@ def include_pkg_dependencies(self): f"Incorporating Julia package dependencies from {dep_name} in installation environment: {dep_env}" ) - # to_remove.add(dep_name) - # trace_msg - dep_toml = self.read_project_toml(dep_env) for section in sections: self.env_toml.setdefault(section, {}).update(dep_toml.get(section, {})) - # if to_remove: - # trace_msg( - # "Removing dependencies from installation environment. " - # "These dependencies will not appear in the resulting module:" - # ) - # for dep_type in ['dependencies', 'builddependencies']: - # to_remove_idx = [] - # ref = self.cfg.get_ref(dep_type) - # for idx, dep in enumerate(ref): - # if dep['name'] in to_remove: - # trace_msg(f'- {dep}') - # to_remove_idx.insert(0, idx) - # for idx in to_remove_idx: - # del ref[idx] - - # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) - # print("Updated dependencies after removing Julia packages: %s", self.cfg.dependencies()) - self.write_env_toml(self.env_toml) - - def install_pkg(self): - """Install Julia package""" - - # determine source type of current installation - if os.path.isdir(os.path.join(self.start_dir, '.git')): - pkg_source = self.start_dir - else: - # copy non-git sources to install directory - pkg_source = os.path.join(self.installdir, 'packages', self.name) - copy_dir(self.start_dir, pkg_source) - - return self.install_pkg_source(pkg_source, self.julia_env_path()) - def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" super().prepare_step(*args, **kwargs) @@ -440,23 +403,14 @@ def test_step(self): load_path = self.get_julia_env("LOAD_PATH") self.log.info("Original DEPOT_PATH for testing: %s", os.pathsep.join(depot_path)) self.log.info("Original LOAD_PATH for testing: %s", os.pathsep.join(load_path)) - # print("Testing with DEPOT_PATH: %s", os.pathsep.join(depot_path)) - # print("Testing with LOAD_PATH: %s", os.pathsep.join(load_path)) depot_path = os.pathsep.join([tmpdir] + depot_path) - # extrapath = " && ".join([ - # f"export JULIA_DEPOT_PATH=\"{depot_path}\"", - # # Ensure TEST dependencies can be downloaded - # # The alternative is to install them as normal dependencies of the package - # # "export JULIA_PKG_OFFLINE=false", - # # "" - # ]) - - extrapath = f"export JULIA_DEPOT_PATH=\"{depot_path}\"" - if self.cfg['test_online']: - extrapath += " && export JULIA_PKG_OFFLINE=false" - extrapath += " && " + extrapath = " && ".join([ + f"export JULIA_DEPOT_PATH=\"{depot_path}\"", + f"export JULIA_PKG_OFFLINE={'false' if self.cfg['test_online'] else 'true'}", + "" + ]) cmd = ' '.join([ extrapath, @@ -467,14 +421,25 @@ def test_step(self): run_shell_cmd(cmd) - def install_step(self): - """Prepare installation environment and install Julia package.""" + def install_source(self): + """Add the Julia package source files in the installation directory.""" + package_dir = os.path.join(self.installdir, 'packages', self.name) + move_file(self.ext_dir if self.is_extension else self.start_dir, package_dir) - self.prepare_julia_env() - self.include_pkg_dependencies() + # Add package to the main environment Project.toml file + self.add_to_env_toml() + def _install_step(self): + """Install step commons between single-package and extensions based installs.""" + self.write_env_toml(self.env_toml) + self.prepare_julia_env() return self.install_pkg() + def install_step(self): + """Prepare installation environment and install Julia package.""" + self.install_source() + return self._install_step() + def install_extension(self): """Install Julia package as an extension.""" @@ -486,47 +451,13 @@ def install_extension(self): self.log.info("Package %s is only needed for testing, skipping installation", self.name) return + # Unpack source into install directory and add package to the main environment Project.toml file ExtensionEasyBlock.install_extension(self, unpack_src=True) + self.install_source() - if not self.cfg['is_main_package']: - self.log.info("Package %s is not the main package of the installation, skipping installation", self.name) - - change_dir(self.master.start_dir) - package_dir = os.path.join(self.installdir, 'packages', self.name) - move_file(self.ext_dir, package_dir) - - self.add_to_env_toml() - else: - self.write_project_toml(self.julia_env_path(base=False), self.env_toml) - - self.prepare_julia_env() - self.install_pkg() - self._test_step() - - @classmethod - def update_toml_data(cls, toml_data, pkg_name, pkg_path): - """Update given toml data with new package and return updated data""" - if os.path.isdir(pkg_path): - pkg_dir = pkg_path - pkg_path = os.path.join(pkg_path, 'Project.toml') - else: - pkg_dir = os.path.dirname(pkg_path) - - if not os.path.isfile(pkg_path): - raise EasyBuildError("Project.toml file not found for package %s in path: %s", pkg_name, pkg_dir) - - project_toml = cls.read_project_toml(pkg_path) - pkg_uuid = project_toml['uuid'] - - toml_data.setdefault('deps', {})[pkg_name] = pkg_uuid - toml_data.setdefault('sources', {})[pkg_name] = {'path': pkg_dir} - - return toml_data - - def add_to_env_toml(self): - """Add package to environment toml file of the installation""" - package_dir = os.path.join(self.installdir, 'packages', self.name) - self.update_toml_data(self.env_toml, self.name, package_dir) + if self.is_last_extension: + self._install_step() + self.test_step() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" @@ -538,6 +469,7 @@ def sanity_check_step(self, *args, **kwargs): # name as rebuilding using PkgA and PkgB as dependenices can retrigger a compilation of PkgB in case e.g. it # was a `weakdep` of PkgA with an associated extension for julia_dep in self.julia_deps: + trace_msg(f"Checking that compile cache of dependency {julia_dep} can still be loaded") custom_commands.append( _COMPILECACHE_CHECK % {'ext_name': julia_dep} ) From 82ae123daf7eff8f503636090c637b3413fbe253 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 7 May 2026 17:20:35 +0200 Subject: [PATCH 05/32] Implement test_dependencies --- easybuild/easyblocks/generic/juliapackage.py | 37 +++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index a97d215a285..72845a404f6 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -77,9 +77,11 @@ setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") """, + # This needs testing, the module generate seems to work fine, but the loading of the module within EB does not + # properly resolve the $::env() and leaves it as a string "Tcl": """ -setenv JULIA_DEPOT_PATH ":$::env(EBJULIA_DEPOT_PATH)" -setenv JULIA_LOAD_PATH "$::env(EBJULIA_LOAD_PATH):" +setenv JULIA_DEPOT_PATH ":\$::env(EBJULIA_DEPOT_PATH)" +setenv JULIA_LOAD_PATH "\$::env(EBJULIA_LOAD_PATH):" """, } @@ -159,6 +161,7 @@ def __init__(self, *args, **kwargs): self._env_toml = {'deps': {}, 'sources': {}} self.julia_deps = [] self._julia_version = None + self._tmp_test_dir = None @property def julia_version(self) -> str: @@ -191,6 +194,16 @@ def is_last_extension(self): return self.master.ext_instances and self.master.ext_instances[-1] is self + @property + def tmp_test_dir(self): + """Temporary path used for running the test step.""" + if not self._tmp_test_dir: + try: + self._tmp_test_dir = tempfile.mkdtemp(suffix='-julia_tests') + except Exception as e: + raise EasyBuildError("Failed to create temporary directory for Julia package testing: %s", str(e)) + return self._tmp_test_dir + @staticmethod def write_project_toml(file_path, project_toml): """Write a Project.toml file with the given content to the given path""" @@ -394,17 +407,12 @@ def test_step(self): if testcmd is None: testcmd = f"julia -e 'using Pkg; Pkg.test(\"{self.name}\")'" - try: - tmpdir = tempfile.mkdtemp() - except OSError as err: - raise EasyBuildError("Failed to create test install dir: %s", err) - depot_path = self.get_julia_env("DEPOT_PATH") load_path = self.get_julia_env("LOAD_PATH") self.log.info("Original DEPOT_PATH for testing: %s", os.pathsep.join(depot_path)) self.log.info("Original LOAD_PATH for testing: %s", os.pathsep.join(load_path)) - depot_path = os.pathsep.join([tmpdir] + depot_path) + depot_path = os.pathsep.join([self.tmp_test_dir] + depot_path) extrapath = " && ".join([ f"export JULIA_DEPOT_PATH=\"{depot_path}\"", @@ -423,7 +431,14 @@ def test_step(self): def install_source(self): """Add the Julia package source files in the installation directory.""" - package_dir = os.path.join(self.installdir, 'packages', self.name) + if self.cfg['is_test_dependency']: + self.log.debug( + f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" + ) + package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) + else: + package_dir = os.path.join(self.installdir, 'packages', self.name) + # This also warks for dirs and will fail if the destination already exists move_file(self.ext_dir if self.is_extension else self.start_dir, package_dir) # Add package to the main environment Project.toml file @@ -447,10 +462,6 @@ def install_extension(self): errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) - if self.cfg['is_test_dependency']: - self.log.info("Package %s is only needed for testing, skipping installation", self.name) - return - # Unpack source into install directory and add package to the main environment Project.toml file ExtensionEasyBlock.install_extension(self, unpack_src=True) self.install_source() From a980e1e2d735755d1b354b4a08efe911e452e8bc Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 3 Jun 2026 16:43:08 +0200 Subject: [PATCH 06/32] Add possibility to also have test-only dependencies --- easybuild/easyblocks/generic/juliapackage.py | 154 ++++++++++++------- 1 file changed, 99 insertions(+), 55 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 72845a404f6..01b3f82673d 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -26,6 +26,7 @@ EasyBuild support for Julia Packages, implemented as an easyblock @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Davide Grassano (CECAM, EPFL) """ import ast import os @@ -126,8 +127,23 @@ def extra_options(extra_vars=None): "Whether this package is only needed for testing and should not be added to installation environment", CUSTOM ], + 'julia_debug': [ + False, + "Whether to set JULIA_DEBUG=all during installation and testing, to get more verbose output.", + CUSTOM + ], + # Arguably this should be set to True by default. In many cases the test environment in Julia packages + # will attempt to re-install the package being tested. If not all required packages to be brought in the + # test environment were specified in test/Project.toml, running Pkg.test() in offline mode will fail. + # Some test tools like Aqua will also re-create their own tmp environment which can be broken even if + # test/Project.toml is properly set up/patched. 'test_online': [ - False, "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", + False, + "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", + CUSTOM + ], + 'subpackages_dirs': [ + None, "List of subdirectories to look for additional packages to add to the environment", CUSTOM ], }) @@ -159,6 +175,7 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._env_toml = {'deps': {}, 'sources': {}} + self._env_toml_test = {'deps': {}, 'sources': {}} self.julia_deps = [] self._julia_version = None self._tmp_test_dir = None @@ -186,6 +203,13 @@ def env_toml(self): return self.master.env_toml return self._env_toml + @property + def env_toml_test(self): + """Property to store content of installation environment Project.toml file.""" + if self.is_extension: + return self.master.env_toml_test + return self._env_toml_test + @property def is_last_extension(self): """Whether this extension is the last one to be installed in the installation.""" @@ -197,6 +221,9 @@ def is_last_extension(self): @property def tmp_test_dir(self): """Temporary path used for running the test step.""" + if self.is_extension: + return self.master.tmp_test_dir + if not self._tmp_test_dir: try: self._tmp_test_dir = tempfile.mkdtemp(suffix='-julia_tests') @@ -207,6 +234,7 @@ def tmp_test_dir(self): @staticmethod def write_project_toml(file_path, project_toml): """Write a Project.toml file with the given content to the given path""" + mkdir(os.path.dirname(file_path), parents=True) with open(file_path, 'w') as env_proj: for section, items in project_toml.items(): env_proj.write(f'[{section}]\n') @@ -225,7 +253,7 @@ def read_project_toml(file_path): return project_toml @classmethod - def update_toml_data(cls, toml_data, pkg_name, pkg_path): + def update_toml_data(cls, toml_data, pkg_path): """Update given toml data with new package and return updated data""" if os.path.isdir(pkg_path): pkg_dir = pkg_path @@ -237,6 +265,7 @@ def update_toml_data(cls, toml_data, pkg_name, pkg_path): raise EasyBuildError("Project.toml file not found for package %s in path: %s", pkg_name, pkg_dir) project_toml = cls.read_project_toml(pkg_path) + pkg_name = project_toml['name'] pkg_uuid = project_toml['uuid'] toml_data.setdefault('deps', {})[pkg_name] = pkg_uuid @@ -244,27 +273,33 @@ def update_toml_data(cls, toml_data, pkg_name, pkg_path): return toml_data - def write_env_toml(self, toml_data): + def write_env_toml(self): """Write environment Project.toml file""" - dir_path = self.julia_env_path() - mkdir(dir_path, parents=True) - self.write_project_toml(self.julia_env_path(base=False), toml_data) + self.write_project_toml(self.julia_env_path(base=False), self.env_toml) + + def write_env_toml_test(self): + """Write test environment Project.toml file""" + self.write_project_toml(self.julia_env_path(base=False, basedir=self.tmp_test_dir), self.env_toml_test) - def add_to_env_toml(self): + def add_to_env_toml(self, package_dir): """Add package to environment toml file of the installation""" - package_dir = os.path.join(self.installdir, 'packages', self.name) - self.update_toml_data(self.env_toml, self.name, package_dir) + self.update_toml_data(self.env_toml, package_dir) - def julia_env_path(self, absolute=True, base=True): + def add_to_env_toml_test(self, package_dir): + """Add package to environment toml file of the test environment""" + self.update_toml_data(self.env_toml_test, package_dir) + + def julia_env_path(self, absolute=True, base=True, basedir=None): """ Return path to installation environment file. """ - julia_version = get_software_version('Julia').split('.') + basedir = basedir or self.installdir + julia_version = self.julia_version.split('.') env_dir = "v{}.{}".format(*julia_version[:2]) project_env = os.path.join("environments", env_dir, "Project.toml") if absolute: - project_env = os.path.join(self.installdir, project_env) + project_env = os.path.join(basedir, project_env) if base: project_env = os.path.dirname(project_env) @@ -273,21 +308,20 @@ def julia_env_path(self, absolute=True, base=True): def set_pkg_offline(self): """Enable offline mode of Julia Pkg""" - if not self.cfg['download_pkg_deps']: - julia_version = get_software_version('Julia') - if LooseVersion(julia_version) >= LooseVersion('1.5'): - # Enable offline mode of Julia Pkg - # https://pkgdocs.julialang.org/v1/api/#Pkg.offline - env.setvar('JULIA_PKG_OFFLINE', 'true') - else: - errmsg = ( - "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " - "Enable easyconfig option 'download_pkg_deps' to allow installation " - "with any extra downloaded dependencies." - ) - raise EasyBuildError(errmsg, julia_version) + julia_version = get_software_version('Julia') + if LooseVersion(julia_version) >= LooseVersion('1.5'): + # Enable offline mode of Julia Pkg + # https://pkgdocs.julialang.org/v1/api/#Pkg.offline + env.setvar('JULIA_PKG_OFFLINE', 'true') + else: + errmsg = ( + "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " + "Enable easyconfig option 'download_pkg_deps' to allow installation " + "with any extra downloaded dependencies." + ) + raise EasyBuildError(errmsg, julia_version) - def prepare_julia_env(self): + def prepare_julia_env(self, basedir=None, online=False): """ 1. Remove user depot and prepend installation directory to DEPOT_PATH. Top directory in Julia DEPOT_PATH is the target installation directory. @@ -303,6 +337,7 @@ def prepare_julia_env(self): 4. Enable automatic precompilation of packages after each build. """ + basedir = basedir or self.installdir # Grab both DEPOT_PATH and LOAD_PATH before any changes are made # given that Julia might automatically update LOAD_PATH from a change on DEPOT_PATH dirty_depot = self.get_julia_env("DEPOT_PATH") @@ -312,33 +347,38 @@ def prepare_julia_env(self): # First set DEPOT_PATH and then LOAD_PATH to avoid any automatic changes made by Julia clean_depot = [path for path in dirty_depot if not USER_DEPOT_PATTERN.search(path) and path != self.installdir] - install_depot = os.pathsep.join([self.installdir] + clean_depot) + install_depot = os.pathsep.join([basedir] + clean_depot) self.log.debug("Preparing Julia 'DEPOT_PATH' for installation: %s", install_depot) env.setvar("JULIA_DEPOT_PATH", install_depot) - project_toml = self.julia_env_path(base=False) + project_toml = self.julia_env_path(base=False, basedir=basedir) clean_load = [path for path in dirty_load if not USER_DEPOT_PATTERN.search(path) and path != project_toml] install_load = os.pathsep.join([project_toml] + clean_load) self.log.debug("Preparing Julia 'LOAD_PATH' for installation: %s", install_load) env.setvar("JULIA_LOAD_PATH", install_load) - if self.julia_env_path(base=False) not in self.get_julia_env("LOAD_PATH"): + if project_toml not in self.get_julia_env("LOAD_PATH"): errmsg = "Failed to prepare Julia environment for installation of: %s" raise EasyBuildError(errmsg, self.name) # Enable offline mode - self.set_pkg_offline() + if not online: + self.set_pkg_offline() + + if self.cfg['julia_debug']: + env.setvar('JULIA_DEBUG', 'all') # Set the maximum number of concurrent package builds env.setvar('JULIA_NUM_PRECOMPILE_TASKS', str(self.cfg.parallel)) # Enable automatic precompilation env.setvar('JULIA_PKG_PRECOMPILE_AUTO', 'true') - def install_pkg(self): + def install_pkg(self, basedir=None): """Execute Julia.Pkg command to install package from its sources""" + basedir = basedir or self.installdir julia_pkg_cmd = [ 'using Pkg', - 'Pkg.activate("%s")' % self.julia_env_path(), + 'Pkg.activate("%s")' % self.julia_env_path(basedir=basedir), # Usage `Pkg.instantiate()` after all sources are in place to let Pkg handle all dependencies. # This has the advantage of letting Pkg deal with the order and parallel installation. 'Pkg.instantiate()', @@ -374,9 +414,9 @@ def include_pkg_dependencies(self): dep_toml = self.read_project_toml(dep_env) for section in sections: - self.env_toml.setdefault(section, {}).update(dep_toml.get(section, {})) - - self.write_env_toml(self.env_toml) + data = dep_toml.get(section, {}) + for toml in [self.env_toml, self.env_toml_test]: + toml.setdefault(section, {}).update(data) def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" @@ -407,21 +447,12 @@ def test_step(self): if testcmd is None: testcmd = f"julia -e 'using Pkg; Pkg.test(\"{self.name}\")'" - depot_path = self.get_julia_env("DEPOT_PATH") - load_path = self.get_julia_env("LOAD_PATH") - self.log.info("Original DEPOT_PATH for testing: %s", os.pathsep.join(depot_path)) - self.log.info("Original LOAD_PATH for testing: %s", os.pathsep.join(load_path)) + # Write test environment Project.toml that also include path reference to test dependencies + self.write_env_toml_test() - depot_path = os.pathsep.join([self.tmp_test_dir] + depot_path) - - extrapath = " && ".join([ - f"export JULIA_DEPOT_PATH=\"{depot_path}\"", - f"export JULIA_PKG_OFFLINE={'false' if self.cfg['test_online'] else 'true'}", - "" - ]) + self.prepare_julia_env(basedir=self.tmp_test_dir, online=self.cfg['test_online']) cmd = ' '.join([ - extrapath, self.cfg['pretestopts'], testcmd, self.cfg['testopts'], @@ -435,20 +466,32 @@ def install_source(self): self.log.debug( f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" ) + trace_msg("Installing as a test dependency...") package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) else: package_dir = os.path.join(self.installdir, 'packages', self.name) # This also warks for dirs and will fail if the destination already exists move_file(self.ext_dir if self.is_extension else self.start_dir, package_dir) - # Add package to the main environment Project.toml file - self.add_to_env_toml() + # Add all packages to the test environment Project.toml file + self.add_to_env_toml_test(package_dir) + if not self.cfg['is_test_dependency']: + self.add_to_env_toml(package_dir) + + subpkg_dirs = self.cfg['subpackages_dirs'] or [] + for subpkg_dir in subpkg_dirs: + subpkg_path = os.path.join(package_dir, subpkg_dir) + if not os.path.isdir(subpkg_path): + self.log.warning("Sub-package directory specified but not found, skipping: %s", subpkg_path) + continue + trace_msg(f"Adding sub-package from specified directory {subpkg_path}") + self.add_to_env_toml(subpkg_path) - def _install_step(self): + def _install_step(self, basedir=None): """Install step commons between single-package and extensions based installs.""" - self.write_env_toml(self.env_toml) - self.prepare_julia_env() - return self.install_pkg() + self.write_env_toml() + self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) + return self.install_pkg(basedir=basedir) def install_step(self): """Prepare installation environment and install Julia package.""" @@ -457,7 +500,6 @@ def install_step(self): def install_extension(self): """Install Julia package as an extension.""" - if not self.src: errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) @@ -472,6 +514,9 @@ def install_extension(self): def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" + if self.cfg['is_test_dependency']: + self.log.debug(f"Package {self.name} is only used for testing, skipping sanity check") + return (True, "Test dependency, skipping sanity check") pkg_dir = os.path.join('packages', self.name) @@ -480,7 +525,6 @@ def sanity_check_step(self, *args, **kwargs): # name as rebuilding using PkgA and PkgB as dependenices can retrigger a compilation of PkgB in case e.g. it # was a `weakdep` of PkgA with an associated extension for julia_dep in self.julia_deps: - trace_msg(f"Checking that compile cache of dependency {julia_dep} can still be loaded") custom_commands.append( _COMPILECACHE_CHECK % {'ext_name': julia_dep} ) From 8fb7dd3881eef20ca602cfe08e2c1ff4f3dc9acb Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 4 Jun 2026 14:02:15 +0200 Subject: [PATCH 07/32] Add tests from 4122 --- test/easyblocks/module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 846c2c3ef33..28f2d11b23f 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -451,7 +451,7 @@ def innertest(self): write_file(os.path.join(TMPDIR, 'modules', 'all', prgenv, '1.2.3'), "#%Module") # add empty module files for dependencies that are required for testing easyblocks - for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1'): + for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1', "Julia/1.6.7"): write_file(os.path.join(TMPDIR, 'modules', 'all', dep_mod_name), "#%Module") for easyblock in easyblocks: @@ -505,6 +505,10 @@ def innertest(self): elif eb_fn in ['python.py', 'tkinter.py']: # custom easyblock for Python (ensurepip) requires version >= 3.4.0 innertest = make_inner_test(easyblock, name=eb_fn.replace('_', '-')[:-3], version='3.4.0') + elif eb_fn in ['juliapackage.py', 'juliabundle.py']: + # Building a Julia package/bundle requires Julia as a dependency + extra_txt = 'dependencies = [("Julia", "1.6.7")]' + innertest = make_inner_test(easyblock, name='julia-stuff', extra_txt=extra_txt) elif eb_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency extra_txt = "dependencies = [('PyTorch', '1.12.1')]" From 53665aff76d83a23f762f248fb38b970c3fe3212 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 4 Jun 2026 15:36:56 +0200 Subject: [PATCH 08/32] Lint --- easybuild/easyblocks/generic/juliapackage.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 01b3f82673d..89b9a3b603c 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -81,8 +81,8 @@ # This needs testing, the module generate seems to work fine, but the loading of the module within EB does not # properly resolve the $::env() and leaves it as a string "Tcl": """ -setenv JULIA_DEPOT_PATH ":\$::env(EBJULIA_DEPOT_PATH)" -setenv JULIA_LOAD_PATH "\$::env(EBJULIA_LOAD_PATH):" +setenv JULIA_DEPOT_PATH ":$::env(EBJULIA_DEPOT_PATH)" +setenv JULIA_LOAD_PATH "$::env(EBJULIA_LOAD_PATH):" """, } @@ -262,7 +262,7 @@ def update_toml_data(cls, toml_data, pkg_path): pkg_dir = os.path.dirname(pkg_path) if not os.path.isfile(pkg_path): - raise EasyBuildError("Project.toml file not found for package %s in path: %s", pkg_name, pkg_dir) + raise EasyBuildError("Project.toml file not found in path: %s", pkg_dir) project_toml = cls.read_project_toml(pkg_path) pkg_name = project_toml['name'] @@ -554,7 +554,10 @@ def make_module_extra(self, *args, **kwargs): # mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) mod += self.module_generator.prepend_paths('EBJULIA_DEPOT_PATH', ['']) - mod += self.module_generator.prepend_paths('EBJULIA_LOAD_PATH', [self.julia_env_path(absolute=False, base=False)]) + mod += self.module_generator.prepend_paths( + 'EBJULIA_LOAD_PATH', + [self.julia_env_path(absolute=False, base=False)] + ) return mod From 8056a5048160ef7a8d6bd93b388040e9a7cec480 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 4 Jun 2026 17:09:56 +0200 Subject: [PATCH 09/32] Add check to avoid duplicate cache from deps packages --- easybuild/easyblocks/generic/juliapackage.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 89b9a3b603c..82d6dd8d1cf 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -413,10 +413,26 @@ def include_pkg_dependencies(self): ) dep_toml = self.read_project_toml(dep_env) + conflicts = [] for section in sections: data = dep_toml.get(section, {}) for toml in [self.env_toml, self.env_toml_test]: - toml.setdefault(section, {}).update(data) + sec_dct = toml.setdefault(section, {}) + for k,v in data.items(): + if k in sec_dct and sec_dct[k] != v: + conflicts.append((k, section, dep_name, sec_dct[k], v)) + sec_dct.update(data) + if conflicts: + error_msg = '\n'.join( + f'- Package "{k}" in section "{section}" `{current_v}` -> `{dep_v}`' + for k, section, _, current_v, dep_v in conflicts + ) + raise EasyBuildError( + "\nConflicts found when merging dependency '%s' into installation environment:\n%s\n" + "Make sure that all dependencies do not specify duplicate packages as extensions, otherwise the " + "compile cache will be broken based on the module load order.", + dep_name, error_msg + ) def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" From c325f44d120ba3059b5f89f0cc341cbfe431a58a Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 4 Jun 2026 17:22:36 +0200 Subject: [PATCH 10/32] Improve sanity checks --- easybuild/easyblocks/generic/juliabundle.py | 4 ---- easybuild/easyblocks/generic/juliapackage.py | 20 +++++++++----------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index d4fa09f8b5c..37be26eb8e4 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -79,10 +79,6 @@ def __init__(self, *args, **kwargs): def prepare_step(self, *args, **kwargs): """Prepare for installing bundle of Julia packages.""" super().prepare_step(*args, **kwargs) - - def install_step(self): - """Prepare installation environment and dd all dependencies to project environment.""" - self.prepare_julia_env() self.include_pkg_dependencies() def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 82d6dd8d1cf..4d87b00c7b7 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -47,12 +47,12 @@ _COMPILECACHE_CHECK = ' | '.join([ "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"))'", - "grep '%(ext_name)s'" + "grep '%(grep_loc)s'" ]) EXTS_FILTER_JULIA_PACKAGES = ( " && ".join([ - _COMPILECACHE_CHECK, + _COMPILECACHE_CHECK.replace('%(grep_loc)s', '%(ext_name)s'), "julia -e 'using %(ext_name)s'", ]), "" @@ -407,7 +407,7 @@ def include_pkg_dependencies(self): self.log.warning("No environment file found in dependency %s, skipping: %s", dep_name, dep_env) continue - self.julia_deps.append(dep_name) + self.julia_deps.append((dep_name, dep_root)) trace_msg( f"Incorporating Julia package dependencies from {dep_name} in installation environment: {dep_env}" ) @@ -418,9 +418,9 @@ def include_pkg_dependencies(self): data = dep_toml.get(section, {}) for toml in [self.env_toml, self.env_toml_test]: sec_dct = toml.setdefault(section, {}) - for k,v in data.items(): - if k in sec_dct and sec_dct[k] != v: - conflicts.append((k, section, dep_name, sec_dct[k], v)) + for key, val in data.items(): + if key in sec_dct and sec_dct[key] != val: + conflicts.append((key, section, dep_name, sec_dct[key], val)) sec_dct.update(data) if conflicts: error_msg = '\n'.join( @@ -537,12 +537,10 @@ def sanity_check_step(self, *args, **kwargs): pkg_dir = os.path.join('packages', self.name) custom_commands = [] - # Check that the compile cache of the dependencies can still be loaded. The check is only w.r.t the package - # name as rebuilding using PkgA and PkgB as dependenices can retrigger a compilation of PkgB in case e.g. it - # was a `weakdep` of PkgA with an associated extension - for julia_dep in self.julia_deps: + # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package + for dep, root in self.julia_deps: custom_commands.append( - _COMPILECACHE_CHECK % {'ext_name': julia_dep} + _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': root} ) kwargs.setdefault('custom_commands', []).extend(custom_commands) From b7754a371d7681d19e38a4cc847a3a46e5913a8b Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Jun 2026 15:52:20 +0200 Subject: [PATCH 11/32] Fix default EB to `JuliaPackage` to disable autodetection --- easybuild/easyblocks/generic/juliabundle.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 37be26eb8e4..d1c69d31435 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -53,6 +53,12 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.cfg['exts_defaultclass'] = 'JuliaPackage' + # Ensure that Julia packages such as LLVM do not try to use the EB_LLVM easyblock + # many packages have names that overlap with EB easyblocks, and would end up using them as default class for extensions, which is not what we want here. We force to always use JuliaPackage unless explicitly specified + # otherwise in the easyconfig file. + self.cfg['exts_default_options'] = { + 'easyblock': 'JuliaPackage', + } self.cfg['exts_filter'] = EXTS_FILTER_JULIA_PACKAGES # need to disable templating to ensure that actual value for exts_default_options is updated... From 3fadc3b4279f0e7358c7a7c1bc7aa808dbec789c Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Jun 2026 15:53:06 +0200 Subject: [PATCH 12/32] Handle DEPOT_PATH and LOAD_PATH in Julia's site startup script --- easybuild/easyblocks/generic/juliapackage.py | 48 +---------- easybuild/easyblocks/j/julia.py | 90 ++++++++++++++++++++ 2 files changed, 94 insertions(+), 44 deletions(-) create mode 100644 easybuild/easyblocks/j/julia.py diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 4d87b00c7b7..4155b3c7f3c 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -45,6 +45,8 @@ from easybuild.tools.utilities import trace_msg from easybuild.tools import tomllib +from easybuild.easyblocks.j.julia import EB_JULIA_DEPOT_PATH_VAR, EB_JULIA_LOAD_PATH_VAR + _COMPILECACHE_CHECK = ' | '.join([ "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"))'", "grep '%(grep_loc)s'" @@ -59,33 +61,6 @@ ) USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") -# JULIA_PATHS_SOFT_INIT = { -# "Lua": """ -# if ( mode() == "load" ) then -# if ( os.getenv("JULIA_DEPOT_PATH") == nil ) then setenv("JULIA_DEPOT_PATH", ":") end -# if ( os.getenv("JULIA_LOAD_PATH") == nil ) then setenv("JULIA_LOAD_PATH", ":") end -# end -# """, -# "Tcl": """ -# if { [ module-info mode load ] } { -# if {![info exists env(JULIA_DEPOT_PATH)]} { setenv JULIA_DEPOT_PATH : } -# if {![info exists env(JULIA_LOAD_PATH)]} { setenv JULIA_LOAD_PATH : } -# """, -# } - -JULIA_MODULE_FOOTER = { - "Lua": """ -setenv("JULIA_DEPOT_PATH", ":" .. os.getenv("EBJULIA_DEPOT_PATH") ) -setenv("JULIA_LOAD_PATH", os.getenv("EBJULIA_LOAD_PATH") .. ":") -""", - # This needs testing, the module generate seems to work fine, but the loading of the module within EB does not - # properly resolve the $::env() and leaves it as a string - "Tcl": """ -setenv JULIA_DEPOT_PATH ":$::env(EBJULIA_DEPOT_PATH)" -setenv JULIA_LOAD_PATH "$::env(EBJULIA_LOAD_PATH):" -""", -} - class JuliaPackage(ExtensionEasyBlock): """ @@ -563,26 +538,11 @@ def make_module_extra(self, *args, **kwargs): See issue easybuilders/easybuild-easyconfigs#17455 """ mod = super().make_module_extra() - # if self.module_generator.SYNTAX: - # mod += JULIA_PATHS_SOFT_INIT[self.module_generator.SYNTAX] - # mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) - mod += self.module_generator.prepend_paths('EBJULIA_DEPOT_PATH', ['']) + mod += self.module_generator.prepend_paths(EB_JULIA_DEPOT_PATH_VAR, ['']) mod += self.module_generator.prepend_paths( - 'EBJULIA_LOAD_PATH', + EB_JULIA_LOAD_PATH_VAR, [self.julia_env_path(absolute=False, base=False)] ) return mod - - def make_module_footer(self): - """ - Extend module footer with statements to set up shell completion for Click-based Python tools. - """ - footer = super().make_module_footer() - - if self.module_generator.SYNTAX: - extra_footer = JULIA_MODULE_FOOTER[self.module_generator.SYNTAX] - footer += '\n' + extra_footer + '\n' - - return footer diff --git a/easybuild/easyblocks/j/julia.py b/easybuild/easyblocks/j/julia.py new file mode 100644 index 00000000000..21f42322ed8 --- /dev/null +++ b/easybuild/easyblocks/j/julia.py @@ -0,0 +1,90 @@ +## +# Copyright 2022-2026 Vrije Universiteit Brussel +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Custom EasyBlock for installing Julia binaries with an extra startup script to give special treatment to. + +@author: Davide Grassano (CECAM) +""" +import os + +from easybuild.tools.filetools import write_file +from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.easyblocks.generic.tarball import Tarball + +EB_JULIA_DEPOT_PATH_VAR = "EBJULIA_DEPOT_PATH" +EB_JULIA_LOAD_PATH_VAR = "EBJULIA_LOAD_PATH" + +STARTUP_CONTENT = rf""" +# Read EB environment variables +function get_env_list(name) + haskey(ENV, name) ? split(ENV[name], ':') : [] +end + +EB_LOAD_PATH = get_env_list("{EB_JULIA_LOAD_PATH_VAR}") +EB_DEPOT_PATH = get_env_list("{EB_JULIA_DEPOT_PATH_VAR}") + +# Append EB_LOAD_PATH to the default LOAD_PATH +if !isempty(EB_LOAD_PATH) + pushfirst!(LOAD_PATH, EB_LOAD_PATH...) +end +# Prepend EB_DEPOT_PATH to the default DEPOT_PATH +if !isempty(EB_DEPOT_PATH) + push!(DEPOT_PATH, EB_DEPOT_PATH...) +end +""" + + +class EB_Julia(Tarball): + """Add custom startup script and sanity checks to Julia installations.""" + + def sanity_check_step(self): + """Custom sanity check for Julia.""" + + shlib_ext = get_shared_lib_ext() + + custom_files = [ + 'bin/julia', 'include/julia/julia.h', f'lib/libjulia.{shlib_ext}', 'etc/julia/startup.jl', + ] + custom_dirs = ['bin', 'etc', 'include', 'lib', 'share'] + custom_commands = [ + "julia --help", + "julia --version", + "julia -E '1+2 == 3' | grep true", + ] + custom_paths = { + 'files': custom_files, + 'dirs': custom_dirs, + } + + super().sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def install_step(self, *args, **kwargs): + """Install procedure for Julia.""" + super().install_step(*args, **kwargs) + + startup_script = os.path.join(self.installdir, 'etc', 'julia', 'startup.jl') + os.makedirs(os.path.dirname(startup_script), exist_ok=True) + + write_file(startup_script, STARTUP_CONTENT) From 366503531ce36ea89841928895961a022706c99a Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Jun 2026 15:59:12 +0200 Subject: [PATCH 13/32] lint --- easybuild/easyblocks/generic/juliabundle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index d1c69d31435..31b31838688 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -54,7 +54,8 @@ def __init__(self, *args, **kwargs): self.cfg['exts_defaultclass'] = 'JuliaPackage' # Ensure that Julia packages such as LLVM do not try to use the EB_LLVM easyblock - # many packages have names that overlap with EB easyblocks, and would end up using them as default class for extensions, which is not what we want here. We force to always use JuliaPackage unless explicitly specified + # many packages have names that overlap with EB easyblocks, and would end up using them as default class for + # extensions, which is not what we want here. We force to always use JuliaPackage unless explicitly specified # otherwise in the easyconfig file. self.cfg['exts_default_options'] = { 'easyblock': 'JuliaPackage', From 48d9bf31893a071e5a9ee5eba37b70eee178cb8c Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 8 Jun 2026 14:28:40 +0200 Subject: [PATCH 14/32] Remove manual handling of TOML files --- easybuild/easyblocks/generic/juliabundle.py | 5 - easybuild/easyblocks/generic/juliapackage.py | 223 +++++++++---------- 2 files changed, 101 insertions(+), 127 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 31b31838688..aa384808bcd 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -83,11 +83,6 @@ def __init__(self, *args, **kwargs): self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) - def prepare_step(self, *args, **kwargs): - """Prepare for installing bundle of Julia packages.""" - super().prepare_step(*args, **kwargs) - self.include_pkg_dependencies() - def sanity_check_step(self, *args, **kwargs): """Custom sanity check for bundle of Julia packages""" custom_paths = { diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 4155b3c7f3c..7a3f7f0fa83 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -40,15 +40,17 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.filetools import mkdir, move_file +from easybuild.tools.filetools import move_file from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import trace_msg -from easybuild.tools import tomllib from easybuild.easyblocks.j.julia import EB_JULIA_DEPOT_PATH_VAR, EB_JULIA_LOAD_PATH_VAR _COMPILECACHE_CHECK = ' | '.join([ - "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"))'", + # "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"), Base.get_world_counter())'", + # compilecache_path is not part of the stable API and changes arguments across versions + # allow internal cache resolution and use debug statemnts to get the path of the loaded cache + "JULIA_DEBUG=loading julia -e 'using %(ext_name)s' 2>&1 1>/dev/null", "grep '%(grep_loc)s'" ]) @@ -149,9 +151,9 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._env_toml = {'deps': {}, 'sources': {}} - self._env_toml_test = {'deps': {}, 'sources': {}} - self.julia_deps = [] + self._pkg_deps = [] + self._julia_deps = {} + self._julia_deps_test = {} self._julia_version = None self._tmp_test_dir = None @@ -172,18 +174,25 @@ def julia_version(self) -> str: return self._julia_version @property - def env_toml(self): - """Property to store content of installation environment Project.toml file.""" + def pkg_deps(self): + """List of Julia dependencies found in this installation.""" if self.is_extension: - return self.master.env_toml - return self._env_toml + return self.master.pkg_deps + return self._pkg_deps @property - def env_toml_test(self): - """Property to store content of installation environment Project.toml file.""" + def julia_deps(self): + """List of Julia dependencies found in this installation.""" if self.is_extension: - return self.master.env_toml_test - return self._env_toml_test + return self.master.julia_deps + return self._julia_deps + + @property + def julia_deps_test(self): + """List of Julia dependencies found in this installation.""" + if self.is_extension: + return self.master.julia_deps_test + return self._julia_deps_test @property def is_last_extension(self): @@ -206,64 +215,6 @@ def tmp_test_dir(self): raise EasyBuildError("Failed to create temporary directory for Julia package testing: %s", str(e)) return self._tmp_test_dir - @staticmethod - def write_project_toml(file_path, project_toml): - """Write a Project.toml file with the given content to the given path""" - mkdir(os.path.dirname(file_path), parents=True) - with open(file_path, 'w') as env_proj: - for section, items in project_toml.items(): - env_proj.write(f'[{section}]\n') - for key, value in items.items(): - value = repr(value) - value = value.replace("'", '"') - value = value.replace(': ', ' = ') - env_proj.write(f'{key} = {value}\n') - env_proj.write('\n') - - @staticmethod - def read_project_toml(file_path): - """Read a Project.toml file and return its content as a dict""" - with open(file_path, 'rb') as env_proj: - project_toml = tomllib.load(env_proj) - return project_toml - - @classmethod - def update_toml_data(cls, toml_data, pkg_path): - """Update given toml data with new package and return updated data""" - if os.path.isdir(pkg_path): - pkg_dir = pkg_path - pkg_path = os.path.join(pkg_path, 'Project.toml') - else: - pkg_dir = os.path.dirname(pkg_path) - - if not os.path.isfile(pkg_path): - raise EasyBuildError("Project.toml file not found in path: %s", pkg_dir) - - project_toml = cls.read_project_toml(pkg_path) - pkg_name = project_toml['name'] - pkg_uuid = project_toml['uuid'] - - toml_data.setdefault('deps', {})[pkg_name] = pkg_uuid - toml_data.setdefault('sources', {})[pkg_name] = {'path': pkg_dir} - - return toml_data - - def write_env_toml(self): - """Write environment Project.toml file""" - self.write_project_toml(self.julia_env_path(base=False), self.env_toml) - - def write_env_toml_test(self): - """Write test environment Project.toml file""" - self.write_project_toml(self.julia_env_path(base=False, basedir=self.tmp_test_dir), self.env_toml_test) - - def add_to_env_toml(self, package_dir): - """Add package to environment toml file of the installation""" - self.update_toml_data(self.env_toml, package_dir) - - def add_to_env_toml_test(self, package_dir): - """Add package to environment toml file of the test environment""" - self.update_toml_data(self.env_toml_test, package_dir) - def julia_env_path(self, absolute=True, base=True, basedir=None): """ Return path to installation environment file. @@ -332,6 +283,10 @@ def prepare_julia_env(self, basedir=None, online=False): self.log.debug("Preparing Julia 'LOAD_PATH' for installation: %s", install_load) env.setvar("JULIA_LOAD_PATH", install_load) + # Disable EB controlled PATHS to enforce the specific configurations + env.setvar(EB_JULIA_DEPOT_PATH_VAR, "") + env.setvar(EB_JULIA_LOAD_PATH_VAR, "") + if project_toml not in self.get_julia_env("LOAD_PATH"): errmsg = "Failed to prepare Julia environment for installation of: %s" raise EasyBuildError(errmsg, self.name) @@ -348,12 +303,29 @@ def prepare_julia_env(self, basedir=None, online=False): # Enable automatic precompilation env.setvar('JULIA_PKG_PRECOMPILE_AUTO', 'true') + def add_package(self, pkg_source, test_only=False): + pkg_name = os.path.basename(pkg_source) + if pkg_name in self.julia_deps and self.julia_deps[pkg_name] != pkg_source: + raise EasyBuildError( + "Conflict detected for package '%s': already added from source '%s', cannot add from source '%s'", + pkg_name, self.julia_deps[pkg_name], pkg_source + ) + self.julia_deps_test[pkg_name] = pkg_source + if not test_only: + self.julia_deps[pkg_name] = pkg_source + def install_pkg(self, basedir=None): """Execute Julia.Pkg command to install package from its sources""" basedir = basedir or self.installdir + env = self.julia_env_path(basedir=basedir) + + package_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps.values()) + julia_pkg_cmd = [ 'using Pkg', - 'Pkg.activate("%s")' % self.julia_env_path(basedir=basedir), + f'Pkg.activate("{env}")', + + f'Pkg.develop([{package_specs}]; preserve=PRESERVE_ALL)', # Usage `Pkg.instantiate()` after all sources are in place to let Pkg handle all dependencies. # This has the advantage of letting Pkg deal with the order and parallel installation. 'Pkg.instantiate()', @@ -369,45 +341,46 @@ def install_pkg(self, basedir=None): return res.output + def _deps_from_project(self, environment): + """Get list of dependencies from a Project.toml file""" + julia_root = get_software_root('Julia') + cmd = "; ".join([ + 'using Pkg', + f'Pkg.activate("{environment}")', + # 'Pkg.status(; mode=PKGMODE_MANIFEST)', + 'for (_, pkg) in Pkg.dependencies()', + 'println("$(pkg.source)")', + 'end' + ]) + + res = run_shell_cmd(f"julia -E '{cmd}'", split_stderr=True, hidden=True) + deps = res.output.strip().splitlines() + + # Filter out Julia's stdlib dependencies + deps = filter(lambda d: not d.startswith(julia_root), deps) + deps = filter(lambda d: d != 'nothing', deps) + + return list(deps) + def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" - sections = ['deps', 'sources'] - # add packages found in dependencies to this installation environment for dep in self.cfg.dependencies(): dep_name = dep['name'] dep_root = get_software_root(dep_name) - dep_env = os.path.join(dep_root, self.julia_env_path(absolute=False, base=False)) - if not os.path.isfile(dep_env): - self.log.warning("No environment file found in dependency %s, skipping: %s", dep_name, dep_env) + dep_env = self.julia_env_path(absolute=True, base=True, basedir=dep_root) + dep_manifest = os.path.join(dep_env, 'Manifest.toml') + if not os.path.isfile(dep_manifest): + self.log.warning("No Manifest.toml file found in dependency %s, skipping: %s", dep_name, dep_env) continue - self.julia_deps.append((dep_name, dep_root)) + self.pkg_deps.append((dep_name, dep_root)) trace_msg( f"Incorporating Julia package dependencies from {dep_name} in installation environment: {dep_env}" ) - dep_toml = self.read_project_toml(dep_env) - conflicts = [] - for section in sections: - data = dep_toml.get(section, {}) - for toml in [self.env_toml, self.env_toml_test]: - sec_dct = toml.setdefault(section, {}) - for key, val in data.items(): - if key in sec_dct and sec_dct[key] != val: - conflicts.append((key, section, dep_name, sec_dct[key], val)) - sec_dct.update(data) - if conflicts: - error_msg = '\n'.join( - f'- Package "{k}" in section "{section}" `{current_v}` -> `{dep_v}`' - for k, section, _, current_v, dep_v in conflicts - ) - raise EasyBuildError( - "\nConflicts found when merging dependency '%s' into installation environment:\n%s\n" - "Make sure that all dependencies do not specify duplicate packages as extensions, otherwise the " - "compile cache will be broken based on the module load order.", - dep_name, error_msg - ) + for pkg_path in self._deps_from_project(dep_env): + self.add_package(pkg_path) def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" @@ -416,13 +389,7 @@ def prepare_step(self, *args, **kwargs): if get_software_root('Julia') is None: raise EasyBuildError("Julia not included as dependency!") - def configure_step(self): - """No separate configuration for JuliaPackage.""" - pass - - def build_step(self): - """No separate build procedure for JuliaPackage.""" - pass + self.include_pkg_dependencies() def test_step(self): """ @@ -436,11 +403,20 @@ def test_step(self): if self.cfg['runtest']: if testcmd is None: - testcmd = f"julia -e 'using Pkg; Pkg.test(\"{self.name}\")'" - - # Write test environment Project.toml that also include path reference to test dependencies - self.write_env_toml_test() - + env = self.julia_env_path(basedir=self.tmp_test_dir) + package_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) + testcmd = [ + 'using Pkg', + f'Pkg.activate("{env}")', + f'Pkg.develop([{package_specs}]; preserve=PRESERVE_ALL)', + f'Pkg.test("{self.name}")', + ] + testcmd = '; '.join(testcmd) + testcmd = f"julia -e '{testcmd}'" + + # Also add normal DEPOT/LOAD paths to environment to try and re-use pre-compiled caches as much as possible + # But also ensure that artifacts for packages that do not need recompilation are still found in the tests + self.prepare_julia_env(online=self.cfg['test_online']) self.prepare_julia_env(basedir=self.tmp_test_dir, online=self.cfg['test_online']) cmd = ' '.join([ @@ -461,26 +437,26 @@ def install_source(self): package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) else: package_dir = os.path.join(self.installdir, 'packages', self.name) + # This also warks for dirs and will fail if the destination already exists move_file(self.ext_dir if self.is_extension else self.start_dir, package_dir) # Add all packages to the test environment Project.toml file - self.add_to_env_toml_test(package_dir) - if not self.cfg['is_test_dependency']: - self.add_to_env_toml(package_dir) + self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) subpkg_dirs = self.cfg['subpackages_dirs'] or [] for subpkg_dir in subpkg_dirs: + # pkg_name = os.path.basename(subpkg_dir) subpkg_path = os.path.join(package_dir, subpkg_dir) if not os.path.isdir(subpkg_path): self.log.warning("Sub-package directory specified but not found, skipping: %s", subpkg_path) continue trace_msg(f"Adding sub-package from specified directory {subpkg_path}") - self.add_to_env_toml(subpkg_path) + + self.add_package(subpkg_path, test_only=self.cfg['is_test_dependency']) def _install_step(self, basedir=None): """Install step commons between single-package and extensions based installs.""" - self.write_env_toml() self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) return self.install_pkg(basedir=basedir) @@ -513,10 +489,13 @@ def sanity_check_step(self, *args, **kwargs): custom_commands = [] # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package - for dep, root in self.julia_deps: - custom_commands.append( - _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': root} - ) + + if not self.is_extension: + for dep, root in self.pkg_deps: + root_comp = os.path.join(root, 'compiled') + custom_commands.append( + _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': root_comp} + ) kwargs.setdefault('custom_commands', []).extend(custom_commands) custom_paths = { @@ -542,7 +521,7 @@ def make_module_extra(self, *args, **kwargs): mod += self.module_generator.prepend_paths(EB_JULIA_DEPOT_PATH_VAR, ['']) mod += self.module_generator.prepend_paths( EB_JULIA_LOAD_PATH_VAR, - [self.julia_env_path(absolute=False, base=False)] + [self.julia_env_path(absolute=False, base=True)] ) return mod From f9f1d89b6231c543996c3c90970698fc832d0698 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 8 Jun 2026 16:50:43 +0200 Subject: [PATCH 15/32] Allow cache recompilation by other packages --- easybuild/easyblocks/generic/juliapackage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 7a3f7f0fa83..7b36b956775 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -51,7 +51,7 @@ # compilecache_path is not part of the stable API and changes arguments across versions # allow internal cache resolution and use debug statemnts to get the path of the loaded cache "JULIA_DEBUG=loading julia -e 'using %(ext_name)s' 2>&1 1>/dev/null", - "grep '%(grep_loc)s'" + "grep 'Loading object cache file .*%(grep_loc)s'" ]) EXTS_FILTER_JULIA_PACKAGES = ( @@ -231,14 +231,14 @@ def julia_env_path(self, absolute=True, base=True, basedir=None): return project_env - def set_pkg_offline(self): + def set_pkg_offline(self, online=False): """Enable offline mode of Julia Pkg""" julia_version = get_software_version('Julia') if LooseVersion(julia_version) >= LooseVersion('1.5'): # Enable offline mode of Julia Pkg # https://pkgdocs.julialang.org/v1/api/#Pkg.offline - env.setvar('JULIA_PKG_OFFLINE', 'true') + env.setvar('JULIA_PKG_OFFLINE', 'false' if online else 'true') else: errmsg = ( "Cannot set offline mode in Julia v%s (needs Julia >= 1.5). " @@ -494,7 +494,9 @@ def sanity_check_step(self, *args, **kwargs): for dep, root in self.pkg_deps: root_comp = os.path.join(root, 'compiled') custom_commands.append( - _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': root_comp} + # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} + _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} + ) kwargs.setdefault('custom_commands', []).extend(custom_commands) From 53c8eef6260f54946c79c9969d3bd42e8918de27 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 8 Jun 2026 18:01:29 +0200 Subject: [PATCH 16/32] Remove `subpackage_dirs` and `is_test_dependency` in favor of `start_dir` and using builddependencies --- easybuild/easyblocks/generic/juliapackage.py | 112 ++++++++----------- 1 file changed, 46 insertions(+), 66 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 7b36b956775..7b1323b6404 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -99,11 +99,6 @@ def extra_options(extra_vars=None): 'download_pkg_deps': [ False, "Let Julia download and bundle all needed dependencies for this installation", CUSTOM ], - 'is_test_dependency': [ - False, - "Whether this package is only needed for testing and should not be added to installation environment", - CUSTOM - ], 'julia_debug': [ False, "Whether to set JULIA_DEBUG=all during installation and testing, to get more verbose output.", @@ -119,10 +114,6 @@ def extra_options(extra_vars=None): "Allow online tests that require downloading dependencies. By default, tests are run in offline mode.", CUSTOM ], - 'subpackages_dirs': [ - None, "List of subdirectories to look for additional packages to add to the environment", - CUSTOM - ], }) return extra_vars @@ -152,6 +143,7 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._pkg_deps = [] + self._pkt_to_test = [] self._julia_deps = {} self._julia_deps_test = {} self._julia_version = None @@ -180,6 +172,13 @@ def pkg_deps(self): return self.master.pkg_deps return self._pkg_deps + @property + def pkg_to_test(self): + """List of Julia dependencies to be included in the test environment.""" + if self.is_extension: + return self.master.pkg_to_test + return self._pkt_to_test + @property def julia_deps(self): """List of Julia dependencies found in this installation.""" @@ -231,8 +230,8 @@ def julia_env_path(self, absolute=True, base=True, basedir=None): return project_env - def set_pkg_offline(self, online=False): - """Enable offline mode of Julia Pkg""" + def set_pkg_remote(self, online=False): + """Enable online/offline mode of Julia Pkg""" julia_version = get_software_version('Julia') if LooseVersion(julia_version) >= LooseVersion('1.5'): @@ -291,9 +290,8 @@ def prepare_julia_env(self, basedir=None, online=False): errmsg = "Failed to prepare Julia environment for installation of: %s" raise EasyBuildError(errmsg, self.name) - # Enable offline mode - if not online: - self.set_pkg_offline() + # Enable/disable offline mode + self.set_pkg_remote(online=online) if self.cfg['julia_debug']: env.setvar('JULIA_DEBUG', 'all') @@ -329,6 +327,8 @@ def install_pkg(self, basedir=None): # Usage `Pkg.instantiate()` after all sources are in place to let Pkg handle all dependencies. # This has the advantage of letting Pkg deal with the order and parallel installation. 'Pkg.instantiate()', + # Ensure operations done only by build.jl are done for all packages if needed + 'Pkg.build()', ] julia_pkg_cmd = '; '.join(julia_pkg_cmd) @@ -364,8 +364,10 @@ def _deps_from_project(self, environment): def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" + build_only_deps = self.cfg.dependencies(build_only=True) # add packages found in dependencies to this installation environment for dep in self.cfg.dependencies(): + test_only = dep in build_only_deps dep_name = dep['name'] dep_root = get_software_root(dep_name) dep_env = self.julia_env_path(absolute=True, base=True, basedir=dep_root) @@ -374,13 +376,18 @@ def include_pkg_dependencies(self): self.log.warning("No Manifest.toml file found in dependency %s, skipping: %s", dep_name, dep_env) continue - self.pkg_deps.append((dep_name, dep_root)) - trace_msg( - f"Incorporating Julia package dependencies from {dep_name} in installation environment: {dep_env}" - ) + if test_only: + trace_msg( + f"Incorporating Julia package in TEST environment {dep_name}: {dep_env}" + ) + else: + self.pkg_deps.append((dep_name, dep_root)) + trace_msg( + f"Incorporating Julia package in INSTALL environment {dep_name}: {dep_env}" + ) for pkg_path in self._deps_from_project(dep_env): - self.add_package(pkg_path) + self.add_package(pkg_path, test_only=dep in build_only_deps) def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" @@ -397,22 +404,18 @@ def test_step(self): :param return_output: return output and exit code of test command """ - testcmd = None - if isinstance(self.cfg['runtest'], str): - testcmd = self.cfg['runtest'] - - if self.cfg['runtest']: - if testcmd is None: - env = self.julia_env_path(basedir=self.tmp_test_dir) - package_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) - testcmd = [ - 'using Pkg', - f'Pkg.activate("{env}")', - f'Pkg.develop([{package_specs}]; preserve=PRESERVE_ALL)', - f'Pkg.test("{self.name}")', - ] - testcmd = '; '.join(testcmd) - testcmd = f"julia -e '{testcmd}'" + if self.pkg_to_test: + env = self.julia_env_path(basedir=self.tmp_test_dir) + pkg_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkg_to_test) + testcmd = [ + 'using Pkg', + f'Pkg.activate("{env}")', + f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)', + f'Pkg.test([{pkg_test}])', + ] + testcmd = '; '.join(testcmd) + testcmd = f"julia -e '{testcmd}'" # Also add normal DEPOT/LOAD paths to environment to try and re-use pre-compiled caches as much as possible # But also ensure that artifacts for packages that do not need recompilation are still found in the tests @@ -429,31 +432,9 @@ def test_step(self): def install_source(self): """Add the Julia package source files in the installation directory.""" - if self.cfg['is_test_dependency']: - self.log.debug( - f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" - ) - trace_msg("Installing as a test dependency...") - package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) - else: - package_dir = os.path.join(self.installdir, 'packages', self.name) - - # This also warks for dirs and will fail if the destination already exists - move_file(self.ext_dir if self.is_extension else self.start_dir, package_dir) - - # Add all packages to the test environment Project.toml file - self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) - - subpkg_dirs = self.cfg['subpackages_dirs'] or [] - for subpkg_dir in subpkg_dirs: - # pkg_name = os.path.basename(subpkg_dir) - subpkg_path = os.path.join(package_dir, subpkg_dir) - if not os.path.isdir(subpkg_path): - self.log.warning("Sub-package directory specified but not found, skipping: %s", subpkg_path) - continue - trace_msg(f"Adding sub-package from specified directory {subpkg_path}") - - self.add_package(subpkg_path, test_only=self.cfg['is_test_dependency']) + package_dir = os.path.join(self.installdir, 'packages', self.name) + move_file(self.start_dir, package_dir) + self.add_package(package_dir) def _install_step(self, basedir=None): """Install step commons between single-package and extensions based installs.""" @@ -475,24 +456,23 @@ def install_extension(self): ExtensionEasyBlock.install_extension(self, unpack_src=True) self.install_source() + if self.cfg['runtest']: + self.pkg_to_test.append(self.name) + if self.is_last_extension: self._install_step() self.test_step() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" - if self.cfg['is_test_dependency']: - self.log.debug(f"Package {self.name} is only used for testing, skipping sanity check") - return (True, "Test dependency, skipping sanity check") - pkg_dir = os.path.join('packages', self.name) custom_commands = [] - # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package + # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package if not self.is_extension: for dep, root in self.pkg_deps: - root_comp = os.path.join(root, 'compiled') + # root_comp = os.path.join(root, 'compiled') custom_commands.append( # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} From e94a2b9e5f116e08b0ede443a7ffd514aa1aeff2 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 14:14:33 +0200 Subject: [PATCH 17/32] Check for all conflicts before failing --- easybuild/easyblocks/generic/juliapackage.py | 70 +++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 7b1323b6404..b39e579ccb8 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -128,11 +128,13 @@ def get_julia_env(env_var): "LOAD_PATH": "julia -E 'Base.load_path()'", } - try: - res = run_shell_cmd(julia_read_cmd[env_var], hidden=True) - except KeyError: + if env_var not in julia_read_cmd: raise EasyBuildError("Unknown Julia environment variable requested: %s", env_var) + cmd = julia_read_cmd[env_var] + + res = run_shell_cmd(cmd, hidden=True) + try: parsed_var = ast.literal_eval(res.output) except SyntaxError: @@ -142,11 +144,12 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._pkg_deps = [] - self._pkt_to_test = [] + self._conflicts = [] self._julia_deps = {} self._julia_deps_test = {} self._julia_version = None + self._pkg_deps = [] + self._pkg_to_test = [] self._tmp_test_dir = None @property @@ -166,18 +169,19 @@ def julia_version(self) -> str: return self._julia_version @property - def pkg_deps(self): - """List of Julia dependencies found in this installation.""" + def conflicts(self): + """List of conflicting software.""" if self.is_extension: - return self.master.pkg_deps - return self._pkg_deps + return self.master.conflicts + return self._conflicts @property - def pkg_to_test(self): - """List of Julia dependencies to be included in the test environment.""" - if self.is_extension: - return self.master.pkg_to_test - return self._pkt_to_test + def is_last_extension(self): + """Whether this extension is the last one to be installed in the installation.""" + if not self.is_extension: + return False + + return self.master.ext_instances and self.master.ext_instances[-1] is self @property def julia_deps(self): @@ -194,12 +198,18 @@ def julia_deps_test(self): return self._julia_deps_test @property - def is_last_extension(self): - """Whether this extension is the last one to be installed in the installation.""" - if not self.is_extension: - return False + def pkg_deps(self): + """List of Julia dependencies found in this installation.""" + if self.is_extension: + return self.master.pkg_deps + return self._pkg_deps - return self.master.ext_instances and self.master.ext_instances[-1] is self + @property + def pkg_to_test(self): + """List of Julia dependencies to be included in the test environment.""" + if self.is_extension: + return self.master.pkg_to_test + return self._pkg_to_test @property def tmp_test_dir(self): @@ -303,17 +313,29 @@ def prepare_julia_env(self, basedir=None, online=False): def add_package(self, pkg_source, test_only=False): pkg_name = os.path.basename(pkg_source) - if pkg_name in self.julia_deps and self.julia_deps[pkg_name] != pkg_source: - raise EasyBuildError( - "Conflict detected for package '%s': already added from source '%s', cannot add from source '%s'", - pkg_name, self.julia_deps[pkg_name], pkg_source - ) + if pkg_name in self.julia_deps: + prev_source = self.julia_deps[pkg_name] + if prev_source != pkg_source: + self.conflicts.append((self.name, pkg_name, prev_source, pkg_source)) + self.julia_deps_test[pkg_name] = pkg_source if not test_only: self.julia_deps[pkg_name] = pkg_source + def _check_conflicts(self): + """Check for conflicts in Julia dependencies and raise an error if any are found.""" + if self.conflicts: + conflict_msgs = [] + for ext_name, pkg_name, prev_source, new_source in self.conflicts: + conflict_msgs.append( + f"Conflict detected for package '{pkg_name}' in extension '{ext_name}': " + f"already added from source '{prev_source}', cannot add from source '{new_source}'" + ) + raise EasyBuildError("Conflicts detected in Julia dependencies:\n" + "\n".join(conflict_msgs)) + def install_pkg(self, basedir=None): """Execute Julia.Pkg command to install package from its sources""" + self._check_conflicts() basedir = basedir or self.installdir env = self.julia_env_path(basedir=basedir) From a3cc48a4505d73da455ac606d4193a461f197a46 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 14:31:03 +0200 Subject: [PATCH 18/32] Re-add possibility to have test-only deps with proper checks and remove them from the final extension list --- easybuild/easyblocks/generic/juliabundle.py | 4 ++ easybuild/easyblocks/generic/juliapackage.py | 42 +++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index aa384808bcd..e4303f91877 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -29,6 +29,7 @@ """ import os +from easybuild.tools.build_log import EasyBuildError from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.juliapackage import EXTS_FILTER_JULIA_PACKAGES, JuliaPackage @@ -81,6 +82,9 @@ def __init__(self, *args, **kwargs): } ] + if self.cfg['is_test_dependency']: + raise EasyBuildError("Test dependencies can only be defined as an extension, not as a bundle itself") + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index b39e579ccb8..a9e30f057e5 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -51,7 +51,7 @@ # compilecache_path is not part of the stable API and changes arguments across versions # allow internal cache resolution and use debug statemnts to get the path of the loaded cache "JULIA_DEBUG=loading julia -e 'using %(ext_name)s' 2>&1 1>/dev/null", - "grep 'Loading object cache file .*%(grep_loc)s'" + "grep -E 'Loading (object )?cache file .*%(grep_loc)s'" ]) EXTS_FILTER_JULIA_PACKAGES = ( @@ -99,6 +99,11 @@ def extra_options(extra_vars=None): 'download_pkg_deps': [ False, "Let Julia download and bundle all needed dependencies for this installation", CUSTOM ], + 'is_test_dependency': [ + False, + "Whether this package is only needed for testing and should not be added to installation environment", + CUSTOM + ], 'julia_debug': [ False, "Whether to set JULIA_DEBUG=all during installation and testing, to get more verbose output.", @@ -339,6 +344,15 @@ def install_pkg(self, basedir=None): basedir = basedir or self.installdir env = self.julia_env_path(basedir=basedir) + if len(self.julia_deps) == 0: + if len(self.julia_deps_test) > 0: + raise EasyBuildError( + "Only test dependencies found for Julia package %s, cannot proceed with installation. " + "Please check that all needed dependencies are properly specified in the easyconfig file.", + self.name + ) + raise EasyBuildError("No Julia packages to install for: %s", self.name) + package_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps.values()) julia_pkg_cmd = [ @@ -369,7 +383,6 @@ def _deps_from_project(self, environment): cmd = "; ".join([ 'using Pkg', f'Pkg.activate("{environment}")', - # 'Pkg.status(; mode=PKGMODE_MANIFEST)', 'for (_, pkg) in Pkg.dependencies()', 'println("$(pkg.source)")', 'end' @@ -454,9 +467,17 @@ def test_step(self): def install_source(self): """Add the Julia package source files in the installation directory.""" - package_dir = os.path.join(self.installdir, 'packages', self.name) + if self.cfg['is_test_dependency']: + self.log.debug( + f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" + ) + trace_msg("Installing as a test dependency...") + package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) + else: + package_dir = os.path.join(self.installdir, 'packages', self.name) + move_file(self.start_dir, package_dir) - self.add_package(package_dir) + self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) def _install_step(self, basedir=None): """Install step commons between single-package and extensions based installs.""" @@ -487,11 +508,15 @@ def install_extension(self): def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" + if self.cfg['is_test_dependency']: + self.log.debug(f"Package {self.name} is only used for testing, skipping sanity check") + return (True, "Test dependency, skipping sanity check") + pkg_dir = os.path.join('packages', self.name) custom_commands = [] - # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package + if not self.is_extension: for dep, root in self.pkg_deps: # root_comp = os.path.join(root, 'compiled') @@ -529,3 +554,10 @@ def make_module_extra(self, *args, **kwargs): ) return mod + + def make_extension_list(self): + """Generate list of extensions while filtering out test dependencies.""" + exts = super().make_extension_list() + test_dep_names = self.julia_deps_test.keys() - self.julia_deps.keys() + + return list(filter(lambda ext: ext[0] not in test_dep_names, exts)) From a401696a0a9c27977debf5911c37f8ab574d1e75 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 15:29:22 +0200 Subject: [PATCH 19/32] Check for compile_cache only for Julia >= 1.8 --- easybuild/easyblocks/generic/juliabundle.py | 7 +--- easybuild/easyblocks/generic/juliapackage.py | 38 ++++++++++++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index e4303f91877..03f26f6ce9f 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -31,7 +31,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.juliapackage import EXTS_FILTER_JULIA_PACKAGES, JuliaPackage +from easybuild.easyblocks.generic.juliapackage import JuliaPackage class JuliaBundle(Bundle, JuliaPackage): @@ -61,7 +61,6 @@ def __init__(self, *args, **kwargs): self.cfg['exts_default_options'] = { 'easyblock': 'JuliaPackage', } - self.cfg['exts_filter'] = EXTS_FILTER_JULIA_PACKAGES # need to disable templating to ensure that actual value for exts_default_options is updated... with self.cfg.disable_templating(): @@ -94,7 +93,3 @@ def sanity_check_step(self, *args, **kwargs): 'dirs': [os.path.join('packages', self.name)], } super().sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self, *args, **kwargs): - """Custom module environment from JuliaPackage""" - return super().make_module_extra(*args, **kwargs) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index a9e30f057e5..792a794b8f5 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -54,13 +54,6 @@ "grep -E 'Loading (object )?cache file .*%(grep_loc)s'" ]) -EXTS_FILTER_JULIA_PACKAGES = ( - " && ".join([ - _COMPILECACHE_CHECK.replace('%(grep_loc)s', '%(ext_name)s'), - "julia -e 'using %(ext_name)s'", - ]), - "" -) USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?(.*\.toml)*$") @@ -433,6 +426,14 @@ def prepare_step(self, *args, **kwargs): self.include_pkg_dependencies() + def configure_step(self): + """Configure step for Julia packages is a no-op, as there is no configuration needed before installation.""" + pass + + def build_step(self): + """Build step for Julia packages is a no-op, as there is no build needed before installation.""" + pass + def test_step(self): """ Test the built Julia package. @@ -486,8 +487,14 @@ def _install_step(self, basedir=None): def install_step(self): """Prepare installation environment and install Julia package.""" + if self.cfg['runtest']: + self.pkg_to_test.append(self.name) + self.install_source() - return self._install_step() + res = self._install_step() + self.test_step() + + return res def install_extension(self): """Install Julia package as an extension.""" @@ -514,10 +521,19 @@ def sanity_check_step(self, *args, **kwargs): pkg_dir = os.path.join('packages', self.name) + ext_filters = ["julia -e 'using %(ext_name)s'"] + + cc_check = False + if LooseVersion(self.julia_version) >= LooseVersion('1.8'): + cc_check = True + ext_filters.insert( + 0, + _COMPILECACHE_CHECK % {'ext_name': self.name, 'grep_loc': self.name} + ) + custom_commands = [] # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package - - if not self.is_extension: + if not self.is_extension and cc_check: for dep, root in self.pkg_deps: # root_comp = os.path.join(root, 'compiled') custom_commands.append( @@ -533,7 +549,7 @@ def sanity_check_step(self, *args, **kwargs): } kwargs.update({'custom_paths': custom_paths}) - return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs) + return ExtensionEasyBlock.sanity_check_step(self, (" && ".join(ext_filters), ""), *args, **kwargs) def make_module_extra(self, *args, **kwargs): """ From df08e0a789533063e90d83089aa36de2c7b1d5af Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 16:35:56 +0200 Subject: [PATCH 20/32] Move compilation+build to `build_step` and allow `test_step` to be skipped or ignored --- easybuild/easyblocks/generic/juliapackage.py | 120 +++++++++++-------- 1 file changed, 73 insertions(+), 47 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 792a794b8f5..9e6e9a59b9b 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -39,17 +39,19 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.filetools import move_file from easybuild.tools.run import run_shell_cmd -from easybuild.tools.utilities import trace_msg +from easybuild.tools.utilities import trace_msg, print_msg +from easybuild.tools.hooks import BUILD_STEP, TEST_STEP from easybuild.easyblocks.j.julia import EB_JULIA_DEPOT_PATH_VAR, EB_JULIA_LOAD_PATH_VAR _COMPILECACHE_CHECK = ' | '.join([ # "julia -E 'Base.compilecache_path(Base.identify_package(\"%(ext_name)s\"), Base.get_world_counter())'", # compilecache_path is not part of the stable API and changes arguments across versions - # allow internal cache resolution and use debug statemnts to get the path of the loaded cache + # allow internal cache resolution and use debug statements to get the path of the loaded cache "JULIA_DEBUG=loading julia -e 'using %(ext_name)s' 2>&1 1>/dev/null", "grep -E 'Loading (object )?cache file .*%(grep_loc)s'" ]) @@ -149,6 +151,7 @@ def __init__(self, *args, **kwargs): self._pkg_deps = [] self._pkg_to_test = [] self._tmp_test_dir = None + self._installdir_created = False @property def julia_version(self) -> str: @@ -177,7 +180,7 @@ def conflicts(self): def is_last_extension(self): """Whether this extension is the last one to be installed in the installation.""" if not self.is_extension: - return False + return True return self.master.ext_instances and self.master.ext_instances[-1] is self @@ -395,15 +398,14 @@ def include_pkg_dependencies(self): build_only_deps = self.cfg.dependencies(build_only=True) # add packages found in dependencies to this installation environment for dep in self.cfg.dependencies(): - test_only = dep in build_only_deps dep_name = dep['name'] dep_root = get_software_root(dep_name) dep_env = self.julia_env_path(absolute=True, base=True, basedir=dep_root) - dep_manifest = os.path.join(dep_env, 'Manifest.toml') - if not os.path.isfile(dep_manifest): - self.log.warning("No Manifest.toml file found in dependency %s, skipping: %s", dep_name, dep_env) + if not os.path.isdir(dep_env): + self.log.warning("No environment directory found in dependency %s, skipping: %s", dep_name, dep_env) continue + test_only = dep in build_only_deps if test_only: trace_msg( f"Incorporating Julia package in TEST environment {dep_name}: {dep_env}" @@ -415,11 +417,19 @@ def include_pkg_dependencies(self): ) for pkg_path in self._deps_from_project(dep_env): - self.add_package(pkg_path, test_only=dep in build_only_deps) + self.add_package(pkg_path, test_only=test_only) + + def make_installdir(self): + """Create new installation directory and ensure this is done only once""" + # Stop installdir from being re-created at the install step + if not self._installdir_created: + super().make_installdir() + self._installdir_created = True def prepare_step(self, *args, **kwargs): """Prepare for Julia package installation.""" super().prepare_step(*args, **kwargs) + self.make_installdir() if get_software_root('Julia') is None: raise EasyBuildError("Julia not included as dependency!") @@ -430,9 +440,32 @@ def configure_step(self): """Configure step for Julia packages is a no-op, as there is no configuration needed before installation.""" pass + def install_source(self): + """Add the Julia package source files in the installation directory.""" + if self.cfg['is_test_dependency']: + self.log.debug( + f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" + ) + trace_msg("Installing as a test dependency...") + package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) + else: + package_dir = os.path.join(self.installdir, 'packages', self.name) + + move_file(self.start_dir, package_dir) + self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) + + def _build_install_step(self, basedir=None): + """Install step commons between single-package and extensions based installs.""" + self.install_source() + if self.is_last_extension: + self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) + return self.install_pkg(basedir=basedir) + else: + trace_msg("Delegating build+install to last extension") + def build_step(self): """Build step for Julia packages is a no-op, as there is no build needed before installation.""" - pass + return self._build_install_step() def test_step(self): """ @@ -440,6 +473,13 @@ def test_step(self): :param return_output: return output and exit code of test command """ + if self.cfg['runtest']: + self.pkg_to_test.append(self.name) + + if not self.is_last_extension: + trace_msg("Delegating testing to last extension") + return + if self.pkg_to_test: env = self.julia_env_path(basedir=self.tmp_test_dir) pkg_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) @@ -464,54 +504,40 @@ def test_step(self): self.cfg['testopts'], ]) - run_shell_cmd(cmd) + res = run_shell_cmd(cmd, fail_on_error=False) - def install_source(self): - """Add the Julia package source files in the installation directory.""" - if self.cfg['is_test_dependency']: - self.log.debug( - f"Package {self.name} is only needed for testing, installing sources in {self.tmp_test_dir}" - ) - trace_msg("Installing as a test dependency...") - package_dir = os.path.join(self.tmp_test_dir, 'packages', self.name) - else: - package_dir = os.path.join(self.installdir, 'packages', self.name) - - move_file(self.start_dir, package_dir) - self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) - - def _install_step(self, basedir=None): - """Install step commons between single-package and extensions based installs.""" - self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) - return self.install_pkg(basedir=basedir) + if res.exit_code != 0: + self.report_test_failure(f"Tests failed for Julia package {self.name}:\n{res.output}") def install_step(self): - """Prepare installation environment and install Julia package.""" - if self.cfg['runtest']: - self.pkg_to_test.append(self.name) - - self.install_source() - res = self._install_step() - self.test_step() - - return res + """Install step for Julia packages is a no-op, as there is no build needed before installation.""" + pass - def install_extension(self): + def install_extension(self, *args, **kwargs): """Install Julia package as an extension.""" if not self.src: errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) - # Unpack source into install directory and add package to the main environment Project.toml file - ExtensionEasyBlock.install_extension(self, unpack_src=True) - self.install_source() + super().install_extension(*args, unpack_src=True) - if self.cfg['runtest']: - self.pkg_to_test.append(self.name) - - if self.is_last_extension: - self._install_step() - self.test_step() + steps = [ + (BUILD_STEP, 'building', [lambda x: x._build_install_step], True), + (TEST_STEP, 'testing', [lambda x: x._test_step], True), + ] + self.skip = False # --skip does not apply here + self.silent = build_option('silent') + # See EasyBlock.run_all_steps + for (step_name, descr, step_methods, skippable) in steps: + if self.skip_step(step_name, skippable): + print_msg("\t%s [skipped]" % descr, log=self.log, silent=self.silent) + else: + if self.dry_run: + self.dry_run_msg("\t%s... [DRY RUN]\n", descr) + else: + print_msg("\t%s..." % descr, log=self.log, silent=self.silent) + for step_method in step_methods: + step_method(self)() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for JuliaPackage""" From 3c3b3cbeed25c848d6c2acb0d9d500ebbe119ee5 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 17:28:16 +0200 Subject: [PATCH 21/32] Automatically create and checks for safety wrappers --- easybuild/easyblocks/j/julia.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/j/julia.py b/easybuild/easyblocks/j/julia.py index 21f42322ed8..60c120e2458 100644 --- a/easybuild/easyblocks/j/julia.py +++ b/easybuild/easyblocks/j/julia.py @@ -28,8 +28,9 @@ @author: Davide Grassano (CECAM) """ import os +import stat -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import write_file, move_file, adjust_permissions from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.easyblocks.generic.tarball import Tarball @@ -55,6 +56,10 @@ end """ +WRAPPER_CONTENT = rf"""#!/bin/sh +LD_LIBRARY_PATH="$EBROOTJULIA/lib:$EBROOTJULIA/lib/julia:$LD_LIBRARY_PATH" julia.bin "$@" +""" + class EB_Julia(Tarball): """Add custom startup script and sanity checks to Julia installations.""" @@ -65,7 +70,8 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() custom_files = [ - 'bin/julia', 'include/julia/julia.h', f'lib/libjulia.{shlib_ext}', 'etc/julia/startup.jl', + 'bin/julia', 'bin/julia.bin', + 'include/julia/julia.h', f'lib/libjulia.{shlib_ext}', 'etc/julia/startup.jl', ] custom_dirs = ['bin', 'etc', 'include', 'lib', 'share'] custom_commands = [ @@ -88,3 +94,10 @@ def install_step(self, *args, **kwargs): os.makedirs(os.path.dirname(startup_script), exist_ok=True) write_file(startup_script, STARTUP_CONTENT) + julia_bin = os.path.join(self.installdir, 'bin', 'julia') + julia_bin_new = os.path.join(self.installdir, 'bin', 'julia.bin') + + # install wrapper with linking safeguards for Julia libraries + move_file(julia_bin, julia_bin_new) + write_file(julia_bin, WRAPPER_CONTENT) + adjust_permissions(julia_bin, stat.S_IRUSR | stat.S_IXUSR) From 8560b91038676693ed24ec85849ebdff454d23b0 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 17:36:22 +0200 Subject: [PATCH 22/32] lint --- easybuild/easyblocks/generic/juliapackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 9e6e9a59b9b..76d5f828976 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -560,7 +560,7 @@ def sanity_check_step(self, *args, **kwargs): custom_commands = [] # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package if not self.is_extension and cc_check: - for dep, root in self.pkg_deps: + for dep, _ in self.pkg_deps: # root_comp = os.path.join(root, 'compiled') custom_commands.append( # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} From bedbc765cf58067f753121e7fb9a0ab0de15dce9 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 17:42:38 +0200 Subject: [PATCH 23/32] lint --- easybuild/easyblocks/j/julia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/j/julia.py b/easybuild/easyblocks/j/julia.py index 60c120e2458..d0c7ebfd2b4 100644 --- a/easybuild/easyblocks/j/julia.py +++ b/easybuild/easyblocks/j/julia.py @@ -56,7 +56,7 @@ end """ -WRAPPER_CONTENT = rf"""#!/bin/sh +WRAPPER_CONTENT = r"""#!/bin/sh LD_LIBRARY_PATH="$EBROOTJULIA/lib:$EBROOTJULIA/lib/julia:$LD_LIBRARY_PATH" julia.bin "$@" """ From 82f890049e9f9205be27877c2cafc65368fc9736 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 17:57:24 +0200 Subject: [PATCH 24/32] Clearer attribute name --- easybuild/easyblocks/generic/juliapackage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 76d5f828976..59ac4da739c 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -149,7 +149,7 @@ def __init__(self, *args, **kwargs): self._julia_deps_test = {} self._julia_version = None self._pkg_deps = [] - self._pkg_to_test = [] + self._pkgs_to_test = [] self._tmp_test_dir = None self._installdir_created = False @@ -206,11 +206,11 @@ def pkg_deps(self): return self._pkg_deps @property - def pkg_to_test(self): + def pkgs_to_test(self): """List of Julia dependencies to be included in the test environment.""" if self.is_extension: - return self.master.pkg_to_test - return self._pkg_to_test + return self.master.pkgs_to_test + return self._pkgs_to_test @property def tmp_test_dir(self): @@ -474,16 +474,16 @@ def test_step(self): :param return_output: return output and exit code of test command """ if self.cfg['runtest']: - self.pkg_to_test.append(self.name) + self.pkgs_to_test.append(self.name) if not self.is_last_extension: trace_msg("Delegating testing to last extension") return - if self.pkg_to_test: + if self.pkgs_to_test: env = self.julia_env_path(basedir=self.tmp_test_dir) pkg_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) - pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkg_to_test) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test) testcmd = [ 'using Pkg', f'Pkg.activate("{env}")', From cc6602fd85e9078790ed295e4c84b1dde28fda4d Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 9 Jun 2026 22:56:39 +0200 Subject: [PATCH 25/32] Fix docstrings --- easybuild/easyblocks/generic/juliapackage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 59ac4da739c..87d373d6e9a 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -186,28 +186,28 @@ def is_last_extension(self): @property def julia_deps(self): - """List of Julia dependencies found in this installation.""" + """List of Julia dependencies found in this installation excluding test dependencies.""" if self.is_extension: return self.master.julia_deps return self._julia_deps @property def julia_deps_test(self): - """List of Julia dependencies found in this installation.""" + """List of Julia dependencies found in this installation including test dependencies.""" if self.is_extension: return self.master.julia_deps_test return self._julia_deps_test @property def pkg_deps(self): - """List of Julia dependencies found in this installation.""" + """List of easybuild runtime dependencies that will need to be sanity-checked.""" if self.is_extension: return self.master.pkg_deps return self._pkg_deps @property def pkgs_to_test(self): - """List of Julia dependencies to be included in the test environment.""" + """List extension wiht `runtest` set to true that should be tested.""" if self.is_extension: return self.master.pkgs_to_test return self._pkgs_to_test From db0bdc537a95bc4141d8fa09eeab09de4989d4c4 Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Wed, 10 Jun 2026 10:04:25 +0200 Subject: [PATCH 26/32] Update easybuild/easyblocks/generic/juliapackage.py Co-authored-by: Alexander Grund --- easybuild/easyblocks/generic/juliapackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 87d373d6e9a..786a9ce0e5b 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -207,7 +207,7 @@ def pkg_deps(self): @property def pkgs_to_test(self): - """List extension wiht `runtest` set to true that should be tested.""" + """List extension with `runtest` set to true that should be tested.""" if self.is_extension: return self.master.pkgs_to_test return self._pkgs_to_test From 8441e74e4cf4c649c048f724a9fd219d0d169bf3 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 10:29:34 +0200 Subject: [PATCH 27/32] Improve typehints and docstrings --- easybuild/easyblocks/generic/juliapackage.py | 101 +++++++++++++------ 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 786a9ce0e5b..47e29f85d38 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -33,6 +33,8 @@ import re import tempfile +from typing import List, Dict, Tuple, Union + from easybuild.tools import LooseVersion import easybuild.tools.environment as env @@ -170,50 +172,64 @@ def julia_version(self) -> str: return self._julia_version @property - def conflicts(self): - """List of conflicting software.""" + def conflicts(self) -> List[Tuple[str, str, str, str]]: + """List of 4-tuple conflicting software (same package included from multiple locations) including: + - extension name where conflict is found + - package name with conflict + - previous source of the package that caused the conflict + - new source of the package that caused the conflict + """ if self.is_extension: return self.master.conflicts return self._conflicts @property - def is_last_extension(self): - """Whether this extension is the last one to be installed in the installation.""" + def is_last_extension(self) -> bool: + """Whether this extension is the last one to be installed.""" if not self.is_extension: return True return self.master.ext_instances and self.master.ext_instances[-1] is self @property - def julia_deps(self): - """List of Julia dependencies found in this installation excluding test dependencies.""" + def julia_deps(self) -> Dict[str, str]: + """List of Julia dependencies found in this installation excluding test dependencies + - key: package name + - value: package source + """ if self.is_extension: return self.master.julia_deps return self._julia_deps @property - def julia_deps_test(self): - """List of Julia dependencies found in this installation including test dependencies.""" + def julia_deps_test(self) -> Dict[str, str]: + """List of Julia dependencies found in this installation including test dependencies + - key: package name + - value: package source + """ if self.is_extension: return self.master.julia_deps_test return self._julia_deps_test @property - def pkg_deps(self): - """List of easybuild runtime dependencies that will need to be sanity-checked.""" + def pkg_deps(self) -> List[Tuple[str, str]]: + """List of easybuild runtime dependencies that will need to be sanity-checked. 2-tuple including: + - dependency name + - dependency root (equivalent to get_software_root) + """ if self.is_extension: return self.master.pkg_deps return self._pkg_deps @property - def pkgs_to_test(self): - """List extension with `runtest` set to true that should be tested.""" + def pkgs_to_test(self) -> List[str]: + """List extension names with `runtest` set to True that should be tested.""" if self.is_extension: return self.master.pkgs_to_test return self._pkgs_to_test @property - def tmp_test_dir(self): + def tmp_test_dir(self) -> str: """Temporary path used for running the test step.""" if self.is_extension: return self.master.tmp_test_dir @@ -225,9 +241,14 @@ def tmp_test_dir(self): raise EasyBuildError("Failed to create temporary directory for Julia package testing: %s", str(e)) return self._tmp_test_dir - def julia_env_path(self, absolute=True, base=True, basedir=None): + def julia_env_path(self, absolute: bool = True, base: bool = True, basedir: Union[str, None] = None) -> str: """ Return path to installation environment file. + + :param absolute: whether to return absolute path to environment file or relative path to `basedir` + :param base: whether to return path to environment file or its parent directory + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. """ basedir = basedir or self.installdir julia_version = self.julia_version.split('.') @@ -241,7 +262,7 @@ def julia_env_path(self, absolute=True, base=True, basedir=None): return project_env - def set_pkg_remote(self, online=False): + def set_pkg_remote(self, online: bool = False): """Enable online/offline mode of Julia Pkg""" julia_version = get_software_version('Julia') @@ -257,7 +278,7 @@ def set_pkg_remote(self, online=False): ) raise EasyBuildError(errmsg, julia_version) - def prepare_julia_env(self, basedir=None, online=False): + def prepare_julia_env(self, basedir: Union[str, None] = None, online: bool = False): """ 1. Remove user depot and prepend installation directory to DEPOT_PATH. Top directory in Julia DEPOT_PATH is the target installation directory. @@ -272,6 +293,12 @@ def prepare_julia_env(self, basedir=None, online=False): 3. Enable offline mode in Julia to avoid automatic downloads of packages. 4. Enable automatic precompilation of packages after each build. + + 5. Enable/disable debug mode in Julia to get more verbose output during installation and testing. + + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. + :param online: whether to allow online operations of Julia Pkg """ basedir = basedir or self.installdir # Grab both DEPOT_PATH and LOAD_PATH before any changes are made @@ -304,8 +331,7 @@ def prepare_julia_env(self, basedir=None, online=False): # Enable/disable offline mode self.set_pkg_remote(online=online) - if self.cfg['julia_debug']: - env.setvar('JULIA_DEBUG', 'all') + env.setvar('JULIA_DEBUG', 'all' if self.cfg['julia_debug'] else '') # Set the maximum number of concurrent package builds env.setvar('JULIA_NUM_PRECOMPILE_TASKS', str(self.cfg.parallel)) @@ -313,6 +339,12 @@ def prepare_julia_env(self, basedir=None, online=False): env.setvar('JULIA_PKG_PRECOMPILE_AUTO', 'true') def add_package(self, pkg_source, test_only=False): + """Add a Julia package to the list of dependencies to be installed. + + :param pkg_source: path to the package source to be added as dependency + :param test_only: whether this package is only needed for testing and should not be added to installation + environment + """ pkg_name = os.path.basename(pkg_source) if pkg_name in self.julia_deps: prev_source = self.julia_deps[pkg_name] @@ -334,8 +366,12 @@ def _check_conflicts(self): ) raise EasyBuildError("Conflicts detected in Julia dependencies:\n" + "\n".join(conflict_msgs)) - def install_pkg(self, basedir=None): - """Execute Julia.Pkg command to install package from its sources""" + def install_pkg(self, basedir: Union[str, None] = None): + """Execute Julia.Pkg command to install all packages in `self.julia_deps` in the install environment. + + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified + """ self._check_conflicts() basedir = basedir or self.installdir env = self.julia_env_path(basedir=basedir) @@ -373,8 +409,11 @@ def install_pkg(self, basedir=None): return res.output - def _deps_from_project(self, environment): - """Get list of dependencies from a Project.toml file""" + def _deps_from_project(self, environment: str) -> List[str]: + """Get list of dependencies from a Julia environment + + :param environment: path to Julia environment to query for dependencies + """ julia_root = get_software_root('Julia') cmd = "; ".join([ 'using Pkg', @@ -454,8 +493,12 @@ def install_source(self): move_file(self.start_dir, package_dir) self.add_package(package_dir, test_only=self.cfg['is_test_dependency']) - def _build_install_step(self, basedir=None): - """Install step commons between single-package and extensions based installs.""" + def _build_install_step(self, basedir: Union[str, None] = None): + """Install step commons between single-package and extensions based installs. + + :param basedir: base directory for the environment, as in env=BASEDIR/environments/v#.#/Project.toml. + Defaults to installation directory of this package if not specified. + """ self.install_source() if self.is_last_extension: self.prepare_julia_env(basedir=basedir, online=self.cfg['download_pkg_deps']) @@ -579,13 +622,9 @@ def sanity_check_step(self, *args, **kwargs): def make_module_extra(self, *args, **kwargs): """ - Module load initializes JULIA_DEPOT_PATH and JULIA_LOAD_PATH with default values if they are not set. - - Path to installation directory is appended to JULIA_DEPOT_PATH. - Path to the environment file of this installation is prepended to JULIA_LOAD_PATH. - This configuration fulfils the rule that user depot has to be the first path in JULIA_DEPOT_PATH, - allowing user to add custom Julia packages while having packages in this installation available. - See issue easybuilders/easybuild-easyconfigs#17455 + Set EB-specific PATH like variables `EB_JULIA_DEPOT_PATH` and `EB_JULIA_LOAD_PATH`. + These are then used by Easybuild's custom Julia startup script (etc/julia/startup.jl) to set JULIA_DEPOT_PATH + and JULIA_LOAD_PATH correctly at runtime. """ mod = super().make_module_extra() From 6e5f0c4da140090b7b7fabb560d13362de511b58 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 10:34:30 +0200 Subject: [PATCH 28/32] Missing author --- easybuild/easyblocks/generic/juliabundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 03f26f6ce9f..3352505ca13 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -26,6 +26,7 @@ EasyBuild support for bundles of Julia packages, implemented as an easyblock @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Davide Grassano (CECAM, EPFL) """ import os From 482c8fb2da2e0d93e254c62e056ecabad690208f Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 12:16:29 +0200 Subject: [PATCH 29/32] Move check into JuliaPackage --- easybuild/easyblocks/generic/juliabundle.py | 4 ---- easybuild/easyblocks/generic/juliapackage.py | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 3352505ca13..fab9d99a06f 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -30,7 +30,6 @@ """ import os -from easybuild.tools.build_log import EasyBuildError from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.juliapackage import JuliaPackage @@ -82,9 +81,6 @@ def __init__(self, *args, **kwargs): } ] - if self.cfg['is_test_dependency']: - raise EasyBuildError("Test dependencies can only be defined as an extension, not as a bundle itself") - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 47e29f85d38..2625ea20320 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -146,6 +146,10 @@ def get_julia_env(env_var): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + + if self.cfg['is_test_dependency'] and not self.is_extension: + raise EasyBuildError("Test dependencies can only be defined as an extension, not as a bundle itself") + self._conflicts = [] self._julia_deps = {} self._julia_deps_test = {} From beaa68fa2f1d31de186fa38e382086d63fdc12b4 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 14:17:51 +0200 Subject: [PATCH 30/32] Allow both online and offline tests --- easybuild/easyblocks/generic/juliapackage.py | 80 ++++++++++++++------ 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 2625ea20320..5005ffb81f4 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -155,7 +155,8 @@ def __init__(self, *args, **kwargs): self._julia_deps_test = {} self._julia_version = None self._pkg_deps = [] - self._pkgs_to_test = [] + self._pkgs_to_test_online = [] + self._pkgs_to_test_offline = [] self._tmp_test_dir = None self._installdir_created = False @@ -226,11 +227,18 @@ def pkg_deps(self) -> List[Tuple[str, str]]: return self._pkg_deps @property - def pkgs_to_test(self) -> List[str]: - """List extension names with `runtest` set to True that should be tested.""" + def pkgs_to_test_online(self) -> List[str]: + """List extension names with `runtest` set to True that should be tested in online mode.""" if self.is_extension: - return self.master.pkgs_to_test - return self._pkgs_to_test + return self.master.pkgs_to_test_online + return self._pkgs_to_test_online + + @property + def pkgs_to_test_offline(self) -> List[str]: + """List extension names with `runtest` set to True that should be tested in offline mode.""" + if self.is_extension: + return self.master.pkgs_to_test_offline + return self._pkgs_to_test_offline @property def tmp_test_dir(self) -> str: @@ -521,33 +529,59 @@ def test_step(self): :param return_output: return output and exit code of test command """ if self.cfg['runtest']: - self.pkgs_to_test.append(self.name) + if self.cfg['test_online']: + self.pkgs_to_test_online.append(self.name) + else: + self.pkgs_to_test_offline.append(self.name) if not self.is_last_extension: trace_msg("Delegating testing to last extension") return - if self.pkgs_to_test: - env = self.julia_env_path(basedir=self.tmp_test_dir) + env = self.julia_env_path(basedir=self.tmp_test_dir) + testcmd = [ + 'using Pkg', + f'Pkg.activate("{env}")', + # f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)', + # 'Pkg.test([{pkg_test}])', + ] + if self.julia_deps_test: pkg_specs = ', '.join(f'PackageSpec(path="{path}")' for path in self.julia_deps_test.values()) - pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test) - testcmd = [ - 'using Pkg', - f'Pkg.activate("{env}")', - f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)', - f'Pkg.test([{pkg_test}])', - ] - testcmd = '; '.join(testcmd) - testcmd = f"julia -e '{testcmd}'" - - # Also add normal DEPOT/LOAD paths to environment to try and re-use pre-compiled caches as much as possible - # But also ensure that artifacts for packages that do not need recompilation are still found in the tests - self.prepare_julia_env(online=self.cfg['test_online']) - self.prepare_julia_env(basedir=self.tmp_test_dir, online=self.cfg['test_online']) + testcmd.append(f'Pkg.develop([{pkg_specs}]; preserve=PRESERVE_ALL)') + testcmd.append('Pkg.test([{pkg_test}])') + testcmd = '; '.join(testcmd) + testcmd = f"julia -e '{testcmd}'" + + # Also add normal DEPOT/LOAD paths to environment to try and re-use pre-compiled caches as much as possible + # But also ensure that artifacts for packages that do not need recompilation are still found in the tests + self.prepare_julia_env() + self.prepare_julia_env(basedir=self.tmp_test_dir) + + # Run offline tests first just in case the online ones could modify the environment + if self.pkgs_to_test_offline: + trace_msg("Running offline tests:") + self.set_pkg_remote(online=False) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test_offline) + + cmd = ' '.join([ + self.cfg['pretestopts'], + testcmd.format(pkg_test=pkg_test), + self.cfg['testopts'], + ]) + + res = run_shell_cmd(cmd, fail_on_error=False) + + if res.exit_code != 0: + self.report_test_failure(f"Offline tests failed for Julia package {self.name}:\n{res.output}") + + if self.pkgs_to_test_online: + trace_msg("Running online tests:") + self.set_pkg_remote(online=True) + pkg_test = ', '.join(f'"{pkg}"' for pkg in self.pkgs_to_test_online) cmd = ' '.join([ self.cfg['pretestopts'], - testcmd, + testcmd.format(pkg_test=pkg_test), self.cfg['testopts'], ]) From 764e92553cdf2ffb360bdff95ac1c776a3af6cd4 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 14:18:28 +0200 Subject: [PATCH 31/32] Bundle itself should not be treated as an extension in the sanity checks: allow arbitrary bundle name --- easybuild/easyblocks/generic/juliabundle.py | 11 ++-- easybuild/easyblocks/generic/juliapackage.py | 54 +++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index fab9d99a06f..835f59e6085 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -81,12 +81,7 @@ def __init__(self, *args, **kwargs): } ] - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + # The name of the bundle can be arbitrary and not necessarily a Julia package + self.cfg['exts_filter'] = None - def sanity_check_step(self, *args, **kwargs): - """Custom sanity check for bundle of Julia packages""" - custom_paths = { - 'files': [], - 'dirs': [os.path.join('packages', self.name)], - } - super().sanity_check_step(custom_paths=custom_paths) + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 5005ffb81f4..4f26ffe0c4e 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -626,37 +626,45 @@ def sanity_check_step(self, *args, **kwargs): self.log.debug(f"Package {self.name} is only used for testing, skipping sanity check") return (True, "Test dependency, skipping sanity check") - pkg_dir = os.path.join('packages', self.name) - - ext_filters = ["julia -e 'using %(ext_name)s'"] - + exts_filter = ["julia -e 'using %(ext_name)s'"] cc_check = False if LooseVersion(self.julia_version) >= LooseVersion('1.8'): cc_check = True - ext_filters.insert( - 0, - _COMPILECACHE_CHECK % {'ext_name': self.name, 'grep_loc': self.name} - ) + exts_filter.insert(0, _COMPILECACHE_CHECK % {'ext_name': self.name, 'grep_loc': self.name}) - custom_commands = [] - # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package - if not self.is_extension and cc_check: - for dep, _ in self.pkg_deps: - # root_comp = os.path.join(root, 'compiled') - custom_commands.append( - # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} - _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} + if self.is_extension: + pkg_dir = os.path.join('packages', self.name) - ) - kwargs.setdefault('custom_commands', []).extend(custom_commands) + custom_commands = [] + # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package + if not self.is_extension and cc_check: + for dep, _ in self.pkg_deps: + # root_comp = os.path.join(root, 'compiled') + custom_commands.append( + # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} + _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} + + ) + kwargs.setdefault('custom_commands', []).extend(custom_commands) + + custom_paths = { + 'files': [], + 'dirs': [pkg_dir], + } + + exts_filter = " && ".join(exts_filter) + exts_filter = (exts_filter, "") + + self.cfg['exts_filter'] = exts_filter + else: + custom_paths = { + 'files': [], + 'dirs': ['packages', 'environments'], + } - custom_paths = { - 'files': [], - 'dirs': [pkg_dir], - } kwargs.update({'custom_paths': custom_paths}) - return ExtensionEasyBlock.sanity_check_step(self, (" && ".join(ext_filters), ""), *args, **kwargs) + return super().sanity_check_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """ From e100f571271aa2ed9ddad1c3e66b14fc3f37f164 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 10 Jun 2026 17:08:13 +0200 Subject: [PATCH 32/32] lint --- easybuild/easyblocks/generic/juliabundle.py | 2 -- easybuild/easyblocks/generic/juliapackage.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 835f59e6085..77dfd5d43eb 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -28,8 +28,6 @@ @author: Alex Domingo (Vrije Universiteit Brussel) @author: Davide Grassano (CECAM, EPFL) """ -import os - from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.juliapackage import JuliaPackage diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 4f26ffe0c4e..f063d3ba1f2 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -636,12 +636,11 @@ def sanity_check_step(self, *args, **kwargs): pkg_dir = os.path.join('packages', self.name) custom_commands = [] - # Check that the compile cache of the dependencies can still be loaded and is coming from the expected package + # Check that the compile cache of the dependencies can still be loaded if not self.is_extension and cc_check: for dep, _ in self.pkg_deps: # root_comp = os.path.join(root, 'compiled') custom_commands.append( - # _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': f'Loading object cache file .*{root_comp}'} _COMPILECACHE_CHECK % {'ext_name': dep, 'grep_loc': dep} )