diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index cdd1df2b..213fc0b2 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,6 +9,6 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00e8d72b..b5dae0e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: rm -r $ZEEKROOT/etc/zkg rm -r $ZEEKROOT/lib/zeek/python/zeekpkg rm -r $ZEEKROOT/var/lib/zkg - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 @@ -58,7 +58,7 @@ jobs: - name: Run tests run: uv run pytest - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: failure() with: name: btest-${{ matrix.repo }}-${{ matrix.version }} @@ -69,7 +69,7 @@ jobs: needs: [test] if: github.repository == 'zeek/package-manager' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check release version # This fails e.g. if VERSION contains a dev commits suffix, # since we don't want to push these to PyPI. Accepts two- diff --git a/.gitignore b/.gitignore index 508fb1a4..ddcfc6c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ *.pyc *.egg-info *.swp +.coverage .state doc/_build build dist +uv.lock diff --git a/.update-changes.cfg b/.update-changes.cfg index 3b59292f..a60426b5 100644 --- a/.update-changes.cfg +++ b/.update-changes.cfg @@ -5,7 +5,7 @@ function replace_python_package_version file=$1 version=$2 - cat $file | sed "s#^\\( *__version__ *= *\\)\"\\([0-9.-]\\{1,\\}\\)\"#\1\"$version\"#g" >$file.tmp + cat $file | sed "s#^\\( *VERSION *= *\\)\"\\([0-9.-]\\{1,\\}\\)\"#\1\"$version\"#g" >$file.tmp cat $file.tmp >$file rm -f $file.tmp git add $file @@ -14,7 +14,7 @@ function replace_python_package_version function new_version_hook { version=$1 - replace_python_package_version zeekpkg/__init__.py $version + replace_python_package_version zeekpkg/consts.py $version make man git add ./doc/man diff --git a/doc/package.rst b/doc/package.rst index c403f06a..4276c444 100644 --- a/doc/package.rst +++ b/doc/package.rst @@ -549,6 +549,10 @@ script components, the "plugin" part is always unconditionally loaded by Zeek, but the "script" components must either be explicitly loaded (e.g. :samp:`@load {}`) or the package marked as :ref:`loaded `. +:program:`zkg` captures the output of the build command into a +`-build.log` file in its logs directory, located in +`/logs/`. + .. _metadata-interpolation: Value Interpolation @@ -652,6 +656,10 @@ An example :file:`zkg.meta`:: The recommended test framework for writing package unit tests is `btest`_. See its documentation for further explanation and examples. +:program:`zkg` captures the output of the test command into a +`-test.log` file in its logs directory, located in +`/logs/`. + .. note:: :program:`zkg` version 2.12.0 introduced two improvements to `test_command`: diff --git a/testing/baselines/tests.aliases-bad-2/.stderr b/testing/baselines/tests.aliases-bad-2/.stderr index 6e82d527..abe6e831 100644 --- a/testing/baselines/tests.aliases-bad-2/.stderr +++ b/testing/baselines/tests.aliases-bad-2/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: invalid package "foo": invalid alias ".hidden" +Error: invalid package "foo": invalid alias ".hidden" diff --git a/testing/baselines/tests.aliases-bad-3/.stderr b/testing/baselines/tests.aliases-bad-3/.stderr index 15176aaa..6450b8cf 100644 --- a/testing/baselines/tests.aliases-bad-3/.stderr +++ b/testing/baselines/tests.aliases-bad-3/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: invalid package "foo": invalid alias "/aaa" +Error: invalid package "foo": invalid alias "/aaa" diff --git a/testing/baselines/tests.aliases-bad/.stderr b/testing/baselines/tests.aliases-bad/.stderr index d3d14772..a3e11074 100644 --- a/testing/baselines/tests.aliases-bad/.stderr +++ b/testing/baselines/tests.aliases-bad/.stderr @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: invalid package "foo": invalid alias "../../../../../../../../bad" +Error: invalid package "foo": invalid alias "../../../../../../../../bad" diff --git a/testing/baselines/tests.aliases-conflict/out b/testing/baselines/tests.aliases-conflict/out index dfec4a06..a9497d7f 100644 --- a/testing/baselines/tests.aliases-conflict/out +++ b/testing/baselines/tests.aliases-conflict/out @@ -3,15 +3,15 @@ Installing "one/alice/foo" Installed "one/alice/foo" (main) Loaded "one/alice/foo" Installing "one/alice/bar" -error: incomplete installation, the follow packages failed to be installed: +Error: incomplete installation, the follow packages failed to be installed: one/alice/bar (master) Failed installing "one/alice/bar": name "bar" conflicts with alias from "one/alice/foo" Installing "one/alice/baz" -error: incomplete installation, the follow packages failed to be installed: +Error: incomplete installation, the follow packages failed to be installed: one/alice/baz (master) Failed installing "one/alice/baz": alias "foo" conflicts with name of installed package "one/alice/foo" Installing "one/bob/corge" -error: incomplete installation, the follow packages failed to be installed: +Error: incomplete installation, the follow packages failed to be installed: one/bob/corge (master) Failed installing "one/bob/corge": alias "bar" conflicts with alias of installed package "one/alice/foo" Removed "one/alice/foo" @@ -22,6 +22,6 @@ Installing "one/bob/corge" Installed "one/bob/corge" (master) Loaded "one/bob/corge" Installing "one/alice/foo" -error: incomplete installation, the follow packages failed to be installed: +Error: incomplete installation, the follow packages failed to be installed: one/alice/foo (main) Failed installing "one/alice/foo": name "foo" conflicts with alias from "one/alice/baz" diff --git a/testing/baselines/tests.builtin-install/out b/testing/baselines/tests.builtin-install/out index 8e298d2f..b49b3afb 100644 --- a/testing/baselines/tests.builtin-install/out +++ b/testing/baselines/tests.builtin-install/out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -cannot install "spicy-plugin": built-in package +Warning: cannot install "spicy-plugin": built-in package diff --git a/testing/baselines/tests.builtin-spicy-version-error/output b/testing/baselines/tests.builtin-spicy-version-error/output index 621a1116..957dcfb8 100644 --- a/testing/baselines/tests.builtin-spicy-version-error/output +++ b/testing/baselines/tests.builtin-spicy-version-error/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: failed to resolve dependencies: unsatisfiable dependency: "zeek-builtin/spicy-plugin" (X.X.X) is installed, but "one/alice/foo" requires = 0.63. Running unit tests for "one/alice/foo" Installing "one/alice/foo" Installed "one/alice/foo" (main) diff --git a/testing/baselines/tests.builtin-test/out b/testing/baselines/tests.builtin-test/out index d0a3761f..50900908 100644 --- a/testing/baselines/tests.builtin-test/out +++ b/testing/baselines/tests.builtin-test/out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -cannot run tests for "spicy-plugin": built-in package +Warning: cannot run tests for "spicy-plugin": built-in package diff --git a/testing/baselines/tests.bundle-duplicate-package-name/.stderr b/testing/baselines/tests.bundle-duplicate-package-name/.stderr index e1b44985..fc872f1f 100644 --- a/testing/baselines/tests.bundle-duplicate-package-name/.stderr +++ b/testing/baselines/tests.bundle-duplicate-package-name/.stderr @@ -2,4 +2,4 @@ Switched to a new branch 'main' Switched to a new branch 'drop-corge' Switched to branch 'master' -error: failed to create bundle: duplicate package name "foo": remove one of "<...>/foo", "<...>/foo" from the manifest +Error: failed to create bundle: duplicate package name "foo": remove one of "<...>/foo", "<...>/foo" from the manifest diff --git a/testing/baselines/tests.bundled-zkg/output b/testing/baselines/tests.bundled-zkg/output deleted file mode 100644 index 09b7c5e9..00000000 --- a/testing/baselines/tests.bundled-zkg/output +++ /dev/null @@ -1,3 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -/tests.bundled-zkg/install-dir/bin/zkg -zkg xxx diff --git a/testing/baselines/tests.install-duplicate-dependency-name/.stderr b/testing/baselines/tests.install-duplicate-dependency-name/.stderr index 40db81b1..de913cf2 100644 --- a/testing/baselines/tests.install-duplicate-dependency-name/.stderr +++ b/testing/baselines/tests.install-duplicate-dependency-name/.stderr @@ -2,4 +2,4 @@ Switched to a new branch 'main' Switched to a new branch 'drop-corge' Switched to branch 'master' -error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo" +Error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo" diff --git a/testing/baselines/tests.install-duplicate-package-name/.stderr b/testing/baselines/tests.install-duplicate-package-name/.stderr index 40db81b1..de913cf2 100644 --- a/testing/baselines/tests.install-duplicate-package-name/.stderr +++ b/testing/baselines/tests.install-duplicate-package-name/.stderr @@ -2,4 +2,4 @@ Switched to a new branch 'main' Switched to a new branch 'drop-corge' Switched to branch 'master' -error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo" +Error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo" diff --git a/testing/baselines/tests.install-force-build-fail/out b/testing/baselines/tests.install-force-build-fail/out index 35887d4a..b994c354 100644 --- a/testing/baselines/tests.install-force-build-fail/out +++ b/testing/baselines/tests.install-force-build-fail/out @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. Installing "one<...>/foo" -error: incomplete installation, the follow packages failed to be installed: +Error: incomplete installation, the follow packages failed to be installed: one<...>/foo (main) Failed installing "one<...>/foo": package build_command failed, see log in <...>/foo-build.log have_load_bar=1 diff --git a/testing/baselines/tests.install-force-test-fail/out b/testing/baselines/tests.install-force-test-fail/out index a8caf1d7..1217fa55 100644 --- a/testing/baselines/tests.install-force-test-fail/out +++ b/testing/baselines/tests.install-force-test-fail/out @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. === install without skiptests -error: failed to run tests for one<...>/foo: test_command failed with exit code 1 +Error: failed to run tests for one<...>/foo: test_command failed with exit code 1 Running unit tests for "one<...>/foo" have_load_foo=0 === install with skiptests diff --git a/testing/baselines/tests.install-invalid/output b/testing/baselines/tests.install-invalid/output index 4e53fe29..3476f2fa 100644 --- a/testing/baselines/tests.install-invalid/output +++ b/testing/baselines/tests.install-invalid/output @@ -1,8 +1,8 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: invalid package ".": Package name ' fronting-whitespace' is not valid. -error: invalid package ".": Package name 'trailing-whitespace ' is not valid. -error: path ./packages/doesntexist is not a git repository -error: path ./packages/notagitrepo is not a git repository -error: local git clone at ./packages/dirtyrepo is dirty -error: local git clone at dirtyrepo is dirty -error: local git clone at ./dirtyrepo is dirty +Error: invalid package ".": Package name ' fronting-whitespace' is not valid. +Error: invalid package ".": Package name 'trailing-whitespace ' is not valid. +Error: path ./packages/doesntexist is not a git repository +Error: path ./packages/notagitrepo is not a git repository +Error: local git clone at ./packages/dirtyrepo is dirty +Error: local git clone at dirtyrepo is dirty +Error: local git clone at ./dirtyrepo is dirty diff --git a/testing/baselines/tests.install-reserved/output b/testing/baselines/tests.install-reserved/output index 4c2a908a..3c4c9c1e 100644 --- a/testing/baselines/tests.install-reserved/output +++ b/testing/baselines/tests.install-reserved/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: invalid package ".": Package name 'packages' is not valid. +Error: invalid package ".": Package name 'packages' is not valid. diff --git a/testing/baselines/tests.installed-dependency-conflict/conflict.out b/testing/baselines/tests.installed-dependency-conflict/conflict.out index 50155b9f..172d6334 100644 --- a/testing/baselines/tests.installed-dependency-conflict/conflict.out +++ b/testing/baselines/tests.installed-dependency-conflict/conflict.out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: failed to resolve dependencies: unsatisfiable dependency: "one/alice/bar" (1.0.0) is installed, but "one/alice/foo" requires =2.0.0 (1.0.0 not in =2.0.0) +Error: failed to resolve dependencies: unsatisfiable dependency: "one/alice/bar" (1.0.0) is installed, but "one/alice/foo" requires =2.0.0 (1.0.0 not in =2.0.0) diff --git a/testing/baselines/tests.template-create/output b/testing/baselines/tests.template-create/output index 16e19565..229f4e3c 100644 --- a/testing/baselines/tests.template-create/output +++ b/testing/baselines/tests.template-create/output @@ -6,6 +6,6 @@ <...>/out1 (installed: master) - TODO: A more detailed description of test1. <...>/out2 (installed: master) - TODO: A more detailed description of test2. <...>/out3 (installed: master) - TODO: A more detailed description of test3. -error: could not determine value of user variable "name", provide via environment or --user-var -error: the following features are unknown: "doesntexist". Template "foo" offers "readme". +Error: could not determine value of user variable "name", provide via environment or --user-var +Error: the following features are unknown: "doesntexist". Template "foo" offers "readme". "foo" will use value of "name" (the name of the package, e.g. "FooBar") from command line: 你こ안مγПनสXש😊©GBCDČM diff --git a/testing/baselines/tests.unsatisfied-version-tag-dependency/fail.out b/testing/baselines/tests.unsatisfied-version-tag-dependency/fail.out index 46e4b722..ca9117a0 100644 --- a/testing/baselines/tests.unsatisfied-version-tag-dependency/fail.out +++ b/testing/baselines/tests.unsatisfied-version-tag-dependency/fail.out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: failed to resolve dependencies: "one/alice/bar" has no version satisfying dependencies: +Error: failed to resolve dependencies: "one/alice/bar" has no version satisfying dependencies: "one/alice/foo" requires: "=1.0.0" diff --git a/testing/baselines/tests.upgrade-test-fail/out b/testing/baselines/tests.upgrade-test-fail/out index 206a270a..15b5b49d 100644 --- a/testing/baselines/tests.upgrade-test-fail/out +++ b/testing/baselines/tests.upgrade-test-fail/out @@ -3,7 +3,7 @@ Installing "one<...>/foo" Installed "one<...>/foo" (1.0.2) Loaded "one<...>/foo" === upgrade without skiptests -error: failed to run tests for one<...>/foo: test_command failed with exit code 1 +Error: failed to run tests for one<...>/foo: test_command failed with exit code 1 Running unit tests for "one<...>/foo" === upgrade with skiptest Upgraded "one<...>/foo" (1.0.3) diff --git a/testing/baselines/tests.user-mode/output b/testing/baselines/tests.user-mode/output index 700f4e56..2a82cea5 100644 --- a/testing/baselines/tests.user-mode/output +++ b/testing/baselines/tests.user-mode/output @@ -1,13 +1,12 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -[sources] -one = <...>/sources/one - [paths] -state_dir = <...>/zeekroot/var/lib/zkg -script_dir = <...>/zeekroot/share/zeek/site -plugin_dir = <...>/zeekroot/lib/zeek/plugins bin_dir = <...>/zeekroot/var/lib/zkg/bin -zeek_dist = +plugin_dir = <...>/zeekroot/lib/zeek/plugins +script_dir = <...>/zeekroot/share/zeek/site +state_dir = <...>/zeekroot/var/lib/zkg + +[sources] +one = <...>/sources/one [templates] default = https://github.com/zeek/package-template @@ -22,14 +21,13 @@ Installed "zeek/alice/bar" (master) Loaded "zeek/alice/bar" zeek/alice/bar (installed: master) Successfully wrote config file to <...>/home/testuser/.zkg/config -[sources] - [paths] -state_dir = <...>/home/testuser/.zkg -script_dir = <...>/home/testuser/.zkg/script_dir -plugin_dir = <...>/home/testuser/.zkg/plugin_dir bin_dir = <...>/home/testuser/.zkg/bin -zeek_dist = +plugin_dir = <...>/home/testuser/.zkg/plugin_dir +script_dir = <...>/home/testuser/.zkg/script_dir +state_dir = <...>/home/testuser/.zkg + +[sources] [templates] default = https://github.com/zeek/package-template diff --git a/testing/btest.cfg b/testing/btest.cfg index 5bc86a1f..a43e9e90 100644 --- a/testing/btest.cfg +++ b/testing/btest.cfg @@ -14,3 +14,4 @@ SCRIPTS=%(testbase)s/scripts SOURCES=%(testbase)s/sources PACKAGES=%(testbase)s/packages TEMPLATES=%(testbase)s/templates +ZKG_DEFAULT_SOURCE= diff --git a/testing/scripts/zkg b/testing/scripts/zkg index 5ff78184..0b471227 100755 --- a/testing/scripts/zkg +++ b/testing/scripts/zkg @@ -2,6 +2,10 @@ force="" +# This is brittle: any test saying e.g. "zkg -vvv install" instead of "zkg +# install" will no longer have --force. This should either be more robustly +# parsed or set force via an env var or dedicated command line option. + case "$1" in install) force="--force" diff --git a/testing/scripts/zkg-zeek b/testing/scripts/zkg-zeek index 79b0fa14..1698e438 100755 --- a/testing/scripts/zkg-zeek +++ b/testing/scripts/zkg-zeek @@ -11,7 +11,4 @@ export HOME=$(pwd)/home/testuser export ZEEK_ZKG_CONFIG_DIR=$(pwd)/zeekroot/etc/zkg export ZEEK_ZKG_STATE_DIR=$(pwd)/zeekroot/var/lib/zkg -# Don't use a default package source in zkg: -export ZKG_DEFAULT_SOURCE= - $TEST_BASE/../zkg "$@" diff --git a/testing/test_manager.py b/testing/test_manager.py index 33be4cf8..3fe91103 100644 --- a/testing/test_manager.py +++ b/testing/test_manager.py @@ -6,18 +6,24 @@ import pytest +from zeekpkg.config import CONFIG from zeekpkg.manager import Manager from zeekpkg.package import Package, PackageInfo @pytest.fixture def manager(tmp_path: Path) -> Manager: + CONFIG.read_dict( + { + "paths": { + "state_dir": str(tmp_path / "plugins"), + "script_dir": str(tmp_path / "scripts"), + "plugin_dir": str(tmp_path / "plugins"), + }, + }, + ) with patch.object(Manager, "discover_builtin_packages", return_value=[]): - return Manager( - state_dir=str(tmp_path / "state"), - script_dir=str(tmp_path / "scripts"), - plugin_dir=str(tmp_path / "plugins"), - ) + return Manager() class TestInfoCache: @@ -84,7 +90,10 @@ def _mock_stat(self, mock_stat: MagicMock, key: dict[str, object]) -> None: mock_stat.return_value.st_size = key["zeek_size"] def test_disk_cache_hit_skips_subprocess(self, manager: Manager) -> None: - cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json") + cache_file = os.path.join( + CONFIG.state_dir(), + "zeek_build_info_cache.json", + ) with open(cache_file, "w") as f: json.dump({"key": self._CACHE_KEY, "build_info": self._FAKE_BUILD_INFO}, f) @@ -118,7 +127,10 @@ def test_disk_cache_miss_runs_subprocess(self, manager: Manager) -> None: mock_check_output.assert_called_once() def test_stale_mtime_reruns_subprocess(self, manager: Manager) -> None: - cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json") + cache_file = os.path.join( + CONFIG.state_dir(), + "zeek_build_info_cache.json", + ) stale_key = {**self._CACHE_KEY, "zeek_mtime": 1000.0} with open(cache_file, "w") as f: @@ -139,7 +151,10 @@ def test_stale_mtime_reruns_subprocess(self, manager: Manager) -> None: mock_check_output.assert_called_once() def test_different_path_reruns_subprocess(self, manager: Manager) -> None: - cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json") + cache_file = os.path.join( + CONFIG.state_dir(), + "zeek_build_info_cache.json", + ) other_key = {**self._CACHE_KEY, "zeek_path": "/opt/zeek/bin/zeek"} with open(cache_file, "w") as f: diff --git a/testing/tests/aliases-bad b/testing/tests/aliases-bad index 9dfbfc4d..45f3d07f 100644 --- a/testing/tests/aliases-bad +++ b/testing/tests/aliases-bad @@ -1,7 +1,7 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC-FAIL: zkg install foo -# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep ^error' btest-diff .stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep ^Error' btest-diff .stderr cd packages/foo echo 'aliases = ../../../../../../../../bad' >> zkg.meta diff --git a/testing/tests/bundled-zkg b/testing/tests/bundled-zkg deleted file mode 100644 index 0508ca44..00000000 --- a/testing/tests/bundled-zkg +++ /dev/null @@ -1,17 +0,0 @@ -# @TEST-DOC: Simulate bundling of zkg in Zeek and ensure zkg works. This got broken during a cleanup previously. -# @TEST-EXEC: bash %INPUT > output -# @TEST-EXEC: btest-diff output - -set -eu - -# Place zkg and zeekpkg into bin and lib/python directories and do -# the templating. -mkdir -p install-dir/{bin,lib/python} -cp -R $TEST_BASE/../zeekpkg ./install-dir/lib/python -cp $TEST_BASE/../zkg ./install-dir/bin -sed -i -e "s,@PY_MOD_INSTALL_DIR@,$(pwd)/install-dir/lib/python," ./install-dir/bin/zkg - -export PATH=$(pwd)/install-dir/bin:$PATH -# Ensure we're using the right zkg -command -v zkg | sed -E 's,^.*/(.*/.*/.*/zkg),/\1,' -zkg --version | sed 's/zkg [0-9.-]*/zkg xxx/' diff --git a/testing/tests/test b/testing/tests/test index 3abdb5e7..18faebcd 100644 --- a/testing/tests/test +++ b/testing/tests/test @@ -7,6 +7,10 @@ # @TEST-EXEC: bash we_need_to_go_deeper # @TEST-EXEC-FAIL: zkg test rot13 +# Verify that the build and test logs exist afterward: +# @TEST-EXEC: test -f state/logs/rot13-build.log +# @TEST-EXEC: test -f state/logs/rot13-test.log + echo "$(pwd)/packages/rot13" >> sources/one/bob/zkg.index cd sources/one git commit -am 'add rot13 package' diff --git a/zeekpkg/__init__.py b/zeekpkg/__init__.py index 20a2d8fd..dff25975 100644 --- a/zeekpkg/__init__.py +++ b/zeekpkg/__init__.py @@ -2,20 +2,39 @@ This package defines a Python interface for installing, managing, querying, and performing other operations on Zeek Packages and Package Sources. The main entry point is the :class:`Manager ` class. - -This package provides a logger named ``LOG`` to which logging stream handlers -may be added in order to help log/debug applications. """ -import logging +from . import ( + cli, + config, + consts, + logs, + manager, + package, + source, + template, + ui, + uservar, +) -__version__ = "3.1.0-64" -__all__ = ["manager", "package", "source", "template", "uservar"] # noqa: F405 +__all__ = [ + "CONFIG", + "LOG", + "UI", + "cli", + "config", + "consts", + "logs", + "manager", + "package", + "source", + "template", + "ui", + "uservar", +] -LOG: logging.Logger = logging.getLogger(__name__) -LOG.addHandler(logging.NullHandler()) +__version__ = consts.VERSION -from .manager import * # noqa: F403 -from .package import * # noqa: F403 -from .source import * # noqa: F403 -from .uservar import * # noqa: F403 +from .config import CONFIG +from .logs import LOG +from .ui import UI diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index 3cc1f936..873ec300 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -1,5 +1,5 @@ """ -These are meant to be private utility methods for internal use. +Private utility methods for zkg's internal use. """ import configparser @@ -18,6 +18,10 @@ import git import semantic_version as semver +from .ui import ( + UI, +) + def make_dir(path: str) -> None: """Create a directory or do nothing if it already exists. @@ -35,6 +39,10 @@ def make_dir(path: str) -> None: raise +def file_is_not_empty(path: str) -> bool: + return os.path.isfile(path) and os.path.getsize(path) > 0 + + def normalize_version_tag(tag: str) -> str: """Given version string "vX.Y.Z", returns "X.Y.Z". Returns other input strings unchanged. @@ -113,6 +121,12 @@ def is_within_directory(directory: str, target: str) -> bool: def find_sentence_end(s: str) -> int: + """Returns first index in given string of the end of a sentence. + + A sentence end is either ". " or a "." at the end of the string. + + Returns zero-based index if successful, -1 otherwise. + """ beg = 0 while True: @@ -302,6 +316,67 @@ def git_remote_urls(repo: git.Repo) -> dict[str, str]: return remotes +def active_git_branch(path: str) -> str | None: + try: + repo = git.Repo(path) + except git.NoSuchPathError: + return None + + if not repo.working_tree_dir: + return None + + try: + rval = repo.active_branch + except TypeError: + # return detached commit + rval = repo.head.commit + + if not rval: + return None + + return str(rval) + + +def is_local_git_repo_url(git_url: str) -> bool: + return ( + git_url.startswith(".") or git_url.startswith("/") or is_local_git_repo(git_url) + ) + + +def is_local_git_repo(git_url: str) -> bool: + try: + # The Repo class takes a file system path as first arg. This + # can fail in two ways: (1) the path doesn't exist or isn't + # accessible, (2) it's not the root directory of a git repo. + git.Repo(git_url) + return True + except (git.InvalidGitRepositoryError, git.NoSuchPathError): + return False + + +def is_local_git_repo_dirty(git_url: str) -> bool: + if not is_local_git_repo(git_url): + return False + try: + repo = git.Repo(git_url) + except git.NoSuchPathError: + return False + + return repo.is_dirty(untracked_files=True) + + +def check_local_git_repo(git_url: str) -> bool: + if is_local_git_repo_url(git_url): + if not is_local_git_repo(git_url): + UI.error(f"path {git_url} is not a git repository") + return False + if is_local_git_repo_dirty(git_url): + UI.error(f"local git clone at {git_url} is dirty") + return False + + return True + + def is_sha1(s: str) -> bool: if len(s) != 40: return False diff --git a/zeekpkg/cli.py b/zeekpkg/cli.py new file mode 100644 index 00000000..18c3bde9 --- /dev/null +++ b/zeekpkg/cli.py @@ -0,0 +1,2708 @@ +"""The command-line interface for zkg. + +This includes the argument parser and its command implementations. +""" + +import argparse +import configparser +import filecmp +import io +import json +import os +import re +import shutil +import subprocess +import sys +from collections import OrderedDict +from typing import Any + +from ._util import ( + active_git_branch, + check_local_git_repo, + delete_path, + find_program, + make_dir, + read_zeek_config_line, +) +from .config import ( + CONFIG, +) +from .consts import ( + VERSION, + ZKG_DEFAULT_SOURCE, + ZKG_DEFAULT_TEMPLATE, +) +from .logs import LOG +from .manager import Manager +from .package import ( + BUILTIN_SCHEME, + TRACKING_METHOD_VERSION, + InstalledPackage, + Package, + PackageInfo, + name_from_path, +) +from .template import ( + InputError, + LoadError, + OutputError, + Template, +) +from .ui import ( + UI, + UiCallable, +) +from .uservar import ( + UserVar, +) + + +def prompt_for_user_vars( + manager: Manager, + args: argparse.Namespace, + pkg_infos: list[PackageInfo], +) -> None: + answers: dict[str, str] = {} + + for info in pkg_infos: + name = info.package.qualified_name() + requested_user_vars = info.user_vars() + + if requested_user_vars is None: + UI.error(f'malformed user_vars in "{name}"') + sys.exit(1) + + for uvar in requested_user_vars: + try: + answers[uvar.name()] = uvar.resolve( + name, + args.user_var, + args.force, + ) + except ValueError: + UI.error( + f'could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", + ) + sys.exit(1) + + for key, value in answers.items(): + CONFIG.set("user_vars", key, value) + + if not args.force and answers: + configfile = CONFIG.find_configfile(args) + if not configfile: + UI.warning("could not find config file to save to.") + return + if CONFIG.save(configfile): + UI.info(f"Saved answers to config file: {configfile}") + + +def get_changed_state( + manager: Manager, + saved_state: dict[str, bool], + pkg_lst: list[str], +) -> str: + """Returns the list of packages that have changed loaded state. + + Args: + saved_state (dict): dictionary of saved load state for installed + packages. + + pkg_lst (list): list of package names to be skipped + + Returns: + dep_listing (str): string installed packages that have changed state + + """ + _lst = [name_from_path(_pkg_path) for _pkg_path in pkg_lst] + dep_listing = "" + + for _pkg_name in sorted(manager.installed_package_dependencies()): + if _pkg_name in _lst: + continue + + _ipkg = manager.find_installed_package(_pkg_name) + + if not _ipkg or _ipkg.status.is_loaded == saved_state[_pkg_name]: + continue + + dep_listing += f" {_pkg_name}\n" + + return dep_listing + + +def cmd_test( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.version and len(args.package) > 1: + UI.error('test --version" may only be used for a single package') + sys.exit(1) + + package_infos: list[tuple[PackageInfo, str]] = [] + + for name in args.package: + if not check_local_git_repo(name): + sys.exit(1) + + # If the package to be tested is included with Zeek, don't allow + # to run tests due to the potential of conflicts. + bpkg_info = manager.find_builtin_package(name) + if bpkg_info is not None: + UI.warning(f'cannot run tests for "{name}": built-in package') + sys.exit(1) + + version = args.version if args.version else active_git_branch(name) + package_info = manager.info(name, version=version, prefer_installed=False) + + if package_info.invalid_reason: + UI.error(f'invalid package "{name}": {package_info.invalid_reason}') + sys.exit(1) + + if not version: + version = package_info.best_version() + + package_infos.append((package_info, version)) + + all_passed = True + + for info, version in package_infos: + name = info.package.qualified_name() + + if "test_command" not in info.metadata: + UI.info(f"{name}: no test_command found in metadata, skipping") + continue + + error_msg, passed, test_dir = manager.test( + name, + version, + test_dependencies=True, + ) + + if error_msg: + all_passed = False + UI.error(f'failed to run tests for "{name}": {error_msg}') + continue + + if passed: + UI.info(f"{name}: all tests passed") + else: + all_passed = False + UI.error( + f'error: package "{name}" tests failed, inspect' + f" {manager.package_test_log(info.package.name)} and" + f" the contents of {test_dir}", + ) + + if not all_passed: + sys.exit(1) + + +def cmd_install( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.version and len(args.package) > 1: + UI.error('"install --version" may only be used for a single package') + sys.exit(1) + + package_infos: list[tuple[PackageInfo, str, bool]] = [] + + # For each package, check validity if local, and determine the + # effective version to install. + for name in args.package: + if not check_local_git_repo(name): + sys.exit(1) + + # Outright prevent installing a package that Zeek has built-in. + bpkg_info = manager.find_builtin_package(name) + if bpkg_info is not None: + UI.warning(f'cannot install "{name}": built-in package') + sys.exit(1) + + version = args.version if args.version else active_git_branch(name) + package_info = manager.info(name, version=version, prefer_installed=False) + + if package_info.invalid_reason: + UI.error(f'invalid package "{name}": {package_info.invalid_reason}') + sys.exit(1) + + if not version: + version = package_info.best_version() + + package_infos.append((package_info, version, False)) + + # We modify package_infos below, so copy current state to preserve list of + # packages requested by caller: + orig_pkgs = package_infos.copy() + new_pkgs: list[tuple[PackageInfo, str, bool]] = [] + + # Resolve dependencies. + if not args.nodeps: + to_validate = [] + for info, version, _ in package_infos: + to_validate.append((info.package.qualified_name(), version)) + invalid_reason, new_pkgs = manager.validate_dependencies( + to_validate, + ignore_suggestions=args.nosuggestions, + ) + + if invalid_reason: + UI.error("failed to resolve dependencies:", invalid_reason) + sys.exit(1) + + # Report what we're about to do and obtain confirmation, unless suppressed. + if not args.force: + package_listing = "" + + for info, version, _ in sorted(package_infos, key=lambda x: x[0].package.name): + name = info.package.qualified_name() + package_listing += f" {name} ({version})\n" + + UI.info("The following packages will be INSTALLED:") + UI.info(package_listing) + + if new_pkgs: + dependency_listing = "" + + for info, version, suggested in sorted( + new_pkgs, + key=lambda x: x[0].package.name, + ): + name = info.package.qualified_name() + dependency_listing += f" {name} ({version})" + + if suggested: + dependency_listing += " (suggested)" + + dependency_listing += "\n" + + UI.info("The following dependencies will be INSTALLED:") + UI.info(dependency_listing) + + allpkgs = package_infos + new_pkgs + extdep_listing = "" + + for info, version, _ in sorted(allpkgs, key=lambda x: x[0].package.name): + name = info.package.qualified_name() + extdeps = info.dependencies(field="external_depends") + + if extdeps is None: + extdep_listing += f" from {name} ({version}):\n \n" + continue + + if extdeps: + extdep_listing += f" from {name} ({version}):\n" + + for extdep, semver in sorted(extdeps.items()): + extdep_listing += f" {extdep} {semver}\n" + + if extdep_listing: + UI.info( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):", + ) + UI.info(extdep_listing) + + if not UI.confirmation_prompt("Proceed?"): + return + + package_infos += new_pkgs + + prompt_for_user_vars( + manager, + args, + [info for info, _, _ in package_infos], + ) + + # Run tests on explicitly requested packages. With --force, this + # exits with error if tests fail. + # TODO: this prints output even with --force + if not args.skiptests: + # Iterate only over the requested packages here, skipping the + # dependencies. We ask the manager to include dependencies below. + for info, version, _ in orig_pkgs: + name = info.package.qualified_name() + + if "test_command" not in info.metadata: + LOG.info( + f'Skipping unit tests for "{name}": no test_command in metadata', + ) + continue + + UI.info(f'Running unit tests for "{name}"') + error_msg = "" + # For testing we always process dependencies, since the tests might + # well fail without them. If the user wants --nodeps and the tests + # fail because of it, they'll also need to say --skiptests. + error, passed, test_dir = manager.test( + name, + version, + test_dependencies=True, + ) + if error: + error_msg = f"failed to run tests for {name}: {error}" + elif not passed: + error_msg = ( + f'"{name}" tests failed, inspect' + f" {manager.package_test_log(info.package.name)} and" + f" the contents of {test_dir}" + ) + + if error_msg: + UI.error(error_msg) + + if args.force: + sys.exit(1) + + if not UI.confirmation_prompt( + "Proceed to install anyway?", + default_to_yes=False, + ): + return + + installs_failed = [] + + # Install packages in reverse order, which likely is supposed to mean + # dependencies-first? + for info, version, _ in reversed(package_infos): + name = info.package.qualified_name() + + is_overwriting = False + ipkg = manager.find_installed_package(name) + + modifications = [] + backup_files = [] + prev_upstream_config_files = [] + + if ipkg: + is_overwriting = True + modifications = manager.modified_config_files(ipkg) + backup_files = manager.backup_modified_files(name, modifications) + prev_upstream_config_files = manager.save_temporary_config_files(ipkg) + + UI.info(f'Installing "{name}"', flush=True) + # Use default arguments here to avoid late-binding closure: + installer: UiCallable = lambda n=name, v=version: manager.install(n, v) + err = UI.call_activity(installer) + + if err: + UI.info(f'Failed installing "{name}": {err}') + installs_failed.append((name, version)) + continue + + ipkg = manager.find_installed_package(name) + UI.info(f'Installed "{name}" ({ipkg.status.current_version if ipkg else ""})') + + if is_overwriting: + for i, mf in enumerate(modifications): + next_upstream_config_file = mf[1] + + if not os.path.isfile(next_upstream_config_file): + UI.info("\tConfig file no longer exists:") + UI.info("\t\t" + next_upstream_config_file) + UI.info("\tPrevious, locally modified version backed up to:") + UI.info("\t\t" + backup_files[i]) + continue + + prev_upstream_config_file = prev_upstream_config_files[i][1] + + if filecmp.cmp(prev_upstream_config_file, next_upstream_config_file): + # Safe to restore user's version + shutil.copy2(backup_files[i], next_upstream_config_file) + continue + + UI.info("\tConfig file has been overwritten with a different version:") + UI.info("\t\t" + next_upstream_config_file) + UI.info("\tPrevious, locally modified version backed up to:") + UI.info("\t\t" + backup_files[i]) + + if ipkg and manager.has_scripts(ipkg): + load_error = manager.load(name) + + if load_error: + UI.info(f'Failed loading "{name}": {load_error}') + else: + UI.info(f'Loaded "{name}"') + + if not args.nodeps: + # Now load runtime dependencies after all dependencies and suggested + # packages have been installed and loaded. + for info, _, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): + _listing, saved_state = "", manager.loaded_package_states() + name = info.package.qualified_name() + + load_error2 = manager.load_with_dependencies( + name_from_path(name), + ) + + for _name, _error in load_error2: + if not _error: + _listing += f" {_name}\n" + + if not _listing: + dep_listing = get_changed_state(manager, saved_state, [name]) + + if dep_listing: + UI.info( + "The following installed packages were additionally " + "loaded to satisfy runtime dependencies", + ) + UI.info(dep_listing) + + else: + UI.info( + "The following installed packages could NOT be loaded " + f'to satisfy runtime dependencies for "{name}"', + ) + UI.info(_listing) + manager.restore_loaded_package_states(saved_state) + + if installs_failed: + UI.error( + "incomplete installation, the follow packages failed to be installed:", + ) + + for n, v in installs_failed: + UI.error(f" {n} ({v})", prefix="") + + sys.exit(1) + + +def cmd_bundle( + manager: Manager, + args: argparse.Namespace, +) -> None: + packages_to_bundle = [] + prefer_existing_clones = False + + if args.manifest: + if len(args.manifest) == 1 and os.path.isfile(args.manifest[0]): + config = configparser.ConfigParser(delimiters="=") + config.optionxform = lambda optionstr: optionstr # type: ignore + + if config.read(args.manifest[0]) and config.has_section("bundle"): + packages = config.items("bundle") + else: + UI.error(f'"{args.manifest[0]}" is not a valid manifest file') + sys.exit(1) + + else: + packages = [(name, "") for name in args.manifest] + + to_validate = [] + new_pkgs: list[tuple[PackageInfo, str, bool]] = [] + + for name, version in packages: + if not check_local_git_repo(name): + sys.exit(1) + + if not version: + if v := active_git_branch(name): + version = v + + info = manager.info(name, version=version, prefer_installed=False) + + if info.invalid_reason: + UI.error(f'invalid package "{name}": {info.invalid_reason}') + sys.exit(1) + + if not version: + version = info.best_version() + + to_validate.append((info.package.qualified_name(), version)) + packages_to_bundle.append( + ( + info.package.qualified_name(), + info.package.git_url, + version, + False, + False, + ), + ) + + if not args.nodeps: + invalid_reason, new_pkgs = manager.validate_dependencies( + to_validate, + ignore_installed_packages=True, + ignore_suggestions=args.nosuggestions, + ) + + if invalid_reason: + UI.error("failed to resolve dependencies:", invalid_reason) + sys.exit(1) + + for info, version, suggested in new_pkgs: + packages_to_bundle.append( + ( + info.package.qualified_name(), + info.package.git_url, + version, + True, + suggested, + ), + ) + else: + prefer_existing_clones = True + + for ipkg in manager.installed_packages(): + assert ipkg.status.current_version + packages_to_bundle.append( + ( + ipkg.package.qualified_name(), + ipkg.package.git_url, + ipkg.status.current_version, + False, + False, + ), + ) + + if not packages_to_bundle: + UI.error("no packages to put in bundle") + sys.exit(1) + + if not args.force: + package_listing = "" + + for name, git_url, version, is_dependency, is_suggestion in packages_to_bundle: + package_listing += f" {name} ({version})" + + if is_suggestion: + package_listing += " (suggested)" + elif is_dependency: + package_listing += " (dependency)" + + if git_url.startswith(BUILTIN_SCHEME): + package_listing += " (built-in)" + + package_listing += "\n" + + UI.info(f"The following packages will be BUNDLED into {args.bundle_filename}:") + UI.info(package_listing) + + if not UI.confirmation_prompt("Proceed?"): + return + + git_urls = [(git_url, version) for _, git_url, version, _, _ in packages_to_bundle] + error = manager.bundle( + args.bundle_filename, + git_urls, + prefer_existing_clones=prefer_existing_clones, + ) + + if error: + UI.error(f"failed to create bundle: {error}") + sys.exit(1) + + UI.info(f"Bundle successfully written: {args.bundle_filename}") + + +def cmd_unbundle( + manager: Manager, + args: argparse.Namespace, +) -> None: + prev_load_status = {} + + for ipkg in manager.installed_packages(): + prev_load_status[ipkg.package.git_url] = ipkg.status.is_loaded + + if args.replace: + cmd_purge(manager, args) + + error, bundle_info = manager.bundle_info(args.bundle_filename) + + if error: + UI.error("failed to unbundle {args.bundle_filename}: {error}") + sys.exit(1) + + for git_url, _, pkg_info in bundle_info: + if pkg_info.invalid_reason: + name = pkg_info.package.qualified_name() + UI.error( + f"bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}", + ) + sys.exit(1) + + if not bundle_info: + UI.info("No packages in bundle.") + return + + if not args.force: + package_listing = "" + builtin_listing = "" + for git_url, version, info in bundle_info: + name = git_url + + if info.is_builtin(): + builtin_listing += f" from {name} ({version}):\n" + continue + + for pkg in manager.source_packages(): + if pkg.git_url == git_url: + name = pkg.qualified_name() + break + + package_listing += f" {name} ({version})\n" + + UI.info("The following packages will be INSTALLED:") + UI.info(package_listing) + + extdep_listing = "" + + for git_url, version, info in bundle_info: + name = git_url + + for pkg in manager.source_packages(): + if pkg.git_url == git_url: + name = pkg.qualified_name() + break + + extdeps = info.dependencies(field="external_depends") + + if extdeps is None: + extdep_listing += f" from {name} ({version}):\n \n" + continue + + if extdeps: + extdep_listing += f" from {name} ({version}):\n" + + for extdep, semver in sorted(extdeps.items()): + extdep_listing += f" {extdep} {semver}\n" + + if extdep_listing: + UI.info( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):", + ) + UI.info(extdep_listing) + + if not UI.confirmation_prompt("Proceed?"): + return + + prompt_for_user_vars( + manager, + args, + [info for _, _, info in bundle_info], + ) + + error = manager.unbundle(args.bundle_filename) + + if error: + UI.error("failed to unbundle {args.bundle_filename}: {error}") + sys.exit(1) + + for git_url, _, _ in bundle_info: + if git_url in prev_load_status: + need_load = prev_load_status[git_url] + else: + need_load = True + + ipkg2 = manager.find_installed_package(git_url) + + if not ipkg2: + UI.info(f'Skipped loading "{git_url}": failed to install') + continue + + name = ipkg2.package.qualified_name() + + if not need_load: + UI.info(f'Skipped loading "{name}"') + continue + + load_error = manager.load(name) + + if load_error: + UI.info(f'Failed loading "{name}": {load_error}') + else: + UI.info(f'Loaded "{name}"') + + UI.info("Unbundling complete.") + + +def cmd_remove( + manager: Manager, + args: argparse.Namespace, +) -> None: + packages_to_remove: list[InstalledPackage] = [] + + def package_will_be_removed(pkg_name: str) -> bool: + for _ipkg in packages_to_remove: + if _ipkg.package.name == pkg_name: + return True + + return False + + for name in args.package: + ipkg = manager.find_installed_package(name) + + if not ipkg: + UI.error(f'package "{name}" is not installed') + sys.exit(1) + + if ipkg.is_builtin(): + UI.info(f'cannot remove "{name}": built-in package') + sys.exit(1) + + packages_to_remove.append(ipkg) + + dependers_to_unload = set() + + if not args.nodeps: + for ipkg in packages_to_remove: + for pkg_name in manager.list_depender_pkgs(ipkg.package.name): + ipkg = manager.find_installed_package(pkg_name) + + if ipkg and not package_will_be_removed(ipkg.package.name): + if ipkg.status.is_loaded: + dependers_to_unload.add(ipkg.package.name) + + if not args.force: + UI.info("The following packages will be REMOVED:") + + for ipkg in packages_to_remove: + UI.info(f" {ipkg.package.qualified_name()}") + + UI.info() + + if dependers_to_unload: + UI.info("The following dependent packages will be UNLOADED:") + + for pkg_name in sorted(dependers_to_unload): + ipkg = manager.find_installed_package(pkg_name) + assert ipkg + UI.info(f" {ipkg.package.qualified_name()}") + + UI.info() + + if not UI.confirmation_prompt("Proceed?"): + return + + for pkg_name in sorted(dependers_to_unload): + ipkg = manager.find_installed_package(pkg_name) + assert ipkg + name = ipkg.package.qualified_name() + + if manager.unload(name): + UI.info(f'Unloaded "{name}"') + else: + # Weird that it failed, but if it's not installed and there's + # nothing to unload, not worth using a non-zero exit-code to + # reflect an overall failure of the package removal operation + UI.info(f'Failed unloading "{name}": no such package installed') + + had_failure = False + + for ipkg in packages_to_remove: + name = ipkg.package.qualified_name() + modifications = manager.modified_config_files(ipkg) + backup_files = manager.backup_modified_files(name, modifications) + + if manager.remove(name): + UI.info(f'Removed "{name}"') + + if backup_files: + UI.info("\tCreated backups of locally modified config files:") + + for backup_file in backup_files: + UI.info("\t" + backup_file) + + else: + UI.info(f'Failed removing "{name}": no such package installed') + had_failure = True + + if had_failure: + sys.exit(1) + + +def cmd_purge( + manager: Manager, + args: argparse.Namespace, +) -> None: + packages_to_remove = manager.installed_packages() + packages_to_remove = [p for p in packages_to_remove if not p.is_builtin()] + + if not packages_to_remove: + UI.info("No packages to remove.") + return + + if not args.force: + package_listing = "" + names_to_remove = [ipkg.package.qualified_name() for ipkg in packages_to_remove] + + for name in names_to_remove: + package_listing += f" {name}\n" + + UI.info("The following packages will be REMOVED:") + UI.info(package_listing) + + if not UI.confirmation_prompt("Proceed?"): + return + + had_failure = False + + for ipkg in packages_to_remove: + name = ipkg.package.qualified_name() + modifications = manager.modified_config_files(ipkg) + backup_files = manager.backup_modified_files(name, modifications) + + if manager.remove(name): + UI.info(f'Removed "{name}"') + + if backup_files: + UI.info("\tCreated backups of locally modified config files:") + + for backup_file in backup_files: + UI.info("\t" + backup_file) + + else: + UI.info(f'Unknown error removing "{name}"') + had_failure = True + + if had_failure: + sys.exit(1) + + +def outdated(manager: Manager) -> list[str]: + return [ + ipkg.package.qualified_name() + for ipkg in manager.installed_packages() + if ipkg.status.is_outdated + ] + + +def cmd_refresh( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.fail_on_aggregate_problems and not args.aggregate: + UI.warning( + "--fail-on-aggregate-problems without --aggregate has no effect.", + ) + + if args.push and not args.aggregate: + UI.error("--push requires --aggregate.") + sys.exit(1) + + if not args.sources: + args.sources = list(manager.sources.keys()) + + had_failure = False + had_aggregation_failure = False + + for source in args.sources: + UI.info(f"Refresh package source: {source}") + + src_pkgs_before = {i.qualified_name() for i in manager.source_packages()} + + error = "" + aggregation_issues = [] + + if args.aggregate: + res = manager.refresh_source(source) + if res: + error = res + else: + aggres = manager.aggregate_source(source, args.push) + error = aggres.error + aggregation_issues = aggres.package_issues + else: + error = manager.refresh_source(source) + + if error: + had_failure = True + UI.error(f'failed to refresh "{source}": {error}') + continue + + src_pkgs_after = {i.qualified_name() for i in manager.source_packages()} + + if src_pkgs_before == src_pkgs_after: + UI.info("\tNo membership changes") + else: + UI.info("\tChanges:") + diff = src_pkgs_before.symmetric_difference(src_pkgs_after) + + for name in diff: + change = "Added" if name in src_pkgs_after else "Removed" + UI.info(f"\t\t{change} {name}") + + if args.aggregate: + if aggregation_issues: + UI.info( + "\tWARNING: Metadata aggregated, but excludes the " + "following packages due to described problems:", + ) + + for url, issue in aggregation_issues: + UI.info(f"\t\t{url}: {issue}") + if args.fail_on_aggregate_problems: + had_aggregation_failure = True + else: + UI.info("\tMetadata aggregated") + + if args.push: + UI.info("\tPushed aggregated metadata") + + outdated_before = set(outdated(manager)) + UI.info("Refresh installed packages") + manager.refresh_installed_packages() + outdated_after = set(outdated(manager)) + + if outdated_before == outdated_after: + UI.info("\tNo new outdated packages") + else: + UI.info("\tNew outdated packages:") + diff = outdated_before.symmetric_difference(outdated_after) + + for name in diff: + ipkg = manager.find_installed_package(name) + if not ipkg: + continue + version_change = version_change_string(manager, ipkg) + UI.info(f"\t\t{name} {version_change}") + + if had_failure: + sys.exit(1) + if had_aggregation_failure: + sys.exit(2) + + +def version_change_string( + manager: Manager, + installed_package: InstalledPackage, +) -> str: + old_version = installed_package.status.current_version + new_version = old_version + version_change = "" + + if installed_package.status.tracking_method == TRACKING_METHOD_VERSION: + versions = manager.package_versions(installed_package) + + if len(versions): + new_version = versions[-1] + + version_change = f"({old_version} -> {new_version})" + else: + version_change = f"({new_version})" + + return version_change + + +def cmd_upgrade( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.package: + pkg_list = args.package + else: + pkg_list = outdated(manager) + + outdated_packages = [] + package_listing = "" + + for name in pkg_list: + ipkg = manager.find_installed_package(name) + + if not ipkg: + UI.error(f'package "{name}" is not installed') + sys.exit(1) + + name = ipkg.package.qualified_name() + + if not ipkg.status.is_outdated: + continue + + if not manager.match_source_packages(name): + name = ipkg.package.git_url + + info = manager.info( + name, + version=ipkg.status.current_version, + prefer_installed=False, + ) + + if info.invalid_reason: + UI.error(f'invalid package "{name}": {info.invalid_reason}') + sys.exit(1) + + next_version = ipkg.status.current_version + + if ipkg.status.tracking_method == TRACKING_METHOD_VERSION and info.versions: + next_version = info.versions[-1] + + assert next_version + + outdated_packages.append((info, next_version, False)) + version_change = version_change_string(manager, ipkg) + package_listing += f" {name} {version_change}\n" + + if not outdated_packages: + UI.info("All packages already up-to-date.") + return + + new_pkgs: list[tuple[PackageInfo, str, bool]] = [] + + if not args.nodeps: + to_validate = [ + (info.package.qualified_name(), next_version) + for info, next_version, _ in outdated_packages + ] + invalid_reason, new_pkgs = manager.validate_dependencies( + to_validate, + ignore_suggestions=args.nosuggestions, + ) + + if invalid_reason: + UI.error("failed to resolve dependencies:", invalid_reason) + sys.exit(1) + + allpkgs = outdated_packages + new_pkgs + + if not args.force: + UI.info("The following packages will be UPGRADED:") + UI.info(package_listing) + + if new_pkgs: + dependency_listing = "" + + for info, version, suggestion in new_pkgs: + name = info.package.qualified_name() + dependency_listing += f" {name} ({version})" + + if suggestion: + dependency_listing += " (suggested)" + + dependency_listing += "\n" + + UI.info("The following dependencies will be INSTALLED:") + UI.info(dependency_listing) + + extdep_listing = "" + + for info, version2, _ in allpkgs: + name = info.package.qualified_name() + extdeps = info.dependencies(field="external_depends") + + if extdeps is None: + extdep_listing += f" from {name} ({version2}):\n \n" + continue + + if extdeps: + extdep_listing += f" from {name} ({version2}):\n" + + for extdep, semver in sorted(extdeps.items()): + extdep_listing += f" {extdep} {semver}\n" + + if extdep_listing: + UI.info( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):", + ) + UI.info(extdep_listing) + + if not UI.confirmation_prompt("Proceed?"): + return + + prompt_for_user_vars( + manager, + args, + [info for info, _, _ in allpkgs], + ) + + if not args.skiptests: + for info, version3, _ in outdated_packages: + name = info.package.qualified_name() + + # Get info object for the next version as there may have been a + # test_command added during the upgrade. + next_info = manager.info(name, version=version3, prefer_installed=False) + if next_info.invalid_reason: + UI.error( + f'invalid package "{name}": {next_info.invalid_reason}', + ) + sys.exit(1) + + if "test_command" not in next_info.metadata: + LOG.info( + 'Skipping unit tests for "%s": no test_command in metadata', + name, + ) + continue + + UI.info(f'Running unit tests for "{name}"') + error_msg = "" + # As in cmd_install, we always process dependencies since the tests + # might well fail without them. If the user wants --nodeps and the + # tests fail because of it, they'll also need to say --skiptests. + error, passed, test_dir = manager.test( + name, + version3, + test_dependencies=True, + ) + + if error: + error_msg = f"failed to run tests for {name}: {error}" + elif not passed: + error_msg = ( + f'"{name}" tests failed, inspect' + f" {manager.package_test_log(info.package.name)} and" + f" the contents of {test_dir}" + ) + + if error_msg: + UI.error(error_msg) + + if args.force: + sys.exit(1) + + if not UI.confirmation_prompt( + "Proceed to install anyway?", + default_to_yes=False, + ): + return + + for info, version, _ in reversed(new_pkgs): + name = info.package.qualified_name() + + UI.info(f'Installing "{name}"', flush=True) + # Use default arguments here to avoid late-binding closure: + installer: UiCallable = lambda n=name, v=version: manager.install(n, v) + err = UI.call_activity(installer) + + if err: + UI.info(f'Failed installing "{name}": {err}') + continue + + ipkg = manager.find_installed_package(name) + assert ipkg + UI.info(f'Installed "{name}" ({ipkg.status.current_version})') + + if manager.has_scripts(ipkg): + load_error = manager.load(name) + + if load_error: + UI.info(f'Failed loading "{name}": {load_error}') + else: + UI.info(f'Loaded "{name}"') + + had_failure = False + + for info, _, _ in outdated_packages: + name = info.package.qualified_name() + + if not manager.match_source_packages(name): + name = info.package.git_url + + ipkg = manager.find_installed_package(name) + assert ipkg + modifications = manager.modified_config_files(ipkg) + backup_files = manager.backup_modified_files(name, modifications) + prev_upstream_config_files = manager.save_temporary_config_files(ipkg) + + res = manager.upgrade(name) + + if res: + UI.info(f'Failed upgrading "{name}": {res}') + had_failure = True + else: + ipkg = manager.find_installed_package(name) + assert ipkg + UI.info(f'Upgraded "{name}" ({ipkg.status.current_version})') + + for i, mf in enumerate(modifications): + next_upstream_config_file = mf[1] + + if not os.path.isfile(next_upstream_config_file): + UI.info("\tConfig file no longer exists:") + UI.info("\t\t" + next_upstream_config_file) + UI.info("\tPrevious, locally modified version backed up to:") + UI.info("\t\t" + backup_files[i]) + continue + + prev_upstream_config_file = prev_upstream_config_files[i][1] + + if filecmp.cmp(prev_upstream_config_file, next_upstream_config_file): + # Safe to restore user's version + shutil.copy2(backup_files[i], next_upstream_config_file) + continue + + UI.info("\tConfig file has been updated to a newer version:") + UI.info("\t\t" + next_upstream_config_file) + UI.info("\tPrevious, locally modified version backed up to:") + UI.info("\t\t" + backup_files[i]) + + if had_failure: + sys.exit(1) + + +def cmd_load( + manager: Manager, + args: argparse.Namespace, +) -> None: + had_failure = False + load_error = False + dep_error_listing = "" + + for name in args.package: + ipkg = manager.find_installed_package(name) + + if not ipkg: + had_failure = True + UI.info(f'Failed to load "{name}": no such package installed') + continue + + if not manager.has_scripts(ipkg): + UI.info(f'The package "{name}" does not contain scripts to load.') + continue + + name = ipkg.package.qualified_name() + + saved_state = {} + + if args.nodeps: + load_error = bool(manager.load(name)) + else: + saved_state = manager.loaded_package_states() + dep_error_listing, load_error = "", False + + loaded_dep_list = manager.load_with_dependencies( + name_from_path(name), + ) + + for _name, _error in loaded_dep_list: + if _error: + load_error = True + dep_error_listing += f" {_name}: {_error}\n" + + if not load_error: + dep_listing = get_changed_state(manager, saved_state, [name]) + + if dep_listing: + UI.info( + "The following installed packages were additionally loaded to satisfy" + f' runtime dependencies for "{name}".', + ) + UI.info(dep_listing) + + if load_error: + had_failure = True + + if not args.nodeps: + if dep_error_listing: + UI.info( + f'The following installed dependencies could not be loaded for "{name}".', + ) + UI.info(dep_error_listing) + manager.restore_loaded_package_states(saved_state) + + UI.info(f'Failed to load "{name}": {load_error}') + else: + UI.info(f'Loaded "{name}"') + + if had_failure: + sys.exit(1) + + +def cmd_unload( + manager: Manager, + args: argparse.Namespace, +) -> None: + had_failure = False + packages_to_unload: list[InstalledPackage] = [] + dependers_to_unload = set() + + def package_will_be_unloaded(pkg_name: str) -> bool: + for _ipkg in packages_to_unload: + if _ipkg.package.name == pkg_name: + return True + + return False + + for name in args.package: + ipkg = manager.find_installed_package(name) + + if not ipkg: + had_failure = True + UI.info(f'Failed to unload "{name}": no such package installed') + continue + + if not ipkg.status.is_loaded: + continue + + # Maybe in the future that's possible, but as of now built-in + # packages are really built-in plugins and there is not a way + # to unload them. + if ipkg.is_builtin(): + UI.info(f'cannot unload "{name}": built-in package') + sys.exit(1) + + packages_to_unload.append(ipkg) + + if not args.nodeps: + for ipkg in packages_to_unload: + for pkg_name in manager.list_depender_pkgs(ipkg.package.name): + ipkg = manager.find_installed_package(pkg_name) + + if ipkg and not package_will_be_unloaded(ipkg.package.name): + if ipkg.status.is_loaded: + dependers_to_unload.add(ipkg.package.name) + + if packages_to_unload and not args.force: + UI.info("The following packages will be UNLOADED:") + + for ipkg in packages_to_unload: + UI.info(f" {ipkg.package.qualified_name()}") + + UI.info() + + if dependers_to_unload: + UI.info("The following dependent packages will be UNLOADED:") + + for pkg_name in sorted(dependers_to_unload): + ipkg = manager.find_installed_package(pkg_name) + assert ipkg + UI.info(f" {ipkg.package.qualified_name()}") + + UI.info() + + if not UI.confirmation_prompt("Proceed?"): + if had_failure: + sys.exit(1) + else: + return + + for pkg_name in sorted(dependers_to_unload): + ipkg = manager.find_installed_package(pkg_name) + assert ipkg + packages_to_unload.append(ipkg) + + for ipkg in packages_to_unload: + name = ipkg.package.qualified_name() + + if manager.unload(name): + UI.info(f'Unloaded "{name}"') + else: + had_failure = True + UI.info(f'Failed unloading "{name}": no such package installed') + + if had_failure: + sys.exit(1) + + +def cmd_pin( + manager: Manager, + args: argparse.Namespace, +) -> None: + had_failure = False + + for name in args.package: + ipkg = manager.find_installed_package(name) + + if not ipkg: + had_failure = True + UI.info(f'Failed to pin "{name}": no such package installed') + continue + + if ipkg.is_builtin(): + had_failure = True + UI.info(f'cannot pin "{name}": built-in package') + continue + + name = ipkg.package.qualified_name() + ipkg = manager.pin(name) + + if ipkg: + UI.info( + f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})', + ) + else: + had_failure = True + UI.info(f'Failed pinning "{name}": no such package installed') + + if had_failure: + sys.exit(1) + + +def cmd_unpin( + manager: Manager, + args: argparse.Namespace, +) -> None: + had_failure = False + + for name in args.package: + ipkg = manager.find_installed_package(name) + + if not ipkg: + had_failure = True + UI.info(f'Failed to unpin "{name}": no such package installed') + continue + + if ipkg.is_builtin(): + had_failure = True + UI.info(f'cannot unpin "{name}": built-in package') + continue + + name = ipkg.package.qualified_name() + ipkg = manager.unpin(name) + + if ipkg: + UI.info( + f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})', + ) + else: + had_failure = True + UI.info(f'Failed unpinning "{name}": no such package installed') + + if had_failure: + sys.exit(1) + + +def _get_filtered_packages( + manager: Manager, + category: str, +) -> ( + dict[str, Package] + | dict[str, InstalledPackage] + | dict[str, Package | InstalledPackage] +): + pkg_dict: dict[str, Package | InstalledPackage] = {} + + for ipkg in manager.installed_packages(): + pkg_dict[ipkg.package.qualified_name()] = ipkg + + for pkg in manager.source_packages(): + pkg_qn = pkg.qualified_name() + + if pkg_qn not in pkg_dict: + pkg_dict[pkg_qn] = pkg + + if category == "all": + filtered_pkgs = pkg_dict + elif category == "installed": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, InstalledPackage) + } + elif category == "not_installed": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if not isinstance(value, InstalledPackage) + } + elif category == "loaded": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, InstalledPackage) and value.status.is_loaded + } + elif category == "unloaded": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, InstalledPackage) and not value.status.is_loaded + } + elif category == "outdated": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, InstalledPackage) and value.status.is_outdated + } + else: + raise NotImplementedError + + return filtered_pkgs + + +def cmd_list( + manager: Manager, + args: argparse.Namespace, +) -> None: + filtered_pkgs = _get_filtered_packages(manager, args.category) + + for pkg_name, val in sorted(filtered_pkgs.items()): + if isinstance(val, InstalledPackage): + pkg = val.package + + if val.is_builtin() and not args.include_builtin: + continue + + out = f"{pkg_name} (installed: {val.status.current_version})" + else: + pkg = val + out = pkg_name + + if not args.nodesc: + desc = pkg.short_description() + + if desc: + out += " - " + desc + + UI.info(out) + + +def cmd_search( + manager: Manager, + args: argparse.Namespace, +) -> None: + src_pkgs = manager.source_packages() + matches = set() + + for search_text in args.search_text: + if search_text[0] == "/" and search_text[-1] == "/": + try: + regex = re.compile(search_text[1:-1]) + except re.error as error: + UI.info(f"invalid regex: {error}") + sys.exit(1) + else: + for pkg in src_pkgs: + if regex.search(pkg.name_with_source_directory()): + matches.add(pkg) + + for tag in pkg.tags(): + if regex.search(tag): + matches.add(pkg) + + else: + for pkg in src_pkgs: + name = pkg.name_with_source_directory() + if name and search_text in name: + matches.add(pkg) + + for tag in pkg.tags(): + if search_text in tag: + matches.add(pkg) + + if matches: + for match in sorted(matches): + out = match.qualified_name() + + ipkg = manager.find_installed_package(match.qualified_name()) + + if ipkg: + out += f" (installed: {ipkg.status.current_version})" + + desc = match.short_description() + + if desc: + out += " - " + desc + + UI.info(out) + + else: + UI.info("no matches") + + +def cmd_info( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.version and len(args.package) > 1: + UI.error('"info --version" may only be used for a single package') + sys.exit(1) + + # Dictionary for storing package info to output as JSON + pkginfo: dict[str, Any] = {} + had_invalid_package = False + + if len(args.package) == 1: + try: + package_names = [] + for pkg_name, info in _get_filtered_packages( + manager, + args.package[0], + ).items(): + if info.is_builtin() and not args.include_builtin: + continue + + package_names.append(pkg_name) + except NotImplementedError: + package_names = args.package + else: + package_names = args.package + + for name in package_names: + info2 = manager.info( + name, + version=args.version, + prefer_installed=(not args.nolocal), + ) + + if info2.package: + name = info2.package.qualified_name() + + if args.json: + pkginfo[name] = {} + pkginfo[name]["metadata"] = {} + else: + UI.info(f'"{name}" info:') + + if info2.invalid_reason: + if args.json: + pkginfo[name]["invalid"] = info2.invalid_reason + else: + UI.info(f"\tinvalid package: {info2.invalid_reason}\n") + + had_invalid_package = True + continue + + if args.json: + pkginfo[name]["url"] = info2.package.git_url + pkginfo[name]["versions"] = info2.versions + else: + UI.info(f"\turl: {info2.package.git_url}") + UI.info(f"\tversions: {info2.versions}") + + if info2.status: + if args.json: + pkginfo[name]["install_status"] = {} + + for key, value in sorted(info2.status.__dict__.items()): + pkginfo[name]["install_status"][key] = value + else: + UI.info("\tinstall status:") + + for key, value in sorted(info2.status.__dict__.items()): + UI.info(f"\t\t{key} = {value}") + + if args.json: + if info2.metadata_file: + pkginfo[name]["metadata_file"] = info2.metadata_file + pkginfo[name]["metadata"][info2.metadata_version] = {} + else: + if info2.metadata_file: + UI.info(f"\tmetadata file: {info2.metadata_file}") + UI.info(f'\tmetadata (from version "{info2.metadata_version}"):') + + if len(info2.metadata) == 0: + if not args.json: + UI.info("\t\t") + elif args.json: + _fill_metadata_version( + pkginfo[name]["metadata"][info2.metadata_version], + info2.metadata, + ) + else: + for key, value in sorted(info2.metadata.items()): + value = value.replace("\n", "\n\t\t\t") + UI.info(f"\t\t{key} = {value}") + + # If --json and --allvers given, check for multiple versions and + # add the metadata for each version to the pkginfo. + if args.json and args.allvers: + for vers in info2.versions: + # Skip the version that was already processed + if vers != info2.metadata_version: + info3 = manager.info( + name, + vers, + prefer_installed=(not args.nolocal), + ) + pkginfo[name]["metadata"][info3.metadata_version] = {} + if info3.metadata_file: + pkginfo[name]["metadata_file"] = info3.metadata_file + _fill_metadata_version( + pkginfo[name]["metadata"][info3.metadata_version], + info3.metadata, + ) + + if not args.json: + UI.info() + + if args.json: + UI.info(json.dumps(pkginfo, indent=args.jsonpretty, sort_keys=True)) + + if had_invalid_package: + sys.exit(1) + + +def _fill_metadata_version( + pkginfo_name_metadata_version: dict[str, Any], + info_metadata: dict[str, str], +) -> None: + """Fill a dict with metadata information. + + This helper function is called by cmd_info to fill metadata information + for a specific package version. + + Args: + pkginfo_name_metadata_version (dict of str -> dict): Corresponds + to pkginfo[name]['metadata'][info.metadata_version] in cmd_info. + + info_metadata (dict of str->str): Corresponds to info.metadata + in cmd_info. + + Side effect: + New dict entries are added to pkginfo_name_metadata_version. + """ + for key, value in info_metadata.items(): + if key in {"depends", "suggests"}: + pkginfo_name_metadata_version[key] = {} + deps = value.split("\n") + + for i in range(1, len(deps)): + deplist = deps[i].split(" ") + pkginfo_name_metadata_version[key][deplist[0]] = deplist[1] + else: + pkginfo_name_metadata_version[key] = value + + +def cmd_config( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.config_param == "all": + out = io.StringIO() + CONFIG.write(out) + UI.info(out.getvalue()) + out.close() + elif args.config_param == "sources": + for key, value in CONFIG.items("sources"): + UI.info(f"{key} = {value}") + elif args.config_param == "user_vars": + for key, value in CONFIG.items("user_vars"): + UI.info(f"{key} = {value}") + else: + UI.info(CONFIG.get("paths", args.config_param)) + + +def cmd_autoconfig( + manager: Manager, + args: argparse.Namespace, +) -> None: + if args.user: + configfile = os.path.join(CONFIG.home_config_dir(), "config") + if CONFIG.save(configfile): + UI.info(f"Successfully wrote config file to {configfile}") + return + + configfile = CONFIG.find_configfile(args) + zeek_config = find_program("zeek-config") + + if not zeek_config: + UI.error('no "zeek-config" in PATH') + sys.exit(1) + + cmd = subprocess.Popen( + [zeek_config, "--site_dir", "--plugin_dir", "--prefix", "--zeek_dist"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) + + assert cmd.stdout + script_dir = read_zeek_config_line(cmd.stdout) + plugin_dir = read_zeek_config_line(cmd.stdout) + bin_dir = os.path.join(read_zeek_config_line(cmd.stdout), "bin") + zeek_dist = read_zeek_config_line(cmd.stdout) + + if configfile: + config_dir = os.path.dirname(configfile) + else: + config_dir = CONFIG.default_config_dir() + configfile = os.path.join(config_dir, "config") + + make_dir(config_dir) + + do_prompt = os.path.isfile(configfile) and not args.force + + CONFIG.set_interactively("paths", "script_dir", script_dir, do_prompt) + CONFIG.set_interactively("paths", "plugin_dir", plugin_dir, do_prompt) + CONFIG.set_interactively("paths", "bin_dir", bin_dir, do_prompt) + CONFIG.set_interactively("paths", "zeek_dist", zeek_dist, do_prompt) + + CONFIG.save(configfile) + + UI.info(f"Successfully wrote config file to {configfile}") + + +def cmd_env( + manager: Manager, + args: argparse.Namespace, +) -> None: + zeek_config = find_program("zeek-config") + zeekpath = os.environ.get("ZEEKPATH") + pluginpath = os.environ.get("ZEEK_PLUGIN_PATH") + + if zeek_config: + cmd = subprocess.Popen( + [zeek_config, "--zeekpath", "--plugin_dir"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) + + assert cmd.stdout + line1 = read_zeek_config_line(cmd.stdout) + line2 = read_zeek_config_line(cmd.stdout) + + if not zeekpath: + zeekpath = line1 + + if not pluginpath: + pluginpath = line2 + + zeekpaths = list(zeekpath.split(":")) if zeekpath else [] + pluginpaths = list(pluginpath.split(":")) if pluginpath else [] + + zeekpaths.append(CONFIG.zeek_path()) + pluginpaths.append(CONFIG.zeek_plugin_path()) + + def remove_redundant_paths(paths: list[str]) -> list[str]: + return list(OrderedDict.fromkeys(paths)) + + zeekpaths = remove_redundant_paths(zeekpaths) + pluginpaths = remove_redundant_paths(pluginpaths) + + if os.environ.get("SHELL", "").endswith("csh"): + UI.info("setenv ZEEKPATH {}".format(":".join(zeekpaths))) + UI.info("setenv ZEEK_PLUGIN_PATH {}".format(":".join(pluginpaths))) + UI.info(f"setenv PATH {CONFIG.bin_dir()}:$PATH") + else: + UI.info("export ZEEKPATH={}".format(":".join(zeekpaths))) + UI.info("export ZEEK_PLUGIN_PATH={}".format(":".join(pluginpaths))) + UI.info(f"export PATH={CONFIG.bin_dir()}:$PATH") + + +def cmd_create( + manager: Manager, + args: argparse.Namespace, +) -> None: + tmplname = ( + args.template + or CONFIG.get("templates", "default", fallback=None) + or ZKG_DEFAULT_TEMPLATE + ) + try: + tmpl = Template.load(tmplname, args.version) + except LoadError as error: + msg = f"problem while loading template {tmplname}: {error}" + LOG.exception(msg) + UI.error(msg) + sys.exit(1) + + try: + package = tmpl.package() + uvars = tmpl.define_user_vars() + uvar_names = set(package.needed_user_vars()) + + # Overlay any requested features onto the package. + if args.features: + # If the user provided comma-separated values, split the + # strings. (Argparse expects space separation.) Also + # filter any duplicates. + fnames = set() + for feat in args.features: + fnames |= {f.strip() for f in feat.split(",") if f} + features = tmpl.features() + for feature in features: + if feature.name() in fnames: + package.add_feature(feature) + uvar_names |= set(feature.needed_user_vars()) + fnames.remove(feature.name()) + if len(fnames) > 0: + # Alert if the user requested an unknown feature. + knowns = ", ".join([f'"{f.name()}"' for f in features]) + unknowns = ", ".join([f'"{name}"' for name in fnames]) + UI.error( + "the following features are unknown: {}." + ' Template "{}" offers {}.'.format( + unknowns, + tmpl.name(), + knowns or "no features", + ), + ) + sys.exit(1) + + # Remove user vars we don't actually require from consideration + uvars = [uvar for uvar in uvars if uvar.name() in uvar_names] + + # Resolve the variables via user input, args, etc + for uvar in uvars: + try: + uvar.resolve(tmpl.name(), args.user_var, args.force) + except ValueError: + UI.error( + f'could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", + ) + sys.exit(1) + + # Apply them to the template. After this, any parameter can be + # retrieved from the template via tmpl.lookup_param(). + tmpl._set_user_vars(uvars) + + # Verify that resulting template parameters are formatted correctly + try: + package.do_validate(tmpl) + except InputError as error: + UI.error("template input invalid, " + str(error)) + sys.exit(1) + + # And finally, instantiate the package. + try: + if os.path.isdir(args.packagedir): + if not args.force: + UI.info(f"Package directory {args.packagedir} already exists.") + if not UI.confirmation_prompt("Delete?"): + sys.exit(1) + try: + delete_path(args.packagedir) + LOG.info( + "Removed existing package directory %s", + args.packagedir, + ) + except OSError as err: + UI.error( + f"could not remove package directory {args.packagedir}: {err}", + ) + sys.exit(1) + + package.do_instantiate(tmpl, args.packagedir, args.force) + except OutputError as error: + UI.error("template instantiation failed, " + str(error)) + sys.exit(1) + except Exception as error: + msg = f"problem during template instantiation: {error}" + LOG.exception(msg) + UI.error(msg) + sys.exit(1) + + +def cmd_template_info( + manager: Manager, + args: argparse.Namespace, +) -> None: + tmplname = ( + args.template + or CONFIG.get("templates", "default", fallback=None) + or ZKG_DEFAULT_TEMPLATE + ) + + try: + tmpl = Template.load(tmplname, args.version) + except LoadError as error: + msg = f"problem while loading template {tmplname}: {error}" + LOG.exception(msg) + UI.error(msg) + sys.exit(1) + + tmplinfo = tmpl.info() + + if args.json: + UI.info(json.dumps(tmplinfo, indent=args.jsonpretty, sort_keys=True)) + else: + UI.info("API version: " + tmplinfo["api_version"]) + UI.info("features: " + ", ".join(tmplinfo["features"])) + UI.info("origin: " + tmplinfo["origin"]) + UI.info("provides package: " + str(tmplinfo["provides_package"]).lower()) + UI.info("user vars:") + for uvar_name, uvar_info in tmplinfo["user_vars"].items(): + UI.info( + "\t{}: {}, {}, used by {}".format( + uvar_name, + uvar_info["description"], + uvar_info["default"] or "no default", + ", ".join(uvar_info["used_by"]) or "not used", + ), + ) + UI.info("versions: " + ", ".join(tmplinfo["versions"])) + + +class BundleHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): + # Workaround for underlying argparse bug: https://bugs.python.org/issue9338 + def _format_args(self, action: argparse.Action, default_metavar: str) -> str: + rval = super()._format_args(action, default_metavar) + + if action.nargs == argparse.ZERO_OR_MORE: + rval += " --" + elif action.nargs == argparse.ONE_OR_MORE: + rval += " --" + + return rval + + +def _top_level_parser() -> argparse.ArgumentParser: + top_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="A command-line package manager for Zeek.", + epilog="Environment Variables:\n\n" + " ``ZKG_CONFIG_FILE``:\t" + "Same as ``--configfile`` option, but has less precedence.\n" + " ``ZKG_DEFAULT_SOURCE``:\t" + f"The default package source to use (normally {ZKG_DEFAULT_SOURCE}).\n" + " ``ZKG_DEFAULT_TEMPLATE``:\t" + f"The default package template to use (normally {ZKG_DEFAULT_TEMPLATE}).\n", + ) + top_parser.add_argument( + "--version", + action="version", + version="%(prog)s " + VERSION, + ) + + group = top_parser.add_mutually_exclusive_group() + group.add_argument( + "--configfile", + metavar="FILE", + help="Path to Zeek Package Manager config file. Precludes --user.", + ) + group.add_argument( + "--user", + action="store_true", + help="Store all state in user's home directory. Precludes --configfile.", + ) + + top_parser.add_argument( + "--verbose", + "-v", + action="count", + default=0, + help="Increase program output to stderr." + " Use multiple times for more output (e.g. -vv).", + ) + top_parser.add_argument( + "--extra-source", + action="append", + metavar="NAME=URL", + help="Add an extra source.", + ) + return top_parser + + +def argparser() -> argparse.ArgumentParser: + pkg_name_help = ( + "The name(s) of package(s) to operate on. The package" + " may be named in several ways. If the package is part" + " of a package source, it may be referred to by the" + " base name of the package (last component of git URL)" + " or its path within the package source." + " If two packages in different package sources" + " have conflicting paths, then the package source" + " name may be prepended to the package path to resolve" + " the ambiguity. A full git URL may also be used to refer" + " to a package that does not belong to a source. E.g. for" + ' a package source called "zeek" that has a package named' + ' "foo" located in "alice/zkg.index", the following' + ' names work: "foo", "alice/foo", "zeek/alice/foo".' + ) + + def add_uservar_args( + parser: argparse.ArgumentParser, + force_help: str | None = None, + ) -> None: + parser.add_argument( + "--force", + action="store_true", + help=force_help or "Don't prompt for confirmation or user variables.", + ) + parser.add_argument( + "--user-var", + action="append", + metavar="NAME=VAL", + type=UserVar.parse_arg, + help="A user variable assignment. This avoids prompting" + " for input and lets you provide a value when using --force." + " Use repeatedly as needed for multiple values.", + ) + + def add_json_args(parser: argparse.ArgumentParser, help_text: str) -> None: + parser.add_argument("--json", action="store_true", help=help_text) + parser.add_argument( + "--jsonpretty", + type=int, + default=None, + metavar="SPACES", + help="Optional number of spaces to indent for pretty-printed JSON output.", + ) + + top_parser = _top_level_parser() + command_parser = top_parser.add_subparsers( + title="commands", + dest="command", + help="See `%(prog)s -h` for per-command usage info.", + ) + command_parser.required = True + + # test + sub_parser = command_parser.add_parser( + "test", + help="Runs unit tests for Zeek packages.", + description="Runs the unit tests for the specified Zeek packages." + ' In most cases, the "zeek" and "zeek-config" programs will' + " need to be in PATH before running this command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_test) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) + sub_parser.add_argument( + "--version", + default=None, + help="The version of the package to test. Only one package may be" + " specified at a time when using this flag. A version tag, branch" + " name, or commit hash may be specified here." + " If the package name refers to a local git repo with a working tree," + " then its currently active branch is used." + " The default for other cases is to use" + " the latest version tag, or if a package has none," + ' the default branch, like "main" or "master".', + ) + + # install + sub_parser = command_parser.add_parser( + "install", + help="Installs Zeek packages.", + description="Installs packages from a configured package source or" + " directly from a git URL. After installing, the package" + ' is marked as being "loaded" (see the ``load`` command).', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_install) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) + sub_parser.add_argument( + "--skiptests", + action="store_true", + help="Skip running unit tests for packages before installation.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) + sub_parser.add_argument( + "--nosuggestions", + action="store_true", + help="Skip automatically installing suggested packages.", + ) + sub_parser.add_argument( + "--version", + default=None, + help="The version of the package to install. Only one package may be" + " specified at a time when using this flag. A version tag, branch" + " name, or commit hash may be specified here." + " If the package name refers to a local git repo with a working tree," + " then its currently active branch is used." + " The default for other cases is to use" + " the latest version tag, or if a package has none," + ' the default branch, like "main" or "master".', + ) + add_uservar_args(sub_parser) + + # bundle + sub_parser = command_parser.add_parser( + "bundle", + help="Creates a bundle file containing a collection of Zeek packages.", + description="This command creates a bundle file containing a collection" + " of Zeek packages. If ``--manifest`` is used, the user" + " supplies the list of packages to put in the bundle, else" + " all currently installed packages are put in the bundle." + " A bundle file can be unpacked on any target system," + " resulting in a repeatable/specific set of packages" + " being installed on that target system (see the" + " ``unbundle`` command). This command may be useful for" + " those that want to manage packages on a system that" + " otherwise has limited network connectivity. E.g. one can" + " use a system with an internet connection to create a" + " bundle, transport that bundle to the target machine" + " using whatever means are appropriate, and finally" + " unbundle/install it on the target machine.", + formatter_class=BundleHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_bundle) + sub_parser.add_argument( + "bundle_filename", + metavar="filename.bundle", + help="The path of the bundle file to create. It will be overwritten" + " if it already exists. Note that if --manifest is used before" + " this filename is specified, you should use a double-dash, --," + " to first terminate that argument list.", + ) + sub_parser.add_argument( + "--force", + action="store_true", + help="Skip the confirmation prompt.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks creating a bundle of packages that is in a" + " broken or unusable state.", + ) + sub_parser.add_argument( + "--nosuggestions", + action="store_true", + help="Skip automatically bundling suggested packages.", + ) + sub_parser.add_argument( + "--manifest", + nargs="+", + help="This may either be a file name or a list of packages to include" + " in the bundle. If a file name is supplied, it should be in INI" + " format with a single ``[bundle]`` section. The keys in that section" + " correspond to package names and their values correspond to git" + " version tags, branch names, or commit hashes. The values may be" + " left blank to indicate that the latest available version should be" + " used.", + ) + + # unbundle + sub_parser = command_parser.add_parser( + "unbundle", + help="Unpacks Zeek packages from a bundle file and installs them.", + description="This command unpacks a bundle file formerly created by the" + " ``bundle`` command and installs all the packages" + " contained within.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_unbundle) + sub_parser.add_argument( + "bundle_filename", + metavar="filename.bundle", + help="The path of the bundle file to install.", + ) + sub_parser.add_argument( + "--replace", + action="store_true", + help="Using this flag first removes all installed packages before then" + " installing the packages from the bundle.", + ) + add_uservar_args(sub_parser) + + # remove + sub_parser = command_parser.add_parser( + "remove", + aliases=["uninstall"], + help="Uninstall a package.", + description="Unloads (see the ``unload`` command) and uninstalls a" + " previously installed package.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_remove) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) + sub_parser.add_argument( + "--force", + action="store_true", + help="Skip the confirmation prompt.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) + + # purge + sub_parser = command_parser.add_parser( + "purge", + help="Uninstall all packages.", + description="Unloads (see the ``unload`` command) and uninstalls all" + " previously installed packages.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_purge) + sub_parser.add_argument( + "--force", + action="store_true", + help="Skip the confirmation prompt.", + ) + + # refresh + sub_parser = command_parser.add_parser( + "refresh", + help="Retrieve updated package metadata.", + description="Retrieve latest package metadata from sources and checks" + " whether any installed packages have available upgrades." + " Note that this does not actually upgrade any packages (see the" + " ``upgrade`` command for that).", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_refresh) + sub_parser.add_argument( + "--aggregate", + action="store_true", + help="Crawls the urls listed in package source zkg.index files and" + " aggregates the metadata found in their zkg.meta (or legacy" + " bro-pkg.meta) files. The aggregated metadata is stored in the local" + " clone of the package source that zkg uses internally for locating" + " package metadata." + " For each package, the metadata is taken from the highest available" + ' git version tag or the default branch, like "main" or "master", if no version tags exist', + ) + sub_parser.add_argument( + "--fail-on-aggregate-problems", + action="store_true", + help="When using --aggregate, exit with error when any packages trigger" + " metadata problems. Normally such problems only cause a warning.", + ) + sub_parser.add_argument( + "--push", + action="store_true", + help="Push local package source aggregations to upstream repos. Requires --aggregate.", + ) + sub_parser.add_argument( + "--sources", + nargs="+", + help="A list of package source names to operate on. If this argument" + " is not used, then the command will operate on all configured" + " sources.", + ) + + # upgrade + sub_parser = command_parser.add_parser( + "upgrade", + help="Upgrade installed packages to latest versions.", + description="Uprades the specified package(s) to latest available" + " version. If no specific packages are specified, then all installed" + " packages that are outdated and not pinned are upgraded. For packages" + " that are installed with ``--version`` using a git branch name, the" + " package is updated to the latest commit on that branch, else the" + " package is updated to the highest available git version tag.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_upgrade) + sub_parser.add_argument("package", nargs="*", default=[], help=pkg_name_help) + sub_parser.add_argument( + "--skiptests", + action="store_true", + help="Skip running unit tests for packages before installation.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) + sub_parser.add_argument( + "--nosuggestions", + action="store_true", + help="Skip automatically installing suggested packages.", + ) + add_uservar_args(sub_parser) + + # load + sub_parser = command_parser.add_parser( + "load", + help="Register packages to be be auto-loaded by Zeek.", + description="The Zeek Package Manager keeps track of all packages that" + ' are marked as "loaded" and maintains a single Zeek script that, when' + " loaded by Zeek (e.g. via ``@load packages``), will load the scripts" + ' from all "loaded" packages at once.' + ' This command adds a set of packages to the "loaded packages" list.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_load) + sub_parser.add_argument( + "package", + nargs="+", + default=[], + help="Name(s) of package(s) to load.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) + + # unload + sub_parser = command_parser.add_parser( + "unload", + help="Unregister packages to be be auto-loaded by Zeek.", + description="The Zeek Package Manager keeps track of all packages that" + ' are marked as "loaded" and maintains a single Zeek script that, when' + ' loaded by Zeek, will load the scripts from all "loaded" packages at' + ' once. This command removes a set of packages from the "loaded' + ' packages" list.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_unload) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) + sub_parser.add_argument( + "--force", + action="store_true", + help="Skip the confirmation prompt.", + ) + sub_parser.add_argument( + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) + + # pin + sub_parser = command_parser.add_parser( + "pin", + help="Prevent packages from being automatically upgraded.", + description="Pinned packages are ignored by the ``upgrade`` command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_pin) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) + + # unpin + sub_parser = command_parser.add_parser( + "unpin", + help="Allows packages to be automatically upgraded.", + description="Packages that are not pinned are automatically upgraded" + " by the ``upgrade`` command", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_unpin) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) + + # list + sub_parser = command_parser.add_parser( + "list", + help="Lists packages.", + description="Outputs a list of packages that match a given category.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_list) + sub_parser.add_argument( + "category", + nargs="?", + default="installed", + choices=["all", "installed", "not_installed", "loaded", "unloaded", "outdated"], + help="Package category used to filter listing.", + ) + sub_parser.add_argument( + "--nodesc", + action="store_true", + help="Do not display description text, just the package name(s).", + ) + sub_parser.add_argument( + "--include-builtin", + action="store_true", + help="Also output packages that Zeek has built-in. By default" + " these are not shown.", + ) + + # search + sub_parser = command_parser.add_parser( + "search", + help="Search packages for matching names.", + description="Perform a substring search on package names and metadata" + " tags. Surround search text with slashes to indicate it is a regular" + " expression (e.g. ``/text/``).", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_search) + sub_parser.add_argument( + "search_text", + nargs="+", + default=[], + help="The text(s) or pattern(s) to look for.", + ) + + # info + sub_parser = command_parser.add_parser( + "info", + help="Display package information.", + description="Shows detailed information/metadata for given packages." + " If the package is currently installed, additional information about" + " the status of it is displayed. E.g. the installed version or whether" + ' it is currently marked as "pinned" or "loaded."', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_info) + sub_parser.add_argument( + "package", + nargs="+", + default=[], + help=pkg_name_help + + " If a single name is given and matches one of the same categories" + ' as the "list" command, then it is automatically expanded to be the' + " names of all packages which match the given category.", + ) + sub_parser.add_argument( + "--version", + default=None, + help="The version of the package metadata to inspect. A version tag," + " branch name, or commit hash and only one package at a time may be" + " given when using this flag. If unspecified, the behavior depends" + " on whether the package is currently installed. If installed," + " the metadata will be pulled from the installed version. If not" + " installed, the latest version tag is used, or if a package has no" + ' version tags, the default branch, like "main" or "master", is used.', + ) + sub_parser.add_argument( + "--nolocal", + action="store_true", + help="Do not read information from locally installed packages." + " Instead read info from remote GitHub.", + ) + sub_parser.add_argument( + "--include-builtin", + action="store_true", + help="Also output packages that Zeek has built-in. By default" + " these are not shown.", + ) + add_json_args(sub_parser, "Output package information as JSON.") + sub_parser.add_argument( + "--allvers", + action="store_true", + help="When outputting package information as JSON, show metadata for" + " all versions. This option can be slow since remote repositories" + " may be cloned multiple times. Also, installed packages will show" + " metadata only for the installed version unless the --nolocal " + " option is given.", + ) + + # config + sub_parser = command_parser.add_parser( + "config", + help="Show Zeek Package Manager configuration info.", + description="The default output of this command is a valid package" + " manager config file that corresponds to the one currently being used," + " but also with any defaulted field values filled in. This command" + " also allows for only the value of a specific field to be output if" + " the name of that field is given as an argument to the command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_config) + sub_parser.add_argument( + "config_param", + nargs="?", + default="all", + choices=[ + "all", + "sources", + "user_vars", + "state_dir", + "script_dir", + "plugin_dir", + "bin_dir", + "zeek_dist", + ], + help="Name of a specific config file field to output.", + ) + + # autoconfig + sub_parser = command_parser.add_parser( + "autoconfig", + help="Generate a Zeek Package Manager configuration file.", + description="The output of this command is a valid package manager" + " config file that is generated by using the ``zeek-config`` script" + " that is installed along with Zeek. It is the suggested configuration" + " to use for most Zeek installations. For this command to work, the" + " ``zeek-config`` script must be in ``PATH``," + " unless the --user option is given, in which case this creates" + " a config that does not touch the Zeek installation.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_autoconfig) + sub_parser.add_argument( + "--force", + action="store_true", + help="Skip any confirmation prompt.", + ) + + # env + sub_parser = command_parser.add_parser( + "env", + help="Show the value of environment variables that need to be set for" + " Zeek to be able to use installed packages.", + description="This command returns shell commands that, when executed," + " will correctly set ``ZEEKPATH`` and ``ZEEK_PLUGIN_PATH`` to use" + " scripts and plugins from packages installed by the package manager." + " For this command to function properly, either have the ``zeek-config``" + " script (installed by zeek) in ``PATH``, or have the ``ZEEKPATH`` and" + " ``ZEEK_PLUGIN_PATH`` environment variables already set so this command" + " can append package-specific paths to them.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.set_defaults(run_cmd=cmd_env) + + # create + sub_parser = command_parser.add_parser( + "create", + help="Create a new Zeek package.", + description="This command creates a new Zeek package in the directory" + " provided via --packagedir. If this directory exists, zkg will not" + " modify it unless you provide --force.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sub_parser.add_argument( + "--packagedir", + metavar="DIR", + required=True, + help="Output directory into which to produce the new package. Required.", + ) + sub_parser.add_argument( + "--version", + help="The template version to use. A version tag, branch name, or" + " commit hash may be specified here. If --template refers to a local" + " git repo with a working tree, then zkg uses it as-is and the version" + " is ignored. The default for other cases is to use the latest" + " version tag, or if a template has none, the default branch, like" + ' "main" or "master".', + ) + sub_parser.add_argument( + "--features", + nargs="+", + metavar="FEATURE", + help="Additional features to include in your package. Use the ``template" + " info`` command for information about available features.", + ) + sub_parser.add_argument( + "--template", + metavar="URL", + help="By default, zkg uses its own package template. This makes it" + " select an alternative.", + ) + sub_parser.set_defaults(run_cmd=cmd_create) + add_uservar_args(sub_parser) + + # Template management commands + + sub_parser = command_parser.add_parser("template", help="Manage package templates.") + + template_command_parser = sub_parser.add_subparsers( + title="template commands", + dest="command", + help="See %(prog)s -h for per-command usage info.", + ) + template_command_parser.required = True + + # template info + + sub_parser = template_command_parser.add_parser( + "info", + help="Shows information about a package template.", + description="This command shows versions and supported features for" + " a given package.", + ) + add_json_args(sub_parser, "Output template information as JSON.") + sub_parser.add_argument( + "--version", + help="The template version to report on. A version tag, branch name," + " or commit hash may be specified here. If the selected template" + " refers to a local git repo, the version is ignored. The default" + " for other cases is to use the latest version tag, or if a template" + ' has none, the default branch, like "main" or "master".', + ) + sub_parser.add_argument( + "template", + metavar="URL", + nargs="?", + help="URL of a package template repository, or local path to one." + " When not provided, the configured default template is used.", + ) + sub_parser.set_defaults(run_cmd=cmd_template_info) + + if argcomplete_ := sys.modules.get("argcomplete"): + argcomplete_.autocomplete(top_parser) + + return top_parser diff --git a/zeekpkg/config.py b/zeekpkg/config.py new file mode 100644 index 00000000..163f79ab --- /dev/null +++ b/zeekpkg/config.py @@ -0,0 +1,395 @@ +"""This module provides a specialized ConfigParser class to store +zkg configuration settings. +""" + +import argparse +import configparser +import os +import sys +from collections import OrderedDict + +from ._util import ( + file_is_not_empty, + std_encoding, +) +from .consts import ( + ZKG_DEFAULT_SOURCE, + ZKG_DEFAULT_TEMPLATE, +) +from .ui import ( + UI, +) + + +class Config(configparser.ConfigParser): + """This class is a minor ConfigParser specialization to store the various + config settings needed by zkg. Like ConfigParser it supports loading from + and saving to ini-style config files, but also defaults to a config file + location, supports overriding values from command-line arguments, and + provides convenience methods for accessing specific settings. + """ + + def __init__(self) -> None: + super().__init__() + + # Operate case-sensitively, which helps particularly with user variables. + # mypy complains about this: Cannot assign to a method [method-assign] + Config.optionxform = str # type: ignore + + # Populate defaults: + self.reset() + + def reset(self, args: argparse.Namespace | None = None) -> None: + """Resets the configuration to a bare-bones, built-in default.""" + if args is not None and args.user: + state_dir = self.home_config_dir() + else: + state_dir = self.default_state_dir() + + self.read_dict( + { + "paths": { + "bin_dir": os.path.join(state_dir, "bin"), + "plugin_dir": os.path.join(state_dir, "plugin_dir"), + "script_dir": os.path.join(state_dir, "script_dir"), + "state_dir": state_dir, + }, + "sources": {}, + "templates": {}, + }, + ) + + if val := os.getenv("ZKG_DEFAULT_SOURCE", ZKG_DEFAULT_SOURCE): + self.set("sources", "zeek", val) + if val := os.getenv("ZKG_DEFAULT_TEMPLATE", ZKG_DEFAULT_TEMPLATE): + self.set("templates", "default", val) + + self.canonicalize() + self.sanity_check() + + def save(self, configfile: str) -> bool: + """Saves the configuration. + + When a filename is given, saves to it; otherwise it uses the default + config file location logic. Returns True when saving succeeded. May + throw IOErrors for opening/writing trouble. + + This always saves the config in canonical order, with sections, and keys + in those sections, ordered alphabetically. + """ + # dict in Python >= 3.7 is ordered -- use OrderedDict anyway for extra + # compatibility. + cfg = configparser.ConfigParser(dict_type=OrderedDict) + + for section in sorted(self.sections()): + cfg.add_section(section) + for option in sorted(self.options(section)): + cfg.set(section, option, self.get(section, option)) + + with open(configfile, "w", encoding=std_encoding(sys.stdout)) as hdl: + cfg.write(hdl) + + return True + + def canonicalize(self) -> None: + """Applies canonicalization to select config settings. + + For any entries in the "paths", "sources", and "templates" section, this + includes tilde and environment variable expansion, as well as path + normalization. + """ + for section in ("paths", "sources", "templates"): + for key, path in self.items(section): + path = os.path.expanduser(path) # Expand ~ + path = os.path.expandvars(path) # Env var substitution + self.set(section, key, path) + + for key, path in self.items("paths"): + if path: + # Canonicalize; no trailing slashes -- but only + # if there is a value, because normpath otherwise + # produces "." + path = os.path.normpath(path) + self.set("paths", key, path) + + def sanity_check(self) -> None: + """Checks the settings for acceptable values and exits upon violations.""" + # Paths must be absolute: + for key, val in self.items("paths"): + if val and not os.path.isabs(val): + UI.error( + "invalid config file value for key" + f' "{key}" in section [paths]: "{val}" is not' + " an absolute path", + ) + sys.exit(1) + + def update_from_file(self, configfile: str) -> None: + """Loads the given config file, adding its content to the current settings. + + This also applies canonicalization and sanity-checking. + """ + if not configfile: + return + if not os.path.isfile(configfile): + UI.error(f'invalid config file "{configfile}"') + sys.exit(1) + + # If we're loading a config file, consider its potential `[sources]` + # section authoritative. This allows config files to adjust (or remove) + # package sources, while defaulting to the standard Zeek package source + # in the absence of a config file. We don't blow away the whole section, + # only its contents, to avoid perturbing the order of sections in the + # ConfigParser (picked up by the tests.user-mode btest). + for source in self.options("sources"): + self.remove_option("sources", source) + + self.read(configfile) + self.canonicalize() + self.sanity_check() + + def update_from_args(self, args: argparse.Namespace) -> None: + """Updates the current settings as per the given command-line arguments. + + This currently only supports providing additional package sources. Also + applies canonicalization and sanity-checking. + """ + for key_val in args.extra_source or []: + if "=" not in key_val: + UI.warning(f'invalid extra source: "{key_val}"') + continue + + key, val = key_val.split("=", 1) + + if not key or not val: + UI.warning(f'invalid extra source: "{key_val}"') + continue + + self.set("sources", key, val) + + self.canonicalize() + self.sanity_check() + + def set( + self, + section: str | configparser._UnnamedSection, # type: ignore + option: str, + val: str | None = None, + ) -> None: + """Like ConfigParser's set(), but creating sections on the fly.""" + if not self.has_section(section): + self.add_section(section) + super().set(section, option, val) + + def set_interactively( + self, + section: str, + option: str, + val: str, + prompt: bool = True, + ) -> None: + """Like set(), but with optional prompting. + + When prompting is requested, this will ask the user to confirm new + options, or altering an already existing config option to a new value. + """ + if not prompt: + self.set(section, option, val) + return + + old_val = self.get(section, option) + + if old_val == val: + return + + msg = f'Set "{option}" config option to: {val} ?' + + if old_val: + msg += f"\n(previous value: {old_val})" + + if UI.confirmation_prompt(msg): + self.set(section, option, val) + + def bin_dir(self) -> str: + """Directory for package binaries. + + zkg links executables into this directory as provided by an + installed package through `executables` (as given by its + :file:`zkg.meta` or :file:`bro-pkg.meta`) + + This is usually $ZEEKROOT/bin. + """ + return self.get("paths", "bin_dir") + + def script_dir(self) -> str: + """Toplevel directory for Zeek's site-specific script files. + + This is usually $ZEEKROOT/share/zeek.site. + """ + return self.get("paths", "script_dir") + + def state_dir(self) -> str: + """Directory where zkg maintains internal state. + + zkg maintains its manifest file, package/source git clones, and other + persistent state it requires to operate in this directory. + + This is usually $ZEEKROOT/var/lib.zkg. + """ + return self.get("paths", "state_dir") + + def plugin_dir(self) -> str: + """Toplevel directory for Zeek's installed plugins. + + This is usually $ZEEKROOT/lib64/zeek/plugins. + """ + return self.get("paths", "plugin_dir") + + def zeek_dist(self) -> str | None: + """Directory of the Zeek source code distribution, if available. + + The path to the Zeek sources, as per zeek-config. May not be available + on the system, in which case this returns None. + """ + return self.get("paths", "zeek_dist", fallback=None) + + # Derived directory paths: + + def backup_dir(self) -> str: + """Directory for backed-up content. + + zkg will store backup files (e.g. locally modified package config files) + in this directory. + """ + return os.path.join(self.state_dir(), "backups") + + def log_dir(self) -> str: + """Directory where zkg stores its own log files. + + zkg places package build logs, test logs, and similar outputs there. + """ + return os.path.join(self.state_dir(), "logs") + + def scratch_dir(self) -> str: + """Directory zkg treats as its own scratch space.""" + return os.path.join(self.state_dir(), "scratch") + + def packages_script_dir(self) -> str: + """Directory into which to install a package's scripts. + + zkg copies each installed package's `script_dir` (as given by its + :file:`zkg.meta` or :file:`bro-pkg.meta`). Each package gets a + subdirectory associated with its name. + """ + return os.path.join(self.script_dir(), "packages") + + def packages_plugin_dir(self) -> str: + """Directory into which to install a package's plugin_dir. + + zkg copies each installed package's `plugin_dir` (as given by its + :file:`zkg.meta` or :file:`bro-pkg.meta`). Each package gets a + subdirectory within `plugin_dir` associated with its name. + """ + return os.path.join(self.plugin_dir(), "packages") + + def sources_clone_dir(self) -> str: + """Directory where zkg clones package "source" repos. + + These are zkg's package inventories. Each source gets a subdirectory + associated with its name. The default source is called "zeek". + """ + return os.path.join(self.state_dir(), "clones", "source") + + def packages_clone_dir(self) -> str: + """Directory where zkg keeps the source tree of installed packages. + + Each package gets a subdirectory associated with its name. + """ + return os.path.join(self.state_dir(), "clones", "package") + + def testing_dir(self) -> str: + """Directory in which zkg runs package tests. + + Each package gets a subdirectory associated with its name. + """ + return os.path.join(self.state_dir(), "testing") + + def zeek_path(self) -> str: + """The path where installed package scripts are located. + + This path can be added to :envvar:`ZEEKPATH` for interoperability with + Zeek. + """ + return os.path.dirname(self.packages_script_dir()) + + def zeek_plugin_path(self) -> str: + """Return the path where installed package plugins are located. + + This path can be added to :envvar:`ZEEK_PLUGIN_PATH` for + interoperability with Zeek. + """ + return os.path.dirname(self.packages_plugin_dir()) + + def user_vars(self) -> dict[str, str]: + """Returns a dictionary of user variables and their values. + + If no user variables are available, returns an empty dictionary. + """ + res = {} + if self.has_section("user_vars"): + for key, val in self.items("user_vars"): + res[key] = val + return res + + def home_config_dir(self) -> str: + """Directory in the user's home in which to store zkg's state. + + This is only used in user mode (--user). + """ + return os.path.join(os.path.expanduser("~"), ".zkg") + + def default_config_dir(self) -> str: + """Default directory in which zkg stores its configuration.""" + return os.getenv("ZEEK_ZKG_CONFIG_DIR") or self.home_config_dir() + + def default_state_dir(self) -> str: + """Default directory in which zkg stores all of its state.""" + return os.getenv("ZEEK_ZKG_STATE_DIR") or self.home_config_dir() + + def find_configfile(self, args: argparse.Namespace) -> str: + """Locator for zkg's config file. + + This function attempts the following locations: first, when the provided + args directly include a config file (`--configfile`), it returns its + path. Second, if `--user` is given, it attempts to locate the config + file in the user's home. Third, it attempts the ZKG_CONFIG_FILE + environment variable, and finally, the default configuration directory. + + Returns the path to the first config file successfully located, and the + empty string if all attempts fail. + """ + if args.configfile and file_is_not_empty(args.configfile): + return str(args.configfile) + + if args.user: + configfile = os.path.join(self.home_config_dir(), "config") + + if configfile and file_is_not_empty(configfile): + return configfile + + return "" + + configfile = os.environ.get("ZKG_CONFIG_FILE", default="") + + if configfile and file_is_not_empty(configfile): + return configfile + + configfile = os.path.join(self.default_config_dir(), "config") + + if file_is_not_empty(configfile): + return configfile + + return "" + + +CONFIG = Config() diff --git a/zeekpkg/consts.py b/zeekpkg/consts.py new file mode 100644 index 00000000..477ef9dc --- /dev/null +++ b/zeekpkg/consts.py @@ -0,0 +1,9 @@ +# The default package source we retrieve from when packages are +# identified by name, not URL: +ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" + +# The default package template repository: +ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" + +# The ZKG version: +VERSION = "3.1.0-64" diff --git a/zeekpkg/logs.py b/zeekpkg/logs.py new file mode 100644 index 00000000..fe6f239f --- /dev/null +++ b/zeekpkg/logs.py @@ -0,0 +1,36 @@ +"""Log management for zkg. + +This package provides a logger named ``LOG`` to which logging stream handlers +may be added in order to help log/debug zkg's operation. +""" + +import logging + +LOG: logging.Logger = logging.getLogger(__name__) +LOG.addHandler(logging.NullHandler()) + + +def configure(verbosity: int = 0) -> None: + """Configures logging of operational information to stderr. + + Args: + verbosity (int): the log level. Values 0-2 are supported. + 0 is the default, causing only critical errors to be + logged. Larger values increase the level of detail. + """ + formatter = logging.Formatter( + "%(asctime)s %(levelname)-8s %(message)s", + "%Y-%m-%d %H:%M:%S", + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + if verbosity == 0: + LOG.setLevel(logging.WARNING) + elif verbosity == 1: + LOG.setLevel(logging.INFO) + elif verbosity >= 2: + LOG.setLevel(logging.DEBUG) + + LOG.addHandler(handler) diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 22c3ad86..c985be79 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -5,6 +5,7 @@ import configparser import copy +import errno import fcntl import filecmp import json @@ -17,17 +18,13 @@ import tarfile import time from collections import deque +from typing import cast from urllib.parse import urlparse import git import semantic_version as semver -from . import ( - LOG, - __version__, -) from ._util import ( - configparser_section_dict, copy_over_path, delete_path, find_program, @@ -46,13 +43,21 @@ safe_tarfile_extractall, std_encoding, ) +from .config import ( + CONFIG, +) +from .consts import ( + VERSION, +) +from .logs import ( + LOG, +) from .package import ( BUILTIN_SCHEME, BUILTIN_SOURCE, LEGACY_METADATA_FILENAME, LEGACY_PLUGIN_MAGIC_FILE, LEGACY_PLUGIN_MAGIC_FILE_DISABLED, - METADATA_FILENAME, PLUGIN_MAGIC_FILE, PLUGIN_MAGIC_FILE_DISABLED, TRACKING_METHOD_BRANCH, @@ -65,13 +70,17 @@ PackageVersion, aliases, canonical_url, + get_package_metadata, + is_valid_name, make_builtin_package, name_from_path, + parse_package_metadata, + pick_metadata_file, ) -from .package import ( - is_valid_name as is_valid_package_name, +from .source import ( + AggregationResults, + Source, ) -from .source import AGGREGATE_DATA_FILE, Source from .uservar import ( UserVar, ) @@ -89,13 +98,13 @@ def __init__(self, manager: "Manager", state_dir: str | None = None) -> None: self.plugin_dir = os.path.join(self.state_dir, "plugins", "packages") self.bin_dir = os.path.join(self.state_dir, "bin") else: - # Stages not given a test directory are essentially a shortcut to + # Stages not given a state directory are essentially a shortcut to # standard functionality; this doesn't require all directories: self.state_dir = None - self.clone_dir = manager.package_clonedir - self.script_dir = manager.script_dir - self.plugin_dir = manager.plugin_dir - self.bin_dir = manager.bin_dir + self.clone_dir = CONFIG.packages_clone_dir() + self.script_dir = CONFIG.packages_script_dir() + self.plugin_dir = CONFIG.packages_plugin_dir() + self.bin_dir = CONFIG.bin_dir() def populate(self) -> None: # If we're staging to a temporary location, blow anything existing there @@ -115,7 +124,7 @@ def populate(self) -> None: # real install folders into the staging one. The subsequent cloning of # the packages under test will remove those links as needed. if self.state_dir: - with os.scandir(self.manager.package_clonedir) as it: + with os.scandir(CONFIG.packages_clone_dir()) as it: for entry in it: if not entry.is_dir(): continue @@ -166,61 +175,13 @@ class Manager: track of package sources, installed packages and their statuses. Attributes: - sources (dict of str -> :class:`.source.Source`): dictionary package + sources (dict of str -> :class:`.source.Source`): dictionary of package sources keyed by the name given to :meth:`add_source()` installed_pkgs (dict of str -> :class:`.package.InstalledPackage`): a dictionary of installed packaged keyed on package names (the last component of the package's git URL) - zeek_dist (str): path to the Zeek source code distribution. This - is needed for packages that contain Zeek plugins that need to be - built from source code. - - state_dir (str): the directory where the package manager will - a maintain manifest file, package/source git clones, and other - persistent state the manager needs in order to operate - - user_vars (dict of str -> str): dictionary of key-value pairs where - the value will be substituted into package build commands in place - of the key. - - backup_dir (str): a directory where the package manager will - store backup files (e.g. locally modified package config files) - - log_dir (str): a directory where the package manager will - store misc. logs files (e.g. package build logs) - - scratch_dir (str): a directory where the package manager performs - miscellaneous/temporary file operations - - script_dir (str): the directory where the package manager will - copy each installed package's `script_dir` (as given by its - :file:`zkg.meta` or :file:`bro-pkg.meta`). Each package gets a - subdirectory within `script_dir` associated with its name. - - plugin_dir (str): the directory where the package manager will - copy each installed package's `plugin_dir` (as given by its - :file:`zkg.meta` or :file:`bro-pkg.meta`). Each package gets a - subdirectory within `plugin_dir` associated with its name. - - bin_dir (str): the directory where the package manager will link - executables into that are provided by an installed package through - `executables` (as given by its :file:`zkg.meta` or - :file:`bro-pkg.meta`) - - source_clonedir (str): the directory where the package manager - will clone package sources. Each source gets a subdirectory - associated with its name. - - package_clonedir (str): the directory where the package manager - will clone installed packages. Each package gets a subdirectory - associated with its name. - - package_testdir (str): the directory where the package manager - will run tests. Each package gets a subdirectory - associated with its name. - manifest (str): the path to the package manager's manifest file. This file maintains a list of installed packages and their status. @@ -236,38 +197,14 @@ class Manager: load all installed packages that have been marked as loaded. """ - def __init__( - self, - state_dir: str, - script_dir: str, - plugin_dir: str, - zeek_dist: str = "", - user_vars: dict[str, str] | None = None, - bin_dir: str = "", - ) -> None: + def __init__(self) -> None: """Creates a package manager instance. - Args: - state_dir (str): value to set the `state_dir` attribute to - - script_dir (str): value to set the `script_dir` attribute to - - plugin_dir (str): value to set the `plugin_dir` attribute to - - zeek_dist (str): value to set the `zeek_dist` attribute to - - user_vars (dict of str -> str): key-value pair substitutions for - use in package build commands. - - bin_dir (str): value to set the `bin_dir` attribute to. If - empty/nil value, defaults to setting `bin_dir` attribute to - `/bin`. - Raises: OSError: when a package manager state directory can't be created IOError: when a package manager state file can't be created """ - LOG.debug("init Manager version %s", __version__) + LOG.debug("init Manager version %s", VERSION) self.sources: dict[str, Source] = {} self.installed_pkgs: dict[str, InstalledPackage] = {} self._builtin_packages: list[PackageInfo] | None = ( @@ -275,39 +212,32 @@ def __init__( ) self._builtin_packages_discovered = False # Flag if discovery even worked. self._info_cache: dict[tuple[str, str | None, bool], PackageInfo] = {} - self.zeek_dist = zeek_dist - self.state_dir = state_dir - self.user_vars = {} if user_vars is None else user_vars - self.backup_dir = os.path.join(self.state_dir, "backups") - self.log_dir = os.path.join(self.state_dir, "logs") - self.scratch_dir = os.path.join(self.state_dir, "scratch") - self._script_dir = script_dir - self.script_dir = os.path.join(script_dir, "packages") - self._plugin_dir = plugin_dir - self.plugin_dir = os.path.join(plugin_dir, "packages") - self.bin_dir = bin_dir or os.path.join(self.state_dir, "bin") - self.source_clonedir = os.path.join(self.state_dir, "clones", "source") - self.package_clonedir = os.path.join(self.state_dir, "clones", "package") - self.package_testdir = os.path.join(self.state_dir, "testing") - self.manifest = os.path.join(self.state_dir, "manifest.json") - self.autoload_script = os.path.join(self.script_dir, "packages.zeek") - self.autoload_package = os.path.join(self.script_dir, "__load__.zeek") - - make_dir(self.state_dir) + + self.manifest = os.path.join(CONFIG.state_dir(), "manifest.json") + self.autoload_script = os.path.join( + CONFIG.packages_script_dir(), + "packages.zeek", + ) + self.autoload_package = os.path.join( + CONFIG.packages_script_dir(), + "__load__.zeek", + ) + + make_dir(CONFIG.bin_dir()) + make_dir(CONFIG.log_dir()) + make_dir(CONFIG.packages_clone_dir()) + make_dir(CONFIG.packages_script_dir()) + make_dir(CONFIG.packages_plugin_dir()) + make_dir(CONFIG.scratch_dir()) + make_dir(CONFIG.sources_clone_dir()) + make_dir(CONFIG.state_dir()) # The lock file lives inside `state_dir`, so we open it after ensuring the dir exists. - self._state_lock = open(os.path.join(self.state_dir, ".lock"), "a") + self._state_lock = open(os.path.join(CONFIG.state_dir(), ".lock"), "a") fcntl.flock(self._state_lock, fcntl.LOCK_EX) - make_dir(self.log_dir) - make_dir(self.scratch_dir) - make_dir(self.source_clonedir) - make_dir(self.package_clonedir) - make_dir(self.script_dir) - make_dir(self.plugin_dir) - make_dir(self.bin_dir) - _create_readme(os.path.join(self.script_dir, "README")) - _create_readme(os.path.join(self.plugin_dir, "README")) + _create_readme(os.path.join(CONFIG.packages_script_dir(), "README")) + _create_readme(os.path.join(CONFIG.packages_plugin_dir(), "README")) if not os.path.exists(self.manifest): self._write_manifest() @@ -326,18 +256,24 @@ def __init__( relocating_bin_dir = False # whether bin_dir has relocated need_manifest_update = False - if os.path.realpath(prev_script_dir) != os.path.realpath(self.script_dir): - LOG.info("relocating script_dir %s -> %s", prev_script_dir, self.script_dir) + if os.path.realpath(prev_script_dir) != os.path.realpath( + CONFIG.packages_script_dir(), + ): + LOG.info( + "relocating script_dir %s -> %s", + prev_script_dir, + CONFIG.packages_script_dir(), + ) if os.path.exists(prev_script_dir): - delete_path(self.script_dir) - shutil.move(prev_script_dir, self.script_dir) + delete_path(CONFIG.packages_script_dir()) + shutil.move(prev_script_dir, CONFIG.packages_script_dir()) prev_zeekpath = os.path.dirname(prev_script_dir) for pkg_name in self.installed_pkgs: old_link = os.path.join(prev_zeekpath, pkg_name) - new_link = os.path.join(self.zeekpath(), pkg_name) + new_link = os.path.join(CONFIG.zeek_path(), pkg_name) if os.path.lexists(old_link): LOG.info("moving package link %s -> %s", old_link, new_link) @@ -348,26 +284,32 @@ def __init__( need_manifest_update = True refresh_bin_dir = True - if os.path.realpath(prev_plugin_dir) != os.path.realpath(self.plugin_dir): - LOG.info("relocating plugin_dir %s -> %s", prev_plugin_dir, self.plugin_dir) + if os.path.realpath(prev_plugin_dir) != os.path.realpath( + CONFIG.packages_plugin_dir(), + ): + LOG.info( + "relocating plugin_dir %s -> %s", + prev_plugin_dir, + CONFIG.packages_plugin_dir(), + ) if os.path.exists(prev_plugin_dir): - delete_path(self.plugin_dir) - shutil.move(prev_plugin_dir, self.plugin_dir) + delete_path(CONFIG.packages_plugin_dir()) + shutil.move(prev_plugin_dir, CONFIG.packages_plugin_dir()) need_manifest_update = True refresh_bin_dir = True if prev_bin_dir and os.path.realpath(prev_bin_dir) != os.path.realpath( - self.bin_dir, + CONFIG.bin_dir(), ): - LOG.info("relocating bin_dir %s -> %s", prev_bin_dir, self.bin_dir) + LOG.info("relocating bin_dir %s -> %s", prev_bin_dir, CONFIG.bin_dir()) need_manifest_update = True refresh_bin_dir = True relocating_bin_dir = True if refresh_bin_dir: - self._refresh_bin_dir(self.bin_dir) + self._refresh_bin_dir(CONFIG.bin_dir()) if relocating_bin_dir: self._clear_bin_dir(prev_bin_dir) @@ -422,7 +364,7 @@ def _write_plugin_magic(self, ipkg: InstalledPackage) -> None: a fallback for recognizing the older name so that newer zkg versions continue to work with older Zeek versions for some time longer. """ - package_dir = pathlib.Path(self.plugin_dir) / ipkg.package.name + package_dir = pathlib.Path(CONFIG.packages_plugin_dir()) / ipkg.package.name magic_paths_enabled = [ package_dir / PLUGIN_MAGIC_FILE, @@ -462,8 +404,11 @@ def _write_plugin_magic(self, ipkg: InstalledPackage) -> None: def _read_manifest(self) -> tuple[str, str, str]: """Read the manifest file containing the list of installed packages. + Updates the manager's internal list of installed packages accordingly. + Returns: - tuple: (previous script_dir, previous plugin_dir, previous bin_dir) + tuple: Directories as per the manifest: (previous script_dir, + previous plugin_dir, previous bin_dir) Raises: IOError: when the manifest file can't be read @@ -510,34 +455,21 @@ def _write_manifest(self) -> None: data = { "manifest_version": 1, - "script_dir": self.script_dir, - "plugin_dir": self.plugin_dir, - "bin_dir": self.bin_dir, + "script_dir": CONFIG.packages_script_dir(), + "plugin_dir": CONFIG.packages_plugin_dir(), + "bin_dir": CONFIG.bin_dir(), "installed_packages": pkg_list, } with open(self.manifest, "w") as f: json.dump(data, f, indent=2, sort_keys=True) - def zeekpath(self) -> str: - """Return the path where installed package scripts are located. - - This path can be added to :envvar:`ZEEKPATH` for interoperability with - Zeek. - """ - return os.path.dirname(self.script_dir) - - def zeek_plugin_path(self) -> str: - """Return the path where installed package plugins are located. - - This path can be added to :envvar:`ZEEK_PLUGIN_PATH` for - interoperability with Zeek. - """ - return os.path.dirname(self.plugin_dir) - def add_source(self, name: str, git_url: str) -> str: """Add a git repository that acts as a source of packages. + This clones the source's git repo if there's no local clone in zkg's + internal state yet. + Args: name (str): a short name that will be used to reference the package source. @@ -547,6 +479,7 @@ def add_source(self, name: str, git_url: str) -> str: Returns: str: empty string if the source is successfully added, else the reason why it failed. + """ if name == BUILTIN_SOURCE: return f"{name} is a reserved source name" @@ -562,7 +495,7 @@ def add_source(self, name: str, git_url: str) -> str: f"source already exists with different URL: {existing_source.git_url}" ) - clone_path = os.path.join(self.source_clonedir, name) + clone_path = os.path.join(CONFIG.sources_clone_dir(), name) # Support @ in the path to denote the "version" to checkout version = None @@ -595,6 +528,8 @@ def add_source(self, name: str, git_url: str) -> str: version=version, ) except git.GitCommandError as error: + # NOTE: seems this could also error when requesting nonexisting + # branches/versions? LOG.warning("failed to clone git repo: %s", error) return "failed to clone git repo" else: @@ -605,7 +540,7 @@ def add_source(self, name: str, git_url: str) -> str: def source_packages(self) -> list[Package]: """Return a list of :class:`.package.Package` within all sources.""" rval = [] - + # NOTE: odd naming, perhaps available_packages or some such. for _, source in self.sources.items(): rval += source.packages() @@ -637,7 +572,7 @@ def discover_builtin_packages(self) -> list[PackageInfo]: "zeek_mtime": stat.st_mtime, "zeek_size": stat.st_size, } - cache_file = os.path.join(self.state_dir, "zeek_build_info_cache.json") + cache_file = os.path.join(CONFIG.state_dir(), "zeek_build_info_cache.json") build_info = None try: @@ -745,7 +680,10 @@ def installed_package_dependencies(self) -> dict[str, dict[str, str] | None]: } def loaded_packages(self) -> list[InstalledPackage]: - """Return list of loaded :class:`.package.InstalledPackage`.""" + """Return list of loaded :class:`.package.InstalledPackage`. + + The returned packages are ordered alphabetically by their qualified name. + """ rval = [] for _, ipkg in sorted(self.installed_pkgs.items()): @@ -757,15 +695,44 @@ def loaded_packages(self) -> list[InstalledPackage]: def package_build_log(self, pkg_path: str) -> str: """Return the path to the package manager's build log for a package. + The build log is of the form -build.log, resides in the log + directory of zkg's internal state, and combines stdout and stderr in + separate sections of the file. + + The function does not check whether the log actually exists, or whether + the package actually requires one. + + Args: + pkg_path (str): the full git URL of a package or the shortened + path/name that refers to it within a package source. E.g. for + a package source called "zeek" with package named "foo" in + :file:`alice/zkg.index`, the following inputs may refer + to the package: "foo", "alice/foo", or "zeek/alice/foo". + + """ + name = name_from_path(pkg_path) + return os.path.join(CONFIG.log_dir(), f"{name}-build.log") + + def package_test_log(self, pkg_path: str) -> str: + """Return the path to the package manager's test log for a package. + + The test log is of the form -test.log, resides in the log + directory of zkg's internal state, and combines stdout and stderr in + separate sections of the file. + + The function does not check whether the log actually exists, or whether + the package actually requires one. + Args: pkg_path (str): the full git URL of a package or the shortened path/name that refers to it within a package source. E.g. for a package source called "zeek" with package named "foo" in :file:`alice/zkg.index`, the following inputs may refer to the package: "foo", "alice/foo", or "zeek/alice/foo". + """ name = name_from_path(pkg_path) - return os.path.join(self.log_dir, f"{name}-build.log") + return os.path.join(CONFIG.log_dir(), f"{name}-test.log") def match_source_packages(self, pkg_path: str) -> list[Package]: """Return a list of :class:`.package.Package` that match a given path. @@ -780,6 +747,8 @@ def match_source_packages(self, pkg_path: str) -> list[Package]: rval = [] canon_url = canonical_url(pkg_path) + # NOTE: This is the right place to populate a package source on-demand. + for pkg in self.source_packages(): if pkg.matches_path(canon_url): rval.append(pkg) @@ -830,7 +799,11 @@ def has_scripts(self, installed_pkg: InstalledPackage) -> bool: Returns: bool: True if the package has installed Zeek scripts. """ - return os.path.exists(os.path.join(self.script_dir, installed_pkg.package.name)) + # Is this better than relying on the presence of the scripts + # directive in zkg.meta? + return os.path.exists( + os.path.join(CONFIG.packages_script_dir(), installed_pkg.package.name), + ) def has_plugin(self, installed_pkg: InstalledPackage) -> bool: """Return whether a :class:`.package.InstalledPackage` installed a plugin. @@ -842,13 +815,18 @@ def has_plugin(self, installed_pkg: InstalledPackage) -> bool: Returns: bool: True if the package has installed a Zeek plugin. """ - return os.path.exists(os.path.join(self.plugin_dir, installed_pkg.package.name)) + # Is this better than relying on the presence of the build_command + # in zkg.meta? + return os.path.exists( + os.path.join(CONFIG.packages_plugin_dir(), installed_pkg.package.name), + ) def save_temporary_config_files( self, installed_pkg: InstalledPackage, ) -> list[tuple[str, str]]: - """Return a list of temporary package config file backups. + """Copies any Zeek scripts the package declares as config files (i.e., + not to be overwritten during updates) into internal scratchspace. Args: installed_pkg(:class:`.package.InstalledPackage`): the installed @@ -858,9 +836,8 @@ def save_temporary_config_files( list of (str, str): tuples that describe the config files backups. The first element is the config file as specified in the package metadata (a file path relative to the package's root directory). - The second element is an absolute file system path to where that - config file has been copied. It should be considered temporary, - so make use of it before doing any further operations on packages. + The second element is an absolute path to the location in zkg's + internal scratch folder where it copied the file to. """ metadata = installed_pkg.package.metadata config_files = re.split(r",\s*", metadata.get("config_files", "")) @@ -869,7 +846,7 @@ def save_temporary_config_files( return [] pkg_name = installed_pkg.package.name - clone_dir = os.path.join(self.package_clonedir, pkg_name) + clone_dir = os.path.join(CONFIG.packages_clone_dir(), pkg_name) rval = [] for config_file in config_files: @@ -883,11 +860,14 @@ def save_temporary_config_files( ) continue - backup_file = os.path.join(self.scratch_dir, "tmpcfg", config_file) + backup_file = os.path.join(CONFIG.scratch_dir(), "tmpcfg", config_file) make_dir(os.path.dirname(backup_file)) shutil.copy2(config_file_path, backup_file) rval.append((config_file, backup_file)) + # NOTE: there's nothing temporary about the files just created, and + # nothing that cleans them. + return rval def modified_config_files( @@ -896,6 +876,10 @@ def modified_config_files( ) -> list[tuple[str, str]]: """Return a list of package config files that the user has modified. + This compares the scripts the package declares as config file in zkg's + internal clone state to the content of these scripts as actually + installed. + Args: installed_pkg(:class:`.package.InstalledPackage`): the installed package to check for whether it has installed any Zeek scripts. @@ -906,6 +890,7 @@ def modified_config_files( metadata (a file path relative to the package's root directory). The second element is an absolute file system path to where that config file is currently installed. + """ metadata = installed_pkg.package.metadata config_files = re.split(r",\s*", metadata.get("config_files", "")) @@ -914,9 +899,9 @@ def modified_config_files( return [] pkg_name = installed_pkg.package.name - script_install_dir = os.path.join(self.script_dir, pkg_name) - plugin_install_dir = os.path.join(self.plugin_dir, pkg_name) - clone_dir = os.path.join(self.package_clonedir, pkg_name) + script_install_dir = os.path.join(CONFIG.packages_script_dir(), pkg_name) + plugin_install_dir = os.path.join(CONFIG.packages_plugin_dir(), pkg_name) + clone_dir = os.path.join(CONFIG.packages_clone_dir(), pkg_name) script_dir = metadata.get("script_dir", "") plugin_dir = metadata.get("plugin_dir", "build") rval = [] @@ -1000,54 +985,38 @@ def backup_modified_files( config_file_dir = os.path.dirname(config_file) install_path = modified_file[1] filename = os.path.basename(install_path) - backup_dir = os.path.join(self.backup_dir, backup_subdir, config_file_dir) + backup_dir = os.path.join( + CONFIG.backup_dir(), + backup_subdir, + config_file_dir, + ) timestamp = time.strftime(".%Y-%m-%d-%H:%M:%S") backup_path = os.path.join(backup_dir, filename + timestamp) make_dir(backup_dir) shutil.copy2(install_path, backup_path) rval.append(backup_path) - return rval - - class SourceAggregationResults: - """The return value of a call to :meth:`.Manager.aggregate_source()`. + # NOTE: odd to have both this method and save_temporary_config_files(), + # since the latter only copies what's in the clone tree? - Attributes: - refresh_error (str): an empty string if no overall error - occurred in the "refresh" operation, else a description of - what wrong - - package_issues (list of (str, str)): a list of reasons for - failing to collect metadata per packages/repository. - The first tuple element gives the repository URL in which - the problem occurred and the second tuple element describes - the failure. - """ + # NOTE: also note that there's no restore_modified_files() because + # restoration isn't the point of these backups, it's to preserve the + # modified version for reference/review (similar to e.g. .rpmsave). - def __init__( - self, - refresh_error: str = "", - package_issues: list[tuple[str, str]] | None = None, - ) -> None: - self.refresh_error = refresh_error - self.package_issues = package_issues if package_issues else [] + return rval def aggregate_source( self, name: str, push: bool = False, - ) -> SourceAggregationResults: + ) -> AggregationResults: """Pull latest git info from a package source and aggregate metadata. - This is like calling :meth:`refresh_source()` with the *aggregate* - arguments set to True. - - This makes the latest pre-aggregated package metadata available or - performs the aggregation locally in order to push it to the actual - package source. Locally aggregated data also takes precedence over - the source's pre-aggregated data, so it can be useful in the case - the operator of the source does not update their pre-aggregated data - at a frequent enough interval. + This performs package aggregation locally in order to push it to the + selected package source. Locally aggregated data also takes precedence + over the source's pre-aggregated data, so it can be useful in the case + the operator of the source does not update their pre-aggregated data at + a frequent enough interval. Args: name(str): the name of the package source. E.g. the same name @@ -1057,248 +1026,37 @@ def aggregate_source( metadata to the remote package source. Returns: - :class:`.Manager.SourceAggregationResults`: the results of the + :class:`AggregationResults`: the results of the refresh/aggregation. + """ - return self._refresh_source(name, True, push) + if name not in self.sources: + return AggregationResults("source name does not exist") + + return self.sources[name].aggregate(push) def refresh_source( self, name: str, - aggregate: bool = False, - push: bool = False, ) -> str: """Pull latest git information from a package source. - This makes the latest pre-aggregated package metadata available or - performs the aggregation locally in order to push it to the actual - package source. Locally aggregated data also takes precedence over - the source's pre-aggregated data, so it can be useful in the case - the operator of the source does not update their pre-aggregated data - at a frequent enough interval. + This makes the latest package metadata, including pre-aggregated + information, available locally. Args: name(str): the name of the package source. E.g. the same name used as a key to :meth:`add_source()`. - aggregate (bool): whether to perform a local metadata aggregation - by crawling all packages listed in the source's index files. - - push (bool): whether to push local changes to the aggregated - metadata to the remote package source. If the `aggregate` - flag is set, the data will be pushed after the aggregation - is finished. - Returns: str: an empty string if no errors occurred, else a description of what went wrong. - """ - res = self._refresh_source(name, aggregate, push) - return res.refresh_error - def _refresh_source( - self, - name: str, - aggregate: bool = False, - push: bool = False, - ) -> SourceAggregationResults: - """Used by :meth:`refresh_source()` and :meth:`aggregate_source()`.""" + """ if name not in self.sources: - return self.SourceAggregationResults("source name does not exist") - - source = self.sources[name] - LOG.debug('refresh "%s": pulling %s', name, source.git_url) - aggregate_file = os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) - agg_file_ours = os.path.join(self.scratch_dir, AGGREGATE_DATA_FILE) - agg_file_their_orig = os.path.join( - self.scratch_dir, - AGGREGATE_DATA_FILE + ".orig", - ) - - delete_path(agg_file_ours) - delete_path(agg_file_their_orig) - - if os.path.isfile(aggregate_file): - shutil.copy2(aggregate_file, agg_file_ours) - - source.clone.git.reset(hard=True) - source.clone.git.clean("-f", "-x", "-d") - - if os.path.isfile(aggregate_file): - shutil.copy2(aggregate_file, agg_file_their_orig) - - try: - source.clone.git.fetch("--recurse-submodules=yes") - git_pull(source.clone) - except git.GitCommandError as error: - LOG.error("failed to pull source %s: %s", name, error) - return self.SourceAggregationResults( - f"failed to pull from remote source: {error}", - ) - - if os.path.isfile(agg_file_ours): - if os.path.isfile(aggregate_file): - # There's a tracked version of the file after pull. - if os.path.isfile(agg_file_their_orig): - # We had local modifications to the file. - if filecmp.cmp(aggregate_file, agg_file_their_orig): - # Their file hasn't changed, use ours. - shutil.copy2(agg_file_ours, aggregate_file) - LOG.debug( - "aggegrate file in source unchanged, restore local one", - ) - else: - # Their file changed, use theirs. - LOG.debug("aggegrate file in source changed, discard local one") - else: - # File was untracked before pull and tracked after, - # use their version. - LOG.debug("new aggegrate file in source, discard local one") - else: - # They don't have the file after pulling, so restore ours. - shutil.copy2(agg_file_ours, aggregate_file) - LOG.debug("no aggegrate file in source, restore local one") - - aggregation_issues = [] - - if aggregate: - parser = configparser.ConfigParser(interpolation=None) - prev_parser = configparser.ConfigParser(interpolation=None) - prev_packages = set() - - if os.path.isfile(aggregate_file): - prev_parser.read(aggregate_file) - prev_packages = set(prev_parser.sections()) + return "source name does not exist" - agg_adds = [] - agg_mods = [] - - for index_file in source.package_index_files(): - with open(index_file) as f: - urls = [line.rstrip("\n") for line in f] - - for url in urls: - pkg_name = name_from_path(url) - clonepath = os.path.join(self.scratch_dir, pkg_name) - delete_path(clonepath) - - try: - clone = git_clone(url, clonepath, shallow=True) - except git.GitCommandError as error: - LOG.warning( - "failed to clone %s, skipping aggregation: %s", - url, - error, - ) - aggregation_issues.append((url, repr(error))) - continue - - version_tags = git_version_tags(clone) - - if len(version_tags): - version = version_tags[-1] - else: - version = git_default_branch(clone) - - try: - git_checkout(clone, version) - except git.GitCommandError as error: - LOG.warning( - 'failed to checkout branch/version "%s" of %s, ' - "skipping aggregation: %s", - version, - url, - error, - ) - msg = ( - f'failed to checkout branch/version "{version}": {error!r}' - ) - aggregation_issues.append((url, msg)) - continue - - metadata_file = _pick_metadata_file(str(clone.working_dir)) - metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata( - metadata_parser, - metadata_file, - ) - - if invalid_reason: - LOG.warning( - "skipping aggregation of %s: bad metadata: %s", - url, - invalid_reason, - ) - aggregation_issues.append((url, invalid_reason)) - continue - - metadata = _get_package_metadata(metadata_parser) - index_dir = os.path.dirname(index_file)[ - len(self.source_clonedir) + len(name) + 2 : - ] - qualified_name = os.path.join(index_dir, pkg_name) - - parser.add_section(qualified_name) - - for key, value in sorted(metadata.items()): - parser.set(qualified_name, key, value) - - parser.set(qualified_name, "url", url) - parser.set(qualified_name, "version", version) - - if qualified_name not in prev_packages: - agg_adds.append(qualified_name) - else: - prev_meta = configparser_section_dict( - prev_parser, - qualified_name, - ) - new_meta = configparser_section_dict(parser, qualified_name) - if prev_meta != new_meta: - agg_mods.append(qualified_name) - - with open(aggregate_file, "w") as f: - parser.write(f) - - agg_dels = list(prev_packages.difference(set(parser.sections()))) - - adds_str = " (" + ", ".join(sorted(agg_adds)) + ")" if agg_adds else "" - mods_str = " (" + ", ".join(sorted(agg_mods)) + ")" if agg_mods else "" - dels_str = " (" + ", ".join(sorted(agg_dels)) + ")" if agg_dels else "" - - LOG.debug( - "metadata refresh: %d additions%s, %d changes%s, %d removals%s", - len(agg_adds), - adds_str, - len(agg_mods), - mods_str, - len(agg_dels), - dels_str, - ) - - if push: - if os.path.isfile( - os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE), - ): - source.clone.git.add(AGGREGATE_DATA_FILE) - - if source.clone.is_dirty(): - # There's an assumption here that the dirty state is - # due to a metadata refresh. This could be incorrect - # if somebody makes local modifications and then runs - # the refresh without --aggregate, but it's not clear - # why one would use zkg for this as opposed to git - # itself. - source.clone.git.commit( - "--no-verify", - "--message", - "Update aggregated metadata.", - ) - LOG.info('committed package source "%s" metadata update', name) - - source.clone.git.push("--no-verify") - - return self.SourceAggregationResults("", aggregation_issues) + return self.sources[name].refresh() def refresh_installed_packages(self) -> None: """Fetch latest git information for installed packages. @@ -1317,7 +1075,7 @@ def refresh_installed_packages(self) -> None: ) continue - clonepath = os.path.join(self.package_clonedir, ipkg.package.name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), ipkg.package.name) clone = git.Repo(clonepath) LOG.debug("fetch package %s", ipkg.package.qualified_name()) @@ -1343,6 +1101,10 @@ def refresh_installed_packages(self) -> None: def upgrade(self, pkg_path: str) -> str: """Upgrade a package to the latest available version. + This does nothing if the package is pinned, or if the package isn't + outdated. The latter determination isn't made by this function, it + relies on the package's is_outdated flag for that purpose. + Args: pkg_path (str): the full git URL of a package or the shortened path/name that refers to it within a package source. E.g. for @@ -1356,6 +1118,7 @@ def upgrade(self, pkg_path: str) -> str: Raises: IOError: if the manifest can't be written + """ pkg_path = canonical_url(pkg_path) LOG.debug('upgrading "%s"', pkg_path) @@ -1373,7 +1136,7 @@ def upgrade(self, pkg_path: str) -> str: LOG.info('upgrading "%s": package not outdated', pkg_path) return "package is not outdated" - clonepath = os.path.join(self.package_clonedir, ipkg.package.name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), ipkg.package.name) clone = git.Repo(clonepath) if ipkg.status.tracking_method == TRACKING_METHOD_VERSION: @@ -1388,6 +1151,10 @@ def upgrade(self, pkg_path: str) -> str: if ipkg.status.tracking_method == TRACKING_METHOD_COMMIT: # The above check for whether the installed package is outdated # also should have already caught this situation. + + # NOTE: This likely refers to the fact that if a package got installed + # to a given commit, there's only that single version, so it cannot + # be outdated. So this is effectively pins the package. return "package is not outdated" raise NotImplementedError @@ -1395,6 +1162,8 @@ def upgrade(self, pkg_path: str) -> str: def remove(self, pkg_path: str) -> bool: """Remove an installed package. + This does not consider potential dependencies/dependees. + Args: pkg_path (str): the full git URL of a package or the shortened path/name that refers to it within a package source. E.g. for @@ -1425,16 +1194,16 @@ def remove(self, pkg_path: str) -> bool: self.unload(pkg_path) pkg_to_remove = ipkg.package - delete_path(os.path.join(self.package_clonedir, pkg_to_remove.name)) - delete_path(os.path.join(self.script_dir, pkg_to_remove.name)) - delete_path(os.path.join(self.plugin_dir, pkg_to_remove.name)) - delete_path(os.path.join(self.zeekpath(), pkg_to_remove.name)) + delete_path(os.path.join(CONFIG.packages_clone_dir(), pkg_to_remove.name)) + delete_path(os.path.join(CONFIG.packages_script_dir(), pkg_to_remove.name)) + delete_path(os.path.join(CONFIG.packages_plugin_dir(), pkg_to_remove.name)) + delete_path(os.path.join(CONFIG.zeek_path(), pkg_to_remove.name)) for alias in pkg_to_remove.aliases(): - delete_path(os.path.join(self.zeekpath(), alias)) + delete_path(os.path.join(CONFIG.zeek_path(), alias)) for exe in self._get_executables(pkg_to_remove.metadata): - link = os.path.join(self.bin_dir, os.path.basename(exe)) + link = os.path.join(CONFIG.bin_dir(), os.path.basename(exe)) if os.path.islink(link): try: LOG.debug("removing link %s", link) @@ -1551,7 +1320,7 @@ def load(self, pkg_path: str) -> str: return "" pkg_load_script = os.path.join( - self.script_dir, + CONFIG.packages_script_dir(), ipkg.package.name, "__load__.zeek", ) @@ -1572,7 +1341,7 @@ def load(self, pkg_path: str) -> str: return "" def loaded_package_states(self) -> dict[str, bool]: - """Save "loaded" state for all installed packages. + """Retrieves "loaded" state for all installed packages. Returns: dict: dictionary of "loaded" status for installed packages @@ -1582,7 +1351,7 @@ def loaded_package_states(self) -> dict[str, bool]: } def restore_loaded_package_states(self, saved_state: dict[str, bool]) -> None: - """Restores state for installed packages. + """Restores load-state for installed packages to the given constellation. Args: saved_state (dict): dictionary of saved "loaded" state for installed @@ -1827,6 +1596,9 @@ def bundle_info( ) -> tuple[str, list[tuple[str, str, PackageInfo]]]: """Retrieves information on all packages contained in a bundle. + To do this, it extracts the bundle into zkg's internal scratch space and + parses the contained manifest's "bundle" section. + Args: bundle_file (str): the path to the bundle to inspect. @@ -1839,9 +1611,10 @@ def bundle_info( the exact git URL and version string from the bundle's manifest along with the package info object retrieved by inspecting git repo contained in the bundle. + """ LOG.debug('getting bundle info for file "%s"', bundle_file) - bundle_dir = os.path.join(self.scratch_dir, "bundle") + bundle_dir = os.path.join(CONFIG.scratch_dir(), "bundle") delete_path(bundle_dir) make_dir(bundle_dir) infos: list[tuple[str, str, PackageInfo]] = [] @@ -1913,7 +1686,7 @@ def info( name = name_from_path(pkg_path) - if not is_valid_package_name(name): + if not is_valid_name(name): reason = f"Package name {name!r} is not valid." return PackageInfo(Package(git_url=pkg_path), invalid_reason=reason) @@ -1939,7 +1712,7 @@ def _info_lookup( if prefer_installed and ipkg: pkg_name = ipkg.package.name - clonepath = os.path.join(self.package_clonedir, pkg_name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), pkg_name) clone = git.Repo(clonepath) return _info_from_clone( clone, @@ -2007,7 +1780,7 @@ def _info( Raises: git.GitCommandError: when failing to clone the package repo """ - clonepath = os.path.join(self.scratch_dir, package.name) + clonepath = os.path.join(CONFIG.scratch_dir(), package.name) delete_path(clonepath) clone = git_clone(package.git_url, clonepath, shallow=True, recursive=False) versions = git_version_tags(clone) @@ -2039,7 +1812,7 @@ def package_versions(self, installed_package: InstalledPackage) -> list[str]: """ name = installed_package.package.name assert name - clonepath = os.path.join(self.package_clonedir, name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), name) clone = git.Repo(clonepath) return git_version_tags(clone) @@ -2260,7 +2033,7 @@ def add_node(node: Node) -> str: node = Node("zkg") node.installed_version = PackageVersion( TRACKING_METHOD_VERSION, - __version__, + VERSION, ) graph["zkg"] = node @@ -2509,7 +2282,7 @@ def bundle( str: empty string if the bundle is successfully created, else an error string explaining what failed. """ - bundle_dir = os.path.join(self.scratch_dir, "bundle") + bundle_dir = os.path.join(CONFIG.scratch_dir(), "bundle") delete_path(bundle_dir) make_dir(bundle_dir) manifest_file = os.path.join(bundle_dir, "manifest.txt") @@ -2557,7 +2330,7 @@ def match_package_url_and_version( ipkg = match_package_url_and_version(git_url, version) if ipkg: - src = os.path.join(self.package_clonedir, ipkg.package.name) + src = os.path.join(CONFIG.packages_clone_dir(), ipkg.package.name) shutil.copytree(src, clonepath, symlinks=True) clone = git.Repo(clonepath) clone.git.reset(hard=True) @@ -2577,7 +2350,7 @@ def match_package_url_and_version( # Record the built-in packages expected by this bundle (or simply # installed on the source system) in a new [meta] section to aid # debugging. This isn't interpreted, but if unbundle produces - # warnings it may proof helpful. + # warnings it may prove helpful. if builtin_packages: config.add_section("meta") entries = [] @@ -2605,7 +2378,7 @@ def unbundle(self, bundle_file: str) -> str: message indicated what went wrong. """ LOG.debug('unbundle "%s"', bundle_file) - bundle_dir = os.path.join(self.scratch_dir, "bundle") + bundle_dir = os.path.join(CONFIG.scratch_dir(), "bundle") delete_path(bundle_dir) make_dir(bundle_dir) @@ -2634,7 +2407,7 @@ def unbundle(self, bundle_file: str) -> str: ) # Prepare the clonepath with the contents from the bundle. - clonepath = os.path.join(self.package_clonedir, package.name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), package.name) delete_path(clonepath) shutil.move(os.path.join(bundle_dir, package.name), clonepath) @@ -2725,7 +2498,7 @@ def test( version = pkg_info.metadata_version package = pkg_info.package - stage = Stage(self, os.path.join(self.package_testdir, package.name)) + stage = Stage(self, os.path.join(CONFIG.testing_dir(), package.name)) stage.populate() request = [(package.qualified_name(), version)] @@ -2789,7 +2562,7 @@ def test( fail_msg = self._stage(info.package, version, clone, stage, env) if fail_msg: - return (fail_msg, False, self.state_dir) + return (fail_msg, False, CONFIG.state_dir()) # Finally, run tests (with correct environment set) if test_dependencies: @@ -2817,13 +2590,10 @@ def test( continue test_command = metadata["test_command"] - cwd = os.path.join(stage.clone_dir, info.package.name) - outfile = os.path.join(cwd, "zkg.test_command.stdout") - errfile = os.path.join(cwd, "zkg.test_command.stderr") LOG.debug( - 'running test_command for %s with cwd="%s", PATH="%s",' + 'testing "%s": running test_command with cwd="%s", PATH="%s",' ' and ZEEKPATH="%s": %s', info.package.name, cwd, @@ -2832,17 +2602,59 @@ def test( test_command, ) - with open(outfile, "w") as test_stdout, open(errfile, "w") as test_stderr: - cmd = subprocess.Popen( - test_command, - shell=True, - cwd=cwd, - env=env, - stdout=test_stdout, - stderr=test_stderr, + bufsize = 4096 + test = subprocess.Popen( + test_command, + shell=True, + cwd=cwd, + env=env, + bufsize=bufsize, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + testlog = self.package_test_log(info.package.name) + try: + with open(testlog, "wb") as f: + LOG.info( + 'testing "%s" in "%s": writing test log: %s', + package, + cwd, + testlog, + ) + + f.write("=== STDERR ===\n".encode(std_encoding(sys.stderr))) + + while True: + assert test.stderr + data = test.stderr.read(bufsize) + + if data: + f.write(data) + else: + break + + f.write("=== STDOUT ===\n".encode(std_encoding(sys.stdout))) + + while True: + assert test.stdout + data = test.stdout.read(bufsize) + + if data: + f.write(data) + else: + break + + except OSError as error: + LOG.warning( + 'testing "%s": failed to write test log %s %s: %s', + package, + testlog, + error.errno, + error.strerror, ) - rc = cmd.wait() + rc = test.wait() if rc != 0: assert stage.state_dir @@ -2898,16 +2710,16 @@ def _stage( """ LOG.debug('staging "%s": version %s', package, version) - metadata_file = _pick_metadata_file(str(clone.working_dir)) + metadata_file = pick_metadata_file(str(clone.working_dir)) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason: str | None = _parse_package_metadata( + invalid_reason: str | None = parse_package_metadata( metadata_parser, metadata_file, ) if invalid_reason: return invalid_reason - metadata = _get_package_metadata(metadata_parser) + metadata = get_package_metadata(metadata_parser) interpolated_metadata, invalid_reason = self._interpolate_package_metadata( metadata, stage, @@ -2983,12 +2795,16 @@ def _stage( script_dir_src = os.path.join(clone.working_dir, pkg_script_dir) script_dir_dst = os.path.join(stage.script_dir, package.name) + # TODO: what if the package has only a plugin, no script_dir? + # Seems conceivable and not an error. if not os.path.exists(script_dir_src): return f"package's 'script_dir' does not exist: {pkg_script_dir}" pkgload = os.path.join(script_dir_src, "__load__.zeek") if os.path.isfile(pkgload): + # In script_dir (share/zeek/site/ in a Zeek install), symlink + # -> packages//. Similarly for any aliases. symlink_path = os.path.join( os.path.dirname(stage.script_dir), package.name, @@ -3013,7 +2829,6 @@ def _stage( "script_dir", script_dir_src, script_dir_dst, - self.scratch_dir, ): return err else: @@ -3047,7 +2862,6 @@ def _stage( "plugin_dir", plugin_dir_src, plugin_dir_dst, - self.scratch_dir, ): return err @@ -3100,7 +2914,7 @@ def install(self, pkg_path: str, version: str = "") -> str: if conflict.qualified_name().endswith(pkg_path): LOG.debug('installing "%s": re-install: %s', pkg_path, conflict) - clonepath = os.path.join(self.package_clonedir, conflict.name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), conflict.name) _clone_package(conflict, clonepath, version) return self._install(conflict, version) @@ -3212,7 +3026,7 @@ def _install( git.GitCommandError: if the git repo is invalid IOError: if the package manifest file can't be written """ - clonepath = os.path.join(self.package_clonedir, package.name) + clonepath = os.path.join(CONFIG.packages_clone_dir(), package.name) ipkg = self.find_installed_package(package.name) if use_existing_clone or ipkg: @@ -3256,14 +3070,14 @@ def _install( status.current_hash = clone.head.object.hexsha status.is_outdated = _is_clone_outdated(clone, version, status.tracking_method) - metadata_file = _pick_metadata_file(str(clone.working_dir)) + metadata_file = pick_metadata_file(str(clone.working_dir)) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata(metadata_parser, metadata_file) + invalid_reason = parse_package_metadata(metadata_parser, metadata_file) if invalid_reason: return invalid_reason - raw_metadata = _get_package_metadata(metadata_parser) + raw_metadata = get_package_metadata(metadata_parser) invalid_reason = self._validate_alias_conflict(package, raw_metadata) @@ -3280,6 +3094,9 @@ def _install( if not package.source: # If installing directly from git URL, see if it actually is found # in a package source and fill in those details. + # + # TODO: This seems strange -- what if this is ambiguous across + # sources? Also we overwrite the metdata below anyway. for pkg in self.source_packages(): if pkg.git_url == package.git_url: package.source = pkg.source @@ -3290,7 +3107,7 @@ def _install( package.metadata = raw_metadata self.installed_pkgs[package.name] = InstalledPackage(package, status) self._write_manifest() - self._refresh_bin_dir(self.bin_dir) + self._refresh_bin_dir(CONFIG.bin_dir()) LOG.debug('installed "%s"', package) return "" @@ -3308,11 +3125,13 @@ def _interpolate_package_metadata( return None, "package has malformed 'user_vars' metadata field" substitutions = { - "zeek_dist": self.zeek_dist, "package_base": stage.clone_dir, } - substitutions.update(self.user_vars) + if CONFIG.zeek_dist(): + substitutions["zeek_dist"] = cast(str, CONFIG.zeek_dist()) + + substitutions.update(CONFIG.user_vars()) for uvar in requested_user_vars: val_from_env = os.environ.get(uvar.name()) @@ -3329,7 +3148,7 @@ def _interpolate_package_metadata( metadata_parser = configparser.ConfigParser(defaults=substitutions) metadata_parser.read_dict({"package": metadata}) - return _get_package_metadata(metadata_parser), None + return get_package_metadata(metadata_parser), None # Ensure we have links in bin_dir for all executables coming with any of # the currently installed packages. @@ -3337,7 +3156,7 @@ def _refresh_bin_dir(self, bin_dir: str) -> None: for ipkg in self.installed_pkgs.values(): for exe in self._get_executables(ipkg.package.metadata): # Put symlinks in place that are missing in current directory - src = os.path.join(self.package_clonedir, ipkg.package.name, exe) + src = os.path.join(CONFIG.packages_clone_dir(), ipkg.package.name, exe) dst = os.path.join(bin_dir, os.path.basename(exe)) if ( @@ -3416,7 +3235,6 @@ def _copy_package_dir( dirname: str, src: str, dst: str, - scratch_dir: str, ) -> str: """Copy a directory from a package to its installation location. @@ -3427,8 +3245,12 @@ def _copy_package_dir( if not os.path.exists(src): return "" + # TODO: instead of magic handling of tarfiles, it'd be nicer + # to have separate keywords for shipping archives. + # Since it only seems to make sense for pre-built plugins + # and their dependencies, perhaps plugin_archive. if os.path.isfile(src) and tarfile.is_tarfile(src): - tmp_dir = os.path.join(scratch_dir, "untar") + tmp_dir = os.path.join(CONFIG.scratch_dir(), "untar") delete_path(tmp_dir) make_dir(tmp_dir) @@ -3509,41 +3331,6 @@ def _clone_package( return git_clone(package.git_url, clonepath, shallow=shallow, recursive=recursive) -def _get_package_metadata(parser: configparser.ConfigParser) -> dict[str, str]: - return {item[0]: item[1] for item in parser.items("package")} - - -def _pick_metadata_file(directory: str) -> str: - rval = os.path.join(directory, METADATA_FILENAME) - - if os.path.exists(rval): - return rval - - return os.path.join(directory, LEGACY_METADATA_FILENAME) - - -def _parse_package_metadata( - parser: configparser.ConfigParser, - metadata_file: str, -) -> str: - """Return string explaining why metadata is invalid, or '' if valid.""" - if not parser.read(metadata_file): - LOG.warning("%s: missing metadata file", metadata_file) - return ( - f"missing {METADATA_FILENAME} (or {LEGACY_METADATA_FILENAME}) metadata file" - ) - - if not parser.has_section("package"): - LOG.warning("%s: metadata missing [package]", metadata_file) - return f"{os.path.basename(metadata_file)} is missing [package] section" - - for a in aliases(_get_package_metadata(parser)): - if not is_valid_package_name(a): - return f'invalid alias "{a}"' - - return "" - - _legacy_metadata_warnings: set[str] = set() @@ -3568,9 +3355,9 @@ def _info_from_clone( else: version_type = TRACKING_METHOD_BRANCH - metadata_file = _pick_metadata_file(str(clone.working_dir)) + metadata_file = pick_metadata_file(str(clone.working_dir)) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata(metadata_parser, metadata_file) + invalid_reason = parse_package_metadata(metadata_parser, metadata_file) if invalid_reason: return PackageInfo( @@ -3597,7 +3384,7 @@ def _info_from_clone( ) _legacy_metadata_warnings.add(package.qualified_name()) - metadata = _get_package_metadata(metadata_parser) + metadata = get_package_metadata(metadata_parser) return PackageInfo( package=package, @@ -3614,3 +3401,29 @@ def _info_from_clone( def _is_reserved_pkg_name(name: str) -> bool: return name in {"zeek", "zkg"} + + +def create_manager() -> Manager | None: + """Helper to create the manager robustly in the presence of file permission errors. + + Returns a manager instance, or None in case of file access errors. + Other types of exceptions are not caught. + """ + try: + return Manager() + except OSError as error: + if error.errno == errno.EACCES: + LOG.error(f"{type(error).__name__}: {error}") + + def check_permission(d: str) -> None: + if not os.access(d, os.W_OK): + LOG.error(f"user does not have write access in {d}") + + check_permission(CONFIG.state_dir()) + check_permission(CONFIG.script_dir()) + check_permission(CONFIG.plugin_dir()) + check_permission(CONFIG.bin_dir()) + + return None + + raise diff --git a/zeekpkg/package.py b/zeekpkg/package.py index ab2c570d..67f43f67 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -3,6 +3,7 @@ the properties and status of Zeek packages. """ +import configparser import os import re from functools import total_ordering @@ -10,6 +11,7 @@ import semantic_version as semver from ._util import find_sentence_end, normalize_version_tag +from .logs import LOG from .uservar import UserVar #: The name of files used by packages to store their metadata. @@ -30,6 +32,41 @@ LEGACY_PLUGIN_MAGIC_FILE_DISABLED = "__bro_plugin__.disabled" +def get_package_metadata(parser: configparser.ConfigParser) -> dict[str, str]: + return {item[0]: item[1] for item in parser.items("package")} + + +def pick_metadata_file(directory: str) -> str: + rval = os.path.join(directory, METADATA_FILENAME) + + if os.path.exists(rval): + return rval + + return os.path.join(directory, LEGACY_METADATA_FILENAME) + + +def parse_package_metadata( + parser: configparser.ConfigParser, + metadata_file: str, +) -> str: + """Return string explaining why metadata is invalid, or '' if valid.""" + if not parser.read(metadata_file): + LOG.warning("%s: missing metadata file", metadata_file) + return ( + f"missing {METADATA_FILENAME} (or {LEGACY_METADATA_FILENAME}) metadata file" + ) + + if not parser.has_section("package"): + LOG.warning("%s: metadata missing [package]", metadata_file) + return f"{os.path.basename(metadata_file)} is missing [package] section" + + for a in aliases(get_package_metadata(parser)): + if not is_valid_name(a): + return f'invalid alias "{a}"' + + return "" + + def name_from_path(path: str) -> str: """Returns the name of a package given a path to its git repository.""" return canonical_url(path).split("/")[-1] @@ -46,7 +83,11 @@ def canonical_url(path: str) -> str: def is_valid_name(name: str) -> bool: - """Returns True if name is a valid package name, else False.""" + """Returns True if name is a valid package name, else False. + + A name is valid if it doesn't contain spaces, slashes, dots, or reserved + words. + """ if name != name.strip(): # Reject names with leading/trailing whitespace return False @@ -87,6 +128,10 @@ def short_description(metadata_dict: dict[str, str]) -> str: return "" description = metadata_dict["description"] + + # Strip any quotes and whitespace around the whole description. + description = description.strip(' "') + lines = description.split("\n") rval = "" diff --git a/zeekpkg/source.py b/zeekpkg/source.py index fccff006..89b1a5fb 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -7,14 +7,32 @@ """ import configparser +import filecmp import os import shutil import git -from . import LOG -from ._util import git_checkout, git_clone, git_default_branch -from .package import Package, name_from_path +from ._util import ( + configparser_section_dict, + delete_path, + git_checkout, + git_clone, + git_default_branch, + git_pull, + git_version_tags, +) +from .config import ( + CONFIG, +) +from .logs import LOG +from .package import ( + Package, + get_package_metadata, + name_from_path, + parse_package_metadata, + pick_metadata_file, +) #: The name of package index files. INDEX_FILENAME = "zkg.index" @@ -23,6 +41,29 @@ AGGREGATE_DATA_FILE = "aggregate.meta" +class AggregationResults: + """The return value of a call to :meth:`.Source.aggregate()`. + + Attributes: + error (str): an empty string if no overall error occurred, + otherwise a description of what went wrong. + + package_issues (list of (str, str)): a list of reasons for + failing to collect metadata per packages/repository. + The first tuple element gives the repository URL in which + the problem occurred and the second tuple element describes + the failure. + """ + + def __init__( + self, + error: str = "", + package_issues: list[tuple[str, str]] | None = None, + ) -> None: + self.error = error + self.package_issues = package_issues if package_issues else [] + + class Source: """A Zeek package source. @@ -90,6 +131,8 @@ def __init__( if current != target: git_checkout(self.clone, target) + self.aggregate_file = os.path.join(self.clone.working_dir, AGGREGATE_DATA_FILE) + def __str__(self) -> str: return self.git_url @@ -158,3 +201,192 @@ def packages(self) -> list[Package]: rval.append(package) return rval + + def refresh(self) -> str: + LOG.debug('refresh "%s": pulling %s', self.name, self.git_url) + + agg_file_ours = os.path.join(CONFIG.scratch_dir(), AGGREGATE_DATA_FILE) + agg_file_their_orig = agg_file_ours + ".orig" + + delete_path(agg_file_ours) + delete_path(agg_file_their_orig) + + if os.path.isfile(self.aggregate_file): + shutil.copy2(self.aggregate_file, agg_file_ours) + + self.clone.git.reset(hard=True) + self.clone.git.clean("-f", "-x", "-d") + + if os.path.isfile(self.aggregate_file): + shutil.copy2(self.aggregate_file, agg_file_their_orig) + + try: + self.clone.git.fetch("--recurse-submodules=yes") + git_pull(self.clone) + except git.GitCommandError as error: + LOG.error("failed to pull source %s: %s", self.name, error) + return f"failed to pull from remote source: {error}" + + if os.path.isfile(agg_file_ours): + if os.path.isfile(self.aggregate_file): + # There's a tracked version of the file after pull. + if os.path.isfile(agg_file_their_orig): + # We had local modifications to the file. + if filecmp.cmp(self.aggregate_file, agg_file_their_orig): + # Their file hasn't changed, use ours. + shutil.copy2(agg_file_ours, self.aggregate_file) + LOG.debug( + "aggregate file in source unchanged, restore local one", + ) + else: + # Their file changed, use theirs. + LOG.debug("aggregate file in source changed, discard local one") + else: + # File was untracked before pull and tracked after, + # use their version. + LOG.debug("new aggregate file in source, discard local one") + else: + # They don't have the file after pulling, so restore ours. + shutil.copy2(agg_file_ours, self.aggregate_file) + LOG.debug("no aggregate file in source, restore local one") + + return "" + + def aggregate(self, push: bool = False) -> AggregationResults: + parser = configparser.ConfigParser(interpolation=None) + prev_parser = configparser.ConfigParser(interpolation=None) + prev_packages = set() + + if os.path.isfile(self.aggregate_file): + prev_parser.read(self.aggregate_file) + prev_packages = set(prev_parser.sections()) + + aggregation_issues = [] + agg_adds, agg_mods, agg_dels = [], [], [] + + for index_file in self.package_index_files(): + urls = [] + + with open(index_file) as f: + urls = [line.rstrip("\n") for line in f] + + for url in urls: + pkg_name = name_from_path(url) + clonepath = os.path.join(CONFIG.scratch_dir(), pkg_name) + delete_path(clonepath) + + try: + clone = git_clone(url, clonepath, shallow=True) + except git.GitCommandError as error: + LOG.warning( + "failed to clone %s, skipping aggregation: %s", + url, + error, + ) + aggregation_issues.append((url, repr(error))) + continue + + version_tags = git_version_tags(clone) + + if len(version_tags): + version = version_tags[-1] + else: + version = git_default_branch(clone) + + try: + git_checkout(clone, version) + except git.GitCommandError as error: + LOG.warning( + 'failed to checkout branch/version "%s" of %s, ' + "skipping aggregation: %s", + version, + url, + error, + ) + msg = f'failed to checkout branch/version "{version}": {error!r}' + aggregation_issues.append((url, msg)) + continue + + metadata_file = pick_metadata_file(str(clone.working_dir)) + metadata_parser = configparser.ConfigParser(interpolation=None) + invalid_reason = parse_package_metadata( + metadata_parser, + metadata_file, + ) + + if invalid_reason: + LOG.warning( + "skipping aggregation of %s: bad metadata: %s", + url, + invalid_reason, + ) + aggregation_issues.append((url, invalid_reason)) + continue + + metadata = get_package_metadata(metadata_parser) + index_dir = os.path.dirname(index_file)[ + len(CONFIG.sources_clone_dir()) + len(self.name) + 2 : + ] + qualified_name = os.path.join(index_dir, pkg_name) + + parser.add_section(qualified_name) + + for key, value in sorted(metadata.items()): + parser.set(qualified_name, key, value) + + parser.set(qualified_name, "url", url) + parser.set(qualified_name, "version", version) + + if qualified_name not in prev_packages: + agg_adds.append(qualified_name) + else: + prev_meta = configparser_section_dict( + prev_parser, + qualified_name, + ) + new_meta = configparser_section_dict(parser, qualified_name) + if prev_meta != new_meta: + agg_mods.append(qualified_name) + + with open(self.aggregate_file, "w") as f: + parser.write(f) + + agg_dels = list(prev_packages.difference(set(parser.sections()))) + + adds_str = " (" + ", ".join(sorted(agg_adds)) + ")" if agg_adds else "" + mods_str = " (" + ", ".join(sorted(agg_mods)) + ")" if agg_mods else "" + dels_str = " (" + ", ".join(sorted(agg_dels)) + ")" if agg_dels else "" + + LOG.debug( + "metadata refresh: %d additions%s, %d changes%s, %d removals%s", + len(agg_adds), + adds_str, + len(agg_mods), + mods_str, + len(agg_dels), + dels_str, + ) + + if push: + if os.path.isfile( + os.path.join(self.clone.working_dir, AGGREGATE_DATA_FILE), + ): + self.clone.git.add(AGGREGATE_DATA_FILE) + + if self.clone.is_dirty(): + # There's an assumption here that the dirty state is + # due to a metadata refresh. This could be incorrect + # if somebody makes local modifications and then runs + # the refresh without --aggregate, but it's not clear + # why one would use zkg for this as opposed to git + # itself. + self.clone.git.commit( + "--no-verify", + "--message", + "Update aggregated metadata.", + ) + LOG.info('committed package source "%s" metadata update', self.name) + + self.clone.git.push("--no-verify") + + return AggregationResults("", aggregation_issues) diff --git a/zeekpkg/template.py b/zeekpkg/template.py index bcadfbe8..ef26fe23 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -3,7 +3,6 @@ """ import abc -import configparser import os import re import shutil @@ -14,12 +13,15 @@ import semantic_version as semver if TYPE_CHECKING: - from zeekpkg import UserVar + from zeekpkg.uservar import UserVar -from . import ( - LOG, - __version__, +from zeekpkg.config import ( + CONFIG, +) +from zeekpkg.consts import ( + VERSION, ) + from ._util import ( delete_path, git_checkout, @@ -31,6 +33,9 @@ load_source, make_dir, ) +from .logs import ( + LOG, +) from .package import ( name_from_path, ) @@ -68,7 +73,6 @@ class Template: @staticmethod def load( - config: configparser.ConfigParser, template: str, version: str | None = None, ) -> "Template": @@ -83,8 +87,6 @@ def load( derivative that must be present in it. Args: - config (configparser.ConfigParser): a zkg configuration - template (str): template source repo, as directory or git URL version (str): if provided, a specific version tag to use. @@ -123,7 +125,7 @@ def load( # zkg state folder's clone space and support version # requests. template_clonedir = os.path.join( - config.get("paths", "state_dir"), + CONFIG.get("paths", "state_dir"), "clones", "template", ) @@ -427,7 +429,7 @@ def info(self) -> dict[str, Any]: "provides_package": False, } - # XXX we should revisit the reported 'origin' value in + # TODO: we should revisit the reported 'origin' value in # API 2.0.0 -- the the ad-hoc strings are less helpful # than simply providing the key only when there's an # actual origin. @@ -792,7 +794,7 @@ def _git_init(self, tmpl: Template) -> None: commit_msg = f"""Initial commit. -zkg {__version__} created this package from template "{tmpl_source}" +zkg {VERSION} created this package from template "{tmpl_source}" using {ver_info}.""" if self._features: diff --git a/zeekpkg/ui.py b/zeekpkg/ui.py new file mode 100644 index 00000000..379e5914 --- /dev/null +++ b/zeekpkg/ui.py @@ -0,0 +1,320 @@ +""" +This module provides abstractions for the output directly resulting from the +command-line arguments. It has (basic) support for progress indication and +confirmation prompts, and supports its own level of log/info/warn output. In +contrast to the logging functionality in zeekpkg/log, the UI output focuses on +stdout. +""" + +import abc +import sys +import threading +from collections.abc import Callable +from typing import Any, TextIO + +# A type for the kind of callable we use for activities: +# they return an error string upon completion. +UiCallable = Callable[..., str] + + +class Activity(abc.ABC): + """An activity conducted by zkg, such as a package install, build, or + package source refresh. + """ + + def __init__(self) -> None: + # An error message to leave in case the activity didn't complete + # successfully. + self.error = "" + + @abc.abstractmethod + def __call__(self) -> None: + pass + + +class CallableActivity(Activity): + """An activity defined by a callable passed on to us.""" + + def __init__(self, call: UiCallable): + super().__init__() + self.call = call + + def __call__(self) -> None: + self.error = self.call() + + +class ProgressActivity(Activity): + """An activity for which progress is quantifiable.""" + + def __init__(self, total: float = 1.0) -> None: + super().__init__() + self.total = total + + @abc.abstractmethod + def __call__(self) -> None: + pass + + def progress(self, delta: float) -> None: + """Callback invoked by the activity as it unfolds, whenever it can + report progress toward completion. + """ + pass + + +class Worker(threading.Thread): + """A worker executes an activity. It runs in the background (in the sense of + Python's threading model, so not truly concurrently), and can be waited + upon. + """ + + def __init__( + self, + activity: Activity, + ) -> None: + super().__init__() + self.activity = activity + + def run(self) -> None: + """Runs the activity in the background.""" + self.activity() + + def wait( + self, + out: TextIO | Any = sys.stdout, + ) -> None: + """Blocks until this activity ends, optionally writing liveness indicators. + + This never returns until this thread dies (i.e., is_alive() is False). + When an output file object is provided, the method also indicates + progress by writing a dot character to it once per second. This happens + only when the file is a TTY. When a message is given, it gets written + out first, regardless of TTY status. Any output always terminates with a + newline. + + Args: + msg (str): a message to write first. + + out (file-like object): the destination to write to. + + """ + is_tty = hasattr(out, "isatty") and out.isatty() + + while True: + self.join(1.0) + if not self.is_alive(): + break + + if out is not None and is_tty: + out.write(".") + out.flush() + + if out is not None and is_tty: + out.write("\n") + out.flush() + + +class UserInterface(abc.ABC): + def __init__(self, verbosity: int = 0) -> None: + self.verbosity = verbosity + + @abc.abstractmethod + def debug( + self, + *msgs: str, + prefix: str = "Debug:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + """Very verbose messaging, usually to stdout.""" + + @abc.abstractmethod + def verbose( + self, + *msgs: str, + prefix: str = "Verbose:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + """Verbose messaging, usually to stdout.""" + + @abc.abstractmethod + def info( + self, + *msgs: str, + prefix: str = "Info:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + """Regular messaging, usually to stdout.""" + + @abc.abstractmethod + def warning( + self, + *msgs: str, + prefix: str = "Warning:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + """Warnings, usually to stderr.""" + + @abc.abstractmethod + def error( + self, + *msgs: str, + prefix: str = "Error:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + """Errors, usually to stderr.""" + + @abc.abstractmethod + def activity(self, act: Activity) -> str: + """Launch an open-ended activity to completion. + + Once the activity completes, the returned string contains an error + message if anything went wrong. otherwise the string is empty. + """ + + @abc.abstractmethod + def call_activity(self, call: Callable[[], str]) -> str: + """Launch an open-ended activity to completion. + + The activity is provided in form of the callable. Once the activity + completes, the returned string contains an error message if anything + went wrong. otherwise the string is empty. + """ + + @abc.abstractmethod + def progress_activity(self, act: ProgressActivity) -> str: + """Launch a trackable activity to completion. + + This applies to completions that can track their own progress, knowing + when they complete. Once the activity completes, the returned string + contains an error message if anything went wrong. otherwise the string + is empty. + """ + + def confirmation_prompt(self, prompt: str, default_to_yes: bool = True) -> bool: + """An interactive prompt to obtain confirmation from the user.""" + yes = {"y", "ye", "yes"} + + if default_to_yes: + prompt += " [Y/n] " + else: + prompt += " [N/y] " + + choice = input(prompt).lower() + + if not choice: + if default_to_yes: + return True + + print("Abort.") + return False + + if choice in yes: + return True + + print("Abort.") + return False + + +class PlainUI(UserInterface): + """A basic plaintext UI, suitable for the console or redirection to files.""" + + def __init__( + self, + verbosity: int = 0, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, + ): + super().__init__(verbosity=verbosity) + self.stdout = stdout + self.stderr = stderr + + def debug( + self, + *msgs: str, + prefix: str = "", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + _ = prefix + if self.verbosity >= 2: + print(*msgs, sep=sep, end=end, file=self.stdout, flush=flush) + + def verbose( + self, + *msgs: str, + prefix: str = "", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + _ = prefix + if self.verbosity >= 1: + print(*msgs, sep=sep, end=end, file=self.stdout, flush=flush) + + def info( + self, + *msgs: str, + prefix: str = "Info:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + print(*msgs, sep=sep, end=end, file=self.stdout, flush=flush) + + def warning( + self, + *msgs: str, + prefix: str = "Warning:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + if prefix: + print(prefix, *msgs, sep=sep, end=end, file=self.stderr, flush=flush) + else: + print(*msgs, sep=sep, end=end, file=self.stderr, flush=flush) + + def error( + self, + *msgs: str, + prefix: str = "Error:", + sep: str = " ", + end: str = "\n", + flush: bool = False, + ) -> None: + if prefix: + print(prefix, *msgs, sep=sep, end=end, file=self.stderr, flush=flush) + else: + print(*msgs, sep=sep, end=end, file=self.stderr, flush=flush) + + def activity(self, act: Activity) -> str: + worker = Worker(act) + worker.start() + worker.wait(self.stdout) + return act.error + + def call_activity(self, call: UiCallable) -> str: + act = CallableActivity(call) + worker = Worker(act) + worker.start() + worker.wait() + return act.error + + def progress_activity(self, act: ProgressActivity) -> str: + worker = Worker(act) + worker.start() + worker.wait() + return act.error + + +UI: UserInterface = PlainUI() diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index decc272a..cb7d3228 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -4,11 +4,14 @@ responses to zkg's input prompting. """ -import configparser import os import re import readline +from zeekpkg.config import ( + CONFIG, +) + def slugify(string: str) -> str: """Returns file-system-safe, lower-case version of the input string. @@ -69,7 +72,6 @@ def default(self) -> str | None: def resolve( self, name: str, - config: configparser.ConfigParser, user_var_args: list["UserVar"] | None = None, force: bool = False, ) -> str: @@ -80,7 +82,7 @@ def resolve( (1) Use any value provided on the command line via --user-var (2) If force is not used, prompt the user for input (3) use an environment variables of the same name, - (4) retrieve from the provided config parser's "user_vars" section, + (4) retrieve from zkg's "user_vars" configuration, (5) use the default value of the user variable. The resolved value is stored with the instance (to be @@ -89,8 +91,6 @@ def resolve( Args: name (str): the requesting entity, e.g. a package name - config (configparser.ConfigParser): the zkg configuration - user_var_args (list of UserVar): user-var instances provided via command line @@ -127,7 +127,7 @@ def resolve( if val is None: # Try to re-use a cached value in the subsequent prompt - val = config.get("user_vars", self._name, fallback=self._default) + val = CONFIG.get("user_vars", self._name, fallback=self._default) if force: if val is None: diff --git a/zkg b/zkg index f5fa9909..588e3106 100755 --- a/zkg +++ b/zkg @@ -3,3179 +3,46 @@ # https://pypi.org/project/argcomplete/#global-completion # PYTHON_ARGCOMPLETE_OK -import argparse -import configparser -import errno -import filecmp -import io -import json -import logging -import os -import re -import shutil -import subprocess import sys -import threading -from collections import OrderedDict -from typing import Any, TextIO - -import argcomplete -import git - -# The default package source we fall back to as needed -ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" - -# The default package template -ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" import zeekpkg -from zeekpkg._util import ( - delete_path, - find_program, - make_dir, - read_zeek_config_line, - std_encoding, -) -from zeekpkg.package import ( - BUILTIN_SCHEME, - TRACKING_METHOD_VERSION, - InstalledPackage, - Package, - PackageInfo, -) -from zeekpkg.template import ( - InputError, - LoadError, - OutputError, - Template, -) -from zeekpkg.uservar import ( - UserVar, -) - - -def confirmation_prompt(prompt: str, default_to_yes: bool = True) -> bool: - yes = {"y", "ye", "yes"} - - if default_to_yes: - prompt += " [Y/n] " - else: - prompt += " [N/y] " - - choice = input(prompt).lower() - - if not choice: - if default_to_yes: - return True - - print("Abort.") - return False - - if choice in yes: - return True - - print("Abort.") - return False - - -def prompt_for_user_vars( - manager: zeekpkg.Manager, - config: configparser.ConfigParser, - configfile: str, - args: argparse.Namespace, - pkg_infos: list[PackageInfo], -) -> None: - answers: dict[str, str] = {} - - for info in pkg_infos: - name = info.package.qualified_name() - requested_user_vars = info.user_vars() - - if requested_user_vars is None: - print_error(f'error: malformed user_vars in "{name}"') - sys.exit(1) - - for uvar in requested_user_vars: - try: - answers[uvar.name()] = uvar.resolve( - name, - config, - args.user_var, - args.force, - ) - except ValueError: - print_error( - f'error: could not determine value of user variable "{uvar.name()}",' - " provide via environment or --user-var", - ) - sys.exit(1) - - if not args.force and answers: - for key, value in answers.items(): - if not config.has_section("user_vars"): - config.add_section("user_vars") - - config.set("user_vars", key, value) - - if configfile: - with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: - config.write(f) - - print(f"Saved answers to config file: {configfile}") - - manager.user_vars = answers - - -def print_error(*args: object, **kwargs: Any) -> None: - print(*args, file=sys.stderr, **kwargs) - - -def config_items( - config: configparser.ConfigParser, - section: str, -) -> list[tuple[str, str]]: - # Same as config.items(section), but exclude default keys. - defaults = {key for key, _ in config.items("DEFAULT")} - items = sorted(config.items(section)) - return [(key, value) for (key, value) in items if key not in defaults] - - -def file_is_not_empty(path: str) -> bool: - return os.path.isfile(path) and os.path.getsize(path) > 0 - - -def find_configfile(args: argparse.Namespace) -> str | None: - if args.user: - configfile: str | None = os.path.join(home_config_dir(), "config") - - if configfile and file_is_not_empty(configfile): - return configfile - - return None - - configfile = os.environ.get("ZKG_CONFIG_FILE") - - if configfile and file_is_not_empty(configfile): - return configfile - - configfile = os.path.join(default_config_dir(), "config") - - if file_is_not_empty(configfile): - return configfile - - return None - - -def home_config_dir() -> str: - return os.path.join(os.path.expanduser("~"), ".zkg") - - -def default_config_dir() -> str: - return os.getenv("ZEEK_ZKG_CONFIG_DIR") or home_config_dir() - - -def default_state_dir() -> str: - return os.getenv("ZEEK_ZKG_STATE_DIR") or home_config_dir() - - -def create_config( - args: argparse.Namespace, - configfile: str | None, -) -> configparser.ConfigParser: - config = configparser.ConfigParser() - - if configfile: - if not os.path.isfile(configfile): - print_error(f'error: invalid config file "{configfile}"') - sys.exit(1) - - config.read(configfile) - - if not config.has_section("sources"): - config.add_section("sources") - - if not config.has_section("paths"): - config.add_section("paths") - - if not config.has_section("templates"): - config.add_section("templates") - - if not configfile: - default = os.getenv("ZKG_DEFAULT_SOURCE", ZKG_DEFAULT_SOURCE) - if default: - config.set("sources", "zeek", default) - - if not configfile or not config.has_option("templates", "default"): - default = os.getenv("ZKG_DEFAULT_TEMPLATE", ZKG_DEFAULT_TEMPLATE) - if default: - config.set("templates", "default", default) - - def config_option_set( - config: configparser.ConfigParser, - section: str, - option: str, - ) -> str | None: - if config.has_option(section, option): - return config.get(section, option) - - return None - - def get_option( - config: configparser.ConfigParser, - section: str, - option: str, - default: str, - ) -> str: - if config_option_set(config, section, option): - return config.get(section, option) - - return default - - if args.user: - def_state_dir = home_config_dir() - else: - def_state_dir = default_state_dir() - - state_dir = get_option(config, "paths", "state_dir", os.path.join(def_state_dir)) - script_dir = get_option( - config, - "paths", - "script_dir", - os.path.join(state_dir, "script_dir"), - ) - plugin_dir = get_option( - config, - "paths", - "plugin_dir", - os.path.join(state_dir, "plugin_dir"), - ) - bin_dir = get_option(config, "paths", "bin_dir", os.path.join(state_dir, "bin")) - zeek_dist = get_option(config, "paths", "zeek_dist", "") - - config.set("paths", "state_dir", state_dir) - config.set("paths", "script_dir", script_dir) - config.set("paths", "plugin_dir", plugin_dir) - config.set("paths", "bin_dir", bin_dir) - config.set("paths", "zeek_dist", zeek_dist) - - def expand_config_values(config: configparser.ConfigParser, section: str) -> None: - for key, value in config.items(section): - value = os.path.expandvars(os.path.expanduser(value)) - config.set(section, key, value) - - expand_config_values(config, "sources") - expand_config_values(config, "paths") - expand_config_values(config, "templates") - - for key, value in config.items("paths"): - if value and not os.path.isabs(value): - print_error( - "error: invalid config file value for key" - f' "{key}" in section [paths]: "{value}" is not' - " an absolute path", - ) - sys.exit(1) - - return config - - -def active_git_branch(path: str) -> str | None: - try: - repo = git.Repo(path) - except git.NoSuchPathError: - return None - - if not repo.working_tree_dir: - return None - - try: - rval = repo.active_branch - except TypeError: - # return detached commit - rval = repo.head.commit - - if not rval: - return None - - return str(rval) - - -def is_local_git_repo_url(git_url: str) -> bool: - return ( - git_url.startswith(".") or git_url.startswith("/") or is_local_git_repo(git_url) - ) - - -def is_local_git_repo(git_url: str) -> bool: - try: - # The Repo class takes a file system path as first arg. This - # can fail in two ways: (1) the path doesn't exist or isn't - # accessible, (2) it's not the root directory of a git repo. - git.Repo(git_url) - return True - except (git.InvalidGitRepositoryError, git.NoSuchPathError): - return False - - -def is_local_git_repo_dirty(git_url: str) -> bool: - if not is_local_git_repo(git_url): - return False - try: - repo = git.Repo(git_url) - except git.NoSuchPathError: - return False - - return repo.is_dirty(untracked_files=True) - - -def check_local_git_repo(git_url: str) -> bool: - if is_local_git_repo_url(git_url): - if not is_local_git_repo(git_url): - print_error(f"error: path {git_url} is not a git repository") - return False - if is_local_git_repo_dirty(git_url): - print_error(f"error: local git clone at {git_url} is dirty") - return False - - return True - - -def create_manager( - args: argparse.Namespace, - config: configparser.ConfigParser, -) -> zeekpkg.Manager: - state_dir = config.get("paths", "state_dir") - script_dir = config.get("paths", "script_dir") - plugin_dir = config.get("paths", "plugin_dir") - bin_dir = config.get("paths", "bin_dir") - zeek_dist = config.get("paths", "zeek_dist") - - try: - manager = zeekpkg.Manager( - state_dir=state_dir, - script_dir=script_dir, - plugin_dir=plugin_dir, - bin_dir=bin_dir, - zeek_dist=zeek_dist, - ) - except OSError as error: - if error.errno == errno.EACCES: - print_error(f"{type(error).__name__}: {error}") - - def check_permission(d: str) -> bool: - if os.access(d, os.W_OK): - return True - - print_error(f"error: user does not have write access in {d}") - return False - - permissions_trouble = not all( - [ - check_permission(state_dir), - check_permission(script_dir), - check_permission(plugin_dir), - check_permission(bin_dir), - ], - ) - - if permissions_trouble and not args.user: - print_error( - f"Consider the --user flag to manage zkg state via {home_config_dir()}/config", - ) - sys.exit(1) - - raise - - extra_sources = [] - - for key_value in args.extra_source or []: - if "=" not in key_value: - print_error(f'warning: invalid extra source: "{key_value}"') - continue - - key, value = key_value.split("=", 1) - - if not key or not value: - print_error(f'warning: invalid extra source: "{key_value}"') - continue - - extra_sources.append((key, value)) - - for key, value in extra_sources + config_items(config, "sources"): - if err := manager.add_source(name=key, git_url=value): - print_error( - f'warning: skipped using package source named "{key}": {err}', - ) - - return manager - - -def get_changed_state( - manager: zeekpkg.Manager, - saved_state: dict[str, bool], - pkg_lst: list[str], -) -> str: - """Returns the list of packages that have changed loaded state. - - Args: - saved_state (dict): dictionary of saved load state for installed - packages. - - pkg_lst (list): list of package names to be skipped - - Returns: - dep_listing (str): string installed packages that have changed state - - """ - _lst = [zeekpkg.package.name_from_path(_pkg_path) for _pkg_path in pkg_lst] - dep_listing = "" - - for _pkg_name in sorted(manager.installed_package_dependencies()): - if _pkg_name in _lst: - continue - - _ipkg = manager.find_installed_package(_pkg_name) - - if not _ipkg or _ipkg.status.is_loaded == saved_state[_pkg_name]: - continue - - dep_listing += f" {_pkg_name}\n" - - return dep_listing - - -class InstallWorker(threading.Thread): - def __init__( - self, - manager: zeekpkg.Manager, - package_name: str, - package_version: str, - ) -> None: - super().__init__() - self.manager = manager - self.package_name = package_name - self.package_version = package_version - self.error = "" - - def run(self) -> None: - self.error = self.manager.install(self.package_name, self.package_version) - - def wait( - self, - msg: str | None = None, - out: TextIO | Any = sys.stdout, - tty_only: bool = True, - ) -> None: - """Blocks until this thread ends, optionally writing liveness indicators. - - This never returns until this thread dies (i.e., is_alive() is False). - When an output file object is provided, the method also indicates - progress by writing a dot character to it once per second. This happens - only when the file is a TTY, unless ``tty_only`` is False. When a - message is given, it gets written out first, regardless of TTY - status. Any output always terminates with a newline. - - Args: - msg (str): a message to write first. - - out (file-like object): the destination to write to. - - tty_only (bool): whether to write progress dots also to non-TTYs. - """ - if out is not None and msg: - out.write(msg) - out.flush() - - is_tty = hasattr(out, "isatty") and out.isatty() - - while True: - self.join(1.0) - if not self.is_alive(): - break - - if out is not None and (is_tty or not tty_only): - out.write(".") - out.flush() - - if out is not None and (msg or is_tty or not tty_only): - out.write("\n") - out.flush() - - -def cmd_test( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.version and len(args.package) > 1: - print_error('error: "test --version" may only be used for a single package') - sys.exit(1) - - package_infos: list[tuple[PackageInfo, str]] = [] - - for name in args.package: - if not check_local_git_repo(name): - sys.exit(1) - - # If the package to be tested is included with Zeek, don't allow - # to run tests due to the potential of conflicts. - bpkg_info = manager.find_builtin_package(name) - if bpkg_info is not None: - print_error(f'cannot run tests for "{name}": built-in package') - sys.exit(1) - - version = args.version if args.version else active_git_branch(name) - package_info = manager.info(name, version=version, prefer_installed=False) - - if package_info.invalid_reason: - print_error( - f'error: invalid package "{name}": {package_info.invalid_reason}', - ) - sys.exit(1) - if not version: - version = package_info.best_version() - package_infos.append((package_info, version)) - - all_passed = True - - for info, version in package_infos: - name = info.package.qualified_name() - - if "test_command" not in info.metadata: - print(f"{name}: no test_command found in metadata, skipping") - continue - - error_msg, passed, test_dir = manager.test( - name, - version, - test_dependencies=True, - ) - - if error_msg: - all_passed = False - print_error(f'error: failed to run tests for "{name}": {error_msg}') - continue - - if passed: - print(f"{name}: all tests passed") - else: - all_passed = False - clone_dir = os.path.join( - os.path.join(test_dir, "clones"), - info.package.name, - ) - print_error( - f'error: package "{name}" tests failed, inspect' - f" contents of {test_dir} for details, especially" - ' any "zkg.test_command.{{stderr,stdout}}"' - f" files within {clone_dir}", - ) - - if not all_passed: - sys.exit(1) - - -def cmd_install( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.version and len(args.package) > 1: - print_error('error: "install --version" may only be used for a single package') - sys.exit(1) - - package_infos: list[tuple[PackageInfo, str, bool]] = [] - - for name in args.package: - if not check_local_git_repo(name): - sys.exit(1) - - # Outright prevent installing a package that Zeek has built-in. - bpkg_info = manager.find_builtin_package(name) - if bpkg_info is not None: - print_error(f'cannot install "{name}": built-in package') - sys.exit(1) - - version = args.version if args.version else active_git_branch(name) - package_info = manager.info(name, version=version, prefer_installed=False) - - if package_info.invalid_reason: - print_error( - f'error: invalid package "{name}": {package_info.invalid_reason}', - ) - sys.exit(1) - - if not version: - version = package_info.best_version() - - package_infos.append((package_info, version, False)) - - # We modify package_infos below, so copy current state to preserve list of - # packages requested by caller: - orig_pkgs = package_infos.copy() - new_pkgs: list[tuple[PackageInfo, str, bool]] = [] - - if not args.nodeps: - to_validate = [] - for info, version, _ in package_infos: - to_validate.append((info.package.qualified_name(), version)) - invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, - ignore_suggestions=args.nosuggestions, - ) - - if invalid_reason: - print_error("error: failed to resolve dependencies:", invalid_reason) - sys.exit(1) - - if not args.force: - package_listing = "" - - for info, version, _ in sorted(package_infos, key=lambda x: x[0].package.name): - name = info.package.qualified_name() - package_listing += f" {name} ({version})\n" - - print("The following packages will be INSTALLED:") - print(package_listing) - - if new_pkgs: - dependency_listing = "" - - for info, version, suggested in sorted( - new_pkgs, - key=lambda x: x[0].package.name, - ): - name = info.package.qualified_name() - dependency_listing += f" {name} ({version})" - - if suggested: - dependency_listing += " (suggested)" - - dependency_listing += "\n" - - print("The following dependencies will be INSTALLED:") - print(dependency_listing) - - allpkgs = package_infos + new_pkgs - extdep_listing = "" - - for info, version, _ in sorted(allpkgs, key=lambda x: x[0].package.name): - name = info.package.qualified_name() - extdeps = info.dependencies(field="external_depends") - - if extdeps is None: - extdep_listing += f" from {name} ({version}):\n \n" - continue - - if extdeps: - extdep_listing += f" from {name} ({version}):\n" - - for extdep, semver in sorted(extdeps.items()): - extdep_listing += f" {extdep} {semver}\n" - - if extdep_listing: - print( - "Verify the following REQUIRED external dependencies:\n" - "(Ensure their installation on all relevant systems before" - " proceeding):", - ) - print(extdep_listing) - - if not confirmation_prompt("Proceed?"): - return - - package_infos += new_pkgs - - prompt_for_user_vars( - manager, - config, - configfile, - args, - [info for info, _, _ in package_infos], - ) - - if not args.skiptests: - # Iterate only over the requested packages here, skipping the - # dependencies. We ask the manager to include dependencies below. - for info, version, _ in orig_pkgs: - name = info.package.qualified_name() - - if "test_command" not in info.metadata: - zeekpkg.LOG.info( - f'Skipping unit tests for "{name}": no test_command in metadata', - ) - continue - - print(f'Running unit tests for "{name}"') - error_msg = "" - # For testing we always process dependencies, since the tests might - # well fail without them. If the user wants --nodeps and the tests - # fail because of it, they'll also need to say --skiptests. - error, passed, test_dir = manager.test( - name, - version, - test_dependencies=True, - ) - if error: - error_msg = f"failed to run tests for {name}: {error}" - elif not passed: - clone_dir = os.path.join( - os.path.join(test_dir, "clones"), - info.package.name, - ) - error_msg = ( - f'"{name}" tests failed, inspect contents of' - f" {test_dir} for details, especially any" - ' "zkg.test_command.{{stderr,stdout}}"' - f" files within {clone_dir}" - ) - - if error_msg: - print_error(f"error: {error_msg}") - - if args.force: - sys.exit(1) - - if not confirmation_prompt( - "Proceed to install anyway?", - default_to_yes=False, - ): - return - - installs_failed = [] - - for info, version, _ in reversed(package_infos): - name = info.package.qualified_name() - - is_overwriting = False - ipkg = manager.find_installed_package(name) - - modifications = [] - backup_files = [] - prev_upstream_config_files = [] - - if ipkg: - is_overwriting = True - modifications = manager.modified_config_files(ipkg) - backup_files = manager.backup_modified_files(name, modifications) - prev_upstream_config_files = manager.save_temporary_config_files(ipkg) - - worker = InstallWorker(manager, name, version) - worker.start() - worker.wait(f'Installing "{name}"') - - if worker.error: - print(f'Failed installing "{name}": {worker.error}') - installs_failed.append((name, version)) - continue - - ipkg = manager.find_installed_package(name) - print(f'Installed "{name}" ({ipkg.status.current_version if ipkg else ""})') - - if is_overwriting: - for i, mf in enumerate(modifications): - next_upstream_config_file = mf[1] - - if not os.path.isfile(next_upstream_config_file): - print("\tConfig file no longer exists:") - print("\t\t" + next_upstream_config_file) - print("\tPrevious, locally modified version backed up to:") - print("\t\t" + backup_files[i]) - continue - - prev_upstream_config_file = prev_upstream_config_files[i][1] - - if filecmp.cmp(prev_upstream_config_file, next_upstream_config_file): - # Safe to restore user's version - shutil.copy2(backup_files[i], next_upstream_config_file) - continue - - print("\tConfig file has been overwritten with a different version:") - print("\t\t" + next_upstream_config_file) - print("\tPrevious, locally modified version backed up to:") - print("\t\t" + backup_files[i]) - - if ipkg and manager.has_scripts(ipkg): - load_error = manager.load(name) - - if load_error: - print(f'Failed loading "{name}": {load_error}') - else: - print(f'Loaded "{name}"') - - if not args.nodeps: - # Now load runtime dependencies after all dependencies and suggested - # packages have been installed and loaded. - for info, _, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): - _listing, saved_state = "", manager.loaded_package_states() - name = info.package.qualified_name() - - load_error2 = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name), - ) - - for _name, _error in load_error2: - if not _error: - _listing += f" {_name}\n" - - if not _listing: - dep_listing = get_changed_state(manager, saved_state, [name]) - - if dep_listing: - print( - "The following installed packages were additionally " - "loaded to satisfy runtime dependencies", - ) - print(dep_listing) - - else: - print( - "The following installed packages could NOT be loaded " - f'to satisfy runtime dependencies for "{name}"', - ) - print(_listing) - manager.restore_loaded_package_states(saved_state) +def main() -> None: + parser = zeekpkg.cli.argparser() + args = parser.parse_args() - if installs_failed: - print_error( - "error: incomplete installation, the follow packages" - " failed to be installed:", - ) + zeekpkg.logs.configure(args.verbose) - for n, v in installs_failed: - print_error(f" {n} ({v})") + configfile = zeekpkg.CONFIG.find_configfile(args) + # Treat explicitly requested config files that don't exist as an error: + if args.configfile and configfile != args.configfile: + zeekpkg.UI.error(f'invalid config file "{args.configfile}"') sys.exit(1) + zeekpkg.CONFIG.reset(args) + zeekpkg.CONFIG.update_from_file(configfile) + zeekpkg.CONFIG.update_from_args(args) -def cmd_bundle( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - packages_to_bundle = [] - prefer_existing_clones = False - - if args.manifest: - if len(args.manifest) == 1 and os.path.isfile(args.manifest[0]): - config = configparser.ConfigParser(delimiters="=") - config.optionxform = lambda optionstr: optionstr # type: ignore - - if config.read(args.manifest[0]) and config.has_section("bundle"): - packages = config.items("bundle") - else: - print_error(f'error: "{args.manifest[0]}" is not a valid manifest file') - sys.exit(1) - - else: - packages = [(name, "") for name in args.manifest] - - to_validate = [] - new_pkgs: list[tuple[PackageInfo, str, bool]] = [] - - for name, version in packages: - if not check_local_git_repo(name): - sys.exit(1) - - if not version: - if v := active_git_branch(name): - version = v - - info = manager.info(name, version=version, prefer_installed=False) + manager = zeekpkg.manager.create_manager() - if info.invalid_reason: - print_error(f'error: invalid package "{name}": {info.invalid_reason}') - sys.exit(1) - - if not version: - version = info.best_version() - - to_validate.append((info.package.qualified_name(), version)) - packages_to_bundle.append( - ( - info.package.qualified_name(), - info.package.git_url, - version, - False, - False, - ), - ) - - if not args.nodeps: - invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, - ignore_installed_packages=True, - ignore_suggestions=args.nosuggestions, - ) - - if invalid_reason: - print_error("error: failed to resolve dependencies:", invalid_reason) - sys.exit(1) - - for info, version, suggested in new_pkgs: - packages_to_bundle.append( - ( - info.package.qualified_name(), - info.package.git_url, - version, - True, - suggested, - ), - ) - else: - prefer_existing_clones = True - - for ipkg in manager.installed_packages(): - assert ipkg.status.current_version - packages_to_bundle.append( - ( - ipkg.package.qualified_name(), - ipkg.package.git_url, - ipkg.status.current_version, - False, - False, - ), + if manager is None: + zeekpkg.UI.error("Permission problems prevent zkg from creating internal state") + if not args.user: + zeekpkg.UI.error( + "Consider the --user flag to manage zkg state via {home_config_dir()}/config", ) - - if not packages_to_bundle: - print_error("error: no packages to put in bundle") - sys.exit(1) - - if not args.force: - package_listing = "" - - for name, git_url, version, is_dependency, is_suggestion in packages_to_bundle: - package_listing += f" {name} ({version})" - - if is_suggestion: - package_listing += " (suggested)" - elif is_dependency: - package_listing += " (dependency)" - - if git_url.startswith(BUILTIN_SCHEME): - package_listing += " (built-in)" - - package_listing += "\n" - - print(f"The following packages will be BUNDLED into {args.bundle_filename}:") - print(package_listing) - - if not confirmation_prompt("Proceed?"): - return - - git_urls = [(git_url, version) for _, git_url, version, _, _ in packages_to_bundle] - error = manager.bundle( - args.bundle_filename, - git_urls, - prefer_existing_clones=prefer_existing_clones, - ) - - if error: - print_error(f"error: failed to create bundle: {error}") - sys.exit(1) - - print(f"Bundle successfully written: {args.bundle_filename}") - - -def cmd_unbundle( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - prev_load_status = {} - - for ipkg in manager.installed_packages(): - prev_load_status[ipkg.package.git_url] = ipkg.status.is_loaded - - if args.replace: - cmd_purge(manager, args, config, configfile) - - error, bundle_info = manager.bundle_info(args.bundle_filename) - - if error: - print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") sys.exit(1) - for git_url, _, pkg_info in bundle_info: - if pkg_info.invalid_reason: - name = pkg_info.package.qualified_name() - print_error( - f"error: bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}", - ) - sys.exit(1) - - if not bundle_info: - print("No packages in bundle.") - return - - if not args.force: - package_listing = "" - builtin_listing = "" - for git_url, version, info in bundle_info: - name = git_url - - if info.is_builtin(): - builtin_listing += f" from {name} ({version}):\n" - continue - - for pkg in manager.source_packages(): - if pkg.git_url == git_url: - name = pkg.qualified_name() - break - - package_listing += f" {name} ({version})\n" - - print("The following packages will be INSTALLED:") - print(package_listing) - - extdep_listing = "" - - for git_url, version, info in bundle_info: - name = git_url - - for pkg in manager.source_packages(): - if pkg.git_url == git_url: - name = pkg.qualified_name() - break - - extdeps = info.dependencies(field="external_depends") - - if extdeps is None: - extdep_listing += f" from {name} ({version}):\n \n" - continue - - if extdeps: - extdep_listing += f" from {name} ({version}):\n" - - for extdep, semver in sorted(extdeps.items()): - extdep_listing += f" {extdep} {semver}\n" - - if extdep_listing: + for name, url in zeekpkg.CONFIG.items("sources"): + if err := manager.add_source(name, url): print( - "Verify the following REQUIRED external dependencies:\n" - "(Ensure their installation on all relevant systems before" - " proceeding):", + f'warning: skipped using package source named "{name}" ({url}): {err}', + file=sys.stderr, ) - print(extdep_listing) - - if not confirmation_prompt("Proceed?"): - return - - prompt_for_user_vars( - manager, - config, - configfile, - args, - [info for _, _, info in bundle_info], - ) - - error = manager.unbundle(args.bundle_filename) - - if error: - print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") - sys.exit(1) - - for git_url, _, _ in bundle_info: - if git_url in prev_load_status: - need_load = prev_load_status[git_url] - else: - need_load = True - - ipkg2 = manager.find_installed_package(git_url) - - if not ipkg2: - print(f'Skipped loading "{git_url}": failed to install') - continue - - name = ipkg2.package.qualified_name() - - if not need_load: - print(f'Skipped loading "{name}"') - continue - - load_error = manager.load(name) - - if load_error: - print(f'Failed loading "{name}": {load_error}') - else: - print(f'Loaded "{name}"') - - print("Unbundling complete.") - - -def cmd_remove( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - packages_to_remove: list[InstalledPackage] = [] - - def package_will_be_removed(pkg_name: str) -> bool: - for _ipkg in packages_to_remove: - if _ipkg.package.name == pkg_name: - return True - - return False - - for name in args.package: - ipkg = manager.find_installed_package(name) - - if not ipkg: - print_error(f'error: package "{name}" is not installed') - sys.exit(1) - - if ipkg.is_builtin(): - print_error(f'cannot remove "{name}": built-in package') - sys.exit(1) - - packages_to_remove.append(ipkg) - - dependers_to_unload = set() - - if not args.nodeps: - for ipkg in packages_to_remove: - for pkg_name in manager.list_depender_pkgs(ipkg.package.name): - ipkg = manager.find_installed_package(pkg_name) - - if ipkg and not package_will_be_removed(ipkg.package.name): - if ipkg.status.is_loaded: - dependers_to_unload.add(ipkg.package.name) - - if not args.force: - print("The following packages will be REMOVED:") - - for ipkg in packages_to_remove: - print(f" {ipkg.package.qualified_name()}") - - print() - - if dependers_to_unload: - print("The following dependent packages will be UNLOADED:") - - for pkg_name in sorted(dependers_to_unload): - ipkg = manager.find_installed_package(pkg_name) - assert ipkg - print(f" {ipkg.package.qualified_name()}") - - print() - - if not confirmation_prompt("Proceed?"): - return - - for pkg_name in sorted(dependers_to_unload): - ipkg = manager.find_installed_package(pkg_name) - assert ipkg - name = ipkg.package.qualified_name() - - if manager.unload(name): - print(f'Unloaded "{name}"') - else: - # Weird that it failed, but if it's not installed and there's - # nothing to unload, not worth using a non-zero exit-code to - # reflect an overall failure of the package removal operation - print(f'Failed unloading "{name}": no such package installed') - - had_failure = False - - for ipkg in packages_to_remove: - name = ipkg.package.qualified_name() - modifications = manager.modified_config_files(ipkg) - backup_files = manager.backup_modified_files(name, modifications) - - if manager.remove(name): - print(f'Removed "{name}"') - - if backup_files: - print("\tCreated backups of locally modified config files:") - - for backup_file in backup_files: - print("\t" + backup_file) - - else: - print(f'Failed removing "{name}": no such package installed') - had_failure = True - - if had_failure: - sys.exit(1) - - -def cmd_purge( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - packages_to_remove = manager.installed_packages() - packages_to_remove = [p for p in packages_to_remove if not p.is_builtin()] - - if not packages_to_remove: - print("No packages to remove.") - return - - if not args.force: - package_listing = "" - names_to_remove = [ipkg.package.qualified_name() for ipkg in packages_to_remove] - - for name in names_to_remove: - package_listing += f" {name}\n" - - print("The following packages will be REMOVED:") - print(package_listing) - - if not confirmation_prompt("Proceed?"): - return - - had_failure = False - - for ipkg in packages_to_remove: - name = ipkg.package.qualified_name() - modifications = manager.modified_config_files(ipkg) - backup_files = manager.backup_modified_files(name, modifications) - - if manager.remove(name): - print(f'Removed "{name}"') - - if backup_files: - print("\tCreated backups of locally modified config files:") - - for backup_file in backup_files: - print("\t" + backup_file) - - else: - print(f'Unknown error removing "{name}"') - had_failure = True - - if had_failure: - sys.exit(1) - - -def outdated(manager: zeekpkg.Manager) -> list[str]: - return [ - ipkg.package.qualified_name() - for ipkg in manager.installed_packages() - if ipkg.status.is_outdated - ] - - -def cmd_refresh( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if not args.sources: - args.sources = list(manager.sources.keys()) - - if args.fail_on_aggregate_problems and not args.aggregate: - print_error( - "warning: --fail-on-aggregate-problems without --aggregate has no effect.", - ) - - had_failure = False - had_aggregation_failure = False - - for source in args.sources: - print(f"Refresh package source: {source}") - - src_pkgs_before = {i.qualified_name() for i in manager.source_packages()} - - error = "" - aggregation_issues = [] - - if args.aggregate: - res = manager.aggregate_source(source, args.push) - error = res.refresh_error - aggregation_issues = res.package_issues - else: - error = manager.refresh_source(source, False, args.push) - - if error: - had_failure = True - print_error(f'error: failed to refresh "{source}": {error}') - continue - - src_pkgs_after = {i.qualified_name() for i in manager.source_packages()} - - if src_pkgs_before == src_pkgs_after: - print("\tNo membership changes") - else: - print("\tChanges:") - diff = src_pkgs_before.symmetric_difference(src_pkgs_after) - - for name in diff: - change = "Added" if name in src_pkgs_after else "Removed" - print(f"\t\t{change} {name}") - - if args.aggregate: - if aggregation_issues: - print( - "\tWARNING: Metadata aggregated, but excludes the " - "following packages due to described problems:", - ) - - for url, issue in aggregation_issues: - print(f"\t\t{url}: {issue}") - if args.fail_on_aggregate_problems: - had_aggregation_failure = True - else: - print("\tMetadata aggregated") - - if args.push: - print("\tPushed aggregated metadata") - - outdated_before = set(outdated(manager)) - print("Refresh installed packages") - manager.refresh_installed_packages() - outdated_after = set(outdated(manager)) - - if outdated_before == outdated_after: - print("\tNo new outdated packages") - else: - print("\tNew outdated packages:") - diff = outdated_before.symmetric_difference(outdated_after) - - for name in diff: - ipkg = manager.find_installed_package(name) - if not ipkg: - continue - version_change = version_change_string(manager, ipkg) - print(f"\t\t{name} {version_change}") - - if had_failure: - sys.exit(1) - if had_aggregation_failure: - sys.exit(2) - - -def version_change_string( - manager: zeekpkg.Manager, - installed_package: InstalledPackage, -) -> str: - old_version = installed_package.status.current_version - new_version = old_version - version_change = "" - - if installed_package.status.tracking_method == TRACKING_METHOD_VERSION: - versions = manager.package_versions(installed_package) - - if len(versions): - new_version = versions[-1] - - version_change = f"({old_version} -> {new_version})" - else: - version_change = f"({new_version})" - - return version_change - - -def cmd_upgrade( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.package: - pkg_list = args.package - else: - pkg_list = outdated(manager) - - outdated_packages = [] - package_listing = "" - - for name in pkg_list: - ipkg = manager.find_installed_package(name) - - if not ipkg: - print_error(f'error: package "{name}" is not installed') - sys.exit(1) - - name = ipkg.package.qualified_name() - - if not ipkg.status.is_outdated: - continue - - if not manager.match_source_packages(name): - name = ipkg.package.git_url - - info = manager.info( - name, - version=ipkg.status.current_version, - prefer_installed=False, - ) - - if info.invalid_reason: - print_error(f'error: invalid package "{name}": {info.invalid_reason}') - sys.exit(1) - - next_version = ipkg.status.current_version - - if ipkg.status.tracking_method == TRACKING_METHOD_VERSION and info.versions: - next_version = info.versions[-1] - - assert next_version - - outdated_packages.append((info, next_version, False)) - version_change = version_change_string(manager, ipkg) - package_listing += f" {name} {version_change}\n" - - if not outdated_packages: - print("All packages already up-to-date.") - return - - new_pkgs: list[tuple[PackageInfo, str, bool]] = [] - - if not args.nodeps: - to_validate = [ - (info.package.qualified_name(), next_version) - for info, next_version, _ in outdated_packages - ] - invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, - ignore_suggestions=args.nosuggestions, - ) - - if invalid_reason: - print_error("error: failed to resolve dependencies:", invalid_reason) - sys.exit(1) - - allpkgs = outdated_packages + new_pkgs - - if not args.force: - print("The following packages will be UPGRADED:") - print(package_listing) - - if new_pkgs: - dependency_listing = "" - - for info, version, suggestion in new_pkgs: - name = info.package.qualified_name() - dependency_listing += f" {name} ({version})" - - if suggestion: - dependency_listing += " (suggested)" - - dependency_listing += "\n" - - print("The following dependencies will be INSTALLED:") - print(dependency_listing) - - extdep_listing = "" - - for info, version2, _ in allpkgs: - name = info.package.qualified_name() - extdeps = info.dependencies(field="external_depends") - - if extdeps is None: - extdep_listing += f" from {name} ({version2}):\n \n" - continue - - if extdeps: - extdep_listing += f" from {name} ({version2}):\n" - - for extdep, semver in sorted(extdeps.items()): - extdep_listing += f" {extdep} {semver}\n" - - if extdep_listing: - print( - "Verify the following REQUIRED external dependencies:\n" - "(Ensure their installation on all relevant systems before" - " proceeding):", - ) - print(extdep_listing) - - if not confirmation_prompt("Proceed?"): - return - - prompt_for_user_vars( - manager, - config, - configfile, - args, - [info for info, _, _ in allpkgs], - ) - - if not args.skiptests: - for info, version3, _ in outdated_packages: - name = info.package.qualified_name() - - # Get info object for the next version as there may have been a - # test_command added during the upgrade. - next_info = manager.info(name, version=version3, prefer_installed=False) - if next_info.invalid_reason: - print_error( - f'error: invalid package "{name}": {next_info.invalid_reason}', - ) - sys.exit(1) - - if "test_command" not in next_info.metadata: - zeekpkg.LOG.info( - 'Skipping unit tests for "%s": no test_command in metadata', - name, - ) - continue - - print(f'Running unit tests for "{name}"') - error_msg = "" - # As in cmd_install, we always process dependencies since the tests - # might well fail without them. If the user wants --nodeps and the - # tests fail because of it, they'll also need to say --skiptests. - error, passed, test_dir = manager.test( - name, - version3, - test_dependencies=True, - ) - - if error: - error_msg = f"failed to run tests for {name}: {error}" - elif not passed: - clone_dir = os.path.join( - os.path.join(test_dir, "clones"), - info.package.name, - ) - error_msg = ( - f'"{name}" tests failed, inspect contents of' - f" {test_dir} for details, especially any" - ' "zkg.test_command.{{stderr,stdout}}"' - f" files within {clone_dir}" - ) - - if error_msg: - print_error(f"error: {error_msg}") - - if args.force: - sys.exit(1) - - if not confirmation_prompt( - "Proceed to install anyway?", - default_to_yes=False, - ): - return - - for info, version, _ in reversed(new_pkgs): - name = info.package.qualified_name() - worker = InstallWorker(manager, name, version) - worker.start() - worker.wait(f'Installing "{name}"') - - if worker.error: - print(f'Failed installing "{name}": {worker.error}') - continue - - ipkg = manager.find_installed_package(name) - assert ipkg - print(f'Installed "{name}" ({ipkg.status.current_version})') - - if manager.has_scripts(ipkg): - load_error = manager.load(name) - - if load_error: - print(f'Failed loading "{name}": {load_error}') - else: - print(f'Loaded "{name}"') - - had_failure = False - - for info, _, _ in outdated_packages: - name = info.package.qualified_name() - - if not manager.match_source_packages(name): - name = info.package.git_url - - ipkg = manager.find_installed_package(name) - assert ipkg - modifications = manager.modified_config_files(ipkg) - backup_files = manager.backup_modified_files(name, modifications) - prev_upstream_config_files = manager.save_temporary_config_files(ipkg) - - res = manager.upgrade(name) - - if res: - print(f'Failed upgrading "{name}": {res}') - had_failure = True - else: - ipkg = manager.find_installed_package(name) - assert ipkg - print(f'Upgraded "{name}" ({ipkg.status.current_version})') - - for i, mf in enumerate(modifications): - next_upstream_config_file = mf[1] - - if not os.path.isfile(next_upstream_config_file): - print("\tConfig file no longer exists:") - print("\t\t" + next_upstream_config_file) - print("\tPrevious, locally modified version backed up to:") - print("\t\t" + backup_files[i]) - continue - - prev_upstream_config_file = prev_upstream_config_files[i][1] - - if filecmp.cmp(prev_upstream_config_file, next_upstream_config_file): - # Safe to restore user's version - shutil.copy2(backup_files[i], next_upstream_config_file) - continue - - print("\tConfig file has been updated to a newer version:") - print("\t\t" + next_upstream_config_file) - print("\tPrevious, locally modified version backed up to:") - print("\t\t" + backup_files[i]) - - if had_failure: - sys.exit(1) - - -def cmd_load( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - had_failure = False - load_error = False - dep_error_listing = "" - - for name in args.package: - ipkg = manager.find_installed_package(name) - - if not ipkg: - had_failure = True - print(f'Failed to load "{name}": no such package installed') - continue - - if not manager.has_scripts(ipkg): - print(f'The package "{name}" does not contain scripts to load.') - continue - - name = ipkg.package.qualified_name() - - saved_state = {} - - if args.nodeps: - load_error = bool(manager.load(name)) - else: - saved_state = manager.loaded_package_states() - dep_error_listing, load_error = "", False - - loaded_dep_list = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name), - ) - - for _name, _error in loaded_dep_list: - if _error: - load_error = True - dep_error_listing += f" {_name}: {_error}\n" - - if not load_error: - dep_listing = get_changed_state(manager, saved_state, [name]) - - if dep_listing: - print( - "The following installed packages were additionally loaded to satisfy" - f' runtime dependencies for "{name}".', - ) - print(dep_listing) - - if load_error: - had_failure = True - - if not args.nodeps: - if dep_error_listing: - print( - f'The following installed dependencies could not be loaded for "{name}".', - ) - print(dep_error_listing) - manager.restore_loaded_package_states(saved_state) - - print(f'Failed to load "{name}": {load_error}') - else: - print(f'Loaded "{name}"') - - if had_failure: - sys.exit(1) - - -def cmd_unload( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - had_failure = False - packages_to_unload: list[InstalledPackage] = [] - dependers_to_unload = set() - - def package_will_be_unloaded(pkg_name: str) -> bool: - for _ipkg in packages_to_unload: - if _ipkg.package.name == pkg_name: - return True - - return False - - for name in args.package: - ipkg = manager.find_installed_package(name) - - if not ipkg: - had_failure = True - print(f'Failed to unload "{name}": no such package installed') - continue - - if not ipkg.status.is_loaded: - continue - - # Maybe in the future that's possible, but as of now built-in - # packages are really built-in plugins and there is not a way - # to unload them. - if ipkg.is_builtin(): - print_error(f'cannot unload "{name}": built-in package') - sys.exit(1) - - packages_to_unload.append(ipkg) - - if not args.nodeps: - for ipkg in packages_to_unload: - for pkg_name in manager.list_depender_pkgs(ipkg.package.name): - ipkg = manager.find_installed_package(pkg_name) - - if ipkg and not package_will_be_unloaded(ipkg.package.name): - if ipkg.status.is_loaded: - dependers_to_unload.add(ipkg.package.name) - - if packages_to_unload and not args.force: - print("The following packages will be UNLOADED:") - - for ipkg in packages_to_unload: - print(f" {ipkg.package.qualified_name()}") - - print() - - if dependers_to_unload: - print("The following dependent packages will be UNLOADED:") - - for pkg_name in sorted(dependers_to_unload): - ipkg = manager.find_installed_package(pkg_name) - assert ipkg - print(f" {ipkg.package.qualified_name()}") - - print() - - if not confirmation_prompt("Proceed?"): - if had_failure: - sys.exit(1) - else: - return - - for pkg_name in sorted(dependers_to_unload): - ipkg = manager.find_installed_package(pkg_name) - assert ipkg - packages_to_unload.append(ipkg) - - for ipkg in packages_to_unload: - name = ipkg.package.qualified_name() - - if manager.unload(name): - print(f'Unloaded "{name}"') - else: - had_failure = True - print(f'Failed unloading "{name}": no such package installed') - - if had_failure: - sys.exit(1) - - -def cmd_pin( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - had_failure = False - - for name in args.package: - ipkg = manager.find_installed_package(name) - - if not ipkg: - had_failure = True - print(f'Failed to pin "{name}": no such package installed') - continue - - if ipkg.is_builtin(): - had_failure = True - print_error(f'cannot pin "{name}": built-in package') - continue - - name = ipkg.package.qualified_name() - ipkg = manager.pin(name) - - if ipkg: - print( - f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})', - ) - else: - had_failure = True - print(f'Failed pinning "{name}": no such package installed') - - if had_failure: - sys.exit(1) - - -def cmd_unpin( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - had_failure = False - - for name in args.package: - ipkg = manager.find_installed_package(name) - - if not ipkg: - had_failure = True - print(f'Failed to unpin "{name}": no such package installed') - continue - - if ipkg.is_builtin(): - had_failure = True - print_error(f'cannot unpin "{name}": built-in package') - continue - - name = ipkg.package.qualified_name() - ipkg = manager.unpin(name) - - if ipkg: - print( - f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})', - ) - else: - had_failure = True - print(f'Failed unpinning "{name}": no such package installed') - - if had_failure: - sys.exit(1) - - -def _get_filtered_packages( - manager: zeekpkg.Manager, - category: str, -) -> ( - dict[str, Package] - | dict[str, InstalledPackage] - | dict[str, Package | InstalledPackage] -): - pkg_dict: dict[str, Package | InstalledPackage] = {} - - for ipkg in manager.installed_packages(): - pkg_dict[ipkg.package.qualified_name()] = ipkg - - for pkg in manager.source_packages(): - pkg_qn = pkg.qualified_name() - - if pkg_qn not in pkg_dict: - pkg_dict[pkg_qn] = pkg - - if category == "all": - filtered_pkgs = pkg_dict - elif category == "installed": - filtered_pkgs = { - key: value - for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) - } - elif category == "not_installed": - filtered_pkgs = { - key: value - for key, value in pkg_dict.items() - if not isinstance(value, zeekpkg.InstalledPackage) - } - elif category == "loaded": - filtered_pkgs = { - key: value - for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) and value.status.is_loaded - } - elif category == "unloaded": - filtered_pkgs = { - key: value - for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) - and not value.status.is_loaded - } - elif category == "outdated": - filtered_pkgs = { - key: value - for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) and value.status.is_outdated - } - else: - raise NotImplementedError - - return filtered_pkgs - - -def cmd_list( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - filtered_pkgs = _get_filtered_packages(manager, args.category) - - for pkg_name, val in sorted(filtered_pkgs.items()): - if isinstance(val, zeekpkg.InstalledPackage): - pkg = val.package - - if val.is_builtin() and not args.include_builtin: - continue - - out = f"{pkg_name} (installed: {val.status.current_version})" - else: - pkg = val - out = pkg_name - - if not args.nodesc: - desc = pkg.short_description() - - if desc: - out += " - " + desc - - print(out) - - -def cmd_search( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - src_pkgs = manager.source_packages() - matches = set() - - for search_text in args.search_text: - if search_text[0] == "/" and search_text[-1] == "/": - try: - regex = re.compile(search_text[1:-1]) - except re.error as error: - print(f"invalid regex: {error}") - sys.exit(1) - else: - for pkg in src_pkgs: - if regex.search(pkg.name_with_source_directory()): - matches.add(pkg) - - for tag in pkg.tags(): - if regex.search(tag): - matches.add(pkg) - - else: - for pkg in src_pkgs: - name = pkg.name_with_source_directory() - if name and search_text in name: - matches.add(pkg) - - for tag in pkg.tags(): - if search_text in tag: - matches.add(pkg) - - if matches: - for match in sorted(matches): - out = match.qualified_name() - - ipkg = manager.find_installed_package(match.qualified_name()) - - if ipkg: - out += f" (installed: {ipkg.status.current_version})" - - desc = match.short_description() - - if desc: - out += " - " + desc - - print(out) - - else: - print("no matches") - - -def cmd_info( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.version and len(args.package) > 1: - print_error('error: "info --version" may only be used for a single package') - sys.exit(1) - - # Dictionary for storing package info to output as JSON - pkginfo: dict[str, Any] = {} - had_invalid_package = False - - if len(args.package) == 1: - try: - package_names = [] - for pkg_name, info in _get_filtered_packages( - manager, - args.package[0], - ).items(): - if info.is_builtin() and not args.include_builtin: - continue - - package_names.append(pkg_name) - except NotImplementedError: - package_names = args.package - else: - package_names = args.package - - for name in package_names: - info2 = manager.info( - name, - version=args.version, - prefer_installed=(not args.nolocal), - ) - - if info2.package: - name = info2.package.qualified_name() - - if args.json: - pkginfo[name] = {} - pkginfo[name]["metadata"] = {} - else: - print(f'"{name}" info:') - - if info2.invalid_reason: - if args.json: - pkginfo[name]["invalid"] = info2.invalid_reason - else: - print(f"\tinvalid package: {info2.invalid_reason}\n") - - had_invalid_package = True - continue - - if args.json: - pkginfo[name]["url"] = info2.package.git_url - pkginfo[name]["versions"] = info2.versions - else: - print(f"\turl: {info2.package.git_url}") - print(f"\tversions: {info2.versions}") - - if info2.status: - if args.json: - pkginfo[name]["install_status"] = {} - - for key, value in sorted(info2.status.__dict__.items()): - pkginfo[name]["install_status"][key] = value - else: - print("\tinstall status:") - - for key, value in sorted(info2.status.__dict__.items()): - print(f"\t\t{key} = {value}") - - if args.json: - if info2.metadata_file: - pkginfo[name]["metadata_file"] = info2.metadata_file - pkginfo[name]["metadata"][info2.metadata_version] = {} - else: - if info2.metadata_file: - print(f"\tmetadata file: {info2.metadata_file}") - print(f'\tmetadata (from version "{info2.metadata_version}"):') - - if len(info2.metadata) == 0: - if not args.json: - print("\t\t") - elif args.json: - _fill_metadata_version( - pkginfo[name]["metadata"][info2.metadata_version], - info2.metadata, - ) - else: - for key, value in sorted(info2.metadata.items()): - value = value.replace("\n", "\n\t\t\t") - print(f"\t\t{key} = {value}") - - # If --json and --allvers given, check for multiple versions and - # add the metadata for each version to the pkginfo. - if args.json and args.allvers: - for vers in info2.versions: - # Skip the version that was already processed - if vers != info2.metadata_version: - info3 = manager.info( - name, - vers, - prefer_installed=(not args.nolocal), - ) - pkginfo[name]["metadata"][info3.metadata_version] = {} - if info3.metadata_file: - pkginfo[name]["metadata_file"] = info3.metadata_file - _fill_metadata_version( - pkginfo[name]["metadata"][info3.metadata_version], - info3.metadata, - ) - - if not args.json: - print() - - if args.json: - print(json.dumps(pkginfo, indent=args.jsonpretty, sort_keys=True)) - - if had_invalid_package: - sys.exit(1) - - -def _fill_metadata_version( - pkginfo_name_metadata_version: dict[str, Any], - info_metadata: dict[str, str], -) -> None: - """Fill a dict with metadata information. - - This helper function is called by cmd_info to fill metadata information - for a specific package version. - - Args: - pkginfo_name_metadata_version (dict of str -> dict): Corresponds - to pkginfo[name]['metadata'][info.metadata_version] in cmd_info. - - info_metadata (dict of str->str): Corresponds to info.metadata - in cmd_info. - - Side effect: - New dict entries are added to pkginfo_name_metadata_version. - """ - for key, value in info_metadata.items(): - if key in {"depends", "suggests"}: - pkginfo_name_metadata_version[key] = {} - deps = value.split("\n") - - for i in range(1, len(deps)): - deplist = deps[i].split(" ") - pkginfo_name_metadata_version[key][deplist[0]] = deplist[1] - else: - pkginfo_name_metadata_version[key] = value - - -def cmd_config( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.config_param == "all": - out = io.StringIO() - config.write(out) - print(out.getvalue()) - out.close() - elif args.config_param == "sources": - for key, value in config_items(config, "sources"): - print(f"{key} = {value}") - elif args.config_param == "user_vars": - if config.has_section("user_vars"): - for key, value in config_items(config, "user_vars"): - print(f"{key} = {value}") - else: - print(config.get("paths", args.config_param)) - - -def cmd_autoconfig( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - if args.user: - configfile = os.path.join(home_config_dir(), "config") - with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: - config.write(f) - print(f"Successfully wrote config file to {configfile}") - return - - zeek_config = find_program("zeek-config") - - if not zeek_config: - print_error('error: no "zeek-config" in PATH') - sys.exit(1) - - cmd = subprocess.Popen( - [zeek_config, "--site_dir", "--plugin_dir", "--prefix", "--zeek_dist"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=1, - universal_newlines=True, - ) - - assert cmd.stdout - script_dir = read_zeek_config_line(cmd.stdout) - plugin_dir = read_zeek_config_line(cmd.stdout) - bin_dir = os.path.join(read_zeek_config_line(cmd.stdout), "bin") - zeek_dist = read_zeek_config_line(cmd.stdout) - - if configfile: - config_dir = os.path.dirname(configfile) - else: - config_dir = default_config_dir() - configfile = os.path.join(config_dir, "config") - - make_dir(config_dir) - - config_file_exists = os.path.isfile(configfile) - - def change_config_value( - config: configparser.ConfigParser, - section: str, - option: str, - new_value: str, - use_prompt: bool, - ) -> None: - if not use_prompt: - config.set(section, option, new_value) - return - - old_value = config.get(section, option) - - if old_value == new_value: - return - - prompt = f'Set "{option}" config option to: {new_value} ?' - - if old_value: - prompt += f"\n(previous value: {old_value})" - - if args.force or confirmation_prompt(prompt): - config.set(section, option, new_value) - - change_config_value(config, "paths", "script_dir", script_dir, config_file_exists) - change_config_value(config, "paths", "plugin_dir", plugin_dir, config_file_exists) - change_config_value(config, "paths", "bin_dir", bin_dir, config_file_exists) - change_config_value(config, "paths", "zeek_dist", zeek_dist, config_file_exists) - - with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: - config.write(f) - - print(f"Successfully wrote config file to {configfile}") - - -def cmd_env( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - zeek_config = find_program("zeek-config") - zeekpath = os.environ.get("ZEEKPATH") - pluginpath = os.environ.get("ZEEK_PLUGIN_PATH") - - if zeek_config: - cmd = subprocess.Popen( - [zeek_config, "--zeekpath", "--plugin_dir"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=1, - universal_newlines=True, - ) - - assert cmd.stdout - line1 = read_zeek_config_line(cmd.stdout) - line2 = read_zeek_config_line(cmd.stdout) - - if not zeekpath: - zeekpath = line1 - - if not pluginpath: - pluginpath = line2 - - zeekpaths = list(zeekpath.split(":")) if zeekpath else [] - pluginpaths = list(pluginpath.split(":")) if pluginpath else [] - - zeekpaths.append(manager.zeekpath()) - pluginpaths.append(manager.zeek_plugin_path()) - - def remove_redundant_paths(paths: list[str]) -> list[str]: - return list(OrderedDict.fromkeys(paths)) - - zeekpaths = remove_redundant_paths(zeekpaths) - pluginpaths = remove_redundant_paths(pluginpaths) - - if os.environ.get("SHELL", "").endswith("csh"): - print("setenv ZEEKPATH {}".format(":".join(zeekpaths))) - print("setenv ZEEK_PLUGIN_PATH {}".format(":".join(pluginpaths))) - print(f"setenv PATH {manager.bin_dir}:$PATH") - else: - print("export ZEEKPATH={}".format(":".join(zeekpaths))) - print("export ZEEK_PLUGIN_PATH={}".format(":".join(pluginpaths))) - print(f"export PATH={manager.bin_dir}:$PATH") - - -def cmd_create( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - tmplname = ( - args.template - or config.get("templates", "default", fallback=None) - or ZKG_DEFAULT_TEMPLATE - ) - try: - tmpl = Template.load(config, tmplname, args.version) - except LoadError as error: - msg = f"problem while loading template {tmplname}: {error}" - zeekpkg.LOG.exception(msg) - print_error("error: " + msg) - sys.exit(1) - - try: - package = tmpl.package() - uvars = tmpl.define_user_vars() - uvar_names = set(package.needed_user_vars()) - - # Overlay any requested features onto the package. - if args.features: - # If the user provided comma-separated values, split the - # strings. (Argparse expects space separation.) Also - # filter any duplicates. - fnames = set() - for feat in args.features: - fnames |= {f.strip() for f in feat.split(",") if f} - features = tmpl.features() - for feature in features: - if feature.name() in fnames: - package.add_feature(feature) - uvar_names |= set(feature.needed_user_vars()) - fnames.remove(feature.name()) - if len(fnames) > 0: - # Alert if the user requested an unknown feature. - knowns = ", ".join([f'"{f.name()}"' for f in features]) - unknowns = ", ".join([f'"{name}"' for name in fnames]) - print_error( - "error: the following features are unknown: {}." - ' Template "{}" offers {}.'.format( - unknowns, - tmpl.name(), - knowns or "no features", - ), - ) - sys.exit(1) - - # Remove user vars we don't actually require from consideration - uvars = [uvar for uvar in uvars if uvar.name() in uvar_names] - - # Resolve the variables via user input, args, etc - for uvar in uvars: - try: - uvar.resolve(tmpl.name(), config, args.user_var, args.force) - except ValueError: - print_error( - f'error: could not determine value of user variable "{uvar.name()}",' - " provide via environment or --user-var", - ) - sys.exit(1) - - # Apply them to the template. After this, any parameter can be - # retrieved from the template via tmpl.lookup_param(). - tmpl._set_user_vars(uvars) - - # Verify that resulting template parameters are formatted correctly - try: - package.do_validate(tmpl) - except InputError as error: - print_error("error: template input invalid, " + str(error)) - sys.exit(1) - - # And finally, instantiate the package. - try: - if os.path.isdir(args.packagedir): - if not args.force: - print(f"Package directory {args.packagedir} already exists.") - if not confirmation_prompt("Delete?"): - sys.exit(1) - try: - delete_path(args.packagedir) - zeekpkg.LOG.info( - "Removed existing package directory %s", - args.packagedir, - ) - except OSError as err: - print_error( - f"error: could not remove package directory {args.packagedir}: {err}", - ) - sys.exit(1) - - package.do_instantiate(tmpl, args.packagedir, args.force) - except OutputError as error: - print_error("error: template instantiation failed, " + str(error)) - sys.exit(1) - except Exception as error: - msg = f"problem during template instantiation: {error}" - zeekpkg.LOG.exception(msg) - print_error("error: " + msg) - sys.exit(1) - - -def cmd_template_info( - manager: zeekpkg.Manager, - args: argparse.Namespace, - config: configparser.ConfigParser, - configfile: str, -) -> None: - tmplname = ( - args.template - or config.get("templates", "default", fallback=None) - or ZKG_DEFAULT_TEMPLATE - ) - - try: - tmpl = Template.load(config, tmplname, args.version) - except LoadError as error: - msg = f"problem while loading template {tmplname}: {error}" - zeekpkg.LOG.exception(msg) - print_error("error: " + msg) - sys.exit(1) - - tmplinfo = tmpl.info() - - if args.json: - print(json.dumps(tmplinfo, indent=args.jsonpretty, sort_keys=True)) - else: - print("API version: " + tmplinfo["api_version"]) - print("features: " + ", ".join(tmplinfo["features"])) - print("origin: " + tmplinfo["origin"]) - print("provides package: " + str(tmplinfo["provides_package"]).lower()) - print("user vars:") - for uvar_name, uvar_info in tmplinfo["user_vars"].items(): - print( - "\t{}: {}, {}, used by {}".format( - uvar_name, - uvar_info["description"], - uvar_info["default"] or "no default", - ", ".join(uvar_info["used_by"]) or "not used", - ), - ) - print("versions: " + ", ".join(tmplinfo["versions"])) - - -class BundleHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): - # Workaround for underlying argparse bug: https://bugs.python.org/issue9338 - def _format_args(self, action: argparse.Action, default_metavar: str) -> str: - rval = super()._format_args(action, default_metavar) - - if action.nargs == argparse.ZERO_OR_MORE: - rval += " --" - elif action.nargs == argparse.ONE_OR_MORE: - rval += " --" - - return rval - - -def top_level_parser() -> argparse.ArgumentParser: - top_parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="A command-line package manager for Zeek.", - epilog="Environment Variables:\n\n" - " ``ZKG_CONFIG_FILE``:\t" - "Same as ``--configfile`` option, but has less precedence.\n" - " ``ZKG_DEFAULT_SOURCE``:\t" - f"The default package source to use (normally {ZKG_DEFAULT_SOURCE}).\n" - " ``ZKG_DEFAULT_TEMPLATE``:\t" - f"The default package template to use (normally {ZKG_DEFAULT_TEMPLATE}).\n", - ) - top_parser.add_argument( - "--version", - action="version", - version="%(prog)s " + zeekpkg.__version__, - ) - - group = top_parser.add_mutually_exclusive_group() - group.add_argument( - "--configfile", - metavar="FILE", - help="Path to Zeek Package Manager config file. Precludes --user.", - ) - group.add_argument( - "--user", - action="store_true", - help="Store all state in user's home directory. Precludes --configfile.", - ) - - top_parser.add_argument( - "--verbose", - "-v", - action="count", - default=0, - help="Increase program output for debugging." - " Use multiple times for more output (e.g. -vv).", - ) - top_parser.add_argument( - "--extra-source", - action="append", - metavar="NAME=URL", - help="Add an extra source.", - ) - return top_parser - - -def argparser() -> argparse.ArgumentParser: - pkg_name_help = ( - "The name(s) of package(s) to operate on. The package" - " may be named in several ways. If the package is part" - " of a package source, it may be referred to by the" - " base name of the package (last component of git URL)" - " or its path within the package source." - " If two packages in different package sources" - " have conflicting paths, then the package source" - " name may be prepended to the package path to resolve" - " the ambiguity. A full git URL may also be used to refer" - " to a package that does not belong to a source. E.g. for" - ' a package source called "zeek" that has a package named' - ' "foo" located in "alice/zkg.index", the following' - ' names work: "foo", "alice/foo", "zeek/alice/foo".' - ) - - def add_uservar_args( - parser: argparse.ArgumentParser, - force_help: str | None = None, - ) -> None: - parser.add_argument( - "--force", - action="store_true", - help=force_help or "Don't prompt for confirmation or user variables.", - ) - parser.add_argument( - "--user-var", - action="append", - metavar="NAME=VAL", - type=UserVar.parse_arg, - help="A user variable assignment. This avoids prompting" - " for input and lets you provide a value when using --force." - " Use repeatedly as needed for multiple values.", - ) - - def add_json_args(parser: argparse.ArgumentParser, help_text: str) -> None: - parser.add_argument("--json", action="store_true", help=help_text) - parser.add_argument( - "--jsonpretty", - type=int, - default=None, - metavar="SPACES", - help="Optional number of spaces to indent for pretty-printed JSON output.", - ) - - top_parser = top_level_parser() - command_parser = top_parser.add_subparsers( - title="commands", - dest="command", - help="See `%(prog)s -h` for per-command usage info.", - ) - command_parser.required = True - - # test - sub_parser = command_parser.add_parser( - "test", - help="Runs unit tests for Zeek packages.", - description="Runs the unit tests for the specified Zeek packages." - ' In most cases, the "zeek" and "zeek-config" programs will' - " need to be in PATH before running this command.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_test) - sub_parser.add_argument("package", nargs="+", help=pkg_name_help) - sub_parser.add_argument( - "--version", - default=None, - help="The version of the package to test. Only one package may be" - " specified at a time when using this flag. A version tag, branch" - " name, or commit hash may be specified here." - " If the package name refers to a local git repo with a working tree," - " then its currently active branch is used." - " The default for other cases is to use" - " the latest version tag, or if a package has none," - ' the default branch, like "main" or "master".', - ) - - # install - sub_parser = command_parser.add_parser( - "install", - help="Installs Zeek packages.", - description="Installs packages from a configured package source or" - " directly from a git URL. After installing, the package" - ' is marked as being "loaded" (see the ``load`` command).', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_install) - sub_parser.add_argument("package", nargs="+", help=pkg_name_help) - sub_parser.add_argument( - "--skiptests", - action="store_true", - help="Skip running unit tests for packages before installation.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks putting your installed package collection into a" - " broken or unusable state.", - ) - sub_parser.add_argument( - "--nosuggestions", - action="store_true", - help="Skip automatically installing suggested packages.", - ) - sub_parser.add_argument( - "--version", - default=None, - help="The version of the package to install. Only one package may be" - " specified at a time when using this flag. A version tag, branch" - " name, or commit hash may be specified here." - " If the package name refers to a local git repo with a working tree," - " then its currently active branch is used." - " The default for other cases is to use" - " the latest version tag, or if a package has none," - ' the default branch, like "main" or "master".', - ) - add_uservar_args(sub_parser) - - # bundle - sub_parser = command_parser.add_parser( - "bundle", - help="Creates a bundle file containing a collection of Zeek packages.", - description="This command creates a bundle file containing a collection" - " of Zeek packages. If ``--manifest`` is used, the user" - " supplies the list of packages to put in the bundle, else" - " all currently installed packages are put in the bundle." - " A bundle file can be unpacked on any target system," - " resulting in a repeatable/specific set of packages" - " being installed on that target system (see the" - " ``unbundle`` command). This command may be useful for" - " those that want to manage packages on a system that" - " otherwise has limited network connectivity. E.g. one can" - " use a system with an internet connection to create a" - " bundle, transport that bundle to the target machine" - " using whatever means are appropriate, and finally" - " unbundle/install it on the target machine.", - formatter_class=BundleHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_bundle) - sub_parser.add_argument( - "bundle_filename", - metavar="filename.bundle", - help="The path of the bundle file to create. It will be overwritten" - " if it already exists. Note that if --manifest is used before" - " this filename is specified, you should use a double-dash, --," - " to first terminate that argument list.", - ) - sub_parser.add_argument( - "--force", - action="store_true", - help="Skip the confirmation prompt.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks creating a bundle of packages that is in a" - " broken or unusable state.", - ) - sub_parser.add_argument( - "--nosuggestions", - action="store_true", - help="Skip automatically bundling suggested packages.", - ) - sub_parser.add_argument( - "--manifest", - nargs="+", - help="This may either be a file name or a list of packages to include" - " in the bundle. If a file name is supplied, it should be in INI" - " format with a single ``[bundle]`` section. The keys in that section" - " correspond to package names and their values correspond to git" - " version tags, branch names, or commit hashes. The values may be" - " left blank to indicate that the latest available version should be" - " used.", - ) - - # unbundle - sub_parser = command_parser.add_parser( - "unbundle", - help="Unpacks Zeek packages from a bundle file and installs them.", - description="This command unpacks a bundle file formerly created by the" - " ``bundle`` command and installs all the packages" - " contained within.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_unbundle) - sub_parser.add_argument( - "bundle_filename", - metavar="filename.bundle", - help="The path of the bundle file to install.", - ) - sub_parser.add_argument( - "--replace", - action="store_true", - help="Using this flag first removes all installed packages before then" - " installing the packages from the bundle.", - ) - add_uservar_args(sub_parser) - - # remove - sub_parser = command_parser.add_parser( - "remove", - aliases=["uninstall"], - help="Uninstall a package.", - description="Unloads (see the ``unload`` command) and uninstalls a" - " previously installed package.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_remove) - sub_parser.add_argument("package", nargs="+", help=pkg_name_help) - sub_parser.add_argument( - "--force", - action="store_true", - help="Skip the confirmation prompt.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks putting your installed package collection into a" - " broken or unusable state.", - ) - - # purge - sub_parser = command_parser.add_parser( - "purge", - help="Uninstall all packages.", - description="Unloads (see the ``unload`` command) and uninstalls all" - " previously installed packages.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_purge) - sub_parser.add_argument( - "--force", - action="store_true", - help="Skip the confirmation prompt.", - ) - - # refresh - sub_parser = command_parser.add_parser( - "refresh", - help="Retrieve updated package metadata.", - description="Retrieve latest package metadata from sources and checks" - " whether any installed packages have available upgrades." - " Note that this does not actually upgrade any packages (see the" - " ``upgrade`` command for that).", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_refresh) - sub_parser.add_argument( - "--aggregate", - action="store_true", - help="Crawls the urls listed in package source zkg.index files and" - " aggregates the metadata found in their zkg.meta (or legacy" - " bro-pkg.meta) files. The aggregated metadata is stored in the local" - " clone of the package source that zkg uses internally for locating" - " package metadata." - " For each package, the metadata is taken from the highest available" - ' git version tag or the default branch, like "main" or "master", if no version tags exist', - ) - sub_parser.add_argument( - "--fail-on-aggregate-problems", - action="store_true", - help="When using --aggregate, exit with error when any packages trigger" - " metadata problems. Normally such problems only cause a warning.", - ) - sub_parser.add_argument( - "--push", - action="store_true", - help="Push all local changes to package sources to upstream repos", - ) - sub_parser.add_argument( - "--sources", - nargs="+", - help="A list of package source names to operate on. If this argument" - " is not used, then the command will operate on all configured" - " sources.", - ) - - # upgrade - sub_parser = command_parser.add_parser( - "upgrade", - help="Upgrade installed packages to latest versions.", - description="Upgrades the specified package(s) to latest available" - " version. If no specific packages are specified, then all installed" - " packages that are outdated and not pinned are upgraded. For packages" - " that are installed with ``--version`` using a git branch name, the" - " package is updated to the latest commit on that branch, else the" - " package is updated to the highest available git version tag.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_upgrade) - sub_parser.add_argument("package", nargs="*", default=[], help=pkg_name_help) - sub_parser.add_argument( - "--skiptests", - action="store_true", - help="Skip running unit tests for packages before installation.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks putting your installed package collection into a" - " broken or unusable state.", - ) - sub_parser.add_argument( - "--nosuggestions", - action="store_true", - help="Skip automatically installing suggested packages.", - ) - add_uservar_args(sub_parser) - - # load - sub_parser = command_parser.add_parser( - "load", - help="Register packages to be be auto-loaded by Zeek.", - description="The Zeek Package Manager keeps track of all packages that" - ' are marked as "loaded" and maintains a single Zeek script that, when' - " loaded by Zeek (e.g. via ``@load packages``), will load the scripts" - ' from all "loaded" packages at once.' - ' This command adds a set of packages to the "loaded packages" list.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_load) - sub_parser.add_argument( - "package", - nargs="+", - default=[], - help="Name(s) of package(s) to load.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks putting your installed package collection into a" - " broken or unusable state.", - ) - - # unload - sub_parser = command_parser.add_parser( - "unload", - help="Unregister packages to be be auto-loaded by Zeek.", - description="The Zeek Package Manager keeps track of all packages that" - ' are marked as "loaded" and maintains a single Zeek script that, when' - ' loaded by Zeek, will load the scripts from all "loaded" packages at' - ' once. This command removes a set of packages from the "loaded' - ' packages" list.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_unload) - sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) - sub_parser.add_argument( - "--force", - action="store_true", - help="Skip the confirmation prompt.", - ) - sub_parser.add_argument( - "--nodeps", - action="store_true", - help="Skip all dependency resolution/checks. Note that using this" - " option risks putting your installed package collection into a" - " broken or unusable state.", - ) - - # pin - sub_parser = command_parser.add_parser( - "pin", - help="Prevent packages from being automatically upgraded.", - description="Pinned packages are ignored by the ``upgrade`` command.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_pin) - sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) - - # unpin - sub_parser = command_parser.add_parser( - "unpin", - help="Allows packages to be automatically upgraded.", - description="Packages that are not pinned are automatically upgraded" - " by the ``upgrade`` command", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_unpin) - sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) - - # list - sub_parser = command_parser.add_parser( - "list", - help="Lists packages.", - description="Outputs a list of packages that match a given category.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_list) - sub_parser.add_argument( - "category", - nargs="?", - default="installed", - choices=["all", "installed", "not_installed", "loaded", "unloaded", "outdated"], - help="Package category used to filter listing.", - ) - sub_parser.add_argument( - "--nodesc", - action="store_true", - help="Do not display description text, just the package name(s).", - ) - sub_parser.add_argument( - "--include-builtin", - action="store_true", - help="Also output packages that Zeek has built-in. By default" - " these are not shown.", - ) - - # search - sub_parser = command_parser.add_parser( - "search", - help="Search packages for matching names.", - description="Perform a substring search on package names and metadata" - " tags. Surround search text with slashes to indicate it is a regular" - " expression (e.g. ``/text/``).", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_search) - sub_parser.add_argument( - "search_text", - nargs="+", - default=[], - help="The text(s) or pattern(s) to look for.", - ) - - # info - sub_parser = command_parser.add_parser( - "info", - help="Display package information.", - description="Shows detailed information/metadata for given packages." - " If the package is currently installed, additional information about" - " the status of it is displayed. E.g. the installed version or whether" - ' it is currently marked as "pinned" or "loaded."', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_info) - sub_parser.add_argument( - "package", - nargs="+", - default=[], - help=pkg_name_help - + " If a single name is given and matches one of the same categories" - ' as the "list" command, then it is automatically expanded to be the' - " names of all packages which match the given category.", - ) - sub_parser.add_argument( - "--version", - default=None, - help="The version of the package metadata to inspect. A version tag," - " branch name, or commit hash and only one package at a time may be" - " given when using this flag. If unspecified, the behavior depends" - " on whether the package is currently installed. If installed," - " the metadata will be pulled from the installed version. If not" - " installed, the latest version tag is used, or if a package has no" - ' version tags, the default branch, like "main" or "master", is used.', - ) - sub_parser.add_argument( - "--nolocal", - action="store_true", - help="Do not read information from locally installed packages." - " Instead read info from remote GitHub.", - ) - sub_parser.add_argument( - "--include-builtin", - action="store_true", - help="Also output packages that Zeek has built-in. By default" - " these are not shown.", - ) - add_json_args(sub_parser, "Output package information as JSON.") - sub_parser.add_argument( - "--allvers", - action="store_true", - help="When outputting package information as JSON, show metadata for" - " all versions. This option can be slow since remote repositories" - " may be cloned multiple times. Also, installed packages will show" - " metadata only for the installed version unless the --nolocal " - " option is given.", - ) - - # config - sub_parser = command_parser.add_parser( - "config", - help="Show Zeek Package Manager configuration info.", - description="The default output of this command is a valid package" - " manager config file that corresponds to the one currently being used," - " but also with any defaulted field values filled in. This command" - " also allows for only the value of a specific field to be output if" - " the name of that field is given as an argument to the command.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_config) - sub_parser.add_argument( - "config_param", - nargs="?", - default="all", - choices=[ - "all", - "sources", - "user_vars", - "state_dir", - "script_dir", - "plugin_dir", - "bin_dir", - "zeek_dist", - ], - help="Name of a specific config file field to output.", - ) - - # autoconfig - sub_parser = command_parser.add_parser( - "autoconfig", - help="Generate a Zeek Package Manager configuration file.", - description="The output of this command is a valid package manager" - " config file that is generated by using the ``zeek-config`` script" - " that is installed along with Zeek. It is the suggested configuration" - " to use for most Zeek installations. For this command to work, the" - " ``zeek-config`` script must be in ``PATH``," - " unless the --user option is given, in which case this creates" - " a config that does not touch the Zeek installation.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_autoconfig) - sub_parser.add_argument( - "--force", - action="store_true", - help="Skip any confirmation prompt.", - ) - - # env - sub_parser = command_parser.add_parser( - "env", - help="Show the value of environment variables that need to be set for" - " Zeek to be able to use installed packages.", - description="This command returns shell commands that, when executed," - " will correctly set ``ZEEKPATH`` and ``ZEEK_PLUGIN_PATH`` to use" - " scripts and plugins from packages installed by the package manager." - " For this command to function properly, either have the ``zeek-config``" - " script (installed by zeek) in ``PATH``, or have the ``ZEEKPATH`` and" - " ``ZEEK_PLUGIN_PATH`` environment variables already set so this command" - " can append package-specific paths to them.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.set_defaults(run_cmd=cmd_env) - - # create - sub_parser = command_parser.add_parser( - "create", - help="Create a new Zeek package.", - description="This command creates a new Zeek package in the directory" - " provided via --packagedir. If this directory exists, zkg will not" - " modify it unless you provide --force.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - sub_parser.add_argument( - "--packagedir", - metavar="DIR", - required=True, - help="Output directory into which to produce the new package. Required.", - ) - sub_parser.add_argument( - "--version", - help="The template version to use. A version tag, branch name, or" - " commit hash may be specified here. If --template refers to a local" - " git repo with a working tree, then zkg uses it as-is and the version" - " is ignored. The default for other cases is to use the latest" - " version tag, or if a template has none, the default branch, like" - ' "main" or "master".', - ) - sub_parser.add_argument( - "--features", - nargs="+", - metavar="FEATURE", - help="Additional features to include in your package. Use the ``template" - " info`` command for information about available features.", - ) - sub_parser.add_argument( - "--template", - metavar="URL", - help="By default, zkg uses its own package template. This makes it" - " select an alternative.", - ) - sub_parser.set_defaults(run_cmd=cmd_create) - add_uservar_args(sub_parser) - - # Template management commands - - sub_parser = command_parser.add_parser("template", help="Manage package templates.") - - template_command_parser = sub_parser.add_subparsers( - title="template commands", - dest="command", - help="See %(prog)s -h for per-command usage info.", - ) - template_command_parser.required = True - - # template info - - sub_parser = template_command_parser.add_parser( - "info", - help="Shows information about a package template.", - description="This command shows versions and supported features for" - " a given package.", - ) - add_json_args(sub_parser, "Output template information as JSON.") - sub_parser.add_argument( - "--version", - help="The template version to report on. A version tag, branch name," - " or commit hash may be specified here. If the selected template" - " refers to a local git repo, the version is ignored. The default" - " for other cases is to use the latest version tag, or if a template" - ' has none, the default branch, like "main" or "master".', - ) - sub_parser.add_argument( - "template", - metavar="URL", - nargs="?", - help="URL of a package template repository, or local path to one." - " When not provided, the configured default template is used.", - ) - sub_parser.set_defaults(run_cmd=cmd_template_info) - - argcomplete.autocomplete(top_parser) - - return top_parser - - -def main() -> None: - args = argparser().parse_args() - - formatter = logging.Formatter( - "%(asctime)s %(levelname)-8s %(message)s", - "%Y-%m-%d %H:%M:%S", - ) - handler = logging.StreamHandler() - handler.setFormatter(formatter) - - if args.verbose == 1: - zeekpkg.LOG.setLevel(logging.INFO) - elif args.verbose >= 2: - zeekpkg.LOG.setLevel(logging.DEBUG) - - zeekpkg.LOG.addHandler(handler) - - configfile = args.configfile - - if not configfile: - configfile = find_configfile(args) - - config = create_config(args, configfile) - manager = create_manager(args, config) - args.run_cmd(manager, args, config, configfile) + args.run_cmd(manager, args) if __name__ == "__main__":