Code reorg

.github/workflows/pre-commit.yml

@@ -9,6 +9,6 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v7
+    - uses: actions/setup-python@v7
     - uses: pre-commit/action@v3.0.0

.github/workflows/test.yml

@@ -47,7 +47,7 @@ jobs:
           rm -r $ZEEKROOT/etc/zkg
           rm -r $ZEEKROOT/lib/zeek/python/zeekpkg
           rm -r $ZEEKROOT/var/lib/zkg
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: Install uv
         uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
@@ -58,7 +58,7 @@ jobs:
       - name: Run tests
         run: uv run pytest
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         if: failure()
         with:
           name: btest-${{ matrix.repo }}-${{ matrix.version }}
@@ -69,7 +69,7 @@ jobs:
     needs: [test]
     if: github.repository == 'zeek/package-manager' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Check release version
         # This fails e.g. if VERSION contains a dev commits suffix,
         # since we don't want to push these to PyPI. Accepts two-

.gitignore

@@ -1,7 +1,9 @@
 *.pyc
 *.egg-info
 *.swp
+.coverage
 .state
 doc/_build
 build
 dist
+uv.lock

.update-changes.cfg

@@ -5,7 +5,7 @@ function replace_python_package_version
     file=$1
     version=$2
 
-    cat $file | sed "s#^\\( *__version__ *= *\\)\"\\([0-9.-]\\{1,\\}\\)\"#\1\"$version\"#g" >$file.tmp
+    cat $file | sed "s#^\\( *VERSION *= *\\)\"\\([0-9.-]\\{1,\\}\\)\"#\1\"$version\"#g" >$file.tmp
     cat $file.tmp >$file
     rm -f $file.tmp
     git add $file
@@ -14,7 +14,7 @@ function replace_python_package_version
 function new_version_hook
 {
     version=$1
-    replace_python_package_version zeekpkg/__init__.py $version
+    replace_python_package_version zeekpkg/consts.py $version
 
     make man
     git add ./doc/man

doc/package.rst

@@ -549,6 +549,10 @@ script components, the "plugin" part is always unconditionally loaded by Zeek,
 but the "script" components must either be explicitly loaded (e.g. :samp:`@load
 {<package_name>}`) or the package marked as :ref:`loaded <load-command>`.
 
+:program:`zkg` captures the output of the build command into a
+`<package>-build.log` file in its logs directory, located in
+`<state_dir>/logs/`.
+
 .. _metadata-interpolation:
 
 Value Interpolation
@@ -652,6 +656,10 @@ An example :file:`zkg.meta`::
 The recommended test framework for writing package unit tests is `btest`_.
 See its documentation for further explanation and examples.
 
+:program:`zkg` captures the output of the test command into a
+`<package>-test.log` file in its logs directory, located in
+`<state_dir>/logs/`.
+
 .. note::
 
    :program:`zkg` version 2.12.0 introduced two improvements to `test_command`:

testing/baselines/tests.aliases-bad-2/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: invalid package "foo": invalid alias ".hidden"
+Error: invalid package "foo": invalid alias ".hidden"

testing/baselines/tests.aliases-bad-3/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: invalid package "foo": invalid alias "/aaa"
+Error: invalid package "foo": invalid alias "/aaa"

testing/baselines/tests.aliases-bad/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: invalid package "foo": invalid alias "../../../../../../../../bad"
+Error: invalid package "foo": invalid alias "../../../../../../../../bad"

testing/baselines/tests.aliases-conflict/out

@@ -3,15 +3,15 @@ Installing "one/alice/foo"
 Installed "one/alice/foo" (main)
 Loaded "one/alice/foo"
 Installing "one/alice/bar"
-error: incomplete installation, the follow packages failed to be installed:
+Error: incomplete installation, the follow packages failed to be installed:
   one/alice/bar (master)
 Failed installing "one/alice/bar": name "bar" conflicts with alias from "one/alice/foo"
 Installing "one/alice/baz"
-error: incomplete installation, the follow packages failed to be installed:
+Error: incomplete installation, the follow packages failed to be installed:
   one/alice/baz (master)
 Failed installing "one/alice/baz": alias "foo" conflicts with name of installed package "one/alice/foo"
 Installing "one/bob/corge"
-error: incomplete installation, the follow packages failed to be installed:
+Error: incomplete installation, the follow packages failed to be installed:
   one/bob/corge (master)
 Failed installing "one/bob/corge": alias "bar" conflicts with alias of installed package "one/alice/foo"
 Removed "one/alice/foo"
@@ -22,6 +22,6 @@ Installing "one/bob/corge"
 Installed "one/bob/corge" (master)
 Loaded "one/bob/corge"
 Installing "one/alice/foo"
-error: incomplete installation, the follow packages failed to be installed:
+Error: incomplete installation, the follow packages failed to be installed:
   one/alice/foo (main)
 Failed installing "one/alice/foo": name "foo" conflicts with alias from "one/alice/baz"

testing/baselines/tests.builtin-install/out

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-cannot install "spicy-plugin": built-in package
+Warning: cannot install "spicy-plugin": built-in package

testing/baselines/tests.builtin-spicy-version-error/output

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: failed to resolve dependencies: unsatisfiable dependency: "zeek-builtin/spicy-plugin" (X.X.X) is installed, but "one/alice/foo" requires <X.X.X (X.X.X not in <X.X.X)
+Error: failed to resolve dependencies: unsatisfiable dependency: "zeek-builtin/spicy-plugin" (X.X.X) is installed, but "one/alice/foo" requires <X.X.X (X.X.X not in <X.X.X)

testing/baselines/tests.builtin-spicy/output

@@ -1,3 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 Running unit tests for "one/alice/foo"
 Installing "one/alice/foo"
 Installed "one/alice/foo" (main)

testing/baselines/tests.builtin-test/out

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-cannot run tests for "spicy-plugin": built-in package
+Warning: cannot run tests for "spicy-plugin": built-in package

testing/baselines/tests.bundle-duplicate-package-name/.stderr

@@ -2,4 +2,4 @@
 Switched to a new branch 'main'
 Switched to a new branch 'drop-corge'
 Switched to branch 'master'
-error: failed to create bundle: duplicate package name "foo": remove one of "<...>/foo", "<...>/foo" from the manifest
+Error: failed to create bundle: duplicate package name "foo": remove one of "<...>/foo", "<...>/foo" from the manifest

testing/baselines/tests.install-duplicate-dependency-name/.stderr

@@ -2,4 +2,4 @@
 Switched to a new branch 'main'
 Switched to a new branch 'drop-corge'
 Switched to branch 'master'
-error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo"
+Error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo"

testing/baselines/tests.install-duplicate-package-name/.stderr

@@ -2,4 +2,4 @@
 Switched to a new branch 'main'
 Switched to a new branch 'drop-corge'
 Switched to branch 'master'
-error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo"
+Error: failed to resolve dependencies: duplicate package name "foo": remove one of "file:<...>/foo", "file:<...>/foo"

testing/baselines/tests.install-force-build-fail/out

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 Installing "one<...>/foo"
-error: incomplete installation, the follow packages failed to be installed:
+Error: incomplete installation, the follow packages failed to be installed:
   one<...>/foo (main)
 Failed installing "one<...>/foo": package build_command failed, see log in <...>/foo-build.log
 have_load_bar=1

testing/baselines/tests.install-force-test-fail/out

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 === install without skiptests
-error: failed to run tests for one<...>/foo: test_command failed with exit code 1
+Error: failed to run tests for one<...>/foo: test_command failed with exit code 1
 Running unit tests for "one<...>/foo"
 have_load_foo=0
 === install with skiptests

testing/baselines/tests.install-invalid/output

@@ -1,8 +1,8 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: invalid package ".": Package name ' fronting-whitespace' is not valid.
-error: invalid package ".": Package name 'trailing-whitespace ' is not valid.
-error: path ./packages/doesntexist is not a git repository
-error: path ./packages/notagitrepo is not a git repository
-error: local git clone at ./packages/dirtyrepo is dirty
-error: local git clone at dirtyrepo is dirty
-error: local git clone at ./dirtyrepo is dirty
+Error: invalid package ".": Package name ' fronting-whitespace' is not valid.
+Error: invalid package ".": Package name 'trailing-whitespace ' is not valid.
+Error: path ./packages/doesntexist is not a git repository
+Error: path ./packages/notagitrepo is not a git repository
+Error: local git clone at ./packages/dirtyrepo is dirty
+Error: local git clone at dirtyrepo is dirty
+Error: local git clone at ./dirtyrepo is dirty

testing/baselines/tests.install-reserved/output

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: invalid package ".": Package name 'packages' is not valid.
+Error: invalid package ".": Package name 'packages' is not valid.

testing/baselines/tests.installed-dependency-conflict/conflict.out

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: failed to resolve dependencies: unsatisfiable dependency: "one/alice/bar" (1.0.0) is installed, but "one/alice/foo" requires =2.0.0 (1.0.0 not in =2.0.0)
+Error: failed to resolve dependencies: unsatisfiable dependency: "one/alice/bar" (1.0.0) is installed, but "one/alice/foo" requires =2.0.0 (1.0.0 not in =2.0.0)

testing/baselines/tests.template-create/output

@@ -6,6 +6,6 @@
 <...>/out1 (installed: master) - TODO: A more detailed description of test1.
 <...>/out2 (installed: master) - TODO: A more detailed description of test2.
 <...>/out3 (installed: master) - TODO: A more detailed description of test3.
-error: could not determine value of user variable "name", provide via environment or --user-var
-error: the following features are unknown: "doesntexist". Template "foo" offers "readme".
+Error: could not determine value of user variable "name", provide via environment or --user-var
+Error: the following features are unknown: "doesntexist". Template "foo" offers "readme".
 "foo" will use value of "name" (the name of the package, e.g. "FooBar") from command line: 你こ안مγПनสXש😊©GBCDČM

testing/baselines/tests.unsatisfied-version-tag-dependency/fail.out

@@ -1,4 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: failed to resolve dependencies: "one/alice/bar" has no version satisfying dependencies:
+Error: failed to resolve dependencies: "one/alice/bar" has no version satisfying dependencies:
 	"one/alice/foo" requires: "=1.0.0"
 

testing/baselines/tests.upgrade-test-fail/out

@@ -3,7 +3,7 @@ Installing "one<...>/foo"
 Installed "one<...>/foo" (1.0.2)
 Loaded "one<...>/foo"
 === upgrade without skiptests
-error: failed to run tests for one<...>/foo: test_command failed with exit code 1
+Error: failed to run tests for one<...>/foo: test_command failed with exit code 1
 Running unit tests for "one<...>/foo"
 === upgrade with skiptest
 Upgraded "one<...>/foo" (1.0.3)

testing/baselines/tests.user-mode/output

@@ -1,13 +1,12 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[sources]
-one = <...>/sources/one
-
 [paths]
-state_dir = <...>/zeekroot/var/lib/zkg
-script_dir = <...>/zeekroot/share/zeek/site
-plugin_dir = <...>/zeekroot/lib/zeek/plugins
 bin_dir = <...>/zeekroot/var/lib/zkg/bin
-zeek_dist = 
+plugin_dir = <...>/zeekroot/lib/zeek/plugins
+script_dir = <...>/zeekroot/share/zeek/site
+state_dir = <...>/zeekroot/var/lib/zkg
+
+[sources]
+one = <...>/sources/one
 
 [templates]
 default = https://github.com/zeek/package-template
@@ -22,14 +21,13 @@ Installed "zeek/alice/bar" (master)
 Loaded "zeek/alice/bar"
 zeek/alice/bar (installed: master)
 Successfully wrote config file to <...>/home/testuser/.zkg/config
-[sources]
-
 [paths]
-state_dir = <...>/home/testuser/.zkg
-script_dir = <...>/home/testuser/.zkg/script_dir
-plugin_dir = <...>/home/testuser/.zkg/plugin_dir
 bin_dir = <...>/home/testuser/.zkg/bin
-zeek_dist = 
+plugin_dir = <...>/home/testuser/.zkg/plugin_dir
+script_dir = <...>/home/testuser/.zkg/script_dir
+state_dir = <...>/home/testuser/.zkg
+
+[sources]
 
 [templates]
 default = https://github.com/zeek/package-template

testing/btest.cfg

@@ -14,3 +14,4 @@ SCRIPTS=%(testbase)s/scripts
 SOURCES=%(testbase)s/sources
 PACKAGES=%(testbase)s/packages
 TEMPLATES=%(testbase)s/templates
+ZKG_DEFAULT_SOURCE=

testing/scripts/zkg

@@ -2,6 +2,10 @@
 
 force=""
 
+# This is brittle: any test saying e.g. "zkg -vvv install" instead of "zkg
+# install" will no longer have --force. This should either be more robustly
+# parsed or set force via an env var or dedicated command line option.
+
 case "$1" in
     install)
         force="--force"

testing/scripts/zkg-zeek

@@ -11,7 +11,4 @@ export HOME=$(pwd)/home/testuser
 export ZEEK_ZKG_CONFIG_DIR=$(pwd)/zeekroot/etc/zkg
 export ZEEK_ZKG_STATE_DIR=$(pwd)/zeekroot/var/lib/zkg
 
-# Don't use a default package source in zkg:
-export ZKG_DEFAULT_SOURCE=
-
 $TEST_BASE/../zkg "$@"

testing/test_manager.py

@@ -6,18 +6,24 @@
 
 import pytest
 
+from zeekpkg.config import CONFIG
 from zeekpkg.manager import Manager
 from zeekpkg.package import Package, PackageInfo
 
 
 @pytest.fixture
 def manager(tmp_path: Path) -> Manager:
+    CONFIG.read_dict(
+        {
+            "paths": {
+                "state_dir": str(tmp_path / "plugins"),
+                "script_dir": str(tmp_path / "scripts"),
+                "plugin_dir": str(tmp_path / "plugins"),
+            },
+        },
+    )
     with patch.object(Manager, "discover_builtin_packages", return_value=[]):
-        return Manager(
-            state_dir=str(tmp_path / "state"),
-            script_dir=str(tmp_path / "scripts"),
-            plugin_dir=str(tmp_path / "plugins"),
-        )
+        return Manager()
 
 
 class TestInfoCache:
@@ -84,7 +90,10 @@ def _mock_stat(self, mock_stat: MagicMock, key: dict[str, object]) -> None:
         mock_stat.return_value.st_size = key["zeek_size"]
 
     def test_disk_cache_hit_skips_subprocess(self, manager: Manager) -> None:
-        cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json")
+        cache_file = os.path.join(
+            CONFIG.state_dir(),
+            "zeek_build_info_cache.json",
+        )
 
         with open(cache_file, "w") as f:
             json.dump({"key": self._CACHE_KEY, "build_info": self._FAKE_BUILD_INFO}, f)
@@ -118,7 +127,10 @@ def test_disk_cache_miss_runs_subprocess(self, manager: Manager) -> None:
         mock_check_output.assert_called_once()
 
     def test_stale_mtime_reruns_subprocess(self, manager: Manager) -> None:
-        cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json")
+        cache_file = os.path.join(
+            CONFIG.state_dir(),
+            "zeek_build_info_cache.json",
+        )
         stale_key = {**self._CACHE_KEY, "zeek_mtime": 1000.0}
 
         with open(cache_file, "w") as f:
@@ -139,7 +151,10 @@ def test_stale_mtime_reruns_subprocess(self, manager: Manager) -> None:
         mock_check_output.assert_called_once()
 
     def test_different_path_reruns_subprocess(self, manager: Manager) -> None:
-        cache_file = os.path.join(manager.state_dir, "zeek_build_info_cache.json")
+        cache_file = os.path.join(
+            CONFIG.state_dir(),
+            "zeek_build_info_cache.json",
+        )
         other_key = {**self._CACHE_KEY, "zeek_path": "/opt/zeek/bin/zeek"}
 
         with open(cache_file, "w") as f:

testing/tests/aliases-bad

@@ -1,7 +1,7 @@
 
 # @TEST-EXEC: bash %INPUT
 # @TEST-EXEC-FAIL: zkg install foo
-# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep ^error' btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep ^Error' btest-diff .stderr
 
 cd packages/foo
 echo 'aliases = ../../../../../../../../bad' >> zkg.meta

testing/tests/test

@@ -7,6 +7,10 @@
 # @TEST-EXEC: bash we_need_to_go_deeper
 # @TEST-EXEC-FAIL: zkg test rot13
 
+# Verify that the build and test logs exist afterward:
+# @TEST-EXEC: test -f state/logs/rot13-build.log
+# @TEST-EXEC: test -f state/logs/rot13-test.log
+
 echo "$(pwd)/packages/rot13" >> sources/one/bob/zkg.index
 cd sources/one
 git commit -am 'add rot13 package'