PMM-14931 Control built-in ClickHouse config via PMM_CLICKHOUSE_CONFIG env var

build/ansible/roles/clickhouse/files/default-config.xml

@@ -590,7 +590,7 @@
     <user_directories>
         <users_xml>
             <!-- Path to configuration file with predefined users. -->
-            <path>users.xml</path>
+            <path>default-users.xml</path>
         </users_xml>
         <local_directory>
             <!-- Path to folder where users created by SQL commands are stored. -->

build/ansible/roles/clickhouse/files/low-memory-config.xml

@@ -590,7 +590,7 @@
     <user_directories>
         <users_xml>
             <!-- Path to configuration file with predefined users. -->
-            <path>users.xml</path>
+            <path>low-memory-users.xml</path>
         </users_xml>
         <local_directory>
             <!-- Path to folder where users created by SQL commands are stored. -->

build/ansible/roles/clickhouse/files/switch-config.sh

@@ -1,55 +1,7 @@
 #!/bin/bash
-# Usage: switch-config.sh [low|default]
-# Switches /etc/clickhouse-server/config.xml
+# switch-config.sh is deprecated and will be removed in a future PMM release.
+# Use the PMM_CLICKHOUSE_CONFIG environment variable instead.
 
-set -e
-CONFIG_DIR="/etc/clickhouse-server"
-PROFILE="$1"
-
-if [ -z "$PROFILE" ]; then
-  echo "Usage: $0 [low|default]" >&2
-  exit 1
-fi
-
-case "$PROFILE" in
-  low)
-    CONFIG_TARGET="low-memory-config.xml"
-    USERS_TARGET="low-memory-users.xml"
-    ;;
-  default)
-    CONFIG_TARGET="default-config.xml"
-    USERS_TARGET="default-users.xml"
-    ;;
-  *)
-    echo "Usage: $0 [low|default]" >&2
-    exit 1
-    ;;
-esac
-
-if [ ! -e "$CONFIG_DIR/$CONFIG_TARGET" ]; then
-  echo "Config profile $CONFIG_TARGET does not exist in $CONFIG_DIR." >&2
-  exit 2
-fi
-if [ ! -e "$CONFIG_DIR/$USERS_TARGET" ]; then
-  echo "Users profile $USERS_TARGET does not exist in $CONFIG_DIR." >&2
-  exit 2
-fi
-
-echo "Stopping clickhouse..."
-if ! supervisorctl stop clickhouse; then
-  echo "Failed to stop clickhouse!" >&2
-  exit 3
-fi
-
-ln -sf "$CONFIG_TARGET" "$CONFIG_DIR/config.xml"
-ln -sf "$USERS_TARGET" "$CONFIG_DIR/users.xml"
-echo "Switched config.xml to $CONFIG_TARGET."
-echo "Switched users.xml to $USERS_TARGET."
-
-echo "Starting clickhouse..."
-if ! supervisorctl start clickhouse; then
-  echo "Failed to start clickhouse!" >&2
-  exit 4
-fi
-
-exit 0
+echo "switch-config.sh is deprecated and will be removed in a future PMM release." >&2
+echo "Set PMM_CLICKHOUSE_CONFIG=default|low-memory" >&2
+exit 1

build/ansible/roles/supervisord/files/pmm.ini

@@ -51,7 +51,7 @@ redirect_stderr = true
 
 [program:clickhouse]
 priority = 2
-command = /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml
+command = /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/default-config.xml
 autorestart = true
 autostart = true
 startretries = 10

documentation/docs/install-pmm/install-pmm-server/deployment-options/docker/env_var.md

@@ -25,6 +25,16 @@ Fine-tune data retention and collection intervals to balance monitoring detail w
 !!! tip "Performance impact"
     Higher resolution (lower values) provides more detailed metrics but increases storage requirements and system load. For high-traffic production environments, consider increasing these values.
 
+### Built-in ClickHouse configuration
+Select the configuration profile for the built-in ClickHouse instance 
+
+| Variable | Default | Description | Example |
+|----------|---------|-------------|----------|
+| `PMM_CLICKHOUSE_CONFIG` | `default` | Configuration profile for the built-in ClickHouse instance. Use `low-memory` for PMM Server environments with less than 16 GB RAM to avoid "memory limit exceeded" errors. | `low-memory` |
+
+!!! note
+    This applies only to the built-in ClickHouse instance. For details, see [ClickHouse memory issues](../../../../troubleshoot/qan_issues.md#clickhouse-memory-issues-in-low-memory-environments).
+
 ### Feature controls
 Enable or disable specific PMM features:
 
@@ -262,4 +272,4 @@ docker run \
 ```
 
 ## Related topic
-[Preview environment variables](../docker/preview_env_var.md)
+[Preview environment variables](../docker/preview_env_var.md)

documentation/docs/troubleshoot/qan_issues.md

@@ -92,32 +92,17 @@ PMM includes two ClickHouse profiles:
 
 ### Switch to low-memory configuration
 
-```bash
-docker exec -it -u pmm pmm-server ./switch-config.sh low
-```
+Select the profile with the `PMM_CLICKHOUSE_CONFIG` environment variable when you create the container:
 
-To switch back:
 ```bash
-docker exec -it -u pmm pmm-server ./switch-config.sh default
+docker run -e PMM_CLICKHOUSE_CONFIG=low-memory ... percona/pmm-server:3
 ```
 
-The script stops ClickHouse, updates the configuration, and restarts the service.
-
-### Persistent configuration
-
-If you run PMM Server with the `--rm` flag, run the switch script each time the container starts. For systemd, add to your unit file:
-```ini
-ExecStartPost=/usr/bin/docker exec -u pmm pmm-server ./switch-config.sh low
-```
-
-See [Install PMM Server with Podman](../install-pmm/install-pmm-server/deployment-options/podman/index.md) for systemd configuration examples.
-
 !!! note "Configuration details"
     Both configuration files are located in `/etc/clickhouse-server/` inside the PMM Server container:
 
     - `default-config.xml`: default profile
     - `low-memory-config.xml`: low-memory profile
 
-    The script is available at `/etc/clickhouse-server/switch-config.sh` or `/opt/switch-config.sh`.
-
-    When switching profiles, both `config.xml` and `users.xml` are updated to point to the selected profile.
+
+The `switch-config.sh` script is deprecated and will be removed in a future PMM release; use `PMM_CLICKHOUSE_CONFIG` instead.

managed/cmd/pmm-managed-init/main.go

@@ -22,7 +22,9 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/percona/pmm/managed/models"
+	"github.com/percona/pmm/managed/services/clickhouse"
 	"github.com/percona/pmm/managed/services/supervisord"
+	"github.com/percona/pmm/managed/utils/env"
 	"github.com/percona/pmm/managed/utils/envvars"
 	"github.com/percona/pmm/utils/logger"
 )
@@ -36,6 +38,10 @@ func main() {
 		logrus.SetLevel(logrus.TraceLevel)
 	}
 	envSettings, errs, warns := envvars.ParseEnvVars(os.Environ())
+	clickHouseConfig, err := clickhouse.GetClickHouseConfig(os.Getenv(env.ClickHouseConfig))
+	if err != nil {
+		errs = append(errs, err)
+	}
 	for _, warn := range warns {
 		logrus.Warnf("Configuration warning: %s", warn)
 	}
@@ -46,8 +52,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err := models.ValidateSettings(envSettings)
-	if err != nil {
+	if err := models.ValidateSettings(envSettings); err != nil {
 		logrus.Errorf("Configuration error: %s.", err)
 		os.Exit(1)
 	}
@@ -56,6 +61,7 @@ func main() {
 	pmmConfigParams["DisableInternalDB"], _ = strconv.ParseBool(os.Getenv("PMM_DISABLE_BUILTIN_POSTGRES"))
 	pmmConfigParams["DisableInternalClickhouse"], _ = strconv.ParseBool(os.Getenv("PMM_DISABLE_BUILTIN_CLICKHOUSE"))
 	pmmConfigParams["AgentConfigFilePath"] = models.AgentConfigFilePath
+	pmmConfigParams["ClickHouseConfig"] = clickHouseConfig
 
 	isHAEnabled, _ := strconv.ParseBool(os.Getenv("PMM_HA_ENABLE"))
 	if isHAEnabled {

managed/services/clickhouse/config.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Package clickhouse provides facilities for working with clickhouse.
+package clickhouse
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+const (
+	defaultClickHouseConfig = "default"
+	clickHouseConfigDir     = "/etc/clickhouse-server"
+)
+
+// GetClickHouseConfig returns the config name if the matching
+// <config>-config.xml files exist on disk.
+// Empty input falls back to defaultClickHouseConfig.
+func GetClickHouseConfig(config string) (string, error) {
+	if config == "" {
+		return defaultClickHouseConfig, nil
+	}
+
+	return config, validateClickHouseConfigAt(config, clickHouseConfigDir)
+}
+
+// validateClickHouseConfigAt returns an error if configuration files are missing for given config
+func validateClickHouseConfigAt(config, dir string) error {
+	availableConfigs, err := availableClickHouseConfigs(dir)
+	if err != nil {
+		return fmt.Errorf("unable to get available ClickHouse configs: %w", err)
+	}
+
+	path := filepath.Join(dir, config+"-config.xml")
+	if _, err := os.Stat(path); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf(
+				"invalid PMM_CLICKHOUSE_CONFIG=%q: %s not found; available configs: %v",
+				config, path, availableConfigs,
+			)
+		}
+		return fmt.Errorf("cannot stat %s: %w", path, err)
+	}
+
+	return nil
+}
+
+// availableClickHouseConfigs lists config names that are present in the dir
+func availableClickHouseConfigs(dir string) ([]string, error) {
+	var configs []string
+
+	matches, err := filepath.Glob(filepath.Join(dir, "*-config.xml"))
+	if err != nil {
+		return nil, err
+	}
+	for _, m := range matches {
+		name := strings.TrimSuffix(filepath.Base(m), "-config.xml")
+		configs = append(configs, name)
+	}
+
+	sort.Strings(configs)
+	return configs, nil
+}

managed/services/clickhouse/config_test.go

@@ -0,0 +1,128 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package clickhouse
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeConfigFiles creates the given files in dir with placeholder content.
+func writeConfigFiles(t *testing.T, dir string, names ...string) {
+	t.Helper()
+	for _, name := range names {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte("<clickhouse/>"), 0o600))
+	}
+}
+
+func TestGetClickHouseConfig(t *testing.T) {
+	t.Parallel()
+
+	// Empty input falls back to the default config.
+	got, err := GetClickHouseConfig("")
+	require.NoError(t, err)
+	assert.Equal(t, defaultClickHouseConfig, got)
+}
+
+func TestValidateClickHouseConfigAt(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	// A config is valid as long as <name>-config.xml exists.
+	writeConfigFiles(
+		t, dir,
+		"default-config.xml",
+		"low-memory-config.xml",
+	)
+
+	tests := []struct {
+		name        string
+		config      string
+		errContains []string
+	}{
+		{name: "default", config: "default"},
+		{name: "low-memory", config: "low-memory"},
+		{
+			name:   "missing",
+			config: "nonexistent",
+			errContains: []string{
+				`invalid PMM_CLICKHOUSE_CONFIG="nonexistent"`,
+				"available configs:",
+				"default", "low-memory",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := validateClickHouseConfigAt(tt.config, dir)
+			if tt.errContains == nil {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			for _, substr := range tt.errContains {
+				assert.Contains(t, err.Error(), substr)
+			}
+		})
+	}
+
+	t.Run("invalid config dir", func(t *testing.T) {
+		t.Parallel()
+
+		base := t.TempDir()
+		// "notdir" is a regular file; using it as the config dir makes os.Stat fail
+		require.NoError(t, os.WriteFile(filepath.Join(base, "notdir"), nil, 0o600))
+
+		err := validateClickHouseConfigAt("default", filepath.Join(base, "notdir"))
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "cannot stat")
+		assert.NotContains(t, err.Error(), "available configs:")
+	})
+}
+
+func TestAvailableClickHouseConfigs(t *testing.T) {
+	t.Parallel()
+
+	t.Run("empty dir", func(t *testing.T) {
+		t.Parallel()
+
+		got, err := availableClickHouseConfigs(t.TempDir())
+		require.NoError(t, err)
+		assert.Empty(t, got)
+	})
+
+	t.Run("lists config names sorted, ignoring non-config files", func(t *testing.T) {
+		t.Parallel()
+
+		dir := t.TempDir()
+		writeConfigFiles(
+			t, dir,
+			"low-memory-config.xml",
+			"default-config.xml",
+			"dhparam.pem", // not a *-config.xml, must be ignored
+		)
+
+		got, err := availableClickHouseConfigs(dir)
+		require.NoError(t, err)
+		assert.Equal(t, []string{"default", "low-memory"}, got)
+	})
+}