Keep kopia's cache on the data directory filesystem and limit its size

packages/umbreld/source/modules/backups/backups.integration.test.ts

@@ -1,10 +1,12 @@
+import nodePath from 'node:path'
 import {setTimeout} from 'node:timers/promises'
 
 import {expect, test, beforeEach, afterEach, vi} from 'vitest'
 import fse from 'fs-extra'
 import yaml from 'js-yaml'
 import {execa} from 'execa'
 import pRetry from 'p-retry'
+import pWaitFor from 'p-wait-for'
 
 import createTestUmbreld from '../test-utilities/create-test-umbreld.js'
 import {BACKUP_RESTORE_FIRST_START_FLAG} from '../../constants.js'
@@ -192,6 +194,7 @@ test('backup() creates a backup successfully', async () => {
 	expect(files).not.toContain('external')
 	expect(files).not.toContain('network')
 	expect(files).not.toContain('thumbnails')
+	expect(files).not.toContain('kopia')
 })
 
 test('backup() throws error for non-existent repository', async () => {
@@ -545,6 +548,55 @@ test('backups respect app backupIgnore glob patterns', async () => {
 	expect(importantDirFiles).toContain('config.json')
 })
 
+test('kopia keeps cache and logs in the data directory with hard cache size limits', async () => {
+	// Create a network share and mount it
+	const backupNetworkSharePath = await createBackupShare(umbreld)
+
+	// Create a new backup repository
+	const repositoryId = await umbreld.client.backups.createRepository.mutate({
+		path: backupNetworkSharePath,
+		password: 'test-password',
+	})
+
+	// Do the backup
+	await expect(umbreld.client.backups.backup.mutate({repositoryId})).resolves.toBe(true)
+
+	// Verify kopia wrote its cache and logs inside the data directory
+	const kopiaDataDirectory = `${umbreld.instance.dataDirectory}/kopia`
+	await expect(fse.readdir(`${kopiaDataDirectory}/cache`)).resolves.not.toHaveLength(0)
+	await expect(fse.readdir(`${kopiaDataDirectory}/logs`)).resolves.not.toHaveLength(0)
+
+	// Verify the kopia directory itself is excluded from the backup
+	const backups = await umbreld.client.backups.listBackups.query({repositoryId})
+	const files = await umbreld.client.backups.listBackupFiles.query({backupId: backups[0].id})
+	expect(files).not.toContain('kopia')
+
+	// Verify cache size limits are pinned in the repository config
+	// Sizes are stored in bytes, connect flags are in MiB (1 MB = 2^20 bytes)
+	// The cache directory is stored relative to the config file directory
+	const kopiaConfig = await fse.readJson(`/kopia/config/${repositoryId}.config`)
+	const cacheDirectory = nodePath.resolve('/kopia/config', kopiaConfig.caching.cacheDirectory)
+	expect(cacheDirectory.startsWith(`${kopiaDataDirectory}/cache/`)).toBe(true)
+	expect(kopiaConfig.caching.maxCacheSize).toBe(500 * 2 ** 20)
+	expect(kopiaConfig.caching.contentCacheSizeLimitBytes).toBe(1000 * 2 ** 20)
+	expect(kopiaConfig.caching.maxMetadataCacheSize).toBe(1000 * 2 ** 20)
+	expect(kopiaConfig.caching.metadataCacheSizeLimitBytes).toBe(2000 * 2 ** 20)
+})
+
+test('stale kopia cache in the legacy location is cleaned up on startup', async () => {
+	// Recreate stale cache leftovers in the legacy location
+	const legacyCacheDirectory = '/kopia/cache/kopia'
+	await fse.ensureDir(`${legacyCacheDirectory}/legacy-repo`)
+	await fse.writeFile(`${legacyCacheDirectory}/legacy-repo/blob`, 'stale-cache-data')
+
+	// Restart umbreld
+	await umbreld.instance.stop()
+	await umbreld.instance.start()
+
+	// Verify the legacy cache is cleaned up (cleanup runs in the background, so poll)
+	await pWaitFor(async () => !(await fse.pathExists(legacyCacheDirectory)), {timeout: 30_000, interval: 100})
+})
+
 test('backups handle disconnected network shares gracefully', async () => {
 	// Set the share watch interval to 100ms and restart umbreld
 	umbreld.instance.files.networkStorage.shareWatchInterval = 100

packages/umbreld/source/modules/backups/backups.ts

@@ -77,6 +77,10 @@ export default class Backups {
 		// Cleanup any left over backup mounts
 		await this.unmountAll().catch((error) => this.logger.error('Error unmounting backups', error))
 
+		// Cleanup any kopia cache from the legacy /kopia/cache location. This can take
+		// a while if the old cache is large so we don't block startup on it.
+		this.cleanupLegacyCache().catch((error) => this.logger.error('Error cleaning up legacy kopia cache', error))
+
 		// Fire off background backup process
 		this.backupJobPromise = this.backupOnInterval().catch((error) =>
 			this.logger.error('Error running backups on interval', error),
@@ -184,8 +188,11 @@ export default class Backups {
 			// Spawn process
 			const env = {
 				KOPIA_CHECK_FOR_UPDATES: 'false',
-				XDG_CACHE_HOME: '/kopia/cache',
+				// Keep the cache and logs on the data directory's filesystem. /kopia lives on
+				// the data partition which on Raspberry Pi installs is too small to hold the cache.
+				XDG_CACHE_HOME: `${this.#umbreld.dataDirectory}/kopia/cache`,
 				XDG_CONFIG_HOME: '/kopia/config',
+				KOPIA_LOG_DIR: `${this.#umbreld.dataDirectory}/kopia/logs`,
 			}
 			const process = execa('kopia', flags, {env})
 
@@ -435,9 +442,38 @@ export default class Backups {
 			// continue to backup, kopia will see these as backups originating from
 			// different machines.
 			'--override-hostname=umbrel',
+			// Limit the size of kopia's local cache. The soft limits are only enforced
+			// by sweeps when the repository is opened, the hard limits are enforced
+			// continuously. Without a hard limit the cache can grow unbounded between
+			// repository opens and exhaust the disk space/inodes of its filesystem.
+			// We're connecting before every repository operation so these limits can
+			// never drift.
+			'--content-cache-size-mb=500',
+			'--content-cache-size-limit-mb=1000',
+			'--metadata-cache-size-mb=1000',
+			'--metadata-cache-size-limit-mb=2000',
 		])
 	}
 
+	// Remove any kopia cache left behind at the legacy /kopia/cache location. The cache
+	// now lives in the data directory so anything there is dead weight. On Raspberry Pi
+	// installs it could hold enough inodes to exhaust the partition it lives on.
+	private async cleanupLegacyCache() {
+		const legacyCacheDirectory = '/kopia/cache/kopia'
+		const deletingDirectory = `${legacyCacheDirectory}.deleting`
+
+		// Remove any leftovers from a previously interrupted cleanup
+		await fse.remove(deletingDirectory)
+
+		if (!(await fse.pathExists(legacyCacheDirectory))) return
+
+		// Rename first so the removal can't race anything writing into the tree it's deleting
+		this.logger.log('Removing kopia cache from legacy location')
+		await fse.move(legacyCacheDirectory, deletingDirectory)
+		await fse.remove(deletingDirectory)
+		this.logger.log('Removed kopia cache from legacy location')
+	}
+
 	// Wrapper for kopia commands that interact with a repository
 	async repository(
 		repositoryId: string,
@@ -591,6 +627,9 @@ export default class Backups {
 		ignoreFileContents.push('app-stores')
 		ignoreFileContents.push(this.#umbreld.files.thumbnails.thumbnailDirectory)
 
+		// Ignore kopia's own cache and logs otherwise backups include the backup cache
+		ignoreFileContents.push('kopia')
+
 		// Ignore temporary migration directory
 		ignoreFileContents.push('.temporary-migration')