feat(pi): unmount external USB on undervoltage and I/O errors

packages/os/overlay-pi/opt/umbrel-external-storage/umbrel-external-storage

@@ -2,15 +2,17 @@
 
 set -euo pipefail
 
-# This script will:
-# - Look for external storage devices
-# - Check if they contain an Umbrel install
-# - If yes
-# - - Mount it
-# - If no
-# - - Format it
-# - - Mount it
-# - - Install Umbrel on it
+# This script ONLY runs when the user has opted in via umbrel-allow-external-format on
+# the SD card boot partition. It migrates Umbrel's data directory to a USB drive.
+#
+# Normal USB drives (media, backups, etc.) are mounted by umbreld under Files → External
+# and must not be handled here.
+#
+# When opted in, this script will:
+# - Look for a single external storage device
+# - Check if it contains an Umbrel install
+# - If yes, mount it
+# - If no, format it (only with opt-in), mount it, and install Umbrel on it
 # - Bind mount the external installation on top of the local installation
 
 UMBREL_ROOT="/home/umbrel/umbrel"
@@ -108,6 +110,71 @@ is_partition_ext4 () {
   blkid -o value -s TYPE "${partition_path}" | grep --quiet '^ext4$'
 }
 
+get_partition_fstype () {
+  partition_path="${1}"
+  sync
+  blkid -o value -s TYPE "${partition_path}" 2>/dev/null || true
+}
+
+# Explicit opt-in is required before we wipe a user's USB drive.
+# See https://github.com/getumbrel/umbrel/issues/1956
+UMBREL_ALLOW_FORMAT_PATHS=(
+  "/boot/firmware/umbrel-allow-external-format"
+  "/boot/umbrel-allow-external-format"
+)
+
+is_external_format_explicitly_allowed () {
+  if [[ "${UMBREL_ALLOW_EXTERNAL_FORMAT:-}" == "1" ]]; then
+    return 0
+  fi
+  for path in "${UMBREL_ALLOW_FORMAT_PATHS[@]}"; do
+    if [[ -f "${path}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+device_has_filesystem_signatures () {
+  device_path="${1}"
+  signatures=$(wipefs --noheadings "${device_path}" 2>/dev/null || true)
+  [[ -n "${signatures}" ]]
+}
+
+refuse_automatic_format () {
+  reason="${1}"
+  echo "ERROR: Refusing to automatically format external storage: ${reason}"
+  echo "Umbrel will continue running from the SD card."
+  echo "To intentionally use this drive as Umbrel data storage (this erases all data), create:"
+  echo "  /boot/firmware/umbrel-allow-external-format"
+  echo "on the SD card boot partition, then reboot."
+  exit 1
+}
+
+assert_safe_to_format () {
+  block_device="${1}"
+  partition_path="${2}"
+  device_path="/dev/${block_device}"
+
+  if is_external_format_explicitly_allowed; then
+    echo "Explicit consent to format external storage detected, continuing..."
+    return 0
+  fi
+
+  if device_has_filesystem_signatures "${device_path}"; then
+    refuse_automatic_format "existing filesystem signatures detected on ${device_path}"
+  fi
+
+  fstype=$(get_partition_fstype "${partition_path}")
+  if [[ -n "${fstype}" && "${fstype}" != "ext4" ]]; then
+    refuse_automatic_format "partition ${partition_path} uses filesystem type '${fstype}'"
+  fi
+
+  if [[ "${fstype}" == "ext4" ]]; then
+    refuse_automatic_format "ext4 partition exists but is not a recognised Umbrel data drive"
+  fi
+}
+
 # Wipes a block device and reformats it with a single EXT4 partition
 format_block_device () {
   device="${1}"
@@ -139,6 +206,8 @@ setup_new_device () {
   block_device="${1}"
   partition_path="${2}"
 
+  assert_safe_to_format "${block_device}" "${partition_path}"
+
   echo "Formatting device..."
   format_block_device $block_device
 
@@ -164,14 +233,22 @@ copy_docker_to_external_storage () {
 main () {
   echo "Running external storage mount script..."
   check_root
-  check_dependencies sed wipefs parted mount sync umount
+  check_dependencies sed wipefs parted mount sync umount blkid
 
   if [[ "$(running_off_sdcard)" == "false" ]]
   then
     echo "This script should only run when umbrelOS boots from an SD card, exiting..."
     exit
   fi
 
+  if ! is_external_format_explicitly_allowed; then
+    echo "No opt-in for Umbrel external data disk (umbrel-allow-external-format); skipping."
+    echo "USB drives are mounted by umbreld under Files → External."
+    exit 0
+  fi
+
+  echo "Opt-in detected: setting up external drive as Umbrel data disk..."
+
   no_of_block_devices=$(list_block_devices | wc -l)
 
   retry_for_block_devices=1

packages/os/umbrelos.Dockerfile

@@ -150,10 +150,7 @@ RUN apt-get install --yes python3 fswatch jq rsync git gettext-base gnupg procps
 RUN systemctl disable smbd wsdd2
 
 # Filessystem support
-RUN apt-get install --yes gdisk parted e2fsprogs exfatprogs
-# For some reason this always fails on arm64 but it's ok since we
-# don't support external storage on Pi anyway.
-RUN [ "${TARGETARCH}" = "amd64" ] && apt-get install --yes ntfs-3g || true
+RUN apt-get install --yes gdisk parted e2fsprogs exfatprogs ntfs-3g
 
 # Install Node.js
 RUN NODE_ARCH=$([ "${TARGETARCH}" = "arm64" ] && echo "arm64" || echo "x64") && \

packages/ui/public/locales/en.json

@@ -835,6 +835,8 @@
   "notifications.backups-failing.description": "Automatic backups have been failing. Check your backup location and review your settings.",
   "notifications.backups-failing.go-to-backups": "Go to Backups",
   "notifications.backups-failing.title": "No Backups in the last 24 hours",
+  "notifications.external-storage-power-fault.description": "Your Raspberry Pi reported low voltage or USB storage errors. External drives were unmounted to protect your data. Use a powered USB hub or a drive with its own power supply, then reconnect the drive.",
+  "notifications.external-storage-power-fault.title": "External drive unmounted (power issue)",
   "notifications.cpu.too-hot": "High CPU temperature",
   "notifications.memory.low": "Your device's memory is low",
   "notifications.new-version-available": "{{update}} is now available to install",

packages/ui/public/locales/ru.json

@@ -835,6 +835,8 @@
   "notifications.backups-failing.description": "Автоматическое резервное копирование завершается с ошибками. Проверь место хранения резервных копий и свои настройки.",
   "notifications.backups-failing.go-to-backups": "Перейти в Backups",
   "notifications.backups-failing.title": "Нет резервных копий за последние 24 часа",
+  "notifications.external-storage-power-fault.description": "Raspberry Pi сообщил о просадке напряжения или ошибках USB-накопителя. Внешние диски отключены, чтобы защитить данные. Используйте USB-хаб с питанием или диск с собственным блоком питания, затем подключите диск снова.",
+  "notifications.external-storage-power-fault.title": "Внешний диск отключён (проблема питания)",
   "notifications.cpu.too-hot": "Высокая температура CPU",
   "notifications.memory.low": "На устройстве мало памяти",
   "notifications.new-version-available": "{{update}} уже доступно для установки",

packages/ui/src/features/files/hooks/use-external-storage.ts

@@ -21,10 +21,8 @@ export function useExternalStorage() {
 	const utils = trpcReact.useUtils()
 	const {add} = useQueryParams()
 
-	// Check device information to determine if external storage is supported (currently not supported on Raspberry Pi)
-	const {data: deviceInfo} = trpcReact.systemNg.device.getIdentity.useQuery()
-
-	const isExternalStorageSupported = deviceInfo?.productName !== 'Raspberry Pi'
+	const {data: isExternalStorageSupported = false} =
+		trpcReact.files.isExternalStorageSupported.useQuery()
 
 	// Subscribe to files:external-storage:change events that fire when devices are mounted/unmounted
 	// and invalidate the external storage queries

packages/ui/src/routes/notifications.tsx

@@ -228,6 +228,13 @@ export function Notifications() {
 			return getMigratedBackThatMacUpContent()
 		}
 
+		if (notification === 'external-storage-power-fault') {
+			return {
+				title: t('notifications.external-storage-power-fault.title'),
+				description: t('notifications.external-storage-power-fault.description'),
+			}
+		}
+
 		// Default fallback for unknown notifications
 		return getDefaultNotificationContent(notification)
 	}

packages/umbreld/source/modules/files/external-storage-power-monitor.ts

@@ -0,0 +1,100 @@
+import {$} from 'execa'
+
+import runEvery from '../utilities/run-every.js'
+
+import type Umbreld from '../../index.js'
+import type ExternalStorage from './external-storage.js'
+
+// throttled=0x1 — undervoltage now; 0x10000 — undervoltage since boot
+const UNDERVOLTAGE_THROTTLE_MASK = 0x0001 | 0x00010000
+
+export const EXTERNAL_STORAGE_POWER_FAULT_NOTIFICATION = 'external-storage-power-fault'
+
+const POLL_INTERVAL = '15 seconds'
+const JOURNAL_LOOKBACK = '30 seconds ago'
+const FAULT_COOLDOWN_MS = 5 * 60 * 1000
+
+export function startExternalStoragePowerMonitor(umbreld: Umbreld, externalStorage: ExternalStorage) {
+	const logger = umbreld.logger.createChildLogger('files:external-storage-power-monitor')
+	let lastFaultHandledAt = 0
+
+	const check = async () => {
+		const mountedDevices = await externalStorage.getMountedExternalDevices()
+		if (mountedDevices.length === 0) return
+
+		const powerFault = await detectPowerRelatedFault(mountedDevices.map((device) => device.id))
+		if (!powerFault) return
+
+		const now = Date.now()
+		if (now - lastFaultHandledAt < FAULT_COOLDOWN_MS) return
+		lastFaultHandledAt = now
+
+		logger.warn(
+			`Raspberry Pi power fault detected (${powerFault}) with external USB storage mounted; unmounting drives`,
+		)
+
+		for (const device of mountedDevices) {
+			await externalStorage
+				.unmountExternalDevice(device.id, {remove: false})
+				.catch((error) => logger.error(`Failed to unmount ${device.id} after power fault`, error))
+		}
+
+		await umbreld.notifications.add(EXTERNAL_STORAGE_POWER_FAULT_NOTIFICATION).catch((error) => {
+			logger.error('Failed to add power fault notification', error)
+		})
+	}
+
+	return runEvery(POLL_INTERVAL, check, {runInstantly: false})
+}
+
+async function detectPowerRelatedFault(deviceIds: string[]) {
+	const [undervoltage, ioErrors] = await Promise.all([
+		hasUndervoltageSignals(),
+		hasRecentIoErrors(deviceIds),
+	])
+
+	if (undervoltage) return 'undervoltage'
+	if (ioErrors) return 'io-error'
+	return false
+}
+
+async function hasUndervoltageSignals() {
+	const [throttled, kernelMessage] = await Promise.all([
+		hasThrottledUndervoltage(),
+		hasRecentKernelUnderVoltageMessage(),
+	])
+	return throttled || kernelMessage
+}
+
+async function hasThrottledUndervoltage() {
+	try {
+		const {stdout} = await $`vcgencmd get_throttled`
+		const match = stdout.match(/throttled=(0x[0-9a-f]+)/i)
+		if (!match?.[1]) return false
+		const value = Number.parseInt(match[1], 16)
+		return (value & UNDERVOLTAGE_THROTTLE_MASK) !== 0
+	} catch {
+		return false
+	}
+}
+
+async function hasRecentKernelUnderVoltageMessage() {
+	try {
+		const {stdout} = await $`journalctl -k --since ${JOURNAL_LOOKBACK} --grep Under-voltage --no-pager -q`
+		return stdout.trim().length > 0
+	} catch {
+		return false
+	}
+}
+
+async function hasRecentIoErrors(deviceIds: string[]) {
+	for (const deviceId of deviceIds) {
+		try {
+			const {stdout} = await $`journalctl -k --since ${JOURNAL_LOOKBACK} --grep ${deviceId} --grep "I/O error" --no-pager -q`
+			if (stdout.trim().length > 0) return true
+		} catch {
+			// Continue checking other devices
+		}
+	}
+	return false
+}

packages/umbreld/source/modules/files/external-storage.integration.test.ts

@@ -82,9 +82,9 @@ describe('enabled', () => {
 		await expect(umbreld.instance.files.externalStorage.supported()).resolves.toBe(true)
 	})
 
-	test('is disabled on Raspberry Pi', async () => {
+	test('is enabled on Raspberry Pi', async () => {
 		isRaspberryPiMockValue = true
-		await expect(umbreld.instance.files.externalStorage.supported()).resolves.toBe(false)
+		await expect(umbreld.instance.files.externalStorage.supported()).resolves.toBe(true)
 	})
 })
 

packages/umbreld/source/modules/files/external-storage.ts

@@ -10,6 +10,8 @@ import PQueue from 'p-queue'
 
 import {isRaspberryPi} from '../system/system.js'
 
+import {startExternalStoragePowerMonitor} from './external-storage-power-monitor.js'
+
 import type Umbreld from '../../index.js'
 
 type BlockDevice = {
@@ -92,6 +94,7 @@ export default class ExternalStorage {
 	logger: Umbreld['logger']
 	#mountQueue = new PQueue({concurrency: 1})
 	#removeDeviceChangeListener?: () => void
+	#stopPowerMonitor?: () => void
 	formatJobs: Set<string> = new Set()
 
 	constructor(umbreld: Umbreld) {
@@ -100,12 +103,10 @@ export default class ExternalStorage {
 		this.logger = umbreld.logger.createChildLogger(`files:${name.toLocaleLowerCase()}`)
 	}
 
-	// Only enable this module on non raspberry pi devices.
-	// We disable on Pi due to unreliable power issues when running USB storage devices
-	// and also due to complexities with the current mount script.
+	// External storage is supported on all devices, including Raspberry Pi.
+	// Use a powered USB hub for hard drives; the UI warns about power limits on Pi.
 	async supported() {
-		const isNotRaspberryPi = !(await isRaspberryPi())
-		return isNotRaspberryPi
+		return true
 	}
 
 	// Add listener
@@ -116,6 +117,12 @@ export default class ExternalStorage {
 
 		this.logger.log('Starting external storage')
 
+		if (await isRaspberryPi()) {
+			this.logger.log('Running UAS driver blacklist check for Raspberry Pi USB stability')
+			const blacklistUas = (await import('../blacklist-uas/blacklist-uas.js')).default
+			await blacklistUas().catch((error) => this.logger.error('UAS blacklist check failed', error))
+		}
+
 		// Safely clean up any left over mount points
 		await this.#cleanLeftOverMountPoints()
 
@@ -129,6 +136,10 @@ export default class ExternalStorage {
 			this.logger.log('Device change detected')
 			await this.#mountExternalDevices()
 		})
+
+		if (await isRaspberryPi()) {
+			this.#stopPowerMonitor = startExternalStoragePowerMonitor(this.#umbreld, this)
+		}
 	}
 
 	// Remove listener
@@ -139,6 +150,8 @@ export default class ExternalStorage {
 
 		this.logger.log('Stopping external storage')
 		this.#removeDeviceChangeListener?.()
+		this.#stopPowerMonitor?.()
+		this.#stopPowerMonitor = undefined
 
 		// Unmount all external devices
 		const ONE_SECOND = 1000
@@ -362,7 +375,23 @@ export default class ExternalStorage {
 		const systemDiskIds = await this.#getSystemDiskIds(blockDevices)
 
 		// Filter out any non-USB devices and disks that back the running system.
-		return blockDevices.filter((device) => device.transport === 'usb' && !systemDiskIds.has(device.id))
+		return blockDevices.filter(
+			(device) =>
+				device.transport === 'usb' &&
+				!systemDiskIds.has(device.id) &&
+				!this.#isUmbrelExternalDataDisk(device),
+		)
+	}
+
+	// The Pi data-disk migration script mounts the Umbrel data volume at /mnt/data.
+	#isUmbrelExternalDataDisk(device: BlockDevice) {
+		const umbrelDataMount = '/mnt/data'
+		return device.partitions.some((partition) =>
+			partition.mountpoints.some(
+				(mountpoint) =>
+					mountpoint === umbrelDataMount || mountpoint.startsWith(`${umbrelDataMount}/`),
+			),
+		)
 	}
 
 	// Get disks used by the running system so they are never treated as external storage.