diff --git a/openwisp_firmware_upgrader/admin.py b/openwisp_firmware_upgrader/admin.py index ea25e5dc5..beb7163b1 100644 --- a/openwisp_firmware_upgrader/admin.py +++ b/openwisp_firmware_upgrader/admin.py @@ -1,6 +1,7 @@ import json import logging from datetime import timedelta +from functools import partial import reversion import swapper @@ -12,6 +13,7 @@ from django.core.exceptions import ValidationError from django.core.paginator import InvalidPage, Paginator from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction from django.forms.formsets import DELETION_FIELD_NAME from django.shortcuts import redirect from django.template.response import TemplateResponse @@ -38,6 +40,7 @@ LocationFilter, ) from .swapper import load_model +from .tasks import extract_firmware_metadata from .utils import get_upgrader_schema_for_device from .widgets import FirmwareSchemaWidget, MassUpgradeSelect2Widget @@ -60,6 +63,45 @@ ) IN_PROGRESS_STATUS = UpgradeOperation.CANCELLABLE_STATUS +_STATUS_CONFIG = { + "unconfirmed": {"label": _("Unconfirmed"), "class": "ow-status-grey"}, + "in_progress": {"label": _("In Progress"), "class": "ow-status-warning"}, + "success": {"label": _("Success"), "class": "ow-status-success"}, + "failed": {"label": _("Failed"), "class": "ow-status-error"}, + "manually_confirmed": { + "label": _("Manually Confirmed"), + "class": "ow-status-success", + }, + "invalid": {"label": _("Invalid"), "class": "ow-status-error"}, +} + +_BUILD_STATUS_CONFIG = { + "analyzing": ("warning", _("Analyzing")), + "success": ("success", _("Success")), + "failed": ("error", _("Failed")), + "invalid": ("error", _("Invalid")), + "manually_confirmed": ("success", _("Manually Confirmed")), +} + +_FAILURE_REASON_TEXT = { + "unsupported_format": _( + "Both fwtool and DTB scan were unable to extract metadata. " + "Fill in the Device Metadata fields below manually." + ), + "out_of_memory": _("Decompression exceeded the configured size or ratio limit."), + "invalid_file": _("This file was rejected as an invalid firmware image."), + "timeout": _("Metadata extraction timed out. Re-upload the image to try again."), +} + + +def _extraction_status_badge(status): + cfg = _STATUS_CONFIG.get(status, {"label": status, "class": "ow-status-grey"}) + return format_html( + '{}', + cfg["class"], + cfg["label"], + ) + class BaseAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin): save_on_top = True @@ -82,6 +124,11 @@ class CategoryAdmin(BaseVersionAdmin): class FirmwareImageInline(TimeReadonlyAdminMixin, admin.StackedInline): model = FirmwareImage extra = 0 + readonly_fields = ["created", "modified", "extraction_status_display"] + + @admin.display(description=_("Extraction Status")) + def extraction_status_display(self, obj): + return _extraction_status_badge(obj.extraction_status) class Media: extra = "" if getattr(settings, "DEBUG", False) else ".min" @@ -108,6 +155,174 @@ def has_change_permission(self, request, obj=None): return True +@admin.register(FirmwareImage) +class FirmwareImageAdmin(BaseAdmin): + list_display = [ + "__str__", + "build", + "type", + "extraction_status_display", + "created", + "modified", + ] + list_filter = ["extraction_status", "build__category"] + search_fields = ["board", "target", "type"] + ordering = ["-created"] + actions = ["re_extract_metadata"] + readonly_fields = [ + "created", + "modified", + "extraction_status_display", + "failure_reason_display", + "extraction_log_display", + "source", + ] + fieldsets = [ + ( + None, + { + "fields": ["build", "file", "type", "created", "modified"], + }, + ), + ( + _("Extraction Status"), + { + "fields": [ + "extraction_status_display", + "failure_reason_display", + "extraction_log_display", + ], + "description": _( + "Metadata is extracted automatically on upload using fwtool " + "(primary) and DTB scan (fallback). " + "If both fail, fill in the Device Metadata fields below manually." + ), + }, + ), + ( + _("Device Metadata"), + { + "fields": [ + "board", + "compatible", + "target", + "fw_version", + "compat_version", + "source", + ], + }, + ), + ] + + def get_readonly_fields(self, request, obj=None): + readonly = list(self.readonly_fields) + if obj: + status = obj.extraction_status + if status in ( + FirmwareImage.STATUS_IN_PROGRESS, + FirmwareImage.STATUS_INVALID, + FirmwareImage.STATUS_MANUALLY_CONFIRMED, + ): + readonly += [ + "board", + "compatible", + "target", + "fw_version", + "compat_version", + ] + elif status == FirmwareImage.STATUS_SUCCESS: + if obj.source == "dtb": + readonly += ["board", "compatible", "compat_version"] + else: + readonly += [ + "board", + "compatible", + "target", + "fw_version", + "compat_version", + ] + return readonly + + @admin.display(description=_("Extraction Status")) + def extraction_status_display(self, obj): + return _extraction_status_badge(obj.extraction_status) + + @admin.display(description=_("Failure Reason")) + def failure_reason_display(self, obj): + if not obj.failure_reason: + return "-" + return _FAILURE_REASON_TEXT.get(obj.failure_reason, obj.failure_reason) + + @admin.display(description=_("Extraction Log")) + def extraction_log_display(self, obj): + if not obj.extraction_log: + return "-" + return format_html( + '
{}',
+ obj.extraction_log,
+ )
+
+ def save_model(self, request, obj, form, change):
+ if change and "file" in form.changed_data:
+ obj.extraction_status = FirmwareImage.STATUS_UNCONFIRMED
+ obj.extraction_log = ""
+ obj.failure_reason = ""
+ obj.board = ""
+ obj.compatible = []
+ obj.target = ""
+ obj.fw_version = ""
+ obj.compat_version = ""
+ obj.source = ""
+ super().save_model(request, obj, form, change)
+ transaction.on_commit(lambda: extract_firmware_metadata.delay(obj.pk))
+ return
+ if change:
+ if obj.extraction_status == FirmwareImage.STATUS_FAILED:
+ metadata_fields = ["board", "target", "fw_version"]
+ if any(f in form.changed_data for f in metadata_fields):
+ obj.extraction_status = FirmwareImage.STATUS_MANUALLY_CONFIRMED
+ obj.source = "manual"
+ elif (
+ obj.extraction_status == FirmwareImage.STATUS_SUCCESS
+ and obj.source == "dtb"
+ and any(
+ field in form.changed_data for field in ["target", "fw_version"]
+ )
+ ):
+ obj.extraction_status = FirmwareImage.STATUS_MANUALLY_CONFIRMED
+ super().save_model(request, obj, form, change)
+
+ @admin.action(
+ description=_("Re-extract metadata from selected images"),
+ permissions=["change"],
+ )
+ def re_extract_metadata(self, request, queryset):
+ image_pks = list(queryset.values_list("pk", flat=True))
+ build_ids = set(queryset.values_list("build_id", flat=True))
+ queryset.update(
+ extraction_status=FirmwareImage.STATUS_UNCONFIRMED,
+ extraction_log="",
+ failure_reason="",
+ board="",
+ compatible=[],
+ target="",
+ fw_version="",
+ compat_version="",
+ source="",
+ )
+ Build.objects.filter(pk__in=build_ids).update(
+ status=Build.BUILD_STATUS_ANALYZING
+ )
+ for pk in image_pks:
+ transaction.on_commit(partial(extract_firmware_metadata.delay, pk))
+ self.message_user(
+ request,
+ _("Metadata re-extraction scheduled for %(count)d image(s).")
+ % {"count": len(image_pks)},
+ messages.SUCCESS,
+ )
+
+
class BatchUpgradeConfirmationForm(forms.ModelForm):
upgrade_options = forms.JSONField(widget=FirmwareSchemaWidget(), required=False)
build = forms.ModelChoiceField(
@@ -172,7 +387,14 @@ class Media:
@admin.register(load_model("Build"))
class BuildAdmin(BaseAdmin):
- list_display = ["__str__", "organization", "category", "created", "modified"]
+ list_display = [
+ "__str__",
+ "organization",
+ "category",
+ "build_status_display",
+ "created",
+ "modified",
+ ]
list_filter = [CategoryOrganizationFilter, CategoryFilter]
list_select_related = ["category", "category__organization"]
search_fields = ["category__name", "version", "os"]
@@ -190,6 +412,15 @@ def organization(self, obj):
organization.short_description = _("organization")
+ @admin.display(description=_("Extraction status"))
+ def build_status_display(self, obj):
+ css, label = _BUILD_STATUS_CONFIG.get(obj.status, ("grey", obj.status))
+ return format_html(
+ '{}',
+ css,
+ label,
+ )
+
@admin.action(
description=_("Mass-upgrade devices related to the selected build"),
permissions=["change"],
diff --git a/openwisp_firmware_upgrader/apps.py b/openwisp_firmware_upgrader/apps.py
index ce76c7187..50b7f31e6 100644
--- a/openwisp_firmware_upgrader/apps.py
+++ b/openwisp_firmware_upgrader/apps.py
@@ -29,6 +29,7 @@ def ready(self, *args, **kwargs):
self.connect_device_signals()
self.connect_upgrade_signals()
self.connect_delete_signals()
+ self.connect_metadata_signals()
def register_menu_groups(self):
register_menu_group(
@@ -54,6 +55,12 @@ def register_menu_groups(self):
"name": "changelist",
"icon": "ow-mass-upgrade",
},
+ 4: {
+ "label": _("Firmware Images"),
+ "model": get_model_name(self.label, "FirmwareImage"),
+ "name": "changelist",
+ "icon": "ow-firmware",
+ },
},
"icon": "ow-firmware",
},
@@ -116,5 +123,14 @@ def connect_delete_signals(self):
dispatch_uid="organization.pre_delete.firmware_files",
)
+ def connect_metadata_signals(self):
+ FirmwareImage = load_model("firmware_upgrader", "FirmwareImage")
+
+ post_save.connect(
+ FirmwareImage.trigger_metadata_extraction,
+ sender=FirmwareImage,
+ dispatch_uid="firmware_image.trigger_metadata_extraction",
+ )
+
del ApiAppConfig
diff --git a/openwisp_firmware_upgrader/base/models.py b/openwisp_firmware_upgrader/base/models.py
index 8bfe6cab2..6f36012e9 100644
--- a/openwisp_firmware_upgrader/base/models.py
+++ b/openwisp_firmware_upgrader/base/models.py
@@ -9,9 +9,12 @@
from django.core.validators import MaxValueValidator
from django.db import models, transaction
from django.db.models import Q
+from django.urls import reverse
from django.utils import timezone
from django.utils.functional import cached_property
+from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
+from openwisp_notifications.signals import notify
from private_storage.fields import PrivateFileField
from openwisp_controller.connection.exceptions import NoWorkingDeviceConnectionError
@@ -38,6 +41,7 @@
batch_upgrade_operation,
create_all_device_firmwares,
create_device_firmware,
+ extract_firmware_metadata,
upgrade_firmware,
)
from ..utils import (
@@ -145,6 +149,28 @@ class AbstractBuild(TimeStampedEditableModel):
),
)
+ BUILD_STATUS_ANALYZING = "analyzing"
+ BUILD_STATUS_SUCCESS = "success"
+ BUILD_STATUS_FAILED = "failed"
+ BUILD_STATUS_INVALID = "invalid"
+ BUILD_STATUS_MANUALLY_CONFIRMED = "manually_confirmed"
+
+ BUILD_STATUS_CHOICES = [
+ (BUILD_STATUS_ANALYZING, _("Analyzing")),
+ (BUILD_STATUS_SUCCESS, _("Success")),
+ (BUILD_STATUS_FAILED, _("Failed")),
+ (BUILD_STATUS_INVALID, _("Invalid")),
+ (BUILD_STATUS_MANUALLY_CONFIRMED, _("Manually Confirmed")),
+ ]
+
+ status = models.CharField(
+ _("extraction status"),
+ max_length=20,
+ choices=BUILD_STATUS_CHOICES,
+ default=BUILD_STATUS_ANALYZING,
+ db_index=True,
+ )
+
class Meta:
abstract = True
verbose_name = _("Firmware Build")
@@ -185,6 +211,20 @@ def batch_upgrade(
self, firmwareless, upgrade_options=None, group=None, location=None
):
upgrade_options = upgrade_options or {}
+ FirmwareImage = load_model("FirmwareImage")
+ unconfirmed = self.firmwareimage_set.exclude(
+ extraction_status__in=[
+ FirmwareImage.STATUS_SUCCESS,
+ FirmwareImage.STATUS_MANUALLY_CONFIRMED,
+ ]
+ )
+ if unconfirmed.exists():
+ raise ValidationError(
+ _(
+ "All firmware images must have confirmed metadata "
+ "before a mass upgrade can be scheduled"
+ )
+ )
# Check if there are any devices to upgrade with the given filters
dry_run_result = load_model("BatchUpgradeOperation").dry_run(
build=self, group=group, location=location
@@ -256,6 +296,76 @@ def _find_firmwareless_devices(self, boards=None, group=None, location=None):
qs = qs.filter(devicelocation__location=location)
return qs.order_by("-created")
+ def _update_extraction_status(self):
+ Build = load_model("Build")
+ FirmwareImage = load_model("FirmwareImage")
+ statuses = set(
+ FirmwareImage.objects.filter(build_id=self.pk).values_list(
+ "extraction_status", flat=True
+ )
+ )
+ if not statuses:
+ return
+ analyzing = {FirmwareImage.STATUS_UNCONFIRMED, FirmwareImage.STATUS_IN_PROGRESS}
+ if statuses & analyzing:
+ new_status = self.BUILD_STATUS_ANALYZING
+ elif FirmwareImage.STATUS_INVALID in statuses:
+ new_status = self.BUILD_STATUS_INVALID
+ elif FirmwareImage.STATUS_FAILED in statuses:
+ new_status = self.BUILD_STATUS_FAILED
+ elif FirmwareImage.STATUS_MANUALLY_CONFIRMED in statuses:
+ new_status = self.BUILD_STATUS_MANUALLY_CONFIRMED
+ else:
+ new_status = self.BUILD_STATUS_SUCCESS
+ rows_updated = Build.objects.filter(
+ pk=self.pk, status=self.BUILD_STATUS_ANALYZING
+ ).update(status=new_status)
+ if rows_updated:
+ self.status = new_status
+ if new_status != self.BUILD_STATUS_ANALYZING:
+ self._notify_extraction_complete(new_status)
+ return
+ Build.objects.filter(pk=self.pk).exclude(status=new_status).update(
+ status=new_status
+ )
+ self.status = new_status
+
+ def _notify_extraction_complete(self, new_status):
+ level = (
+ "info"
+ if new_status
+ in (
+ self.BUILD_STATUS_SUCCESS,
+ self.BUILD_STATUS_MANUALLY_CONFIRMED,
+ )
+ else "warning"
+ )
+ status_display = dict(self.BUILD_STATUS_CHOICES)[new_status]
+ try:
+ opts = self.__class__._meta
+ admin_url = reverse(
+ f"admin:{opts.app_label}_{opts.model_name}_change",
+ args=[str(self.pk)],
+ )
+ notify.send(
+ sender=self,
+ type="generic_message",
+ level=level,
+ url=admin_url,
+ target=self,
+ message=format_html(
+ _(
+ 'Metadata extraction for build {build} '
+ "completed with status: {status}."
+ ),
+ url=admin_url,
+ build=self,
+ status=status_display,
+ ),
+ )
+ except Exception:
+ logger.exception("Failed to send build extraction completion notification")
+
def get_build_directory(instance, filename):
build_pk = str(instance.build.pk)
@@ -282,6 +392,65 @@ class AbstractFirmwareImage(TimeStampedEditableModel):
),
)
+ STATUS_UNCONFIRMED = "unconfirmed"
+ STATUS_IN_PROGRESS = "in_progress"
+ STATUS_SUCCESS = "success"
+ STATUS_FAILED = "failed"
+ STATUS_MANUALLY_CONFIRMED = "manually_confirmed"
+ STATUS_INVALID = "invalid"
+ LOCKED_STATUSES = (STATUS_SUCCESS, STATUS_MANUALLY_CONFIRMED)
+
+ FAILURE_UNSUPPORTED = "unsupported_format"
+ FAILURE_OOM = "out_of_memory"
+ FAILURE_INVALID = "invalid_file"
+ FAILURE_TIMEOUT = "timeout"
+
+ EXTRACTION_STATUS_CHOICES = [
+ (STATUS_UNCONFIRMED, _("Unconfirmed")),
+ (STATUS_IN_PROGRESS, _("In Progress")),
+ (STATUS_SUCCESS, _("Success")),
+ (STATUS_FAILED, _("Failed")),
+ (STATUS_MANUALLY_CONFIRMED, _("Manually Confirmed")),
+ (STATUS_INVALID, _("Invalid")),
+ ]
+
+ FAILURE_REASON_CHOICES = [
+ (FAILURE_UNSUPPORTED, _("Unsupported format")),
+ (FAILURE_OOM, _("Out of memory")),
+ (FAILURE_INVALID, _("Invalid file")),
+ (FAILURE_TIMEOUT, _("Extraction timed out")),
+ ]
+
+ extraction_status = models.CharField(
+ _("extraction status"),
+ max_length=20,
+ choices=EXTRACTION_STATUS_CHOICES,
+ default=STATUS_UNCONFIRMED,
+ db_index=True,
+ )
+ extraction_log = models.TextField(_("extraction log"), blank=True)
+ failure_reason = models.CharField(
+ _("failure reason"),
+ max_length=20,
+ choices=FAILURE_REASON_CHOICES,
+ blank=True,
+ default="",
+ )
+ board = models.CharField(_("board"), max_length=200, blank=True)
+ compatible = models.JSONField(_("compatible"), default=list, blank=True)
+ target = models.CharField(_("target"), max_length=100, blank=True)
+ fw_version = models.CharField(_("firmware version"), max_length=50, blank=True)
+ compat_version = models.CharField(_("compat version"), max_length=10, blank=True)
+ source = models.CharField(_("source"), max_length=20, blank=True)
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._original_extraction_status = self.extraction_status
+
+ def save(self, *args, **kwargs):
+ super().save(*args, **kwargs)
+ self._original_extraction_status = self.extraction_status
+
class Meta:
abstract = True
verbose_name = _("Firmware Image")
@@ -299,12 +468,9 @@ def boards(self):
def clean(self):
self._clean_type()
+ self._validate_locked()
self._validate_file_header()
self._validate_rootfs()
- try:
- self.boards
- except KeyError:
- raise ValidationError({"type": "Could not find boards for this type"})
def delete(self, *args, **kwargs):
super().delete(*args, **kwargs)
@@ -342,15 +508,58 @@ def _remove_file(cls, file_path):
return True
def _clean_type(self):
- """
- auto determine type if missing
- """
if self.type:
return
filename = self.file.name
- # removes leading prefix
self.type = "-".join(filename.split("-")[1:])
+ @classmethod
+ def trigger_metadata_extraction(cls, instance, created, **kwargs):
+ if not created:
+ return
+ Build = load_model("Build")
+ Build.objects.filter(pk=instance.build_id).update(
+ status=Build.BUILD_STATUS_ANALYZING
+ )
+ transaction.on_commit(lambda: extract_firmware_metadata.delay(str(instance.pk)))
+
+ def _validate_locked(self):
+ if self._state.adding or not self.pk:
+ return
+ original = (
+ self.__class__.objects.filter(pk=self.pk)
+ .values(
+ "extraction_status",
+ "board",
+ "compatible",
+ "target",
+ "fw_version",
+ "compat_version",
+ "source",
+ )
+ .first()
+ )
+ if not original:
+ return
+ if (
+ original["extraction_status"] not in self.LOCKED_STATUSES
+ and self.extraction_status not in self.LOCKED_STATUSES
+ ):
+ return
+ for field in (
+ "board",
+ "compatible",
+ "target",
+ "fw_version",
+ "compat_version",
+ "source",
+ ):
+ original_val = original[field]
+ if original_val and getattr(self, field) != original_val:
+ raise ValidationError(
+ _("Metadata fields are read-only after confirmation.")
+ )
+
def _validate_file_header(self):
if not self.file:
return
@@ -449,6 +658,16 @@ class Meta:
def clean(self):
if not hasattr(self, "image") or not hasattr(self, "device"):
return
+ if self.image.extraction_status not in self.image.LOCKED_STATUSES:
+ raise ValidationError(
+ {
+ "image": _(
+ "This firmware image's metadata has not been confirmed yet. "
+ "Metadata extraction must complete successfully "
+ "before it can be used for upgrades."
+ )
+ }
+ )
if (
self.image.build.category.organization is not None
and self.image.build.category.organization != self.device.organization
@@ -567,9 +786,18 @@ def auto_add_device_firmware_to_device(cls, instance, created, **kwargs):
@classmethod
def auto_create_device_firmwares(cls, instance, created, **kwargs):
if created:
- transaction.on_commit(
- partial(create_all_device_firmwares.delay, instance.pk)
- )
+ return
+ confirmed = (
+ instance.STATUS_SUCCESS,
+ instance.STATUS_MANUALLY_CONFIRMED,
+ )
+ if instance.extraction_status not in confirmed:
+ return
+ if instance._original_extraction_status in confirmed:
+ return
+ transaction.on_commit(
+ partial(create_all_device_firmwares.delay, str(instance.pk))
+ )
@classmethod
def get_image_queryset_for_device(cls, device, device_firmware=None):
diff --git a/openwisp_firmware_upgrader/migrations/0018_build_status_firmwareimage_board_and_more.py b/openwisp_firmware_upgrader/migrations/0018_build_status_firmwareimage_board_and_more.py
new file mode 100644
index 000000000..9a13e189d
--- /dev/null
+++ b/openwisp_firmware_upgrader/migrations/0018_build_status_firmwareimage_board_and_more.py
@@ -0,0 +1,100 @@
+# Generated by Django 5.2.13 on 2026-05-22 16:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("firmware_upgrader", "0017_alter_batchupgradeoperation_status"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="build",
+ name="status",
+ field=models.CharField(
+ choices=[
+ ("analyzing", "Analyzing"),
+ ("success", "Success"),
+ ("failed", "Failed"),
+ ("invalid", "Invalid"),
+ ("manually_confirmed", "Manually Confirmed"),
+ ],
+ db_index=True,
+ default="analyzing",
+ max_length=20,
+ verbose_name="extraction status",
+ ),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="board",
+ field=models.CharField(blank=True, max_length=200),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="compat_version",
+ field=models.CharField(blank=True, max_length=10),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="compatible",
+ field=models.JSONField(blank=True, default=list),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="extraction_log",
+ field=models.TextField(blank=True),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="extraction_status",
+ field=models.CharField(
+ choices=[
+ ("unconfirmed", "Unconfirmed"),
+ ("in_progress", "In Progress"),
+ ("success", "Success"),
+ ("failed", "Failed"),
+ ("manually_confirmed", "Manually Confirmed"),
+ ("invalid", "Invalid"),
+ ],
+ db_index=True,
+ default="unconfirmed",
+ max_length=20,
+ verbose_name="extraction status",
+ ),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="failure_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("unsupported_format", "Unsupported format"),
+ ("out_of_memory", "Out of memory"),
+ ("invalid_file", "Invalid file"),
+ ("timeout", "Extraction timed out"),
+ ],
+ default="",
+ max_length=20,
+ ),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="fw_version",
+ field=models.CharField(
+ blank=True, max_length=50, verbose_name="firmware version"
+ ),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="source",
+ field=models.CharField(blank=True, max_length=20),
+ ),
+ migrations.AddField(
+ model_name="firmwareimage",
+ name="target",
+ field=models.CharField(blank=True, max_length=100),
+ ),
+ ]
diff --git a/openwisp_firmware_upgrader/migrations/0019_backfill_extraction_status.py b/openwisp_firmware_upgrader/migrations/0019_backfill_extraction_status.py
new file mode 100644
index 000000000..e1a4ba8e3
--- /dev/null
+++ b/openwisp_firmware_upgrader/migrations/0019_backfill_extraction_status.py
@@ -0,0 +1,30 @@
+from django.db import migrations
+
+
+def backfill_firmware_image_status(apps, schema_editor):
+ FirmwareImage = apps.get_model("firmware_upgrader", "FirmwareImage")
+ FirmwareImage.objects.filter(extraction_status="unconfirmed").update(
+ extraction_status="manually_confirmed",
+ source="manual",
+ )
+
+
+def backfill_build_status(apps, schema_editor):
+ Build = apps.get_model("firmware_upgrader", "Build")
+ Build.objects.filter(status="analyzing").update(status="manually_confirmed")
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("firmware_upgrader", "0018_build_status_firmwareimage_board_and_more"),
+ ]
+ operations = [
+ migrations.RunPython(
+ backfill_firmware_image_status,
+ reverse_code=migrations.RunPython.noop,
+ ),
+ migrations.RunPython(
+ backfill_build_status,
+ reverse_code=migrations.RunPython.noop,
+ ),
+ ]
diff --git a/openwisp_firmware_upgrader/migrations/0020_alter_firmwareimage_board_and_more.py b/openwisp_firmware_upgrader/migrations/0020_alter_firmwareimage_board_and_more.py
new file mode 100644
index 000000000..87056a3e5
--- /dev/null
+++ b/openwisp_firmware_upgrader/migrations/0020_alter_firmwareimage_board_and_more.py
@@ -0,0 +1,61 @@
+# Generated by Django 5.2.14 on 2026-06-15 20:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("firmware_upgrader", "0019_backfill_extraction_status"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="board",
+ field=models.CharField(blank=True, max_length=200, verbose_name="board"),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="compat_version",
+ field=models.CharField(
+ blank=True, max_length=10, verbose_name="compat version"
+ ),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="compatible",
+ field=models.JSONField(blank=True, default=list, verbose_name="compatible"),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="extraction_log",
+ field=models.TextField(blank=True, verbose_name="extraction log"),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="failure_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("unsupported_format", "Unsupported format"),
+ ("out_of_memory", "Out of memory"),
+ ("invalid_file", "Invalid file"),
+ ("timeout", "Extraction timed out"),
+ ],
+ default="",
+ max_length=20,
+ verbose_name="failure reason",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="source",
+ field=models.CharField(blank=True, max_length=20, verbose_name="source"),
+ ),
+ migrations.AlterField(
+ model_name="firmwareimage",
+ name="target",
+ field=models.CharField(blank=True, max_length=100, verbose_name="target"),
+ ),
+ ]
diff --git a/openwisp_firmware_upgrader/tasks.py b/openwisp_firmware_upgrader/tasks.py
index 38ad83bdd..c1abf7dee 100644
--- a/openwisp_firmware_upgrader/tasks.py
+++ b/openwisp_firmware_upgrader/tasks.py
@@ -4,12 +4,17 @@
from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded
from django.core.exceptions import ObjectDoesNotExist
+from django.urls import reverse
+from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
+from openwisp_notifications.signals import notify
from openwisp_utils.tasks import OpenwispCeleryTask
from . import settings as app_settings
from .exceptions import RecoverableFailure
+from .extractors.exceptions import DecompressionLimitExceeded, UnsupportedImageError
+from .extractors.openwrt import OpenWrtMetadataExtractor
from .swapper import load_model
logger = logging.getLogger(__name__)
@@ -97,3 +102,157 @@ def delete_firmware_files(files_to_delete):
FirmwareImage = load_model("FirmwareImage")
for file_path in files_to_delete:
FirmwareImage._remove_file(file_path)
+
+
+def _compat_blocks_pairing(compat_version):
+ try:
+ major, minor = (int(x) for x in str(compat_version).split("."))
+ return (major, minor) > (1, 0)
+ except (ValueError, AttributeError, TypeError):
+ return False
+
+
+@shared_task(bind=True, soft_time_limit=app_settings.TASK_TIMEOUT)
+def extract_firmware_metadata(self, image_pk):
+ FirmwareImage = load_model("FirmwareImage")
+
+ try:
+ image = FirmwareImage.objects.get(pk=image_pk)
+ except FirmwareImage.DoesNotExist:
+ logger.warning(
+ "extract_firmware_metadata: FirmwareImage pk=%s not found, skipping",
+ image_pk,
+ )
+ return
+
+ updated = FirmwareImage.objects.filter(
+ pk=image_pk,
+ extraction_status=FirmwareImage.STATUS_UNCONFIRMED,
+ ).update(extraction_status=FirmwareImage.STATUS_IN_PROGRESS)
+ if not updated:
+ return
+ log_lines = [f"[+] Analyzing: {image.file.name}"]
+ update = {}
+
+ try:
+ extractor_class = getattr(
+ image.build.category.__class__,
+ "metadata_extractor_class",
+ OpenWrtMetadataExtractor,
+ )
+ meta = extractor_class(image.file.path).extract()
+ log_lines.append("[+] extraction: success")
+ update = {
+ "extraction_status": FirmwareImage.STATUS_SUCCESS,
+ "extraction_log": "\n".join(log_lines),
+ "board": meta.get("model", ""),
+ "compatible": meta.get("compatible", []),
+ "target": meta.get("target", ""),
+ "fw_version": meta.get("version", ""),
+ "compat_version": meta.get("compat_version", ""),
+ "source": meta.get("source", "fwtool"),
+ }
+
+ except SoftTimeLimitExceeded:
+ log_lines.append(f"[!] Task timed out after {app_settings.TASK_TIMEOUT}s.")
+ update = {
+ "extraction_status": FirmwareImage.STATUS_FAILED,
+ "failure_reason": FirmwareImage.FAILURE_TIMEOUT,
+ "extraction_log": "\n".join(log_lines),
+ }
+ logger.warning(
+ "extract_firmware_metadata: soft time limit exceeded for pk=%s",
+ image_pk,
+ )
+
+ except DecompressionLimitExceeded as exc:
+ log_lines.append(f"[!] {exc}")
+ update = {
+ "extraction_status": FirmwareImage.STATUS_FAILED,
+ "failure_reason": FirmwareImage.FAILURE_OOM,
+ "extraction_log": "\n".join(log_lines),
+ }
+ logger.warning(
+ "extract_firmware_metadata: decompression limit exceeded for pk=%s - %s",
+ image_pk,
+ exc,
+ )
+
+ except UnsupportedImageError as exc:
+ log_lines.append(f"[-] fwtool: {exc}")
+ log_lines.append("[!] Extraction failed. Manual input required.")
+ update = {
+ "extraction_status": FirmwareImage.STATUS_FAILED,
+ "failure_reason": FirmwareImage.FAILURE_UNSUPPORTED,
+ "extraction_log": "\n".join(log_lines),
+ }
+ logger.warning(
+ "extract_firmware_metadata: unsupported image pk=%s - %s",
+ image_pk,
+ exc,
+ )
+
+ except Exception:
+ log_lines.append("[!] Unexpected error during extraction.")
+ update = {
+ "extraction_status": FirmwareImage.STATUS_INVALID,
+ "failure_reason": FirmwareImage.FAILURE_INVALID,
+ "extraction_log": "\n".join(log_lines),
+ }
+ logger.exception(
+ "extract_firmware_metadata: unhandled exception for pk=%s",
+ image_pk,
+ )
+
+ FirmwareImage.objects.filter(pk=image_pk).update(**update)
+
+ if update.get("extraction_status") not in (
+ FirmwareImage.STATUS_SUCCESS,
+ FirmwareImage.STATUS_IN_PROGRESS,
+ ):
+ try:
+ image = FirmwareImage.objects.select_related(
+ "build", "build__category"
+ ).get(pk=image_pk)
+ build_opts = image.build._meta
+ admin_url = reverse(
+ f"admin:{build_opts.app_label}_{build_opts.model_name}_change",
+ args=[str(image.build_id)],
+ )
+ notify.send(
+ sender=image,
+ type="generic_message",
+ level="error",
+ url=admin_url,
+ target=image.build,
+ message=format_html(
+ _(
+ 'Metadata extraction failed for {image}: '
+ "{reason}. You can manually enter metadata or re-upload the image."
+ ),
+ url=admin_url,
+ image=image,
+ reason=update.get("failure_reason", "unknown error"),
+ ),
+ )
+ except Exception:
+ logger.exception("Failed to send extraction failure notification")
+
+ try:
+ fresh = FirmwareImage.objects.select_related("build").get(pk=image_pk)
+ fresh.build._update_extraction_status()
+ except Exception:
+ logger.exception(
+ "Failed to update build extraction status for image %s", image_pk
+ )
+
+ if update.get("extraction_status") == FirmwareImage.STATUS_SUCCESS:
+ compat = update.get("compat_version", "")
+ if _compat_blocks_pairing(compat):
+ logger.info(
+ "Auto-pairing skipped for image %s: compat_version %s > 1.0",
+ image_pk,
+ compat,
+ )
+ else:
+ create_all_device_firmwares.delay(str(image_pk))
diff --git a/openwisp_firmware_upgrader/tests/base.py b/openwisp_firmware_upgrader/tests/base.py
index 79614f3b2..fe32466bb 100644
--- a/openwisp_firmware_upgrader/tests/base.py
+++ b/openwisp_firmware_upgrader/tests/base.py
@@ -77,6 +77,8 @@ def _create_build(self, **kwargs):
def _create_firmware_image(self, **kwargs):
opts = dict(type=self.TPLINK_4300_IMAGE)
opts.update(kwargs)
+ if "extraction_status" not in opts:
+ opts["extraction_status"] = FirmwareImage.STATUS_SUCCESS
category_opts = {}
if "organization" in opts:
category_opts["organization"] = opts.pop("organization")
diff --git a/openwisp_firmware_upgrader/tests/test_admin.py b/openwisp_firmware_upgrader/tests/test_admin.py
index 344218a01..50fd7db6e 100644
--- a/openwisp_firmware_upgrader/tests/test_admin.py
+++ b/openwisp_firmware_upgrader/tests/test_admin.py
@@ -21,6 +21,7 @@
DeviceFirmwareForm,
DeviceFirmwareInline,
DeviceUpgradeOperationInline,
+ FirmwareImageAdmin,
FirmwareImageInline,
admin,
)
@@ -213,6 +214,120 @@ def test_firmware_image_has_change_permission(self):
self.assertIs(inline.has_change_permission(request), True)
self.assertIs(inline.has_change_permission(request, obj=env["image1a"]), False)
+ def test_firmware_image_save_model_clears_compat_version_on_file_change(self):
+ fw = self._create_firmware_image()
+ FirmwareImage.objects.filter(pk=fw.pk).update(
+ board="TP-Link WDR4300",
+ compat_version="21.09",
+ extraction_status=FirmwareImage.STATUS_SUCCESS,
+ )
+ fw.refresh_from_db()
+ request = MockRequest()
+ request.user = User.objects.first()
+ form = mock.MagicMock()
+ form.changed_data = ["file"]
+ fw_admin = FirmwareImageAdmin(FirmwareImage, admin.site)
+ with mock.patch("django.db.transaction.on_commit"):
+ fw_admin.save_model(request, fw, form, change=True)
+ fw.refresh_from_db()
+ self.assertEqual(fw.board, "")
+ self.assertEqual(fw.compat_version, "")
+
+ def test_firmware_image_save_model_dtb_no_flip_without_changed_fields(self):
+ fw = self._create_firmware_image()
+ FirmwareImage.objects.filter(pk=fw.pk).update(
+ source="dtb",
+ board="Orange Pi Zero",
+ extraction_status=FirmwareImage.STATUS_SUCCESS,
+ )
+ fw.refresh_from_db()
+ request = MockRequest()
+ request.user = User.objects.first()
+ form = mock.MagicMock()
+ form.changed_data = ["board"]
+ fw_admin = FirmwareImageAdmin(FirmwareImage, admin.site)
+ with mock.patch("django.db.transaction.on_commit"):
+ fw_admin.save_model(request, fw, form, change=True)
+ fw.refresh_from_db()
+ self.assertEqual(fw.extraction_status, FirmwareImage.STATUS_SUCCESS)
+
+ def test_re_extract_metadata_action(self):
+ self._login()
+ image = self._create_firmware_image()
+ FirmwareImage.objects.filter(pk=image.pk).update(
+ extraction_status=FirmwareImage.STATUS_FAILED,
+ failure_reason=FirmwareImage.FAILURE_UNSUPPORTED,
+ extraction_log="log output",
+ board="TP-Link WDR4300",
+ compatible=["tplink,tl-wdr4300-v1"],
+ target="ath79/generic",
+ fw_version="23.05.5",
+ compat_version="1.0",
+ source="fwtool",
+ )
+ Build.objects.filter(pk=image.build_id).update(status=Build.BUILD_STATUS_FAILED)
+ url = reverse(f"admin:{self.app_label}_firmwareimage_changelist")
+ with mock.patch(
+ "openwisp_firmware_upgrader.tasks.extract_firmware_metadata.delay"
+ ) as mocked_delay:
+ with self.captureOnCommitCallbacks(execute=True):
+ r = self.client.post(
+ url,
+ {
+ "action": "re_extract_metadata",
+ ACTION_CHECKBOX_NAME: (str(image.pk),),
+ },
+ follow=True,
+ )
+ self.assertEqual(r.status_code, 200)
+ image.refresh_from_db()
+ self.assertEqual(image.extraction_status, FirmwareImage.STATUS_UNCONFIRMED)
+ self.assertEqual(image.extraction_log, "")
+ self.assertEqual(image.failure_reason, "")
+ self.assertEqual(image.board, "")
+ self.assertEqual(image.compatible, [])
+ self.assertEqual(image.target, "")
+ self.assertEqual(image.fw_version, "")
+ self.assertEqual(image.compat_version, "")
+ self.assertEqual(image.source, "")
+ image.build.refresh_from_db()
+ self.assertEqual(image.build.status, Build.BUILD_STATUS_ANALYZING)
+ mocked_delay.assert_called_once_with(image.pk)
+
+ def test_re_extract_metadata_action_multiple(self):
+ self._login()
+ build = self._create_build()
+ image1 = self._create_firmware_image(build=build, type=self.TPLINK_4300_IMAGE)
+ image2 = self._create_firmware_image(
+ build=build, type=self.TPLINK_4300_IL_IMAGE
+ )
+ FirmwareImage.objects.filter(pk__in=[image1.pk, image2.pk]).update(
+ extraction_status=FirmwareImage.STATUS_FAILED,
+ )
+ Build.objects.filter(pk=build.pk).update(status=Build.BUILD_STATUS_FAILED)
+ url = reverse(f"admin:{self.app_label}_firmwareimage_changelist")
+ with mock.patch(
+ "openwisp_firmware_upgrader.tasks.extract_firmware_metadata.delay"
+ ) as mocked_delay:
+ with self.captureOnCommitCallbacks(execute=True):
+ self.client.post(
+ url,
+ {
+ "action": "re_extract_metadata",
+ ACTION_CHECKBOX_NAME: (str(image1.pk), str(image2.pk)),
+ },
+ follow=True,
+ )
+ self.assertEqual(mocked_delay.call_count, 2)
+ called_pks = {call.args[0] for call in mocked_delay.call_args_list}
+ self.assertEqual(called_pks, {image1.pk, image2.pk})
+ image1.refresh_from_db()
+ image2.refresh_from_db()
+ self.assertEqual(image1.extraction_status, FirmwareImage.STATUS_UNCONFIRMED)
+ self.assertEqual(image2.extraction_status, FirmwareImage.STATUS_UNCONFIRMED)
+ build.refresh_from_db()
+ self.assertEqual(build.status, Build.BUILD_STATUS_ANALYZING)
+
def test_device_firmware_inline_has_add_permission(self):
device_fw = self._create_device_firmware()
device = device_fw.device
@@ -548,8 +663,7 @@ def test_deactivated_firmware_image_inline(self):
# is displayed as readonly in the admin interface.
self.assertContains(
response,
- '