ME-29: Enable REST-triggered export jobs with named collections & ZIPdownload

README.md

@@ -99,6 +99,65 @@ The export is built in two separated stages:
 2. Export. Each `DomainExporter` writes its bucket in its own format. The service holds a registry
    of these and contains no per-domain logic.
 
+Export packages (REST)
+----------------------
+Besides the export-everything-on-startup behaviour, named *export packages* can be defined and
+built over REST. A package describes what to export — a list of entries, each an Initializer
+domain optionally narrowed to specific item uuids (empty list = the whole domain) — so e.g. a
+"Site A locations" package exports just one site's locations (plus dependency closure). A
+package with *no entries at all* exports every registered domain; `GET /domains` lists which
+domains are registered on the server. Package
+definitions are stored in the database; every build of a package gets an incrementing version, a
+status (`QUEUED` → `RUNNING` → `COMPLETED`/`FAILED`), and a downloadable zip containing the
+`configuration/` tree plus a `package.json` manifest recording exactly what was exported.
+
+Builds run asynchronously on a daemon thread; trigger, then poll. Endpoints (all under
+`/openmrs/ws/rest/v1/metadataexport`, plain Spring controllers — the webservices.rest module is
+not required, but its authentication filter covers these URLs when it is installed):
+
+| Method | Path                         | Action                                     |
+|--------|------------------------------|--------------------------------------------|
+| GET    | `/domains`                   | list the registered, exportable domains    |
+| GET    | `/packages?includeRetired=`  | list packages                              |
+| POST   | `/packages`                  | create a package (201)                     |
+| GET    | `/packages/{uuid}`           | fetch one, incl. its latest build          |
+| PUT    | `/packages/{uuid}`           | update name/description/entries            |
+| DELETE | `/packages/{uuid}?reason=`   | retire (204)                               |
+| POST   | `/packages/{uuid}/builds`    | trigger a build (202; 409 if one is active)|
+| GET    | `/packages/{uuid}/builds`    | build history, newest first                |
+| GET    | `/builds/{uuid}`             | poll status, incl. manifest when done      |
+| GET    | `/builds/{uuid}/download`    | the zip (409 unless COMPLETED, 410 if gone)|
+
+Example flow:
+
+```bash
+# define a package scoped to two locations
+curl -u admin:pw -H 'Content-Type: application/json' -d '{
+  "name": "Site A locations",
+  "description": "Everything Site A needs",
+  "entries": [ { "domain": "LOCATIONS", "itemUuids": ["<uuid-1>", "<uuid-2>"] } ]
+}' http://localhost:8080/openmrs/ws/rest/v1/metadataexport/packages
+
+# trigger a build, poll until COMPLETED, then download
+curl -u admin:pw -X POST .../packages/<pkg-uuid>/builds
+curl -u admin:pw .../builds/<build-uuid>
+curl -u admin:pw -OJ .../builds/<build-uuid>/download
+```
+
+Reads require the `Get Metadata Export Packages` privilege; creating, updating, retiring,
+triggering and downloading require `Manage Metadata Export Packages`. The curl examples use
+basic auth, which is provided by webservices.rest's filter — without that module, authenticate
+with a session instead.
+
+If the server restarts mid-build, the activator marks any stranded QUEUED/RUNNING builds as
+FAILED on startup so they never block future builds of their package.
+
+Zips and their unzipped working copies accumulate under
+`<app data dir>/metadataexport/packages/<package-uuid>/<version>/` — there is no retention policy
+yet, so clean up old builds manually if disk space matters. Note also that these endpoints are
+session-authenticated but not CSRF-protected (nothing under `/ws/*` is); treat them as an
+admin-only API.
+
 Requirements
 ------------
 The Initializer module must be installed (declared in `config.xml` `require_modules`); this module
@@ -201,8 +260,8 @@ Known limitations
 -----------------
 * Concept description UUIDs and index-term names are not round-trip-able (Initializer
   format/loader limitations), so they are not preserved or re-loadable.
-* Selection currently exports all instances of the registered domains; instance-level seed
-  selection is not yet exposed.
+* The startup export always exports all instances of the registered domains; instance-level
+  selection is available through export packages (see "Export packages (REST)").
 * Cross-domain closure only pulls in objects whose domain has a registered exporter.
 
 Building from source

api/src/main/java/org/openmrs/module/metadataexport/MetadataExportActivator.java

@@ -16,6 +16,7 @@
 import org.openmrs.module.DaemonToken;
 import org.openmrs.module.DaemonTokenAware;
 import org.openmrs.module.metadataexport.api.ExporterService;
+import org.openmrs.module.metadataexport.api.MetadataExportService;
 import org.openmrs.util.OpenmrsUtil;
 
 import java.io.File;
@@ -28,11 +29,28 @@ public class MetadataExportActivator extends BaseModuleActivator implements Daem
 	@Override
 	public void setDaemonToken(DaemonToken token) {
 		this.daemonToken = token;
+		MetadataExportDaemonToken.set(token);
 	}
 
 	@Override
 	public void started() {
-		Daemon.runInDaemonThreadWithoutResult(this::exportAllMetadata, daemonToken);
+		Daemon.runInDaemonThreadWithoutResult(() -> {
+			recoverStrandedBuilds();
+			exportAllMetadata();
+		}, daemonToken);
+	}
+
+	private void recoverStrandedBuilds() {
+		try {
+			int recovered = Context.getService(MetadataExportService.class)
+			        .failStrandedBuilds("Interrupted by a server restart");
+			if (recovered > 0) {
+				log.warn("Metadata Export: marked {} stranded QUEUED/RUNNING build(s) as FAILED", recovered);
+			}
+		}
+		catch (Exception e) {
+			log.error("Metadata Export: failed to recover stranded builds on startup", e);
+		}
 	}
 
 	private void exportAllMetadata() {

api/src/main/java/org/openmrs/module/metadataexport/MetadataExportDaemonToken.java

@@ -0,0 +1,32 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.metadataexport;
+
+import org.openmrs.module.DaemonToken;
+
+/**
+ * Holds the module's {@link DaemonToken} (handed to the activator by the module framework) so that
+ * Spring components can start daemon threads without depending on the activator instance.
+ */
+public final class MetadataExportDaemonToken {
+
+	private static volatile DaemonToken token;
+
+	private MetadataExportDaemonToken() {
+	}
+
+	public static void set(DaemonToken daemonToken) {
+		token = daemonToken;
+	}
+
+	public static DaemonToken get() {
+		return token;
+	}
+}

api/src/main/java/org/openmrs/module/metadataexport/api/ActiveBuildException.java

@@ -0,0 +1,27 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.metadataexport.api;
+
+import org.openmrs.api.APIException;
+
+/**
+ * A build was requested for a package that already has a QUEUED or RUNNING build. The REST layer
+ * maps exactly this type to 409; other {@link APIException}s keep their generic handling.
+ */
+public class ActiveBuildException extends APIException {
+
+	public ActiveBuildException(String message) {
+		super(message);
+	}
+
+	public ActiveBuildException(String message, Throwable cause) {
+		super(message, cause);
+	}
+}

api/src/main/java/org/openmrs/module/metadataexport/api/ExportJobRunner.java

@@ -0,0 +1,124 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.metadataexport.api;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.hibernate.exception.ConstraintViolationException;
+import org.openmrs.api.APIException;
+import org.openmrs.api.context.Context;
+import org.openmrs.api.context.Daemon;
+import org.openmrs.module.metadataexport.MetadataExportDaemonToken;
+import org.openmrs.module.metadataexport.api.model.ExportBuild;
+import org.openmrs.module.metadataexport.api.model.ExportPackage;
+import org.openmrs.module.metadataexport.api.model.ExportStatus;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Triggers export builds and runs them on a daemon thread. Deliberately not transactional: the
+ * QUEUED build must be committed before the daemon thread starts, and each status change goes
+ * through the transactional {@link MetadataExportService} proxy so polling clients see progress. A
+ * build must never be left QUEUED/RUNNING with nothing running it — that would block every future
+ * build of its package — so every failure path here ends in a FAILED build or a logged error, and
+ * the activator sweeps stranded builds on startup.
+ */
+@Slf4j
+@Component
+public class ExportJobRunner {
+
+	public ExportBuild trigger(String packageUuid) {
+		MetadataExportService service = service();
+		ExportPackage exportPackage = service.getPackageByUuid(packageUuid);
+		if (exportPackage == null) {
+			throw new APIException("No export package with uuid " + packageUuid);
+		}
+		List<ExportBuild> builds = service.getBuilds(packageUuid);
+		for (ExportBuild existing : builds) {
+			if (!existing.getExportStatus().isTerminal()) {
+				throw new ActiveBuildException("Build v" + existing.getVersion() + " of '" + exportPackage.getName()
+				        + "' is already " + existing.getExportStatus());
+			}
+		}
+		ExportBuild build = new ExportBuild();
+		build.setExportPackage(exportPackage);
+		build.setVersion(builds.isEmpty() ? 1 : builds.get(0).getVersion() + 1);
+		build.setExportStatus(ExportStatus.QUEUED);
+		final ExportBuild queued;
+		try {
+			queued = service.saveExportBuild(build);
+		}
+		catch (RuntimeException e) {
+			// two concurrent triggers can both pass the scan above; the unique
+			// (package_id, version) constraint catches the loser
+			if (ExceptionUtils.indexOfType(e, ConstraintViolationException.class) != -1) {
+				throw new ActiveBuildException(
+				        "A build of '" + exportPackage.getName() + "' was just triggered concurrently", e);
+			}
+			throw e;
+		}
+
+		try {
+			Daemon.runInDaemonThreadWithoutResult(() -> execute(queued.getUuid()), MetadataExportDaemonToken.get());
+		}
+		catch (Exception e) {
+			failBuild(queued.getUuid(),
+			    "Could not start the export daemon thread: " + ExceptionUtils.getRootCauseMessage(e));
+			throw new APIException("Could not start the export daemon thread for build " + queued.getUuid(), e);
+		}
+		return queued;
+	}
+
+	// package-private so context-sensitive tests can run a build synchronously
+	void execute(String buildUuid) {
+		try {
+			MetadataExportService service = service();
+			ExportBuild build = service.getBuildByUuid(buildUuid);
+			build.setExportStatus(ExportStatus.RUNNING);
+			build.setDateStarted(new Date());
+			service.saveExportBuild(build);
+
+			service.runBuild(buildUuid);
+		}
+		catch (Throwable t) {
+			log.error("Metadata Export: build {} failed", buildUuid, t);
+			failBuild(buildUuid, ExceptionUtils.getRootCauseMessage(t));
+			if (t instanceof Error) {
+				throw (Error) t;
+			}
+		}
+	}
+
+	private void failBuild(String buildUuid, String errorMessage) {
+		try {
+			MetadataExportService service = service();
+			ExportBuild build = service.getBuildByUuid(buildUuid);
+			if (build == null) {
+				log.error("Metadata Export: cannot mark unknown build {} as FAILED", buildUuid);
+				return;
+			}
+			build.setExportStatus(ExportStatus.FAILED);
+			build.setDateCompleted(new Date());
+			build.setErrorMessage(errorMessage);
+			service.saveExportBuild(build);
+		}
+		catch (Exception e) {
+			log.error("Metadata Export: build {} failed AND could not be marked FAILED; it will block future builds"
+			        + " of its package until the module restarts",
+			    buildUuid, e);
+		}
+	}
+
+	private MetadataExportService service() {
+		return Context.getService(MetadataExportService.class);
+	}
+}

api/src/main/java/org/openmrs/module/metadataexport/api/ExporterService.java

@@ -9,7 +9,9 @@
  */
 package org.openmrs.module.metadataexport.api;
 
+import org.openmrs.OpenmrsObject;
 import org.openmrs.module.initializer.Domain;
+import org.openmrs.module.metadataexport.select.ExportManifest;
 
 import java.io.File;
 import java.io.IOException;
@@ -24,4 +26,7 @@ public interface ExporterService {
 	 * itself.
 	 */
 	void export(File outDir, Collection<Domain> domains) throws IOException;
+
+	ExportManifest exportSeeds(File outDir, Collection<? extends OpenmrsObject> seeds) throws IOException;
+
 }

api/src/main/java/org/openmrs/module/metadataexport/api/MetadataExportService.java

@@ -0,0 +1,42 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.metadataexport.api;
+
+import org.openmrs.api.OpenmrsService;
+import org.openmrs.module.metadataexport.api.model.ExportBuild;
+import org.openmrs.module.metadataexport.api.model.ExportPackage;
+
+import java.io.File;
+import java.util.List;
+
+public interface MetadataExportService extends OpenmrsService {
+
+	ExportPackage saveExportPackage(ExportPackage exportPackage);
+
+	ExportPackage getPackageByUuid(String uuid);
+
+	List<ExportPackage> getAllPackages(boolean includeRetired);
+
+	ExportPackage retireExportPackage(ExportPackage exportPackage, String reason);
+
+	ExportBuild saveExportBuild(ExportBuild build);
+
+	ExportBuild getBuildByUuid(String uuid);
+
+	List<ExportBuild> getBuilds(String packageUuid);
+
+	ExportBuild getLatestBuild(ExportPackage exportPackage);
+
+	ExportBuild runBuild(String buildUuid);
+
+	int failStrandedBuilds(String reason);
+
+	File getBuildZip(ExportBuild build);
+}

api/src/main/java/org/openmrs/module/metadataexport/api/db/MetadataExportDao.java

@@ -0,0 +1,37 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.openmrs.module.metadataexport.api.db;
+
+import org.openmrs.module.metadataexport.api.model.ExportBuild;
+import org.openmrs.module.metadataexport.api.model.ExportPackage;
+
+import java.util.List;
+
+public interface MetadataExportDao {
+
+	ExportPackage savePackage(ExportPackage exportPackage);
+
+	ExportPackage getPackageByUuid(String uuid);
+
+	ExportPackage getPackageByName(String name);
+
+	List<ExportPackage> getAllPackages(boolean includeRetired);
+
+	ExportBuild saveBuild(ExportBuild exportBuild);
+
+	ExportBuild getBuildByUuid(String uuid);
+
+	List<ExportBuild> getBuilds(ExportPackage exportPackage);
+
+	ExportBuild getLatestBuild(ExportPackage exportPackage);
+
+	List<ExportBuild> getActiveBuilds();
+
+}