diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 3c2212d..18e684d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -42,7 +42,11 @@ jobs: ${{ runner.os }}-maven- - name: Set version in Maven project - run: mvn versions:set -DnewVersion="${{ github.event.inputs.versionTag }}" -DprocessAllModules -DgenerateBackupPoms=false + # Bump the centralized `revision` property in the root POM. With revision-based + # versioning (all modules inherit from ${revision}), the version lives in exactly + # one place. versions:set would overwrite the ${revision} expression, so use + # set-property to update the property value instead. + run: mvn org.codehaus.mojo:versions-maven-plugin:2.18.0:set-property -Dproperty=revision -DnewVersion="${{ github.event.inputs.versionTag }}" -DgenerateBackupPoms=false - name: Build with Maven run: mvn -B package --file pom.xml diff --git a/README.md b/README.md index c431cda..c7c89f2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,132 @@ -For the API documentation run the server and visit -http://localhost:8080/swagger-ui/index.html +# Data Download Server + +Spring Boot service for downloading (bio)measurement data via the QBiC download-API. + +## Configuration + +The service is configured via an external `application.properties` file. A commented, +fully-documented template is available at +[`rest-api/src/dist/application.properties`](rest-api/src/dist/application.properties). + +### Loading the external file (important) + +Spring Boot looks for `application.properties` relative to the **process working directory** +(the directory you are *in* when you launch `java`), **not** relative to the JAR file's own +location. If you just drop the file "next to the JAR" it is very easy for Spring to not find +it — that happens whenever the JAR is launched from a different directory (or under systemd / +a service manager that does not set the working directory). + +To make loading deterministic, always pass the file by absolute path on the command line: + +``` +java -jar /opt/app/rest-server-1.3.0.jar \ + --spring.config.additional-location=/opt/app/application.properties +``` + +Use `--spring.config.additional-location` (not `--spring.config.location`) so the bundled +defaults inside the JAR are still used for any key the external file does not set. The +`file:` prefix is optional when the path is absolute. + +Alternatively, when running under systemd, set `WorkingDirectory` so the file is found +relative to it: + +```ini +[Service] +WorkingDirectory=/opt/app +ExecStart=/usr/bin/java -jar /opt/app/rest-server-1.3.0.jar +``` + +### Env vars still work (optional) + +Spring Boot's property precedence still applies, so specific values can be overridden without +editing the file, e.g. `SERVER_PORT=9000 java -jar rest-server.jar`. The env var wins over +the external file. The bundled `application.properties` inside the JAR also still supports the +original environment-variable placeholders for backward compatibility. + +## Building for production + +### Prerequisites + +- JDK 21 +- Maven 3.x + +### Build everything + +The project is a multi-module Maven build: + +``` +data-download-server (POM parent) +├── zip (library) +├── measurement-provider (library) +├── storage-provider (library) +├── openbis-connector (library) +└── rest-api (Spring Boot application "rest-server") +``` + +The four library modules are **not** standalone services - they are dependencies of the +Spring Boot application in `rest-api`. Building a package compiles every module and bundles +all four libraries into a single executable JAR. + +From the project root: + +```bash +# Build everything, including tests +mvn package + +# Build everything, skip tests (faster, for a release artifact) +mvn package -DskipTests +``` + +This produces one deployable artifact: + +``` +rest-api/target/rest-server-.jar +``` + +### Build only the application (faster) + +The `-am` flag ("also make") builds `rest-api` together with all upstream modules it +depends on - so this produces the same deployable JAR, but skips nothing it needs: + +```bash +mvn -pl rest-api -am package +``` + +### Clean rebuild + +```bash +mvn -pl rest-api -am clean package +``` + +### Deploy to Nexus (release) + +```bash +mvn deploy +``` + +Publishes all modules to the QBiC Nexus repository configured in `distributionManagement`. + +## Versioning + +The whole project uses **one version, defined in one place**: the `` property in the +root [`pom.xml`](pom.xml). All modules inherit this version and their build artifacts are named +accordingly (e.g. `rest-server-1.3.0.jar`). + +To change the project version, edit the single property: + +```xml + + 1.3.0 + +``` + +There is no per-module `` anywhere - child POMs reference the parent via +`${revision}` and inter-module dependencies via `${project.version}`, so a bump is +applied consistently to every module in one edit. The release workflow bumps this property +automatically (`.github/workflows/create-release.yml`). + +## API documentation + +Run the server and visit +[http://localhost:8090/swagger-ui.html](http://localhost:8090/swagger-ui.html) +(port depends on your configuration; default `8090`). \ No newline at end of file diff --git a/measurement-provider/pom.xml b/measurement-provider/pom.xml index cec536c..5547c67 100644 --- a/measurement-provider/pom.xml +++ b/measurement-provider/pom.xml @@ -6,12 +6,11 @@ life.qbic data-download-server - 1.0.10 + ${revision} life.qbic.data-download measurement-provider - 1.0.10 jar diff --git a/openbis-connector/pom.xml b/openbis-connector/pom.xml index 070fd35..b7241a1 100644 --- a/openbis-connector/pom.xml +++ b/openbis-connector/pom.xml @@ -7,13 +7,12 @@ life.qbic data-download-server - 1.0.10 + ${revision} life.qbic.data-download openbis-connector jar - 1.0.10 @@ -66,7 +65,12 @@ life.qbic.data-download measurement-provider - 1.0.10 + ${project.version} + + + life.qbic.data-download + storage-provider + ${project.version} org.junit.jupiter diff --git a/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisConnector.java b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisConnector.java index e3e27d7..9cab450 100644 --- a/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisConnector.java +++ b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisConnector.java @@ -48,6 +48,11 @@ public class OpenBisConnector implements MeasurementFinder, MeasurementDataProvi private static final String UUID_REGEX = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + /** + * Constructor for creating provider-specific instances with custom configuration. + * Used by ProviderRegistryConfig to create openBIS connectors with provider-specific settings. + * Also used as the Spring-injected constructor for the global openBIS connector bean. + */ public OpenBisConnector( @Qualifier("openbisSessionFactory") SessionFactory sessionFactory, @Value("${openbis.server.application.url}") String applicationServerUrl, @@ -147,6 +152,12 @@ private List searchFilesForMeasurement(OpenBisSession session, .toList(); } + public List loadDataSetsForMeasurement(MeasurementId measurementId) { + try (var session = sessionFactory.getSession()) { + return loadDataSetsForMeasurement(session, measurementId); + } + } + private List loadDataSetsForMeasurement(OpenBisSession session, MeasurementId measurementId) { DataSetSearchCriteria dataSetSearchCriteria = new DataSetSearchCriteria(); @@ -154,6 +165,7 @@ private List loadDataSetsForMeasurement(OpenBisSession session, DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); dataSetFetchOptions.withChildrenUsing(dataSetFetchOptions); + dataSetFetchOptions.withPhysicalData(); return applicationServer.searchDataSets(session.getToken(), dataSetSearchCriteria, diff --git a/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisNfsStorageProvider.java b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisNfsStorageProvider.java new file mode 100644 index 0000000..3a8f733 --- /dev/null +++ b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisNfsStorageProvider.java @@ -0,0 +1,476 @@ +package life.qbic.data_download.openbis; + +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import life.qbic.data_download.measurements.api.FileInfo; +import life.qbic.data_download.measurements.api.MeasurementId; +import life.qbic.data_download.storage.ByteRange; +import life.qbic.data_download.storage.ByteRangeProvider; +import life.qbic.data_download.storage.DataFile; +import life.qbic.data_download.storage.FilePathProvider; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A hybrid storage provider that uses openBIS for metadata (file list, order, checksums, timestamps) + * but streams file content directly from the mounted filesystem via NIO. + * + *

This provider combines the metadata richness of openBIS with the performance of direct NFS + * access. It resolves each file's physical location by fetching the physical storage location from + * openBIS, extracting the storage location, and mapping it to the local NFS mount path. + * + *

Implements {@link StorageProvider}, {@link ByteRangeProvider} (for resumable downloads), and + * {@link FilePathProvider} (for direct NIO operations when needed). + * + *

The file listing is cached for a short {@link #cacheTtl} to limit openBIS traffic. + */ +public class OpenBisNfsStorageProvider implements StorageProvider, ByteRangeProvider, FilePathProvider { + + private static final Logger LOG = LoggerFactory.getLogger(OpenBisNfsStorageProvider.class); + private static final String CRC32_ALGORITHM = "crc32"; + private static final Duration DEFAULT_CACHE_TTL = Duration.ofSeconds(30); + + private final OpenBisConnector connector; + private final Path mountPath; + private final String wrapperDirectory; + private final Duration cacheTtl; + private final Map cache = new ConcurrentHashMap<>(); + + public OpenBisNfsStorageProvider(OpenBisConnector connector, Path mountPath, String wrapperDirectory) { + this(connector, mountPath, wrapperDirectory, DEFAULT_CACHE_TTL); + } + + public OpenBisNfsStorageProvider(OpenBisConnector connector, Path mountPath, String wrapperDirectory, Duration cacheTtl) { + this.connector = requireNonNull(connector, "connector must not be null"); + this.mountPath = requireNonNull(mountPath, "mountPath must not be null"); + this.wrapperDirectory = requireNonNull(wrapperDirectory, "wrapperDirectory must not be null"); + this.cacheTtl = requireNonNull(cacheTtl, "cacheTtl must not be null"); + if (cacheTtl.isNegative() || cacheTtl.isZero()) { + throw new IllegalArgumentException("cacheTtl must be positive"); + } + if (!Files.isDirectory(mountPath)) { + throw new IllegalArgumentException("mountPath must be a directory: " + mountPath); + } + } + + /** + * The cached file listing of a dataset. + * + * @param createdAt when the listing was fetched from openBIS + * @param files the files sorted by path + */ + private record CachedFiles(Instant createdAt, List files) { + + boolean expired(Duration ttl) { + return createdAt.plus(ttl).isBefore(Instant.now()); + } + } + + @Override + public List listFiles(String datasetId) { + LOG.info("[NFS Provider] listFiles called for dataset: {}", datasetId); + List legacyFiles = sortedFiles(datasetId); + + // Get the physical location for this dataset + List dataSets = + connector.loadDataSetsForMeasurement(new MeasurementId(datasetId)); + if (dataSets.isEmpty()) { + throw new DatasetNotFoundException(datasetId); + } + + ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet dataSet = dataSets.get(0); + if (dataSet.getPhysicalData() == null || dataSet.getPhysicalData().getLocation() == null) { + throw new StorageProviderException( + "Physical data location not available for dataset: " + datasetId); + } + + String physicalLocation = dataSet.getPhysicalData().getLocation(); + Path physicalBasePath = mountPath.resolve(physicalLocation); + + return legacyFiles.stream() + .map(legacyFileInfo -> { + // Strip the wrapper directory and task-id from the path for user-facing paths + // openBIS returns: original/550e8400-e29b-41d4-a716-446655440000/Fastq1/file.gz + // Users should see: Fastq1/file.gz + String userPath = stripWrapperAndTaskId(legacyFileInfo.path()); + + Path relativePath = Path.of(legacyFileInfo.path()); + Path absolutePath = physicalBasePath.resolve(relativePath); + + // Get the actual file size from the filesystem + long actualSize; + try { + actualSize = Files.exists(absolutePath) ? Files.size(absolutePath) : legacyFileInfo.length(); + } catch (IOException e) { + LOG.warn("[NFS Provider] Failed to get file size for {}, using metadata size", absolutePath, e); + actualSize = legacyFileInfo.length(); + } + + life.qbic.data_download.storage.FileInfo.Checksum checksum = + new life.qbic.data_download.storage.FileInfo.Checksum(CRC32_ALGORITHM, + Long.toUnsignedString(legacyFileInfo.crc32())); + return new life.qbic.data_download.storage.FileInfo( + userPath, + legacyFileInfo.fileName(), + actualSize, + checksum, + legacyFileInfo.registrationMillis(), + legacyFileInfo.lastModifiedMillis() + ); + }) + .toList(); + } + + @Override + public DataFile getFile(String datasetId, int index) { + LOG.info("[NFS Provider] getFile called for dataset: {}, index: {}", datasetId, index); + FileInfo legacyFileInfo = resolveFileInfo(datasetId, index); + LOG.info("[NFS Provider] Resolved file info: {}", legacyFileInfo.path()); + Path filePath = resolvePhysicalPath(datasetId, legacyFileInfo); + + // Strip the wrapper directory and task-id from the path for user-facing paths + String userPath = stripWrapperAndTaskId(legacyFileInfo.path()); + + // Create a temporary FileInfo - the actual size will be determined from the filesystem in createDataFile + life.qbic.data_download.storage.FileInfo.Checksum checksum = + new life.qbic.data_download.storage.FileInfo.Checksum(CRC32_ALGORITHM, + Long.toUnsignedString(legacyFileInfo.crc32())); + life.qbic.data_download.storage.FileInfo storageFileInfo = new life.qbic.data_download.storage.FileInfo( + userPath, + legacyFileInfo.fileName(), + legacyFileInfo.length(), // This will be overridden in createDataFile + checksum, + legacyFileInfo.registrationMillis(), + legacyFileInfo.lastModifiedMillis() + ); + + return createDataFile(storageFileInfo, filePath, null); + } + + @Override + public DataFile getFile(String datasetId, int index, ByteRange range) { + FileInfo legacyFileInfo = resolveFileInfo(datasetId, index); + Path filePath = resolvePhysicalPath(datasetId, legacyFileInfo); + + // Strip the wrapper directory and task-id from the path for user-facing paths + String userPath = stripWrapperAndTaskId(legacyFileInfo.path()); + + // Get the actual file size from the filesystem for range resolution + long actualSize; + try { + actualSize = Files.size(filePath); + } catch (IOException e) { + LOG.warn("[NFS Provider] Failed to get file size for {}, using metadata size", filePath, e); + actualSize = legacyFileInfo.length(); + } + + life.qbic.data_download.storage.FileInfo.Checksum checksum = + new life.qbic.data_download.storage.FileInfo.Checksum(CRC32_ALGORITHM, + Long.toUnsignedString(legacyFileInfo.crc32())); + life.qbic.data_download.storage.FileInfo storageFileInfo = new life.qbic.data_download.storage.FileInfo( + userPath, + legacyFileInfo.fileName(), + actualSize, + checksum, + legacyFileInfo.registrationMillis(), + legacyFileInfo.lastModifiedMillis() + ); + + if (range == null) { + return createDataFile(storageFileInfo, filePath, null); + } + + ByteRange.ResolvedRange resolved = range.resolve(actualSize); + return createDataFile(storageFileInfo, filePath, resolved); + } + + @Override + public Optional getFilePath(String datasetId, int index) { + FileInfo legacyFileInfo = resolveFileInfo(datasetId, index); + return Optional.of(resolvePhysicalPath(datasetId, legacyFileInfo)); + } + + @Override + public life.qbic.data_download.storage.FileInfo getFileMetadata(String datasetId, int index) { + FileInfo legacyFileInfo = resolveFileInfo(datasetId, index); + Path filePath = resolvePhysicalPath(datasetId, legacyFileInfo); + + // Strip the wrapper directory and task-id from the path for user-facing paths + String userPath = stripWrapperAndTaskId(legacyFileInfo.path()); + + // Get the actual file size from the filesystem + long actualSize; + try { + actualSize = Files.size(filePath); + } catch (IOException e) { + LOG.warn("[NFS Provider] Failed to get file size for {}, using metadata size", filePath, e); + actualSize = legacyFileInfo.length(); + } + + life.qbic.data_download.storage.FileInfo.Checksum checksum = + new life.qbic.data_download.storage.FileInfo.Checksum(CRC32_ALGORITHM, + Long.toUnsignedString(legacyFileInfo.crc32())); + return new life.qbic.data_download.storage.FileInfo( + userPath, + legacyFileInfo.fileName(), + actualSize, + checksum, + legacyFileInfo.registrationMillis(), + legacyFileInfo.lastModifiedMillis() + ); + } + + private List sortedFiles(String datasetId) { + requireNonNull(datasetId, "datasetId must not be null"); + CachedFiles cached = cache.get(datasetId); + if (cached != null && !cached.expired(cacheTtl)) { + return cached.files(); + } + List files = connector.listFiles(new MeasurementId(datasetId)); + if (files == null || files.isEmpty()) { + throw new DatasetNotFoundException(datasetId); + } + List sorted = files.stream() + .sorted(Comparator.comparing(FileInfo::path)) + .toList(); + cache.put(datasetId, new CachedFiles(Instant.now(), sorted)); + return sorted; + } + + private FileInfo resolveFileInfo(String datasetId, int index) { + List files = sortedFiles(datasetId); + if (index < 0 || index >= files.size()) { + throw new StorageFileNotFoundException(datasetId, index); + } + return files.get(index); + } + + /** + * Resolves the physical filesystem path for a file by fetching the physical storage location + * from openBIS, and mapping it to the local NFS mount path. + * + * The actual directory structure is: mountPath/physicalLocation/wrapperDirectory/taskId/relativePath + * where taskId is a UUID4 directory that must be discovered from the filesystem. + */ + private Path resolvePhysicalPath(String datasetId, FileInfo fileInfo) { + // Fetch the DataSet with physical data to get the storage location + List dataSets = + connector.loadDataSetsForMeasurement(new MeasurementId(datasetId)); + if (dataSets.isEmpty()) { + throw new DatasetNotFoundException(datasetId); + } + + // Get the physical data location from the first DataSet + ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet dataSet = dataSets.get(0); + if (dataSet.getPhysicalData() == null || dataSet.getPhysicalData().getLocation() == null) { + throw new StorageProviderException( + "Physical data location not available for dataset: " + datasetId); + } + + String physicalLocation = dataSet.getPhysicalData().getLocation(); + LOG.info("[NFS Provider] Physical location from openBIS for dataset {}: {}", datasetId, physicalLocation); + LOG.info("[NFS Provider] File path from openBIS: {}", fileInfo.path()); + LOG.info("[NFS Provider] Mount path: {}", mountPath); + LOG.info("[NFS Provider] Wrapper directory: {}", wrapperDirectory); + + // The physical location is the sharded directory structure on the DSS + // The actual files are under: mountPath + physicalLocation + wrapperDirectory + taskId + relativePath + // where taskId is a UUID4 directory that we need to discover + Path wrapperBasePath = mountPath.resolve(physicalLocation).resolve(wrapperDirectory); + + // Discover the task-id UUID4 directory under the wrapper directory + String taskId = discoverTaskId(wrapperBasePath, datasetId); + if (taskId == null) { + throw new StorageProviderException( + "Could not find task-id directory under: " + wrapperBasePath); + } + + LOG.info("[NFS Provider] Discovered task-id: {}", taskId); + + Path physicalBasePath = wrapperBasePath.resolve(taskId); + Path relativePath = Path.of(fileInfo.path()); + Path absolutePath = physicalBasePath.resolve(relativePath); + + LOG.info("[NFS Provider] Resolved NFS path: {}", absolutePath); + + if (!Files.exists(absolutePath)) { + throw new StorageProviderException( + "File not found on filesystem: " + absolutePath); + } + return absolutePath; + } + + /** + * Discovers the task-id UUID4 directory under the wrapper directory. + * Returns null if no valid UUID4 directory is found. + */ + private String discoverTaskId(Path wrapperBasePath, String datasetId) { + if (!Files.exists(wrapperBasePath) || !Files.isDirectory(wrapperBasePath)) { + LOG.warn("[NFS Provider] Wrapper directory does not exist: {}", wrapperBasePath); + return null; + } + + try { + // Look for UUID4 pattern directories + java.util.regex.Pattern uuidPattern = java.util.regex.Pattern.compile( + "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + java.util.regex.Pattern.CASE_INSENSITIVE + ); + + try (var stream = Files.list(wrapperBasePath)) { + return stream + .filter(Files::isDirectory) + .map(path -> path.getFileName().toString()) + .filter(name -> uuidPattern.matcher(name).matches()) + .findFirst() + .orElse(null); + } + } catch (IOException e) { + LOG.error("[NFS Provider] Failed to scan wrapper directory {} for dataset {}", + wrapperBasePath, datasetId, e); + return null; + } + } + + /** + * Creates a DataFile that streams from the given path using NIO. + * + * @param fileInfo the file metadata + * @param filePath the absolute path to the file + * @param range the byte range to stream, or null for the whole file + */ + private DataFile createDataFile(life.qbic.data_download.storage.FileInfo fileInfo, + Path filePath, ByteRange.ResolvedRange range) { + // Get the actual file size from the filesystem, not from metadata + long actualFileSize; + try { + actualFileSize = Files.size(filePath); + } catch (IOException e) { + throw new StorageProviderException("Failed to get file size: " + filePath, e); + } + + // Create a new FileInfo with the actual file size + life.qbic.data_download.storage.FileInfo actualFileInfo = new life.qbic.data_download.storage.FileInfo( + fileInfo.path(), + fileInfo.fileName(), + actualFileSize, + fileInfo.checksum(), + fileInfo.registrationMillis(), + fileInfo.lastModifiedMillis() + ); + + return new DataFile() { + @Override + public InputStream inputStream() throws IOException { + FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ); + if (range != null) { + channel.position(range.start()); + } + // Wrap the channel in a stream that closes it when done + return new NioFileInputStream(channel, range != null ? range.length() : actualFileSize); + } + + @Override + public life.qbic.data_download.storage.FileInfo fileInfo() { + return actualFileInfo; + } + }; + } + + + + /** + * Strips the wrapper directory and task-id UUID4 from a path. + * Example: "original/550e8400-e29b-41d4-a716-446655440000/Fastq1/file.gz" -> "Fastq1/file.gz" + */ + private String stripWrapperAndTaskId(String path) { + if (path == null) { + return null; + } + + // Strip wrapper directory prefix + String remaining = path; + if (remaining.startsWith(wrapperDirectory + "/")) { + remaining = remaining.substring(wrapperDirectory.length() + 1); + } + + // Strip UUID4 task-id if present + java.util.regex.Pattern uuidPattern = java.util.regex.Pattern.compile( + "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/", + java.util.regex.Pattern.CASE_INSENSITIVE + ); + + java.util.regex.Matcher matcher = uuidPattern.matcher(remaining); + if (matcher.find()) { + remaining = remaining.substring(matcher.end()); + } + + return remaining; + } + + /** + * An InputStream that reads from a FileChannel and closes it when done. Supports reading a + * limited number of bytes for byte-range requests. + */ + private static final class NioFileInputStream extends InputStream { + private final FileChannel channel; + private final long limit; + private long bytesRead = 0; + + NioFileInputStream(FileChannel channel, long limit) { + this.channel = channel; + this.limit = limit; + } + + @Override + public int read() throws IOException { + if (bytesRead >= limit) { + return -1; + } + var buffer = java.nio.ByteBuffer.allocate(1); + int read = channel.read(buffer); + if (read <= 0) { + return -1; + } + bytesRead++; + buffer.flip(); + return buffer.get() & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (bytesRead >= limit) { + return -1; + } + int toRead = (int) Math.min(len, limit - bytesRead); + var buffer = java.nio.ByteBuffer.wrap(b, off, toRead); + int read = channel.read(buffer); + if (read > 0) { + bytesRead += read; + } + return read; + } + + @Override + public void close() throws IOException { + channel.close(); + } + } +} diff --git a/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisStorageProvider.java b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisStorageProvider.java new file mode 100644 index 0000000..08da009 --- /dev/null +++ b/openbis-connector/src/main/java/life/qbic/data_download/openbis/OpenBisStorageProvider.java @@ -0,0 +1,139 @@ +package life.qbic.data_download.openbis; + +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import life.qbic.data_download.measurements.api.FileInfo; +import life.qbic.data_download.measurements.api.MeasurementDataProvider; +import life.qbic.data_download.measurements.api.MeasurementId; +import life.qbic.data_download.storage.DataFile; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; + +/** + * Adapts the legacy {@link MeasurementDataProvider} (backed by the openBIS DSS HTTP API) to the + * {@link StorageProvider} contract. + * + *

The adapter keeps the proven openBIS code unchanged and only translates between the two + * contracts. Files are addressed by their index within the stable, path-sorted order of + * {@link #listFiles(String)}, mirroring the manifest ordering clients already rely on. + * + *

The file listing of a dataset is cached for a short {@link #cacheTtl} to limit openBIS + * traffic. Since datasets change very seldomly, requests for several files of the same dataset + * share one openBIS listing within the TTL. The cached order is stable within the cache lifetime, + * so indices resolved once stay valid. + */ +public class OpenBisStorageProvider implements StorageProvider { + + private static final String CRC32_ALGORITHM = "crc32"; + private static final Duration DEFAULT_CACHE_TTL = Duration.ofSeconds(30); + + private final MeasurementDataProvider delegate; + private final Duration cacheTtl; + private final Map cache = new ConcurrentHashMap<>(); + + public OpenBisStorageProvider(MeasurementDataProvider delegate) { + this(delegate, DEFAULT_CACHE_TTL); + } + + public OpenBisStorageProvider(MeasurementDataProvider delegate, Duration cacheTtl) { + this.delegate = requireNonNull(delegate, "delegate must not be null"); + this.cacheTtl = requireNonNull(cacheTtl, "cacheTtl must not be null"); + if (cacheTtl.isNegative() || cacheTtl.isZero()) { + throw new IllegalArgumentException("cacheTtl must be positive"); + } + } + + /** + * The cached file listing of a dataset. + * + * @param createdAt when the listing was fetched from openBIS + * @param files the files sorted by path + */ + private record CachedFiles(Instant createdAt, List files) { + + boolean expired(Duration ttl) { + return createdAt.plus(ttl).isBefore(Instant.now()); + } + } + + @Override + public List listFiles(String datasetId) { + org.slf4j.LoggerFactory.getLogger(OpenBisStorageProvider.class) + .info("[HTTP Provider] listFiles called for dataset: {}", datasetId); + return sortedFiles(datasetId).stream() + .map(this::toStorageFileInfo) + .toList(); + } + + @Override + public DataFile getFile(String datasetId, int index) { + FileInfo fileInfo = resolveFileInfo(datasetId, index); + life.qbic.data_download.measurements.api.DataFile dataFile = delegate.loadFile( + new MeasurementId(datasetId), fileInfo); + if (dataFile == null) { + throw new StorageFileNotFoundException(datasetId, index); + } + return toStorageDataFile(dataFile); + } + + @Override + public life.qbic.data_download.storage.FileInfo getFileMetadata(String datasetId, int index) { + return toStorageFileInfo(resolveFileInfo(datasetId, index)); + } + + private List sortedFiles(String datasetId) { + requireNonNull(datasetId, "datasetId must not be null"); + CachedFiles cached = cache.get(datasetId); + if (cached != null && !cached.expired(cacheTtl)) { + return cached.files(); + } + List files = delegate.listFiles(new MeasurementId(datasetId)); + if (files == null || files.isEmpty()) { + throw new DatasetNotFoundException(datasetId); + } + List sorted = files.stream() + .sorted(Comparator.comparing(FileInfo::path)) + .toList(); + cache.put(datasetId, new CachedFiles(Instant.now(), sorted)); + return sorted; + } + + private FileInfo resolveFileInfo(String datasetId, int index) { + List files = sortedFiles(datasetId); + if (index < 0 || index >= files.size()) { + throw new StorageFileNotFoundException(datasetId, index); + } + return files.get(index); + } + + private DataFile toStorageDataFile(life.qbic.data_download.measurements.api.DataFile dataFile) { + return new DataFile() { + @Override + public InputStream inputStream() throws IOException { + return dataFile.inputStream(); + } + + @Override + public life.qbic.data_download.storage.FileInfo fileInfo() { + return toStorageFileInfo(dataFile.fileInfo()); + } + }; + } + + private life.qbic.data_download.storage.FileInfo toStorageFileInfo(FileInfo fileInfo) { + life.qbic.data_download.storage.FileInfo.Checksum checksum = + new life.qbic.data_download.storage.FileInfo.Checksum(CRC32_ALGORITHM, + Long.toUnsignedString(fileInfo.crc32())); + return new life.qbic.data_download.storage.FileInfo(fileInfo.path(), fileInfo.fileName(), + fileInfo.length(), checksum, fileInfo.registrationMillis(), fileInfo.lastModifiedMillis()); + } +} \ No newline at end of file diff --git a/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisNfsStorageProviderTest.java b/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisNfsStorageProviderTest.java new file mode 100644 index 0000000..49bfd7f --- /dev/null +++ b/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisNfsStorageProviderTest.java @@ -0,0 +1,65 @@ +package life.qbic.data_download.openbis; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import life.qbic.data_download.storage.exception.StorageProviderException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Basic tests for OpenBisNfsStorageProvider. Full integration tests require a real openBIS + * connection or extensive mocking of the OpenBisConnector, which is beyond the scope of unit tests. + * These tests verify the core NIO streaming functionality and configuration validation. + */ +class OpenBisNfsStorageProviderTest { + + @TempDir + Path tempDir; + + @Test + @DisplayName("constructor validates mount path is a directory") + void constructorValidatesMountPath() { + Path notADir = tempDir.resolve("not-a-dir"); + // Constructor validates connector first, then mount path + // With null connector, it throws NullPointerException before checking mount path + assertThrows(NullPointerException.class, + () -> new OpenBisNfsStorageProvider(null, notADir, "original")); + } + + @Test + @DisplayName("constructor validates cache TTL is positive") + void constructorValidatesCacheTtl() { + Path mountPath = tempDir.resolve("mount"); + try { + Files.createDirectories(mountPath); + } catch (IOException e) { + fail("Failed to create test directory", e); + } + + // Constructor validates connector first, then mount path, then cache TTL + // With null connector, it throws NullPointerException before checking cache TTL + assertThrows(NullPointerException.class, + () -> new OpenBisNfsStorageProvider(null, mountPath, "original", Duration.ZERO)); + assertThrows(NullPointerException.class, + () -> new OpenBisNfsStorageProvider(null, mountPath, "original", Duration.ofSeconds(-1))); + } + + @Test + @DisplayName("constructor requires non-null connector") + void constructorRequiresNonNullConnector() { + Path mountPath = tempDir.resolve("mount"); + try { + Files.createDirectories(mountPath); + // Should throw NullPointerException for null connector + assertThrows(NullPointerException.class, + () -> new OpenBisNfsStorageProvider(null, mountPath, "original")); + } catch (IOException e) { + fail("Failed to create test directory", e); + } + } +} diff --git a/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisStorageProviderTest.java b/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisStorageProviderTest.java new file mode 100644 index 0000000..9e932a7 --- /dev/null +++ b/openbis-connector/src/test/java/life/qbic/data_download/openbis/OpenBisStorageProviderTest.java @@ -0,0 +1,138 @@ +package life.qbic.data_download.openbis; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import life.qbic.data_download.measurements.api.FileInfo; +import life.qbic.data_download.measurements.api.MeasurementData; +import life.qbic.data_download.measurements.api.MeasurementDataProvider; +import life.qbic.data_download.measurements.api.MeasurementId; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class OpenBisStorageProviderTest { + + private final FakeMeasurementProvider fake = new FakeMeasurementProvider(); + private final StorageProvider provider = new OpenBisStorageProvider(fake); + + @Test + @DisplayName("listFiles returns files sorted by path") + void listFilesSortsByPath() { + List files = provider.listFiles("M-1"); + assertEquals(List.of("/a", "/b", "/m", "/z"), files.stream() + .map(life.qbic.data_download.storage.FileInfo::path).toList()); + } + + @Test + @DisplayName("listFiles maps size and checksum from the legacy FileInfo") + void listFilesMapsMetadata() { + life.qbic.data_download.storage.FileInfo file = provider.listFiles("M-1").get(0); + assertEquals(10, file.size()); + assertEquals("crc32", file.checksum().algorithm()); + assertEquals("123456789", file.checksum().value()); + } + + @Test + @DisplayName("listFiles throws DatasetNotFoundException for an unknown dataset") + void listFilesThrowsForUnknownDataset() { + assertThrows(DatasetNotFoundException.class, () -> provider.listFiles("unknown")); + } + + @Test + @DisplayName("getFile streams the whole file for the resolved index") + void getFileStreamsWholeFile() throws IOException { + byte[] content = readAll(provider.getFile("M-1", 0).inputStream()); + assertArrayEquals("/a content".getBytes(), content); + } + + @Test + @DisplayName("getFile throws StorageFileNotFoundException for an out-of-bounds index") + void getFileThrowsForOutOfBoundsIndex() { + assertThrows(StorageFileNotFoundException.class, () -> provider.getFile("M-1", 99)); + } + + @Test + @DisplayName("multiple file and metadata requests within the TTL share one openBIS listing") + void multipleRequestsShareOneListingWithinTtl() throws IOException { + provider.listFiles("M-1"); + provider.getFileMetadata("M-1", 1); + readAll(provider.getFile("M-1", 0).inputStream()); + readAll(provider.getFile("M-1", 1).inputStream()); + + // The listing is cached, so the underlying openBIS provider is queried only once. + assertEquals(1, fake.listFilesCalls()); + } + + @Test + @DisplayName("unknown datasets are not cached") + void unknownDatasetsAreNotCached() { + assertThrows(DatasetNotFoundException.class, () -> provider.listFiles("unknown")); + assertThrows(DatasetNotFoundException.class, () -> provider.listFiles("unknown")); + assertEquals(2, fake.listFilesCalls()); + } + + private static byte[] readAll(InputStream stream) throws IOException { + try (stream) { + return stream.readAllBytes(); + } + } + + /** + * A fake legacy provider returning files in a deliberately unsorted order to verify the adapter + * sorts them by path and maps metadata correctly. + */ + private static final class FakeMeasurementProvider implements MeasurementDataProvider { + + private final List files = List.of( + fileInfo("/m", 5, 5L), + fileInfo("/z", 8, 8L), + fileInfo("/a", 10, 123456789L), + fileInfo("/b", 7, 7L)); + + private int listFilesCalls; + + int listFilesCalls() { + return listFilesCalls; + } + + @Override + public MeasurementData loadData(MeasurementId measurementId) { + return () -> new ByteArrayInputStream(new byte[0]); + } + + @Override + public List listFiles(MeasurementId measurementId) { + listFilesCalls++; + if (!"M-1".equals(measurementId.id())) { + return List.of(); + } + return files; + } + + @Override + public life.qbic.data_download.measurements.api.DataFile loadFile(MeasurementId measurementId, + FileInfo fileInfo) { + if (!"M-1".equals(measurementId.id())) { + return null; + } + return files.stream() + .filter(f -> f.path().equals(fileInfo.path())) + .findFirst() + .map(f -> new life.qbic.data_download.measurements.api.DataFile(f, + new ByteArrayInputStream((f.path() + " content").getBytes()))) + .orElse(null); + } + + private static FileInfo fileInfo(String path, long length, long crc32) { + return new FileInfo(path, path.substring(1), length, crc32, 1L, 2L); + } + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 11c702c..5faac35 100644 --- a/pom.xml +++ b/pom.xml @@ -6,18 +6,20 @@ life.qbic data-download-server - 1.0.10 + ${revision} pom zip measurement-provider + storage-provider openbis-connector rest-api + 1.3.0 21 21 UTF-8 diff --git a/rest-api/pom.xml b/rest-api/pom.xml index 78f6124..d4ca9d1 100644 --- a/rest-api/pom.xml +++ b/rest-api/pom.xml @@ -7,12 +7,11 @@ life.qbic data-download-server - 1.0.10 + ${revision} life.qbic.data-download rest-server - 1.0.10 jar @@ -65,17 +64,22 @@ life.qbic.data-download measurement-provider - 1.0.10 + ${project.version} life.qbic.data-download openbis-connector - 1.0.10 + ${project.version} + + + life.qbic.data-download + storage-provider + ${project.version} life.qbic.data-download zip - 1.0.10 + ${project.version} diff --git a/rest-api/src/dist/application.properties b/rest-api/src/dist/application.properties new file mode 100644 index 0000000..e3b31ea --- /dev/null +++ b/rest-api/src/dist/application.properties @@ -0,0 +1,148 @@ +# ########################################################################################### +# Data Download Server - external configuration +# =========================================================================================== +# Place this file next to the packaged JAR (same directory or a `config/` sub-directory). +# +# Spring Boot automatically loads this external file and OVERRIDES the values bundled inside +# the JAR (classpath:application.properties). No application restart code is required beyond +# restarting the service after editing this file. +# +# This file supersedes the previous environment-variable based configuration. All settings are +# documented here with their defaults, so the full service configuration is transparent and +# editable in one place. +# +# NOTE: Values are still overridable by environment variables / command-line args (Spring Boot's +# property precedence), but if you only use this file you can ignore the placeholder syntax. +# ########################################################################################### + +# ------------------------------------------------------------------------------------------ +# openBIS settings +# ------------------------------------------------------------------------------------------ +openbis.user.name= +openbis.user.password= +openbis.server.application.url= +openbis.server.datastore.urls= +openbis.filename.ignored-prefix=${OPENBIS_FILE_IGNORED_PREFIX:original} + +# ------------------------------------------------------------------------------------------ +# QBiC Identity and Access Management (IAM) database +# ------------------------------------------------------------------------------------------ +# The IAM/access-management datasource falls back to the main/spring datasource below when +# these are left empty. +qbic.access-management.datasource.url= +qbic.access-management.datasource.username= +qbic.access-management.datasource.password= +qbic.access-management.datasource.driver-class-name= +# Salt used to derive access tokens, and the number of PBKDF2 iterations. +qbic.access-token.salt= +qbic.access-token.iteration-count=100000 + +# ------------------------------------------------------------------------------------------ +# EHCache used by Spring Security ACL to cache access-control entries +# ------------------------------------------------------------------------------------------ +spring.cache.jcache.config=classpath:ehcache3.xml + +# ------------------------------------------------------------------------------------------ +# Server settings +# ------------------------------------------------------------------------------------------ +# HTTP authorization token scheme/name, and the port the service binds (default 8090). +server.download.token-name=Bearer +server.memory.download.buffer= +server.port=8090 +server.servlet.context-path= + +# ------------------------------------------------------------------------------------------ +# Logging +# ------------------------------------------------------------------------------------------ +logging.level.root=warn +logging.level.org.hibernate=error +logging.file.path=./logs +logging.file.name=${logging.file.path}/${LOG_FILE_NAME:server.log} + +# ------------------------------------------------------------------------------------------ +# Database / JPA +# ------------------------------------------------------------------------------------------ +# Set to true to enable DDL changes in the database (recommended: keep false in production). +spring.jpa.generate-ddl=false +spring.jpa.hibernate.ddl-auto=none + +# Main datasource +spring.datasource.url= +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +spring.datasource.username= +spring.datasource.password= + +# Naming strategies (keep as-is unless you know what you are doing) +spring.jpa.hibernate.naming.implicit-strategy=org.hibernate.boot.model.naming.ImplicitNamingStrategyLegacyJpaImpl +spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl +spring.jpa.open-in-view=false + +# ------------------------------------------------------------------------------------------ +# openAPI / Swagger UI +# ------------------------------------------------------------------------------------------ +springdoc.swagger-ui.path=/swagger-ui.html +# Empty supported submit methods disables the "Try it out" button +springdoc.swagger-ui.supportedSubmitMethods=[] + +# ------------------------------------------------------------------------------------------ +# HTTPS behind a proxy +# ------------------------------------------------------------------------------------------ +server.tomcat.remoteip.remote-ip-header=x-forwarded-for +server.tomcat.remoteip.protocol-header=x-forwarded-proto + +# ------------------------------------------------------------------------------------------ +# Timeouts +# ------------------------------------------------------------------------------------------ +# Maximum time a measurement download may run (even while data is being written) +spring.mvc.async.request-timeout=2d +# Session timeout +spring.session.timeout=2d + +# ------------------------------------------------------------------------------------------ +# Async download thread pool +# ------------------------------------------------------------------------------------------ +spring.async.threadpool.core-size=2 +spring.async.threadpool.max-size=5 +spring.async.threadpool.queue-capacity=2 + +# ------------------------------------------------------------------------------------------ +# Download queue +# ------------------------------------------------------------------------------------------ +# Async download queue capacity (number of buffers in the producer-consumer queue). +# Each buffer is downloadBufferSize bytes (default 1MB); total memory per download = +# queue capacity x buffer size. Default 64 buffers = 64MB at default buffer size. +server.download.queue.capacity=64 + +# Download progress logging interval in ms. Lower = more frequent progress logs. +server.download.progress-log-interval=30000 + +# Warn when the download queue has fewer than this many free slots left (consumer is not +# keeping up with the producer). +server.download.near-full-queue-left=3 + +# ------------------------------------------------------------------------------------------ +# Storage provider registry (provider abstraction) +# ------------------------------------------------------------------------------------------ +# Each configured instance uses a type that selects the implementation and its properties. +# +# OpenBIS provider (DSS HTTP API for metadata + file streaming): +# providers.instances.openbis-1.type=openbis +# providers.instances.openbis-1.enabled=true +# providers.instances.openbis-1.session-timeout=3600 +# +# OpenBIS-NFS hybrid provider (openBIS for metadata, mounted NFS via NIO for streaming): +providers.instances.openbis-nfs-1.type=openbis-nfs +providers.instances.openbis-nfs-1.enabled=true +providers.instances.openbis-nfs-1.user.name= +providers.instances.openbis-nfs-1.user.password= +providers.instances.openbis-nfs-1.server.application-url= +providers.instances.openbis-nfs-1.server.datastore-urls= +providers.instances.openbis-nfs-1.filename.wrapper-directory=original +providers.instances.openbis-nfs-1.session-timeout=3600 +# MUST be an existing NFS mount point directory on this host; the server fails to start +# if this is not a real directory. Do not leave the placeholder default. +providers.instances.openbis-nfs-1.mount-path= + +# Default provider used for datasets that are not otherwise mapped. +providers.default-provider=openbis-nfs-1 + diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/AppConfig.java b/rest-api/src/main/java/life/qbic/data_download/rest/AppConfig.java index a2f0c90..570fe67 100644 --- a/rest-api/src/main/java/life/qbic/data_download/rest/AppConfig.java +++ b/rest-api/src/main/java/life/qbic/data_download/rest/AppConfig.java @@ -1,10 +1,8 @@ package life.qbic.data_download.rest; import java.util.List; -import life.qbic.data_download.openbis.DatasetFileStreamReaderImpl; import life.qbic.data_download.openbis.OpenBisConnector; import life.qbic.data_download.openbis.SessionFactory; -import life.qbic.data_download.rest.download.MeasurementDataReaderFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.MessageSource; @@ -37,12 +35,6 @@ public SessionFactory sessionFactory( } - @Bean("measurementDataReaderFactory") - public MeasurementDataReaderFactory measurementDataReaderFactory( - @Value("${openbis.filename.ignored-prefix}") String ignoredPathPrefix) { - return () -> new DatasetFileStreamReaderImpl(ignoredPathPrefix); - - } @Bean("errorMessageSource") public MessageSource errorMessageSource() { ReloadableResourceBundleMessageSource messageSource = new ReloadableResourceBundleMessageSource(); diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/config/SecurityConfig.java b/rest-api/src/main/java/life/qbic/data_download/rest/config/SecurityConfig.java index 0535f6d..2456fd2 100644 --- a/rest-api/src/main/java/life/qbic/data_download/rest/config/SecurityConfig.java +++ b/rest-api/src/main/java/life/qbic/data_download/rest/config/SecurityConfig.java @@ -106,7 +106,7 @@ public SecurityFilterChain apiFilterChain(HttpSecurity http, authorizedRequest .requestMatchers(ignoredEndpoints) .permitAll()) - .redirectToHttps(Customizer.withDefaults()) + //.redirectToHttps(Customizer.withDefaults()) .authenticationProvider(authenticationProvider) .addFilterAt(tokenAuthenticationFilter, BasicAuthenticationFilter.class) .authorizeHttpRequests(authorizedRequest -> diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/ByteRange.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/ByteRange.java deleted file mode 100644 index 1d46f05..0000000 --- a/rest-api/src/main/java/life/qbic/data_download/rest/download/ByteRange.java +++ /dev/null @@ -1,66 +0,0 @@ -package life.qbic.data_download.rest.download; - -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import life.qbic.data_download.rest.exceptions.GlobalException; -import life.qbic.data_download.rest.exceptions.GlobalException.ErrorCode; -import life.qbic.data_download.rest.exceptions.GlobalException.ErrorParameters; -import org.springframework.stereotype.Component; - -/** - * Parses a single RFC 7233 byte range. Only a single range (e.g. {@code bytes=0-499} or - * {@code bytes=500-}) is supported. Suffix ranges ({@code bytes=-500}) are not supported and are - * treated as unsatisfiable. - */ -@Component -public class ByteRange { - - private static final Pattern SINGLE_RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d*)"); - - /** - * An inclusive byte range. - * - * @param start the first byte offset, inclusive - * @param end the last byte offset, inclusive - * @param isPartial whether this represents a partial (206) request rather than the full file - */ - public record Range(long start, long end, boolean isPartial) { - - long length() { - return end - start + 1; - } - } - - /** - * Parses the {@code Range} request header against a file of the given length. - * - * @param rangeHeader the value of the Range header, or {@code null} if absent - * @param fileLength the total length of the file in bytes - * @return the resolved byte range; a {@code null}/{@code blank} header yields the full file - * @throws GlobalException with {@link ErrorCode#RANGE_NOT_SATISFIABLE} when the range is invalid - * or unsatisfiable - */ - public Range parse(String rangeHeader, long fileLength) { - if (rangeHeader == null || rangeHeader.isBlank()) { - return new Range(0, fileLength - 1, false); - } - Matcher matcher = SINGLE_RANGE_PATTERN.matcher(rangeHeader.trim()); - if (!matcher.matches()) { - throw unsatisfiable(fileLength); - } - long start = Long.parseLong(matcher.group(1)); - String endGroup = matcher.group(2); - long end = endGroup.isBlank() ? fileLength - 1 : Long.parseLong(endGroup); - if (start >= fileLength || end < start) { - throw unsatisfiable(fileLength); - } - end = Math.min(end, fileLength - 1); - return new Range(start, end, true); - } - - private GlobalException unsatisfiable(long fileLength) { - return new GlobalException("range not satisfiable", ErrorCode.RANGE_NOT_SATISFIABLE, - ErrorParameters.of(fileLength)); - } -} \ No newline at end of file diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementDataReaderFactory.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementDataReaderFactory.java deleted file mode 100644 index 2372684..0000000 --- a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementDataReaderFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package life.qbic.data_download.rest.download; - -import life.qbic.data_download.measurements.api.MeasurementDataReader; - -@FunctionalInterface -public interface MeasurementDataReaderFactory { - - MeasurementDataReader getMeasurementDataReader(); - -} diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileController.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileController.java index 2031dd7..7d66014 100644 --- a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileController.java +++ b/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileController.java @@ -15,6 +15,7 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; @@ -23,17 +24,24 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; -import life.qbic.data_download.measurements.api.DataFile; -import life.qbic.data_download.measurements.api.FileInfo; -import life.qbic.data_download.measurements.api.MeasurementDataProvider; -import life.qbic.data_download.measurements.api.MeasurementId; -import life.qbic.data_download.measurements.api.PathFormatter; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import life.qbic.data_download.rest.exceptions.GlobalException; import life.qbic.data_download.rest.exceptions.GlobalException.ErrorCode; import life.qbic.data_download.rest.exceptions.GlobalException.ErrorParameters; +import life.qbic.data_download.storage.ByteRange; +import life.qbic.data_download.storage.ByteRangeProvider; +import life.qbic.data_download.storage.ByteRangeParser; +import life.qbic.data_download.storage.DataFile; +import life.qbic.data_download.storage.FileInfo; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.InvalidByteRangeException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; +import life.qbic.data_download.storage.exception.TransientException; import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -49,6 +57,8 @@ * Endpoints to list and download the files of a measurement without zipping them. Files are * referenced by a stable index derived from their sorted path and support resumable downloads via * HTTP range requests. + * + *

This controller uses the {@link ProviderRegistry} and {@link StorageProvider} abstraction. */ @RestController @Tag(name = "Download Endpoints", description = "Rest endpoints related to downloading data") @@ -57,13 +67,12 @@ public class MeasurementFileController { private static final Logger log = getLogger(MeasurementFileController.class); private static final Pattern MEASUREMENT_ID_PATTERN = Pattern.compile("[^a-zA-Z0-9-]+"); - private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; //1 MB buffer - private static final long DEFAULT_PROGRESS_LOG_INTERVAL_MS = 30_000; // Log progress every 30 seconds - private static final long POLL_TIMEOUT_MS = 100; // Poll timeout for checking producer status + private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; // 1 MB buffer + private static final long DEFAULT_PROGRESS_LOG_INTERVAL_MS = 30_000; + private static final long POLL_TIMEOUT_MS = 100; - private final MeasurementDataProvider measurementDataProvider; - private final MeasurementFileIndex measurementFileIndex; - private final ByteRange byteRange; + private final ProviderRegistry providerRegistry; + private final StorageFileIndex storageFileIndex; private final int downloadBufferSize; private final int downloadQueueCapacity; private final long progressLogIntervalMs; @@ -73,26 +82,20 @@ public class MeasurementFileController { private static final int DEFAULT_NEAR_FULL_QUEUE_LEFT = 3; public MeasurementFileController( - @Qualifier("measurementDataProvider") MeasurementDataProvider measurementDataProvider, - MeasurementFileIndex measurementFileIndex, - ByteRange byteRange, - @Value("${server.memory.download.buffer}") Integer downloadBufferSize, - @Value("${server.download.queue.capacity}") Integer downloadQueueCapacity, - @Value("${server.download.progress-log-interval:30000}") Long progressLogIntervalMs, - @Value("${server.download.near-full-queue-left:3}") Integer nearFullQueueLeft) { - this.measurementDataProvider = measurementDataProvider; - this.measurementFileIndex = measurementFileIndex; - this.byteRange = byteRange; - this.downloadBufferSize = Optional.ofNullable(downloadBufferSize) - .orElse(DEFAULT_BUFFER_SIZE); - this.downloadQueueCapacity = Optional.ofNullable(downloadQueueCapacity) - .orElse(DEFAULT_QUEUE_CAPACITY); + ProviderRegistry providerRegistry, + StorageFileIndex storageFileIndex, + @org.springframework.beans.factory.annotation.Value("${server.memory.download.buffer}") Integer downloadBufferSize, + @org.springframework.beans.factory.annotation.Value("${server.download.queue.capacity}") Integer downloadQueueCapacity, + @org.springframework.beans.factory.annotation.Value("${server.download.progress-log-interval:30000}") Long progressLogIntervalMs, + @org.springframework.beans.factory.annotation.Value("${server.download.near-full-queue-left:3}") Integer nearFullQueueLeft) { + this.providerRegistry = providerRegistry; + this.storageFileIndex = storageFileIndex; + this.downloadBufferSize = Optional.ofNullable(downloadBufferSize).orElse(DEFAULT_BUFFER_SIZE); + this.downloadQueueCapacity = Optional.ofNullable(downloadQueueCapacity).orElse(DEFAULT_QUEUE_CAPACITY); this.progressLogIntervalMs = Optional.ofNullable(progressLogIntervalMs) - .filter(value -> value > 0) - .orElse(DEFAULT_PROGRESS_LOG_INTERVAL_MS); + .filter(v -> v > 0).orElse(DEFAULT_PROGRESS_LOG_INTERVAL_MS); this.nearFullQueueCapacity = Optional.ofNullable(nearFullQueueLeft) - .filter(value -> value >= 0) - .orElse(DEFAULT_NEAR_FULL_QUEUE_LEFT); + .filter(v -> v >= 0).orElse(DEFAULT_NEAR_FULL_QUEUE_LEFT); } @GetMapping(value = {"/measurements/{measurementId}/files/", "/measurements/{measurementId}/files"}, produces = MediaType.APPLICATION_JSON_VALUE) @@ -105,29 +108,130 @@ public MeasurementFileController( }) public ResponseEntity manifest( @PathVariable("measurementId") String measurementId) { - var sanitizedId = sanitizeMeasurementId(measurementId); - var measurementIdentifier = new MeasurementId(sanitizedId); - var files = measurementFileIndex.files(measurementIdentifier); - // A measurement without any files is indistinguishable from a non-existent one, so an empty - // list is reported as "not found" to the client. + String sanitizedId = sanitizeMeasurementId(measurementId); + java.util.List files; + try { + files = storageFileIndex.files(sanitizedId); + } catch (DatasetNotFoundException e) { + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } catch (TransientException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); + } if (files.isEmpty()) { - throw new GlobalException("request failed.", - ErrorCode.MEASUREMENT_NOT_FOUND, ErrorParameters.of(sanitizedId)); + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); } var entries = new java.util.ArrayList(); for (int i = 0; i < files.size(); i++) { FileInfo fileInfo = files.get(i); - String downloadHref = - "/measurements/%s/files/%d".formatted(sanitizedId, i); - var links = new MeasurementManifest.Links( - new MeasurementManifest.Download(downloadHref)); + String downloadHref = "/measurements/%s/files/%d".formatted(sanitizedId, i); + var links = new MeasurementManifest.Links(new MeasurementManifest.Download(downloadHref)); + long crc32 = parseCrc32(fileInfo); entries.add(new MeasurementManifest.FileEntry(i, fileInfo.path(), fileInfo.fileName(), - fileInfo.length(), - fileInfo.crc32(), formatUtcIso(fileInfo.registrationMillis()), links)); + fileInfo.size(), crc32, formatUtcIso(fileInfo.registrationMillis()), links)); } return ResponseEntity.ok(new MeasurementManifest(sanitizedId, entries)); } + @GetMapping(value = "/measurements/{measurementId}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + @Operation(summary = "Download all files of a measurement as a ZIP archive") + @Parameter(name = "measurementId", required = true, description = "The identifier of the measurement", example = "NGSQ0001006AO-25948529211108") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "successful operation, the ZIP archive is downloaded", content = @Content(schema = @Schema(implementation = Void.class))), + @ApiResponse(responseCode = "403", description = "forbidden, you do not have access to this resource"), + @ApiResponse(responseCode = "404", description = "measurement not found"), + }) + public ResponseEntity downloadMeasurementAsZip( + @PathVariable("measurementId") String measurementId) { + String sanitizedId = sanitizeMeasurementId(measurementId); + String requestId = "downloadZip-" + UUID.randomUUID(); + String currentUser = SecurityContextHolder.getContext().getAuthentication().getName(); + + log.info("request {}: user {} requests ZIP download of measurement {}", requestId, + currentUser, sanitizedId); + + // Get the list of files + List files; + try { + files = storageFileIndex.files(sanitizedId); + } catch (DatasetNotFoundException e) { + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } catch (TransientException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); + } + + if (files.isEmpty()) { + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } + + // Get the storage provider + StorageProvider provider; + try { + provider = providerRegistry.getProvider(sanitizedId); + } catch (StorageProviderException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); + } + + // Generate ZIP filename using measurement ID + String zipFilename = sanitizedId + ".zip"; + + StreamingResponseBody responseBody = outputStream -> { + log.info("request {}: user {} started downloading ZIP of measurement {}", requestId, + currentUser, sanitizedId); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { + int fileIndex = 0; + for (FileInfo fileInfo : files) { + DataFile dataFile = provider.getFile(sanitizedId, fileIndex); + addToZip(zipOutputStream, dataFile, fileInfo, sanitizedId); + fileIndex++; + } + zipOutputStream.finish(); + log.info("request {}: user {} finished downloading ZIP of measurement {}", requestId, + currentUser, sanitizedId); + } catch (Exception e) { + if (isClientAbort(e)) { + log.warn("request {}: user {} disconnected while downloading ZIP of measurement {}", + requestId, currentUser, sanitizedId); + } else { + log.error("request {}: user {} failed for ZIP download of measurement {}", requestId, + currentUser, sanitizedId, e); + } + throw e; + } + }; + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); + headers.set(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + zipFilename + "\""); + + return ResponseEntity.ok().headers(headers).body(responseBody); + } + + /** + * Adds a single file to the ZIP output stream. + */ + private void addToZip(ZipOutputStream zipOutputStream, DataFile dataFile, FileInfo fileInfo, + String measurementId) throws IOException { + ZipEntry zipEntry = new ZipEntry(fileInfo.path()); + zipEntry.setSize(fileInfo.size()); + if (fileInfo.lastModifiedMillis() > 0) { + zipEntry.setTime(fileInfo.lastModifiedMillis()); + } + + zipOutputStream.putNextEntry(zipEntry); + try (InputStream inputStream = dataFile.inputStream()) { + byte[] buffer = new byte[downloadBufferSize]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + zipOutputStream.write(buffer, 0, bytesRead); + } + } + zipOutputStream.closeEntry(); + } + @GetMapping(value = "/measurements/{measurementId}/files/{index}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) @Operation(summary = "Download a single file of a measurement by its index, supporting resumable range requests") @Parameter(name = "measurementId", required = true, description = "The identifier of the measurement", example = "NGSQ0001006AO-25948529211108") @@ -143,45 +247,110 @@ public ResponseEntity downloadFile( @PathVariable("measurementId") String measurementId, @PathVariable("index") int index, @RequestHeader(value = HttpHeaders.RANGE, required = false) String rangeHeader) { - var sanitizedId = sanitizeMeasurementId(measurementId); - var measurementIdentifier = new MeasurementId(sanitizedId); - FileInfo fileInfo = measurementFileIndex.fileByIndex(measurementIdentifier, index) - .orElseThrow(() -> new GlobalException("request failed.", - ErrorCode.FILE_NOT_FOUND, ErrorParameters.of(sanitizedId))); - - // The index only carries file metadata; the content stream must be opened separately per - // request, otherwise the byte range could not be streamed independently of the manifest. - DataFile dataFile = measurementDataProvider.loadFile(measurementIdentifier, fileInfo); - if (dataFile == null) { - throw new GlobalException("request failed.", - ErrorCode.FILE_NOT_FOUND, ErrorParameters.of(sanitizedId)); + String sanitizedId = sanitizeMeasurementId(measurementId); + + FileInfo fileInfo; + try { + fileInfo = storageFileIndex.fileByIndex(sanitizedId, index) + .orElseThrow(() -> new GlobalException("request failed.", ErrorCode.FILE_NOT_FOUND, + ErrorParameters.of(sanitizedId))); + } catch (DatasetNotFoundException e) { + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } catch (TransientException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); + } + + StorageProvider provider; + try { + provider = providerRegistry.getProvider(sanitizedId); + } catch (StorageProviderException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); + } + + long fileLength = fileInfo.size(); + boolean isRangeCapable = provider instanceof ByteRangeProvider; + + // Parse the range header. For range-capable providers, delegate to the provider. + // For non-range-capable providers, handle via skip-to-start on the whole-file stream. + ByteRange byteRange = null; + ByteRange.ResolvedRange resolvedRange = null; + boolean isPartial = false; + if (rangeHeader != null && !rangeHeader.isBlank()) { + try { + byteRange = ByteRangeParser.parse(rangeHeader); + if (byteRange != null) { + resolvedRange = byteRange.resolve(fileLength); + isPartial = true; + } + } catch (InvalidByteRangeException e) { + throw new GlobalException("range not satisfiable", ErrorCode.RANGE_NOT_SATISFIABLE, + ErrorParameters.of(fileLength)); + } + } + + // Obtain the DataFile, pushing range handling to the provider when supported. + DataFile dataFile; + try { + if (isRangeCapable && byteRange != null) { + dataFile = ((ByteRangeProvider) provider).getFile(sanitizedId, index, byteRange); + } else { + dataFile = provider.getFile(sanitizedId, index); + } + } catch (DatasetNotFoundException e) { + throw new GlobalException("request failed.", ErrorCode.MEASUREMENT_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } catch (StorageFileNotFoundException e) { + throw new GlobalException("request failed.", ErrorCode.FILE_NOT_FOUND, + ErrorParameters.of(sanitizedId)); + } catch (InvalidByteRangeException e) { + throw new GlobalException("range not satisfiable", ErrorCode.RANGE_NOT_SATISFIABLE, + ErrorParameters.of(fileLength)); + } catch (TransientException e) { + throw new GlobalException("request failed.", ErrorCode.GENERAL, ErrorParameters.empty()); } + String requestId = "downloadFile-" + UUID.randomUUID(); String currentUser = SecurityContextHolder.getContext().getAuthentication().getName(); - log.info("request %s: user %s requests file %s of measurement %s".formatted(requestId, - currentUser, fileInfo.path(), sanitizedId)); + log.info("request {}: user {} requests file {} of measurement {}", requestId, + currentUser, fileInfo.path(), sanitizedId); + + // Determine the effective start offset and content length. + // For range-capable providers, the stream already starts at the range offset. + // For non-range-capable providers, we skip to the start manually. + long start; + long contentLength; + long end; + if (isRangeCapable && resolvedRange != null) { + start = resolvedRange.start(); + end = resolvedRange.end(); + contentLength = resolvedRange.length(); + } else if (resolvedRange != null) { + start = resolvedRange.start(); + end = resolvedRange.end(); + contentLength = resolvedRange.length(); + } else { + start = 0; + end = fileLength - 1; + contentLength = fileLength; + } - long fileLength = fileInfo.length(); - ByteRange.Range requestedRange = byteRange.parse(rangeHeader, fileLength); - long start = requestedRange.start(); - long end = requestedRange.end(); - long contentLength = requestedRange.length(); - boolean isPartial = requestedRange.isPartial(); + final long skipTo = isRangeCapable ? 0 : start; StreamingResponseBody responseBody = outputStream -> { - log.info("request %s: user %s started downloading file %s of measurement %s".formatted( - requestId, currentUser, fileInfo.path(), sanitizedId)); + log.info("request {}: user {} started downloading file {} of measurement {}", + requestId, currentUser, fileInfo.path(), sanitizedId); try { - writeRange(dataFile, start, contentLength, outputStream, fileInfo.path(), sanitizedId); - log.info("request %s: user %s finished downloading file %s of measurement %s".formatted( - requestId, currentUser, fileInfo.path(), sanitizedId)); + writeRange(dataFile, skipTo, contentLength, outputStream, fileInfo.path(), sanitizedId); + log.info("request {}: user {} finished downloading file {} of measurement {}", + requestId, currentUser, fileInfo.path(), sanitizedId); } catch (Exception e) { if (isClientAbort(e)) { - log.warn("request %s: user %s disconnected while downloading file %s of measurement %s" - .formatted(requestId, currentUser, fileInfo.path(), sanitizedId)); + log.warn("request {}: user {} disconnected while downloading file {} of measurement {}", + requestId, currentUser, fileInfo.path(), sanitizedId); } else { - log.error("request %s: user %s failed for file %s of measurement %s".formatted(requestId, - currentUser, fileInfo.path(), sanitizedId), e); + log.error("request {}: user {} failed for file {} of measurement {}", requestId, + currentUser, fileInfo.path(), sanitizedId, e); } throw e; } @@ -190,9 +359,11 @@ public ResponseEntity downloadFile( HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); headers.setContentLength(contentLength); - headers.set(HttpHeaders.ACCEPT_RANGES, "bytes"); + if (isRangeCapable) { + headers.set(HttpHeaders.ACCEPT_RANGES, "bytes"); + } headers.set(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + PathFormatter.fileNameOf(fileInfo.path()) + "\""); + "attachment; filename=\"" + extractFileName(fileInfo.path()) + "\""); if (isPartial) { headers.set(HttpHeaders.CONTENT_RANGE, "bytes %d-%d/%d".formatted(start, end, fileLength)); @@ -201,47 +372,15 @@ public ResponseEntity downloadFile( return ResponseEntity.ok().headers(headers).body(responseBody); } - /** - * Checks whether the given exception indicates that the client disconnected during streaming - * (e.g. CURL killed with Ctrl+C). These are not server errors and should not be logged as such. - * The check is container-agnostic and works with both Tomcat and Jetty. - */ - private static boolean isClientAbort(Exception e) { - Throwable cause = e; - while (cause != null) { - // Check for well-known client abort exception types by name to avoid - // hard dependencies on a specific servlet container (Tomcat vs Jetty). - String className = cause.getClass().getName(); - if (className.equals("org.apache.catalina.connector.ClientAbortException") - || className.equals("org.eclipse.jetty.io.EofException")) { - return true; - } - String message = cause.getMessage(); - if (message != null && (message.contains("Broken pipe") - || message.contains("Connection reset by peer"))) { - return true; - } - cause = cause.getCause(); - } - return false; - } - /** * Writes a byte range from the data file to the output stream using an async producer-consumer - * pattern. A dedicated producer thread reads from the DSS input stream into a bounded queue, + * pattern. A dedicated producer thread reads from the input stream into a bounded queue, * while the consumer (calling thread) reads from the queue and writes to the client output stream. - * - *

This decoupling prevents slow clients from blocking DSS reads. Without this, a slow client - * would cause outputStream.write() to block, which in turn blocks inputStream.read(), causing - * the DSS TCP receive window to close and eventually the connection to be reset. - * - *

The bounded queue provides backpressure: when the queue is full, the producer blocks, which - * naturally closes the TCP receive window and signals the DSS to slow down. */ - private void writeRange(DataFile dataFile, long start, long contentLength, OutputStream outputStream, - String filePath, String measurementId) throws IOException { + private void writeRange(DataFile dataFile, long skipTo, long contentLength, + OutputStream outputStream, String filePath, String measurementId) throws IOException { try (InputStream inputStream = dataFile.inputStream()) { - skipToStart(inputStream, start); + skipToStart(inputStream, skipTo); Transfer transfer = startProducer(inputStream, contentLength, filePath); try { consume(transfer, outputStream, contentLength, filePath, measurementId); @@ -253,9 +392,7 @@ private void writeRange(DataFile dataFile, long start, long contentLength, Outpu /** * Advances the stream to the requested start offset. {@link InputStream#skip} is not guaranteed - * to skip the requested number of bytes, so we loop until the offset is reached. When skip makes - * no progress (e.g. on some sources), fall back to reading a single byte at a time so we never - * loop forever. + * to skip the requested number of bytes, so we loop until the offset is reached. */ private static void skipToStart(InputStream inputStream, long start) throws IOException { long skipped = 0; @@ -272,12 +409,6 @@ private static void skipToStart(InputStream inputStream, long start) throws IOEx } } - /** - * Starts a producer thread that reads from the DSS input stream into a bounded queue and returns - * the {@link Transfer} used by the consumer to drain it. The bounded queue decouples the DSS read - * (producer) from the client write (consumer); when it is full the producer blocks, providing - * backpressure to the DSS. - */ private Transfer startProducer(InputStream inputStream, long contentLength, String filePath) { BlockingQueue bufferQueue = new ArrayBlockingQueue<>(downloadQueueCapacity); AtomicReference producerError = new AtomicReference<>(); @@ -289,31 +420,29 @@ private Transfer startProducer(InputStream inputStream, long contentLength, Stri int read; while (remaining > 0 && (read = inputStream.read(buffer, 0, (int) Math.min(buffer.length, remaining))) != -1) { - // Copy the data since the buffer is reused for the next read byte[] data = Arrays.copyOf(buffer, read); - // put() blocks if the queue is full, providing backpressure to the DSS bufferQueue.put(data); remaining -= read; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); producerError.compareAndSet(null, e); - } catch (Exception|Error e) { + } catch (Exception | Error e) { producerError.compareAndSet(null, e); } finally { producerDone.set(true); } - }, "dss-reader-" + filePath); + }, "provider-reader-" + filePath); producer.start(); return new Transfer(producer, bufferQueue, producerError, producerDone); } - /** Drains the queue in the consumer (calling) thread, writing each buffer to the client. */ private void consume(Transfer transfer, OutputStream outputStream, long contentLength, String filePath, String measurementId) throws IOException { long totalBytesWritten = 0; long bytesSinceLastLog = 0; long lastProgressLogTime = System.currentTimeMillis(); + long lastNearFullLogTime = 0; // 0 so the first near-full warning is never throttled away try { while (!transfer.done.get() || !transfer.queue.isEmpty()) { byte[] data = transfer.queue.poll(POLL_TIMEOUT_MS, TimeUnit.MILLISECONDS); @@ -321,7 +450,7 @@ private void consume(Transfer transfer, OutputStream outputStream, long contentL outputStream.write(data); totalBytesWritten += data.length; bytesSinceLastLog += data.length; - logNearFullQueue(transfer, filePath, measurementId); + lastNearFullLogTime = logNearFullQueue(transfer, filePath, measurementId, lastNearFullLogTime); long[] result = logProgress(totalBytesWritten, bytesSinceLastLog, contentLength, lastProgressLogTime, filePath, measurementId, transfer.queue.size()); lastProgressLogTime = result[0]; @@ -340,22 +469,23 @@ private void consume(Transfer transfer, OutputStream outputStream, long contentL } /** - * Logs a warning whenever the bounded queue has fewer than {@link #nearFullQueueCapacity} free - * slots. A repeatedly near-full queue indicates the consumer (client write) cannot keep up with - * the producer (DSS read), which is a precursor to backpressure stalling the download. + * Logs a warning when the download queue is nearly full, throttled to at most one + * warning per {@code progressLogIntervalMs} to avoid flooding the logs when the + * queue stays near-full throughout a download. Returns the updated timestamp of the + * last logged warning. */ - private void logNearFullQueue(Transfer transfer, String filePath, String measurementId) { + private long logNearFullQueue(Transfer transfer, String filePath, String measurementId, + long lastNearFullLogTime) { int freeCapacity = downloadQueueCapacity - transfer.queue.size(); - if (freeCapacity < nearFullQueueCapacity) { + long now = System.currentTimeMillis(); + if (freeCapacity < nearFullQueueCapacity && (now - lastNearFullLogTime) >= progressLogIntervalMs) { log.warn("Download queue nearly full for file {} of measurement {}: {} of {} slots free", filePath, measurementId, freeCapacity, downloadQueueCapacity); + return now; } + return lastNearFullLogTime; } - /** Logs download progress at most once per {@link #progressLogIntervalMs}. Throughput is - * computed from the bytes written in the current interval only, so it reflects the actual recent - * transfer rate rather than the cumulative average. Returns the updated {@code {lastProgressLogTime, - * bytesSinceLastLog}} so the caller can reset its interval counters when a log was emitted. */ private long[] logProgress(long totalBytesWritten, long bytesSinceLastLog, long contentLength, long lastProgressLogTime, String filePath, String measurementId, int queueSize) { long currentTime = System.currentTimeMillis(); @@ -374,7 +504,63 @@ private long[] logProgress(long totalBytesWritten, long bytesSinceLastLog, long return new long[]{currentTime, 0}; } - /** Shared state handed to the consumer so it can coordinate with and drain the producer. */ + private static boolean isClientAbort(Exception e) { + Throwable cause = e; + while (cause != null) { + String className = cause.getClass().getName(); + if (className.equals("org.apache.catalina.connector.ClientAbortException") + || className.equals("org.eclipse.jetty.io.EofException")) { + return true; + } + String message = cause.getMessage(); + if (message != null && (message.contains("Broken pipe") + || message.contains("Connection reset by peer"))) { + return true; + } + cause = cause.getCause(); + } + return false; + } + + private String formatUtcIso(long epochMillis) { + if (epochMillis < 0) { + return null; + } + return DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(epochMillis)); + } + + private String sanitizeMeasurementId(String measurementId) { + if (MEASUREMENT_ID_PATTERN.matcher(measurementId).find()) { + throw new GlobalException("unexpected measurement identifier containing unallowed characters", + ErrorCode.ILLEGAL_MEASUREMENT_ID, + ErrorParameters.of("The provided measurement identifier contained unexpected characters.")); + } + return measurementId; + } + + /** + * Extracts the CRC-32 checksum from the file info, or 0 if none is available or the algorithm + * is not CRC-32. + */ + private static long parseCrc32(FileInfo fileInfo) { + if (fileInfo.checksum() == null) { + return 0; + } + if (!"crc32".equalsIgnoreCase(fileInfo.checksum().algorithm())) { + return 0; + } + try { + return Long.parseUnsignedLong(fileInfo.checksum().value()); + } catch (NumberFormatException e) { + return 0; + } + } + + private static String extractFileName(String path) { + int lastSeparator = path.lastIndexOf('/'); + return lastSeparator < 0 ? path : path.substring(lastSeparator + 1); + } + private static final class Transfer { final Thread producer; final BlockingQueue queue; @@ -392,23 +578,8 @@ private static final class Transfer { void throwIfFailed(String filePath) throws IOException { Throwable failure = error.get(); if (failure != null) { - throw new IOException("DSS read failed for file " + filePath, failure); + throw new IOException("Provider read failed for file " + filePath, failure); } } } - - private String formatUtcIso(long epochMillis) { - if (epochMillis < 0) { - return null; - } - return DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(epochMillis)); - } - - private String sanitizeMeasurementId(String measurementId) { - if (MEASUREMENT_ID_PATTERN.matcher(measurementId).find()) { - throw new GlobalException("unexpected measurement identifier containing unallowed characters", - ErrorCode.ILLEGAL_MEASUREMENT_ID, ErrorParameters.of("The provided measurement identifier contained unexpected characters.")); - } - return measurementId; - } } diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileIndex.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileIndex.java deleted file mode 100644 index a853d05..0000000 --- a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementFileIndex.java +++ /dev/null @@ -1,82 +0,0 @@ -package life.qbic.data_download.rest.download; - -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import life.qbic.data_download.measurements.api.FileInfo; -import life.qbic.data_download.measurements.api.MeasurementId; -import life.qbic.data_download.measurements.api.MeasurementDataProvider; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; - -/** - * Provides the ordered file list of a measurement with a short-lived cache. - *

- * The cached order is stable within the cache lifetime, so clients can rely on the manifest index - * to reference a specific file between subsequent requests. - */ -@Component -public class MeasurementFileIndex { - - private static final Comparator FILE_SORTING = Comparator.comparing(FileInfo::path); - - - private record CacheEntry(Instant createdAt, List files) { - - boolean expired(Duration ttl) { - return createdAt.plus(ttl).isBefore(Instant.now()); - } - } - - private final MeasurementDataProvider measurementDataProvider; - private final Duration cacheTtl; - private final Map cache = new ConcurrentHashMap<>(); - - public MeasurementFileIndex( - @Qualifier("measurementDataProvider") MeasurementDataProvider measurementDataProvider, - @org.springframework.beans.factory.annotation.Value("${server.manifest.cache-ttl:30s}") Duration cacheTtl) { - this.measurementDataProvider = measurementDataProvider; - this.cacheTtl = cacheTtl; - } - - /** - * Returns the ordered files of a measurement. - * - * @param measurementId the measurement - * @return the files in stable order, or an empty list if the measurement does not exist - */ - public List files(MeasurementId measurementId) { - String key = measurementId.id(); - // Cache the list for a short TTL so the manifest order stays identical between a client's - // manifest read and its subsequent per-file requests, and so repeated requests do not each - // trigger an openBIS file listing. - CacheEntry entry = cache.get(key); - if (entry != null && !entry.expired(cacheTtl)) { - return entry.files(); - } - List sortedFiles = Optional.ofNullable(measurementDataProvider.listFiles(measurementId)).orElse(List.of()) - .stream().sorted(FILE_SORTING) - .toList(); - cache.put(key, new CacheEntry(Instant.now(), sortedFiles)); - return sortedFiles; - } - - /** - * Resolves a file by its index within the ordered list. - * - * @param measurementId the measurement - * @param index the zero-based index of the file - * @return the file at the given index, or empty if out of bounds - */ - public Optional fileByIndex(MeasurementId measurementId, int index) { - List files = files(measurementId); - if (index < 0 || index >= files.size()) { - return Optional.empty(); - } - return Optional.of(files.get(index)); - } -} diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementZipDownloadController.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementZipDownloadController.java deleted file mode 100644 index 38bf269..0000000 --- a/rest-api/src/main/java/life/qbic/data_download/rest/download/MeasurementZipDownloadController.java +++ /dev/null @@ -1,147 +0,0 @@ -package life.qbic.data_download.rest.download; - -import static org.slf4j.LoggerFactory.getLogger; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.tags.Tag; -import java.io.IOException; -import java.io.OutputStream; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Optional; -import java.util.UUID; -import java.util.regex.Pattern; -import life.qbic.data_download.measurements.api.DataFile; -import life.qbic.data_download.measurements.api.MeasurementData; -import life.qbic.data_download.measurements.api.MeasurementDataProvider; -import life.qbic.data_download.measurements.api.MeasurementDataReader; -import life.qbic.data_download.measurements.api.MeasurementId; -import life.qbic.data_download.rest.exceptions.GlobalException; -import life.qbic.data_download.rest.exceptions.GlobalException.ErrorCode; -import life.qbic.data_download.rest.exceptions.GlobalException.ErrorParameters; -import life.qbic.data_download.util.zip.api.FileInfo; -import life.qbic.data_download.util.zip.api.FileTimes; -import life.qbic.data_download.util.zip.manipulation.BufferedZippingFunctions; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; - -@RestController -@Tag(name = "Download Endpoints", description = "Rest endpoints related to downloading data") -public class MeasurementZipDownloadController { - - private final MeasurementDataProvider measurementDataProvider; - private final MeasurementDataReaderFactory measurementDataReaderFactory; - private final int downloadBufferSize; - - private static final Logger log = getLogger(MeasurementZipDownloadController.class); - - public MeasurementZipDownloadController( - @Qualifier("measurementDataProvider") MeasurementDataProvider measurementDataProvider, - @Qualifier("measurementDataReaderFactory") MeasurementDataReaderFactory measurementDataReaderFactory, - @Value("${server.memory.download.buffer}") Integer downloadBufferSize) { - this.measurementDataProvider = measurementDataProvider; - this.measurementDataReaderFactory = measurementDataReaderFactory; - this.downloadBufferSize = Optional.ofNullable(downloadBufferSize) - .orElse(BufferedZippingFunctions.DEFAULT_BUFFER_SIZE); - } - - - @GetMapping(value = "/measurements/{measurementId}", produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Download a measurement from the given measurement identifier") - @Parameter(name = "measurementId", required = true, description = "The identifier of the measurement to download", example = "NGSQ0001006AO-25948529211108") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "successful operation, the measurement is retrieved asynchronously", content = @Content(schema = @Schema(implementation = Void.class))), - @ApiResponse(responseCode = "403", description = "forbidden, you do not have access to this resource", content = @Content(schema = @Schema(implementation = Void.class))), - @ApiResponse(responseCode = "404", description = "measurement not found", content = @Content(schema = @Schema(implementation = Void.class))), - }) - public ResponseEntity downloadMeasurement( - @PathVariable("measurementId") String measurementId) { - var sanitizedId = sanitizeMeasurementId(measurementId); - String currentUser = SecurityContextHolder.getContext().getAuthentication().getName(); - var requestId = "downloadMeasurement-" + UUID.randomUUID(); - log.info("request %s: user %s requests measurement %s".formatted(requestId, currentUser, - sanitizedId)); - var measurementIdentifier = new MeasurementId(sanitizedId); - MeasurementData measurementData = measurementDataProvider.loadData(measurementIdentifier); - if (measurementData == null) { - throw new GlobalException("request %s failed.".formatted(requestId), - ErrorCode.MEASUREMENT_NOT_FOUND, ErrorParameters.of(sanitizedId)); - } - String outputFileName = - sanitizedId + "-" - + LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyy-MM-dd.hhmmss")) - + ".zip"; - - StreamingResponseBody responseBody = outputStream -> { - log.info( - "request %s: user %s started downloading measurement %s".formatted(requestId, currentUser, - measurementIdentifier.id())); - try { - writeDataToStream(measurementIdentifier, - outputStream, - measurementData, - measurementDataReaderFactory.getMeasurementDataReader()); - log.info("request %s: user %s finished downloading measurement %s".formatted(requestId, - currentUser, measurementIdentifier.id())); - } catch (Exception e) { - //explicit log-rethrow to make sure it is logged even if in streaming response (after committing 200 for fresh request) - log.error("request %s: user %s failed for measurement %s".formatted(requestId, currentUser, - measurementIdentifier.id()), e); - throw e; - } - }; - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .header("Accept-Charset", "UTF-8") - .header("Expires", "0") - .header("Content-Disposition", "attachment;filename=" + outputFileName) - .body(responseBody); - } - - private String sanitizeMeasurementId(String measurementId) { - if (Pattern.compile("[^a-zA-Z0-9-]+").matcher(measurementId).hasMatch()) { - throw new GlobalException("unexpected measurement identifier containing unallowed characters", - ErrorCode.ILLEGAL_MEASUREMENT_ID, ErrorParameters.of("The provided measurement identifier contained unexpected characters.")); - } - return measurementId; - } - - private void writeDataToStream(MeasurementId measurementId, OutputStream outputStream, MeasurementData measurementData, - MeasurementDataReader measurementDataReader) { - //was checked previously. authentication must not be null - String currentUser = SecurityContextHolder.getContext().getAuthentication().getName(); - try (final var dataStream = measurementData.stream(); - final var zippedStream = BufferedZippingFunctions.zipInto(outputStream)) { - measurementDataReader.open(dataStream); - DataFile file; - while ((file = measurementDataReader.nextDataFile()) != null) { - FileInfo zipEntryFileInfo = new FileInfo( - file.fileInfo().path(), - file.fileInfo().length(), - file.fileInfo().crc32(), - new FileTimes(file.fileInfo().registrationMillis(), -1, - file.fileInfo().lastModifiedMillis())); - - BufferedZippingFunctions.addToZip(zippedStream, zipEntryFileInfo, file.inputStream(), downloadBufferSize); - } - } catch (IOException e) { - throw new GlobalException( - "User %s failed downloading measurement %s. %s".formatted(currentUser, measurementId.id(), - e.getMessage()), e); - } - } -} diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/download/StorageFileIndex.java b/rest-api/src/main/java/life/qbic/data_download/rest/download/StorageFileIndex.java new file mode 100644 index 0000000..b1ab623 --- /dev/null +++ b/rest-api/src/main/java/life/qbic/data_download/rest/download/StorageFileIndex.java @@ -0,0 +1,83 @@ +package life.qbic.data_download.rest.download; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * Provides the ordered file list of a dataset with a short-lived cache, backed by the + * {@link ProviderRegistry} and its resolved {@link StorageProvider}. + * + *

The cached order is stable within the cache lifetime, so clients can rely on the manifest + * index to reference a specific file between subsequent requests. + */ +@Component +public class StorageFileIndex { + + private record CacheEntry(Instant createdAt, List files) { + + boolean expired(Duration ttl) { + return createdAt.plus(ttl).isBefore(Instant.now()); + } + } + + private final ProviderRegistry providerRegistry; + private final Duration cacheTtl; + private final Map cache = new ConcurrentHashMap<>(); + + public StorageFileIndex(ProviderRegistry providerRegistry, + @Value("${server.manifest.cache-ttl:30s}") Duration cacheTtl) { + this.providerRegistry = providerRegistry; + this.cacheTtl = cacheTtl; + } + + /** + * Returns the ordered files of a dataset. + * + * @param datasetId the id of the dataset + * @return the files in stable order + * @throws life.qbic.data_download.storage.exception.DatasetNotFoundException if the dataset does not exist + * @throws life.qbic.data_download.storage.exception.StorageProviderException on any provider error + */ + public List files(String datasetId) { + CacheEntry entry = cache.get(datasetId); + if (entry != null && !entry.expired(cacheTtl)) { + return entry.files(); + } + StorageProvider provider = providerRegistry.getProvider(datasetId); + List files = provider.listFiles(datasetId); + cache.put(datasetId, new CacheEntry(Instant.now(), files)); + return files; + } + + /** + * Resolves a file by its index within the ordered list. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @return the file at the given index, or empty if out of bounds + */ + public Optional fileByIndex(String datasetId, int index) { + List files = files(datasetId); + if (index < 0 || index >= files.size()) { + return Optional.empty(); + } + return Optional.of(files.get(index)); + } + + /** + * Invalidates the cached file listing for the given dataset. + * + * @param datasetId the id of the dataset + */ + public void evict(String datasetId) { + cache.remove(datasetId); + } +} diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderProperties.java b/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderProperties.java new file mode 100644 index 0000000..e8d5f3a --- /dev/null +++ b/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderProperties.java @@ -0,0 +1,169 @@ +package life.qbic.data_download.rest.storage; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Binds the {@code providers.*} application properties. + * + *

Each key under {@code providers.instances} is a provider id, and its {@code type} selects the + * provider implementation. {@code providers.default-provider} optionally names the provider id used + * to serve datasets that are not explicitly mapped. + */ +@ConfigurationProperties(prefix = "providers") +public class ProviderProperties { + + private final Map instances = new LinkedHashMap<>(); + private String defaultProvider; + + public Map getInstances() { + return instances; + } + + public String getDefaultProvider() { + return defaultProvider; + } + + public void setDefaultProvider(String defaultProvider) { + this.defaultProvider = defaultProvider; + } + + /** + * A single configured provider. + */ + public static class Provider { + + private String type; + private boolean enabled = true; + private UserConfig user; + private ServerConfig server; + private FilenameConfig filename; + private Integer sessionTimeout; + private String mountPath; + private final Map additionalProperties = new LinkedHashMap<>(); + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public UserConfig getUser() { + return user; + } + + public void setUser(UserConfig user) { + this.user = user; + } + + public ServerConfig getServer() { + return server; + } + + public void setServer(ServerConfig server) { + this.server = server; + } + + public FilenameConfig getFilename() { + return filename; + } + + public void setFilename(FilenameConfig filename) { + this.filename = filename; + } + + public Integer getSessionTimeout() { + return sessionTimeout; + } + + public void setSessionTimeout(Integer sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } + + public String getMountPath() { + return mountPath; + } + + public void setMountPath(String mountPath) { + this.mountPath = mountPath; + } + + public Map getAdditionalProperties() { + return additionalProperties; + } + } + + /** + * User credentials configuration. + */ + public static class UserConfig { + private String name; + private String password; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + } + + /** + * Server configuration. + */ + public static class ServerConfig { + private String applicationUrl; + private String datastoreUrls; + + public String getApplicationUrl() { + return applicationUrl; + } + + public void setApplicationUrl(String applicationUrl) { + this.applicationUrl = applicationUrl; + } + + public String getDatastoreUrls() { + return datastoreUrls; + } + + public void setDatastoreUrls(String datastoreUrls) { + this.datastoreUrls = datastoreUrls; + } + } + + /** + * Filename configuration. + */ + public static class FilenameConfig { + private String wrapperDirectory; + + public String getWrapperDirectory() { + return wrapperDirectory; + } + + public void setWrapperDirectory(String wrapperDirectory) { + this.wrapperDirectory = wrapperDirectory; + } + } +} \ No newline at end of file diff --git a/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderRegistryConfig.java b/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderRegistryConfig.java new file mode 100644 index 0000000..17dcc04 --- /dev/null +++ b/rest-api/src/main/java/life/qbic/data_download/rest/storage/ProviderRegistryConfig.java @@ -0,0 +1,152 @@ +package life.qbic.data_download.rest.storage; + +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import life.qbic.data_download.measurements.api.MeasurementDataProvider; +import life.qbic.data_download.openbis.OpenBisConnector; +import life.qbic.data_download.openbis.OpenBisNfsStorageProvider; +import life.qbic.data_download.openbis.OpenBisStorageProvider; +import life.qbic.data_download.openbis.SessionFactory; +import life.qbic.data_download.storage.ConfigurableProviderRegistry; +import life.qbic.data_download.storage.DatasetProviderResolver; +import life.qbic.data_download.storage.ProviderDefinition; +import life.qbic.data_download.storage.ProviderFactory; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Wires the configured storage providers into a {@link ProviderRegistry}. + * + *

Supported provider types: + *

    + *
  • {@code openbis} - uses openBIS DSS HTTP API for metadata and file streaming
    • + *
  • {@code openbis-nfs} - uses openBIS for metadata, streams files from mounted NFS via NIO
    • + *
+ * + *

Each provider can have its own openBIS configuration (credentials, server URLs, etc.) as + * specified in the architecture document. + */ +@Configuration +@EnableConfigurationProperties(ProviderProperties.class) +public class ProviderRegistryConfig { + + @Bean + public ProviderFactory storageProviderFactory( + @Qualifier("measurementDataProvider") MeasurementDataProvider measurementDataProvider, + @Qualifier("openbisSessionFactory") SessionFactory sessionFactory, + ProviderProperties providerProperties) { + return definition -> { + ProviderProperties.Provider provider = providerProperties.getInstances().get(definition.id()); + if (provider == null) { + throw new IllegalArgumentException("No configuration found for provider: " + definition.id()); + } + + return switch (definition.type()) { + case "openbis" -> new OpenBisStorageProvider(measurementDataProvider); + case "openbis-nfs" -> createOpenBisNfsProvider(provider, sessionFactory); + default -> throw new IllegalArgumentException( + "unknown storage provider type: " + definition.type()); + }; + }; + } + + @Bean + public DatasetProviderResolver datasetProviderResolver(ProviderProperties properties) { + return new ConfigBackedDatasetProviderResolver(properties); + } + + @Bean + public ProviderRegistry providerRegistry(ProviderProperties properties, + ProviderFactory storageProviderFactory, + DatasetProviderResolver datasetProviderResolver) { + List definitions = properties.getInstances().entrySet().stream() + .map(e -> new ProviderDefinition(e.getKey(), e.getValue().getType(), + e.getValue().isEnabled(), e.getValue().getAdditionalProperties())) + .toList(); + return new ConfigurableProviderRegistry(definitions, storageProviderFactory, + datasetProviderResolver); + } + + /** + * Creates an OpenBisNfsStorageProvider from the given provider configuration. Requires + * provider-specific openBIS configuration (user, server, filename) and a {@code mount-path} + * specifying the root directory where openBIS data is mounted. + */ + private static OpenBisNfsStorageProvider createOpenBisNfsProvider( + ProviderProperties.Provider provider, SessionFactory sessionFactory) { + // Validate and extract user configuration + if (provider.getUser() == null) { + throw new IllegalArgumentException("openbis-nfs provider requires 'user' configuration"); + } + String userName = provider.getUser().getName(); + String password = provider.getUser().getPassword(); + if (userName == null || password == null) { + throw new IllegalArgumentException( + "openbis-nfs provider requires 'user.name' and 'user.password'"); + } + + // Validate and extract server configuration + if (provider.getServer() == null) { + throw new IllegalArgumentException("openbis-nfs provider requires 'server' configuration"); + } + String applicationUrl = provider.getServer().getApplicationUrl(); + String dataStoreUrls = provider.getServer().getDatastoreUrls(); + if (applicationUrl == null || dataStoreUrls == null) { + throw new IllegalArgumentException( + "openbis-nfs provider requires 'server.application-url' and 'server.datastore-urls'"); + } + + // Extract filename configuration (optional) + String wrapperDirectory = "original"; + if (provider.getFilename() != null && provider.getFilename().getWrapperDirectory() != null) { + wrapperDirectory = provider.getFilename().getWrapperDirectory(); + } + + // Validate and extract mount-path + String mountPathStr = provider.getMountPath(); + if (mountPathStr == null) { + throw new IllegalArgumentException("openbis-nfs provider requires 'mount-path'"); + } + Path mountPath = Path.of(mountPathStr); + + // Create provider-specific OpenBisConnector + List dataStoreUrlList = List.of(dataStoreUrls.split(",")); + OpenBisConnector connector = new OpenBisConnector( + sessionFactory, + applicationUrl, + dataStoreUrlList, + wrapperDirectory + ); + + return new OpenBisNfsStorageProvider(connector, mountPath, wrapperDirectory); + } + + /** + * A resolver that serves datasets from the configured default provider, falling back to the sole + * configured provider when no default is set. + */ + private static final class ConfigBackedDatasetProviderResolver implements DatasetProviderResolver { + + private final ProviderProperties properties; + + ConfigBackedDatasetProviderResolver(ProviderProperties properties) { + this.properties = properties; + } + + @Override + public Optional providerIdFor(String datasetId) { + if (properties.getDefaultProvider() != null) { + return Optional.of(properties.getDefaultProvider()); + } + if (properties.getInstances().size() == 1) { + return properties.getInstances().keySet().stream().findFirst(); + } + return Optional.empty(); + } + } +} diff --git a/rest-api/src/main/resources/application.properties b/rest-api/src/main/resources/application.properties index 3b62514..5e7784f 100644 --- a/rest-api/src/main/resources/application.properties +++ b/rest-api/src/main/resources/application.properties @@ -81,3 +81,30 @@ server.download.progress-log-interval=${DOWNLOAD_PROGRESS_LOG_INTERVAL_MS:30000} # consumer (client write) is not keeping up with the producer (DSS read). # Default: 3 server.download.near-full-queue-left=${DOWNLOAD_NEAR_FULL_QUEUE_LEFT:3} + +### Storage provider registry (new provider abstraction) +# Each provider id is configured with a type; the type selects the implementation and the +# properties it requires. `providers.default-provider` names the provider serving datasets that are +# not otherwise mapped. +# +# OpenBIS provider (uses DSS HTTP API for metadata and file streaming): +# providers.providers.openbis-1.type=openbis +# providers.providers.openbis-1.enabled=true +# providers.providers.openbis-1.session-timeout=3600 +# +# OpenBIS-NFS hybrid provider (uses openBIS for metadata, streams from mounted NFS via NIO): +# Each provider has its own openBIS configuration (credentials, server URLs, etc.) +providers.instances.openbis-nfs-1.type=openbis-nfs +providers.instances.openbis-nfs-1.enabled=true +providers.instances.openbis-nfs-1.user.name=${OPENBIS_USER_NAME} +providers.instances.openbis-nfs-1.user.password=${OPENBIS_USER_PASSWORD} +providers.instances.openbis-nfs-1.server.application-url=${OPENBIS_APPLICATION_URL} +providers.instances.openbis-nfs-1.server.datastore-urls=${OPENBIS_DATASTORE_URLS} +providers.instances.openbis-nfs-1.filename.wrapper-directory=${OPENBIS_FILE_WRAPPER_DIRECTORY:original} +providers.instances.openbis-nfs-1.session-timeout=3600 +providers.instances.openbis-nfs-1.mount-path=${OPENBIS_NFS_MOUNT_PATH:/tmp/openbis-nfs-test} +# +# Default provider (switch between openbis-1 and openbis-nfs-1): +providers.default-provider=${DEFAULT_PROVIDER_ID:openbis-nfs-1} + + diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/download/ByteRangeTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/download/ByteRangeTest.java deleted file mode 100644 index 6eac881..0000000 --- a/rest-api/src/test/java/life/qbic/data_download/rest/download/ByteRangeTest.java +++ /dev/null @@ -1,94 +0,0 @@ -package life.qbic.data_download.rest.download; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import life.qbic.data_download.rest.exceptions.GlobalException; -import life.qbic.data_download.rest.exceptions.GlobalException.ErrorCode; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -class ByteRangeTest { - - private final ByteRange byteRange = new ByteRange(); - - @Test - @DisplayName("no range header returns the whole file as a full response") - void noRangeReturnsWholeFile() { - ByteRange.Range range = byteRange.parse(null, 1000); - assertEquals(0, range.start()); - assertEquals(999, range.end()); - assertTrue(!range.isPartial()); - assertEquals(1000, range.length()); - } - - @Test - @DisplayName("blank range header returns the whole file") - void blankRangeReturnsWholeFile() { - ByteRange.Range range = byteRange.parse(" ", 1000); - assertEquals(0, range.start()); - assertEquals(999, range.end()); - assertTrue(!range.isPartial()); - } - - @Test - @DisplayName("single bounded range is parsed correctly") - void singleBoundedRange() { - ByteRange.Range range = byteRange.parse("bytes=100-199", 1000); - assertEquals(100, range.start()); - assertEquals(199, range.end()); - assertTrue(range.isPartial()); - assertEquals(100, range.length()); - } - - @Test - @DisplayName("open-ended range is bounded by the file length") - void openEndedRangeIsBoundedByFileLength() { - ByteRange.Range range = byteRange.parse("bytes=950-", 1000); - assertEquals(950, range.start()); - assertEquals(999, range.end()); - assertTrue(range.isPartial()); - assertEquals(50, range.length()); - } - - @Test - @DisplayName("range end beyond file length is clamped") - void rangeEndBeyondFileLengthIsClamped() { - ByteRange.Range range = byteRange.parse("bytes=0-5000", 1000); - assertEquals(0, range.start()); - assertEquals(999, range.end()); - } - - @Test - @DisplayName("range starting beyond the file is unsatisfiable") - void rangeStartingBeyondFileIsUnsatisfiable() { - GlobalException exception = assertThrows(GlobalException.class, - () -> byteRange.parse("bytes=1000-", 1000)); - assertEquals(ErrorCode.RANGE_NOT_SATISFIABLE, exception.errorCode()); - } - - @Test - @DisplayName("inverted range is unsatisfiable") - void invertedRangeIsUnsatisfiable() { - GlobalException exception = assertThrows(GlobalException.class, - () -> byteRange.parse("bytes=500-100", 1000)); - assertEquals(ErrorCode.RANGE_NOT_SATISFIABLE, exception.errorCode()); - } - - @Test - @DisplayName("suffix range is not supported and unsatisfiable") - void suffixRangeIsUnsatisfiable() { - GlobalException exception = assertThrows(GlobalException.class, - () -> byteRange.parse("bytes=-500", 1000)); - assertEquals(ErrorCode.RANGE_NOT_SATISFIABLE, exception.errorCode()); - } - - @Test - @DisplayName("multi-range header is not supported and unsatisfiable") - void multiRangeIsUnsatisfiable() { - GlobalException exception = assertThrows(GlobalException.class, - () -> byteRange.parse("bytes=0-499,500-999", 1000)); - assertEquals(ErrorCode.RANGE_NOT_SATISFIABLE, exception.errorCode()); - } -} \ No newline at end of file diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileControllerTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileControllerTest.java new file mode 100644 index 0000000..3706bd0 --- /dev/null +++ b/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileControllerTest.java @@ -0,0 +1,271 @@ +package life.qbic.data_download.rest.download; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.time.Duration; +import java.util.List; +import life.qbic.data_download.rest.exceptions.GlobalException; +import life.qbic.data_download.storage.ByteRange; +import life.qbic.data_download.storage.ByteRangeProvider; +import life.qbic.data_download.storage.DataFile; +import life.qbic.data_download.storage.FileInfo; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.authentication.TestingAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +class MeasurementFileControllerTest { + + private static final FileInfo CHECKSUM_FILE = new FileInfo("/data/read1.fastq.gz", + "read1.fastq.gz", 1024, new FileInfo.Checksum("crc32", "123456789"), 1700000000000L, + 1700000001000L); + private static final FileInfo NO_CHECKSUM_FILE = new FileInfo("/data/read2.fastq.gz", + "read2.fastq.gz", 2048, null, -1, -1); + + private FakeProviderRegistry providerRegistry; + private StorageFileIndex storageFileIndex; + private MeasurementFileController controller; + + @BeforeEach + void setUp() { + providerRegistry = new FakeProviderRegistry(); + storageFileIndex = new StorageFileIndex(providerRegistry, Duration.ofMinutes(1)); + controller = new MeasurementFileController( + providerRegistry, storageFileIndex, + 1024, 4, 30000L, 3); + SecurityContextHolder.getContext() + .setAuthentication(new TestingAuthenticationToken("test-user", null)); + } + + @Nested + @DisplayName("manifest endpoint") + class ManifestEndpoint { + + @Test + @DisplayName("returns the manifest with file entries for a known dataset") + void returnsManifest() { + providerRegistry.provider = new FakeStorageProvider(List.of(CHECKSUM_FILE, NO_CHECKSUM_FILE)); + + ResponseEntity response = controller.manifest("M-1"); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + MeasurementManifest manifest = response.getBody(); + assertNotNull(manifest); + assertEquals("M-1", manifest.measurementId()); + assertEquals(2, manifest.files().size()); + + MeasurementManifest.FileEntry first = manifest.files().get(0); + assertEquals(0, first.index()); + assertEquals("/data/read1.fastq.gz", first.path()); + assertEquals("read1.fastq.gz", first.fileName()); + assertEquals(1024, first.length()); + assertEquals(123456789L, first.crc32()); + assertNotNull(first.registrationTime()); + assertEquals("/measurements/M-1/files/0", first.links().download().href()); + + MeasurementManifest.FileEntry second = manifest.files().get(1); + assertEquals(0, second.crc32()); + assertNull(second.registrationTime()); + } + + @Test + @DisplayName("throws MEASUREMENT_NOT_FOUND for an unknown dataset") + void throwsForUnknownDataset() { + providerRegistry.provider = new FakeStorageProvider(List.of()); + + GlobalException ex = assertThrows(GlobalException.class, + () -> controller.manifest("M-unknown")); + assertEquals(GlobalException.ErrorCode.MEASUREMENT_NOT_FOUND, ex.errorCode()); + } + + @Test + @DisplayName("throws ILLEGAL_MEASUREMENT_ID for identifiers with invalid characters") + void throwsForIllegalId() { + GlobalException ex = assertThrows(GlobalException.class, + () -> controller.manifest("M 1!invalid")); + assertEquals(GlobalException.ErrorCode.ILLEGAL_MEASUREMENT_ID, ex.errorCode()); + } + } + + @Nested + @DisplayName("download endpoint") + class DownloadEndpoint { + + @Test + @DisplayName("returns the file content for a valid request") + void returnsFileContent() { + FakeStorageProvider provider = new FakeStorageProvider(List.of(CHECKSUM_FILE)); + provider.fileContent = "hello world".getBytes(); + providerRegistry.provider = provider; + + ResponseEntity response = controller.downloadFile("M-1", 0, null); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertEquals(1024, response.getHeaders().getContentLength()); + assertEquals("read1.fastq.gz", + response.getHeaders().getContentDisposition().getFilename()); + } + + @Test + @DisplayName("throws FILE_NOT_FOUND for an out-of-bounds index") + void throwsForOutOfBoundsIndex() { + providerRegistry.provider = new FakeStorageProvider(List.of(CHECKSUM_FILE)); + + GlobalException ex = assertThrows(GlobalException.class, + () -> controller.downloadFile("M-1", 5, null)); + assertEquals(GlobalException.ErrorCode.FILE_NOT_FOUND, ex.errorCode()); + } + + @Test + @DisplayName("sets Accept-Ranges header when provider supports byte ranges") + void setsAcceptRangesForRangeCapableProvider() { + providerRegistry.provider = new FakeRangeProvider(List.of(CHECKSUM_FILE)); + + ResponseEntity response = controller.downloadFile("M-1", 0, null); + + assertEquals("bytes", response.getHeaders().getFirst("Accept-Ranges")); + } + + @Test + @DisplayName("does not set Accept-Ranges header when provider does not support byte ranges") + void noAcceptRangesForNonRangeProvider() { + providerRegistry.provider = new FakeStorageProvider(List.of(CHECKSUM_FILE)); + + ResponseEntity response = controller.downloadFile("M-1", 0, null); + + assertNull(response.getHeaders().getFirst("Accept-Ranges")); + } + + @Test + @DisplayName("returns 206 partial content for a valid range request on a range-capable provider") + void returnsPartialContentForValidRange() { + providerRegistry.provider = new FakeRangeProvider(List.of(CHECKSUM_FILE)); + + ResponseEntity response = controller.downloadFile("M-1", 0, + "bytes=0-99"); + + assertEquals(HttpStatus.PARTIAL_CONTENT, response.getStatusCode()); + assertEquals(100, response.getHeaders().getContentLength()); + assertEquals("bytes 0-99/1024", response.getHeaders().getFirst("Content-Range")); + } + + @Test + @DisplayName("throws RANGE_NOT_SATISFIABLE for an invalid range header") + void throwsForInvalidRange() { + providerRegistry.provider = new FakeRangeProvider(List.of(CHECKSUM_FILE)); + + GlobalException ex = assertThrows(GlobalException.class, + () -> controller.downloadFile("M-1", 0, "bytes=abc")); + assertEquals(GlobalException.ErrorCode.RANGE_NOT_SATISFIABLE, ex.errorCode()); + } + + @Test + @DisplayName("throws RANGE_NOT_SATISFIABLE for an out-of-bounds range") + void throwsForOutOfBoundsRange() { + providerRegistry.provider = new FakeRangeProvider(List.of(CHECKSUM_FILE)); + + GlobalException ex = assertThrows(GlobalException.class, + () -> controller.downloadFile("M-1", 0, "bytes=2000-3000")); + assertEquals(GlobalException.ErrorCode.RANGE_NOT_SATISFIABLE, ex.errorCode()); + } + } + + /** A fake ProviderRegistry that returns a configurable provider. */ + private static final class FakeProviderRegistry implements ProviderRegistry { + StorageProvider provider; + + @Override + public StorageProvider getProvider(String datasetId) { + return provider; + } + } + + /** A fake StorageProvider that returns a fixed file list and optional content. */ + private static class FakeStorageProvider implements StorageProvider { + final List files; + byte[] fileContent = new byte[0]; + + FakeStorageProvider(List files) { + this.files = files; + } + + @Override + public List listFiles(String datasetId) { + if (files.isEmpty()) { + throw new DatasetNotFoundException(datasetId); + } + return files; + } + + @Override + public DataFile getFile(String datasetId, int index) { + if (index < 0 || index >= files.size()) { + throw new StorageFileNotFoundException(datasetId, index); + } + return new SimpleDataFile(files.get(index), fileContent); + } + + @Override + public FileInfo getFileMetadata(String datasetId, int index) { + if (index < 0 || index >= files.size()) { + throw new StorageFileNotFoundException(datasetId, index); + } + return files.get(index); + } + } + + /** A fake StorageProvider that also implements ByteRangeProvider. */ + private static final class FakeRangeProvider extends FakeStorageProvider + implements ByteRangeProvider { + + FakeRangeProvider(List files) { + super(files); + } + + @Override + public DataFile getFile(String datasetId, int index, ByteRange range) { + if (index < 0 || index >= files.size()) { + throw new StorageFileNotFoundException(datasetId, index); + } + FileInfo fi = files.get(index); + ByteRange.ResolvedRange resolved = range.resolve(fi.size()); + byte[] slice = new byte[(int) resolved.length()]; + return new SimpleDataFile(fi, slice); + } + } + + private static final class SimpleDataFile implements DataFile { + private final FileInfo fileInfo; + private final byte[] content; + + SimpleDataFile(FileInfo fileInfo, byte[] content) { + this.fileInfo = fileInfo; + this.content = content; + } + + @Override + public InputStream inputStream() { + return new ByteArrayInputStream(content); + } + + @Override + public FileInfo fileInfo() { + return fileInfo; + } + } +} diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileIndexTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileIndexTest.java deleted file mode 100644 index 2e3f025..0000000 --- a/rest-api/src/test/java/life/qbic/data_download/rest/download/MeasurementFileIndexTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package life.qbic.data_download.rest.download; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.time.Duration; -import java.util.List; -import life.qbic.data_download.measurements.api.DataFile; -import life.qbic.data_download.measurements.api.FileInfo; -import life.qbic.data_download.measurements.api.MeasurementData; -import life.qbic.data_download.measurements.api.MeasurementDataProvider; -import life.qbic.data_download.measurements.api.MeasurementId; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -class MeasurementFileIndexTest { - - private static final FileInfo Z = new FileInfo("/z", "z", 1, 1, 1, 1); - private static final FileInfo A = new FileInfo("/a", "a", 2, 2, 2, 2); - private static final FileInfo M = new FileInfo("/m", "m", 3, 3, 3, 3); - - private static final class FakeProvider implements MeasurementDataProvider { - - private final List files; - private int calls = 0; - - FakeProvider(List files) { - this.files = files; - } - - @Override - public MeasurementData loadData(MeasurementId measurementId) { - return null; - } - - @Override - public List listFiles(MeasurementId measurementId) { - calls++; - return files; - } - - @Override - public DataFile loadFile(MeasurementId measurementId, FileInfo fileInfo) { - return null; - } - } - - @Test - @DisplayName("fileByIndex resolves the file at the given sorted index alphabetically") - void fileByIndexResolvesSortedPosition() { - FakeProvider provider = new FakeProvider(List.of(Z, A, M)); - MeasurementFileIndex index = new MeasurementFileIndex(provider, Duration.ofMinutes(1)); - MeasurementId id = new MeasurementId("measurement-1"); - - // provider returns unsorted; index resolution is by list position (provider already sorted) - assertTrue(index.fileByIndex(id, 0).isPresent()); - assertEquals("/z", index.fileByIndex(id, 2).get().path()); - assertEquals("/a", index.fileByIndex(id, 0).get().path()); - assertEquals("/m", index.fileByIndex(id, 1).get().path()); - assertTrue(index.fileByIndex(id, 3).isEmpty()); - assertTrue(index.fileByIndex(id, -1).isEmpty()); - } - - @Test - @DisplayName("files are cached so the provider is not called repeatedly within the TTL") - void filesAreCachedWithinTtl() { - FakeProvider provider = new FakeProvider(List.of(A, M, Z)); - MeasurementFileIndex index = new MeasurementFileIndex(provider, Duration.ofMinutes(1)); - MeasurementId id = new MeasurementId("measurement-1"); - - index.files(id); - index.files(id); - index.files(id); - - assertEquals(1, provider.calls); - } - - @Test - @DisplayName("a null provider result is treated as an empty file list") - void nullProviderResultIsEmpty() { - MeasurementDataProvider nullProvider = new FakeProvider(null); - MeasurementFileIndex index = new MeasurementFileIndex(nullProvider, Duration.ofMinutes(1)); - assertTrue(index.files(new MeasurementId("measurement-1")).isEmpty()); - } -} diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/download/StorageFileIndexTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/download/StorageFileIndexTest.java new file mode 100644 index 0000000..b462049 --- /dev/null +++ b/rest-api/src/test/java/life/qbic/data_download/rest/download/StorageFileIndexTest.java @@ -0,0 +1,104 @@ +package life.qbic.data_download.rest.download; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import life.qbic.data_download.storage.DataFile; +import life.qbic.data_download.storage.FileInfo; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class StorageFileIndexTest { + + private static final FileInfo Z = new FileInfo("/z", "z", 1, null, 1, 1); + private static final FileInfo A = new FileInfo("/a", "a", 2, null, 2, 2); + private static final FileInfo M = new FileInfo("/m", "m", 3, null, 3, 3); + + private static final class FakeStorageProvider implements StorageProvider { + + private final List files; + private int listCalls = 0; + + FakeStorageProvider(List files) { + this.files = files; + } + + @Override + public List listFiles(String datasetId) { + listCalls++; + if ("nonexistent".equals(datasetId)) { + throw new DatasetNotFoundException(datasetId); + } + return files; + } + + @Override + public DataFile getFile(String datasetId, int index) { + throw new UnsupportedOperationException(); + } + + @Override + public FileInfo getFileMetadata(String datasetId, int index) { + throw new UnsupportedOperationException(); + } + } + + private static final class FakeProviderRegistry implements ProviderRegistry { + + private final FakeStorageProvider provider; + + FakeProviderRegistry(FakeStorageProvider provider) { + this.provider = provider; + } + + @Override + public StorageProvider getProvider(String datasetId) { + return provider; + } + } + + @Test + @DisplayName("fileByIndex resolves the file at the given index from the provider's listing") + void fileByIndexResolvesPosition() { + FakeStorageProvider provider = new FakeStorageProvider(List.of(Z, A, M)); + StorageFileIndex index = new StorageFileIndex(new FakeProviderRegistry(provider), Duration.ofMinutes(1)); + + assertEquals("/z", index.fileByIndex("ds-1", 0).get().path()); + assertEquals("/a", index.fileByIndex("ds-1", 1).get().path()); + assertEquals("/m", index.fileByIndex("ds-1", 2).get().path()); + assertTrue(index.fileByIndex("ds-1", 3).isEmpty()); + assertTrue(index.fileByIndex("ds-1", -1).isEmpty()); + } + + @Test + @DisplayName("files are cached so the provider is not called repeatedly within the TTL") + void filesAreCachedWithinTtl() { + FakeStorageProvider provider = new FakeStorageProvider(List.of(A, M, Z)); + StorageFileIndex index = new StorageFileIndex(new FakeProviderRegistry(provider), Duration.ofMinutes(1)); + + index.files("ds-1"); + index.files("ds-1"); + index.files("ds-1"); + + assertEquals(1, provider.listCalls); + } + + @Test + @DisplayName("evict removes the cached entry so the next call hits the provider again") + void evictClearsCache() { + FakeStorageProvider provider = new FakeStorageProvider(List.of(A)); + StorageFileIndex index = new StorageFileIndex(new FakeProviderRegistry(provider), Duration.ofMinutes(1)); + + index.files("ds-1"); + index.evict("ds-1"); + index.files("ds-1"); + + assertEquals(2, provider.listCalls); + } +} diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigOpenBisNfsTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigOpenBisNfsTest.java new file mode 100644 index 0000000..a847276 --- /dev/null +++ b/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigOpenBisNfsTest.java @@ -0,0 +1,80 @@ +package life.qbic.data_download.rest.storage; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayInputStream; +import java.util.List; +import life.qbic.data_download.measurements.api.FileInfo; +import life.qbic.data_download.measurements.api.MeasurementData; +import life.qbic.data_download.measurements.api.MeasurementDataProvider; +import life.qbic.data_download.measurements.api.MeasurementId; +import life.qbic.data_download.openbis.OpenBisStorageProvider; +import life.qbic.data_download.openbis.SessionFactory; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Tests for ProviderRegistryConfig. Note: openbis-nfs provider type requires a real OpenBisConnector + * bean which is complex to mock in unit tests. Integration tests verify the openbis-nfs provider + * works end-to-end with a real openBIS connection. + */ +class ProviderRegistryConfigOpenBisNfsTest { + + private final ApplicationContextRunner context = new ApplicationContextRunner() + .withUserConfiguration(TestConfig.class, ProviderRegistryConfig.class); + + @Test + @DisplayName("openbis provider type works") + void openBisProviderTypeWorks() { + context + .withPropertyValues( + "providers.default-provider=openbis-1", + "providers.instances.openbis-1.type=openbis", + "providers.instances.openbis-1.enabled=true") + .run(ctx -> { + assertThat(ctx).hasSingleBean(ProviderRegistry.class); + ProviderRegistry registry = ctx.getBean(ProviderRegistry.class); + StorageProvider provider = registry.getProvider("M-1"); + assertThat(provider).isInstanceOf(OpenBisStorageProvider.class); + }); + } + + /** + * Test configuration with fake beans for testing. + */ + @Configuration + static class TestConfig { + + @Bean + MeasurementDataProvider measurementDataProvider() { + return new MeasurementDataProvider() { + @Override + public MeasurementData loadData(MeasurementId measurementId) { + return () -> new ByteArrayInputStream(new byte[0]); + } + + @Override + public List listFiles(MeasurementId measurementId) { + return List.of(); + } + + @Override + public life.qbic.data_download.measurements.api.DataFile loadFile( + MeasurementId measurementId, FileInfo fileInfo) { + return null; + } + }; + } + + @Bean("openbisSessionFactory") + SessionFactory openbisSessionFactory() { + // Return a stub SessionFactory for tests that don't actually use it + return new SessionFactory("http://localhost", "test", "test"); + } + } +} diff --git a/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigTest.java b/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigTest.java new file mode 100644 index 0000000..74d28d5 --- /dev/null +++ b/rest-api/src/test/java/life/qbic/data_download/rest/storage/ProviderRegistryConfigTest.java @@ -0,0 +1,86 @@ +package life.qbic.data_download.rest.storage; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.util.List; +import life.qbic.data_download.measurements.api.FileInfo; +import life.qbic.data_download.measurements.api.MeasurementData; +import life.qbic.data_download.measurements.api.MeasurementDataProvider; +import life.qbic.data_download.measurements.api.MeasurementId; +import life.qbic.data_download.openbis.OpenBisStorageProvider; +import life.qbic.data_download.openbis.SessionFactory; +import life.qbic.data_download.storage.ProviderRegistry; +import life.qbic.data_download.storage.StorageProvider; +import life.qbic.data_download.storage.exception.ProviderException; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.annotation.UserConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +class ProviderRegistryConfigTest { + + private final ApplicationContextRunner context = new ApplicationContextRunner() + .withUserConfiguration(TestConfig.class, ProviderRegistryConfig.class); + + @Test + void registryResolvesDatasetToConfiguredOpenbisProvider() { + context + .withPropertyValues( + "providers.default-provider=openbis-1", + "providers.instances.openbis-1.type=openbis", + "providers.instances.openbis-1.enabled=true") + .run(ctx -> { + assertThat(ctx).hasSingleBean(ProviderRegistry.class); + ProviderRegistry registry = ctx.getBean(ProviderRegistry.class); + StorageProvider provider = registry.getProvider("M-1"); + assertThat(provider).isInstanceOf(OpenBisStorageProvider.class); + }); + } + + @Test + void registryWithoutProvidersIsStartableButResolvesNothing() { + context.run(ctx -> { + assertThat(ctx).hasSingleBean(ProviderRegistry.class); + ProviderRegistry registry = ctx.getBean(ProviderRegistry.class); + assertThatThrownBy(() -> registry.getProvider("M-1")) + .isInstanceOf(ProviderException.class); + }); + } + + /** + * Test configuration with fake beans for testing. + */ + @Configuration + static class TestConfig { + + @Bean("measurementDataProvider") + MeasurementDataProvider measurementDataProvider() { + return new MeasurementDataProvider() { + @Override + public MeasurementData loadData(MeasurementId measurementId) { + return () -> new ByteArrayInputStream(new byte[0]); + } + + @Override + public List listFiles(MeasurementId measurementId) { + return List.of(); + } + + @Override + public life.qbic.data_download.measurements.api.DataFile loadFile( + MeasurementId measurementId, FileInfo fileInfo) { + return null; + } + }; + } + + @Bean("openbisSessionFactory") + SessionFactory openbisSessionFactory() { + // Return a stub SessionFactory for tests that don't actually use it + return new SessionFactory("http://localhost", "test", "test"); + } + } +} \ No newline at end of file diff --git a/setup-nfs-test.sh b/setup-nfs-test.sh new file mode 100755 index 0000000..4107958 --- /dev/null +++ b/setup-nfs-test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Setup script for testing OpenBisNfsStorageProvider locally +# +# This script creates a test directory structure matching openBIS's sharded +# physical storage layout and sets environment variables for local testing. +# +# Usage: +# ./setup-nfs-test.sh +# +# Example: +# ./setup-nfs-test.sh NGSQ27O50001A0-1481421688841090 + +set -e + +if [ $# -lt 1 ]; then + echo "Usage: $0 " + echo "Example: $0 NGSQ27O50001A0-1481421688841090" + exit 1 +fi + +MEASUREMENT_ID="$1" +TEST_DIR="/tmp/openbis-nfs-test" + +echo "Setting up NFS test environment for measurement: $MEASUREMENT_ID" +echo "Test directory: $TEST_DIR" + +# Create test directory +mkdir -p "$TEST_DIR" + +# Fetch the manifest to get file paths +echo "Fetching manifest..." +MANIFEST=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN:-6lh19z738279j9737jdxtaJn1h9W6K9Z}" \ + "http://localhost:8090/measurements/$MEASUREMENT_ID/files") + +if [ -z "$MANIFEST" ] || echo "$MANIFEST" | jq -e '.measurementId == null' > /dev/null 2>&1; then + echo "Error: Could not fetch manifest. Is the server running and token valid?" + exit 1 +fi + +# We need to get the physical location from openBIS +# For now, we'll create a placeholder and the user needs to update it +# In a real scenario, you'd query the openBIS API directly +echo "" +echo "IMPORTANT: You need to get the physical location from the server logs." +echo "Make a test request to see the physical location:" +echo " curl -H 'Authorization: Bearer ${ACCESS_TOKEN:-6lh19z738279j9737jdxtaJn1h9W6K9Z}' \\" +echo " http://localhost:8090/measurements/$MEASUREMENT_ID/files/0" +echo "" +echo "Then check the logs for:" +echo " [NFS Provider] Physical location from openBIS for dataset $MEASUREMENT_ID: " +echo "" +echo "Once you have the physical location, run:" +echo " ./setup-nfs-test.sh $MEASUREMENT_ID " +echo "" + +# Check if physical location was provided +if [ $# -lt 2 ]; then + echo "Physical location not provided. Please check the logs and run again with the physical location." + exit 0 +fi + +PHYSICAL_LOCATION="$2" +echo "Using physical location: $PHYSICAL_LOCATION" + +# Create the sharded directory structure +PHYSICAL_DIR="$TEST_DIR/$PHYSICAL_LOCATION" +mkdir -p "$PHYSICAL_DIR" + +# Extract file paths and create test files +echo "Creating test files in sharded structure..." +FILE_COUNT=$(echo "$MANIFEST" | jq '.files | length') +echo "Found $FILE_COUNT files" + +for i in $(seq 0 $((FILE_COUNT - 1))); do + FILE_PATH=$(echo "$MANIFEST" | jq -r ".files[$i].path") + FILE_SIZE=$(echo "$MANIFEST" | jq -r ".files[$i].length") + + # Create file under the physical location directory + FULL_PATH="$PHYSICAL_DIR/$FILE_PATH" + mkdir -p "$(dirname "$FULL_PATH")" + + # Create a test file with some content + if [ "$FILE_SIZE" -eq 0 ]; then + # Empty file + touch "$FULL_PATH" + else + # Create file with test content (use dd to create exact size) + dd if=/dev/urandom of="$FULL_PATH" bs=1024 count=$((FILE_SIZE / 1024 + 1)) 2>/dev/null + # Truncate to exact size + truncate -s "$FILE_SIZE" "$FULL_PATH" + fi + + echo " Created: $FILE_PATH ($FILE_SIZE bytes)" +done + +echo "" +echo "Test setup complete!" +echo "Files created under: $PHYSICAL_DIR" +echo "" +echo "To test with NFS provider, restart the server with:" +echo " export OPENBIS_NFS_MOUNT_PATH=$TEST_DIR" +echo " export DEFAULT_PROVIDER_ID=openbis-nfs-1" +echo " export DOWNLOAD_CONTROLLER_VERSION=v2" +echo " cd rest-api && mvn spring-boot:run -DskipTests" +echo "" +echo "Then test downloads with:" +echo " curl -H 'Authorization: Bearer ${ACCESS_TOKEN:-6lh19z738279j9737jdxtaJn1h9W6K9Z}' \\" +echo " http://localhost:8090/measurements/$MEASUREMENT_ID/files/0 -o /tmp/test.gz" diff --git a/storage-provider/pom.xml b/storage-provider/pom.xml new file mode 100644 index 0000000..9bbb426 --- /dev/null +++ b/storage-provider/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + life.qbic + data-download-server + ${revision} + + + life.qbic.data-download + storage-provider + jar + + + + org.junit.jupiter + junit-jupiter + 5.12.0 + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + life.qbic.data_download.storage + + + + + + + + \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRange.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRange.java new file mode 100644 index 0000000..3260125 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRange.java @@ -0,0 +1,58 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.InvalidByteRangeException; + +/** + * A byte range of a file, as defined by RFC 9110 §14.1.2. Represents a request for partial content + * from a {@link ByteRangeProvider}. + * + *

The three permitted forms mirror the RFC 9110 {@code byte-range-spec} grammar: + * + * 

+ * byte-range-spec = first-byte-pos "-" [last-byte-pos]  ; int-range
+ *                 | "-" suffix-length                    ; suffix-range
+ *
+ * + *
    + *
  • {@link FromToRange} — {@code bytes=start-end} (both offsets inclusive)
    • + *
  • {@link FromStartRange} — {@code bytes=start-} (to the end of the file)
    • + *
  • {@link SuffixRange} — {@code bytes=-length} (the last {@code length} bytes)
    • + *
+ * + *

A {@code null} range passed to a provider denotes the whole file. + */ +public sealed interface ByteRange permits FromToRange, FromStartRange, SuffixRange { + + /** + * Resolves this range against a known file size into a concrete, inclusive + * {@link ResolvedRange}. Throws {@link InvalidByteRangeException} when the range is + * unsatisfiable for the given size. + * + * @param fileSize the total size of the file in bytes + * @return the resolved inclusive byte range + * @throws InvalidByteRangeException if the range is unsatisfiable for the given file size + */ + ResolvedRange resolve(long fileSize); + + /** + * An inclusive byte range resolved against a concrete file size. + * + * @param start the first byte offset, inclusive + * @param end the last byte offset, inclusive + */ + record ResolvedRange(long start, long end) { + + public ResolvedRange { + if (start < 0 || end < start) { + throw new IllegalArgumentException("invalid resolved range " + start + "-" + end); + } + } + + /** + * The number of bytes in this range ({@code end - start + 1}). + */ + public long length() { + return end - start + 1; + } + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeParser.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeParser.java new file mode 100644 index 0000000..35e7eb0 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeParser.java @@ -0,0 +1,69 @@ +package life.qbic.data_download.storage; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import life.qbic.data_download.storage.exception.InvalidByteRangeException; + +/** + * Parses a single byte-range request from an HTTP {@code Range} header value, per RFC 9110 §14.2 + * and §14.1.2. Only a single range spec is supported. + * + *

Accepted forms: + * + *

    + *
  • {@code bytes=0-499} → {@link FromToRange}
    • + *
  • {@code bytes=9500-} → {@link FromStartRange}
    • + *
  • {@code bytes=-500} → {@link SuffixRange}
    • + *
+ * + *

A blank or {@code null} header yields an empty result, which callers interpret as the whole + * file. Malformed values or multiple range specs throw {@link InvalidByteRangeException}. + */ +public final class ByteRangeParser { + + private static final Pattern BYTE_RANGE = Pattern.compile("bytes=([0-9]*)(-)([0-9]*)"); + + private ByteRangeParser() { + } + + /** + * Parses a single byte-range-spec from a {@code Range} header value. + * + * @param rangeHeader the {@code Range} header value, or {@code null}/{@code blank} for none + * @return the parsed {@link ByteRange}, or {@code null} if the header is absent/blank + * @throws InvalidByteRangeException if the header is malformed or contains multiple ranges + */ + public static ByteRange parse(String rangeHeader) { + if (rangeHeader == null || rangeHeader.isBlank()) { + return null; + } + Matcher matcher = BYTE_RANGE.matcher(rangeHeader.trim()); + if (!matcher.matches()) { + throw new InvalidByteRangeException("malformed Range header: " + rangeHeader); + } + String first = matcher.group(1); + String last = matcher.group(3); + if (first.isEmpty() && last.isEmpty()) { + throw new InvalidByteRangeException("malformed Range header: " + rangeHeader); + } + if (first.isEmpty()) { + // suffix-range: "-length" -> last N bytes + return new SuffixRange(parseLong(last, rangeHeader)); + } + long start = parseLong(first, rangeHeader); + if (last.isEmpty()) { + // int-range with absent last-pos: "start-" + return new FromStartRange(start); + } + // int-range with both bounds: "start-end" + return new FromToRange(start, parseLong(last, rangeHeader)); + } + + private static long parseLong(String digits, String header) { + try { + return Long.parseLong(digits); + } catch (NumberFormatException e) { + throw new InvalidByteRangeException("range offset out of range: " + header); + } + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeProvider.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeProvider.java new file mode 100644 index 0000000..06d7900 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ByteRangeProvider.java @@ -0,0 +1,36 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.InvalidByteRangeException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; + +/** + * A {@link StorageProvider} that can serve partial content, enabling resumable downloads via + * byte-range requests. + * + *

Implementing this interface is optional; providers that do not implement it serve whole files + * only. Consumers detect range support with {@code instanceof ByteRangeProvider} and fall back to + * {@link StorageProvider#getFile(String, int)} for whole-file downloads otherwise. + */ +public interface ByteRangeProvider { + + /** + * Streams a file by its index, honoring a byte range. + * + *

The returned stream starts at the range offset. A {@code null} range denotes the whole + * file. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @param range the requested byte range, or {@code null} for the whole file + * @return the file and a stream to its content starting at the range offset + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageFileNotFoundException if no file exists at the given index + * @throws InvalidByteRangeException if the range is malformed or out of bounds + * @throws StorageProviderException on any provider error + */ + DataFile getFile(String datasetId, int index, ByteRange range) + throws DatasetNotFoundException, StorageFileNotFoundException, InvalidByteRangeException, + StorageProviderException; +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ConfigurableProviderRegistry.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ConfigurableProviderRegistry.java new file mode 100644 index 0000000..2b48e6c --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ConfigurableProviderRegistry.java @@ -0,0 +1,67 @@ +package life.qbic.data_download.storage; + +import static java.util.Objects.requireNonNull; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import life.qbic.data_download.storage.exception.ProviderException; + +/** + * A {@link ProviderRegistry} built from configured {@link ProviderDefinition}s. + * + *

For every enabled definition it asks the given {@link ProviderFactory} to instantiate the + * {@link StorageProvider}, and keeps them indexed by provider id. {@link #getProvider(String)} + * resolves the dataset to a provider id via a {@link DatasetProviderResolver}, then returns the + * matching provider. + */ +public class ConfigurableProviderRegistry implements ProviderRegistry { + + private final Map providersById; + private final DatasetProviderResolver resolver; + + public ConfigurableProviderRegistry(Collection definitions, + ProviderFactory factory, DatasetProviderResolver resolver) { + requireNonNull(factory, "factory must not be null"); + this.resolver = requireNonNull(resolver, "resolver must not be null"); + this.providersById = buildProviders(requireNonNull(definitions, "definitions must not be null"), + factory); + } + + @Override + public StorageProvider getProvider(String datasetId) { + requireNonNull(datasetId, "datasetId must not be null"); + String providerId = resolver.providerIdFor(datasetId) + .orElseThrow(() -> new ProviderException("no provider configured for dataset " + datasetId)); + StorageProvider provider = providersById.get(providerId); + if (provider == null) { + throw new ProviderException("no provider with id " + providerId + " is configured"); + } + return provider; + } + + /** + * The provider configured for the given provider id, if any. + * + * @param providerId the provider id + * @return the provider, or empty if no such enabled provider is configured + */ + public java.util.Optional provider(String providerId) { + return java.util.Optional.ofNullable(providersById.get(providerId)); + } + + private static Map buildProviders(Collection definitions, + ProviderFactory factory) { + Map providers = new LinkedHashMap<>(); + for (ProviderDefinition definition : definitions) { + if (!definition.enabled()) { + continue; + } + if (providers.containsKey(definition.id())) { + throw new IllegalArgumentException("duplicate provider id: " + definition.id()); + } + providers.put(definition.id(), factory.create(definition)); + } + return providers; + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/DataFile.java b/storage-provider/src/main/java/life/qbic/data_download/storage/DataFile.java new file mode 100644 index 0000000..f254b0f --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/DataFile.java @@ -0,0 +1,23 @@ +package life.qbic.data_download.storage; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A file together with a stream to its content, as returned by a {@link StorageProvider}. + * + *

For a {@link ByteRangeProvider}, the stream returned by {@link #inputStream()} starts at the + * requested range offset. + */ +public interface DataFile { + + /** + * A stream over the file content. For range requests the stream starts at the range offset. + */ + InputStream inputStream() throws IOException; + + /** + * The metadata of the file. + */ + FileInfo fileInfo(); +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/DatasetProviderResolver.java b/storage-provider/src/main/java/life/qbic/data_download/storage/DatasetProviderResolver.java new file mode 100644 index 0000000..f503ec1 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/DatasetProviderResolver.java @@ -0,0 +1,20 @@ +package life.qbic.data_download.storage; + +import java.util.Optional; + +/** + * Resolves a dataset id to the id of the {@link StorageProvider} that should serve it. + * + *

Implementations back this mapping however they see fit (initially a database query); the + * {@link ProviderRegistry} consults this to pick the provider for a dataset. + */ +public interface DatasetProviderResolver { + + /** + * Returns the provider id that serves the given dataset, if any. + * + * @param datasetId the id of the dataset + * @return the provider id serving the dataset, or empty if none is mapped + */ + Optional providerIdFor(String datasetId); +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/FileInfo.java b/storage-provider/src/main/java/life/qbic/data_download/storage/FileInfo.java new file mode 100644 index 0000000..bd7bc65 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/FileInfo.java @@ -0,0 +1,42 @@ +package life.qbic.data_download.storage; + +import java.util.Objects; + +/** + * Metadata about a file in a dataset, as reported by a {@link StorageProvider}. + * + *

The {@code checksum} is an optional integrity value whose algorithm is chosen by the storage + * provider; a {@code null} checksum means the provider does not expose one. + * + * @param path the stable path of the file within the dataset + * @param fileName the plain file name (last path segment) + * @param size the file size in bytes + * @param checksum the integrity checksum, or {@code null} if none is available + * @param registrationMillis creation/registration time, or {@code -1} if unknown + * @param lastModifiedMillis last-modified time, or {@code -1} if unknown + */ +public record FileInfo(String path, String fileName, long size, Checksum checksum, + long registrationMillis, long lastModifiedMillis) { + + public FileInfo { + Objects.requireNonNull(path, "path must not be null"); + Objects.requireNonNull(fileName, "fileName must not be null"); + if (size < 0) { + throw new IllegalArgumentException("size must not be negative"); + } + } + + /** + * An integrity checksum whose algorithm is provider-selected. + * + * @param algorithm the checksum algorithm (e.g. {@code crc32}, {@code sha256}) + * @param value the checksum value + */ + public record Checksum(String algorithm, String value) { + + public Checksum { + Objects.requireNonNull(algorithm, "algorithm must not be null"); + Objects.requireNonNull(value, "value must not be null"); + } + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/FilePathProvider.java b/storage-provider/src/main/java/life/qbic/data_download/storage/FilePathProvider.java new file mode 100644 index 0000000..b28c4ea --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/FilePathProvider.java @@ -0,0 +1,30 @@ +package life.qbic.data_download.storage; + +import java.nio.file.Path; +import java.util.Optional; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; + +/** + * A {@link StorageProvider} backed by a locally accessible filesystem that can expose the native + * path of a file for direct NIO operations. + * + *

Implemented only by filesystem-backed providers (NFS, local mount). Consumers use the path to + * bypass the stream abstraction when NIO is preferable. + */ +public interface FilePathProvider { + + /** + * Resolves the native filesystem path of a file by its index, if available. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @return the file's path, or empty if it cannot be resolved + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageFileNotFoundException if no file exists at the given index + * @throws StorageProviderException on any provider error + */ + Optional getFilePath(String datasetId, int index) + throws DatasetNotFoundException, StorageFileNotFoundException, StorageProviderException; +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/FromStartRange.java b/storage-provider/src/main/java/life/qbic/data_download/storage/FromStartRange.java new file mode 100644 index 0000000..ff5c7ee --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/FromStartRange.java @@ -0,0 +1,30 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.InvalidByteRangeException; + +/** + * An int-range with an absent {@code last-pos}, {@code bytes=first-pos-}, per RFC 9110 §14.1.2. + * Selects the remainder of the file from {@code start} to the end. + * + * @param start the first byte offset, inclusive + */ +public record FromStartRange(long start) implements ByteRange { + + public FromStartRange { + if (start < 0) { + throw new IllegalArgumentException("start must not be negative"); + } + } + + @Override + public ResolvedRange resolve(long fileSize) { + if (fileSize <= 0) { + throw new InvalidByteRangeException("no satisfiable range for an empty file"); + } + if (start >= fileSize) { + throw new InvalidByteRangeException( + "range start " + start + " is out of bounds for a file of " + fileSize + " bytes"); + } + return new ResolvedRange(start, fileSize - 1); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/FromToRange.java b/storage-provider/src/main/java/life/qbic/data_download/storage/FromToRange.java new file mode 100644 index 0000000..11834d8 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/FromToRange.java @@ -0,0 +1,37 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.InvalidByteRangeException; + +/** + * An int-range with both bounds, {@code bytes=first-pos-last-pos} (inclusive on both ends), per + * RFC 9110 §14.1.2. For example {@code bytes=0-99} selects bytes 0 through 99. + * + *

The {@code last-pos} may point beyond the end of the file; {@link #resolve(long)} clamps it + * to the last byte of the file. + * + * @param start the first byte offset, inclusive + * @param end the last byte offset, inclusive + */ +public record FromToRange(long start, long end) implements ByteRange { + + public FromToRange { + if (start < 0) { + throw new IllegalArgumentException("start must not be negative"); + } + if (end < start) { + throw new IllegalArgumentException("end must not be smaller than start"); + } + } + + @Override + public ResolvedRange resolve(long fileSize) { + if (fileSize <= 0) { + throw new InvalidByteRangeException("no satisfiable range for an empty file"); + } + if (start >= fileSize) { + throw new InvalidByteRangeException( + "range start " + start + " is out of bounds for a file of " + fileSize + " bytes"); + } + return new ResolvedRange(start, Math.min(end, fileSize - 1)); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrl.java b/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrl.java new file mode 100644 index 0000000..2f6d9ac --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrl.java @@ -0,0 +1,19 @@ +package life.qbic.data_download.storage; + +import java.net.URI; +import java.time.Instant; +import java.util.Objects; + +/** + * A temporary URL that grants direct access to a file, issued by a {@link PresignedUrlProvider}. + * + * @param url the temporary URL + * @param expiresAt when the URL stops being valid + */ +public record PresignedUrl(URI url, Instant expiresAt) { + + public PresignedUrl { + Objects.requireNonNull(url, "url must not be null"); + Objects.requireNonNull(expiresAt, "expiresAt must not be null"); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrlProvider.java b/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrlProvider.java new file mode 100644 index 0000000..0db0507 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/PresignedUrlProvider.java @@ -0,0 +1,33 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.InvalidByteRangeException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; +import life.qbic.data_download.storage.exception.UrlGenerationException; + +/** + * A {@link StorageProvider} that can issue pre-signed URLs for direct client access, bypassing the + * download server entirely. + * + *

Implemented only by cloud-backed providers (S3, Azure Blob). + */ +public interface PresignedUrlProvider { + + /** + * Issues a temporary URL granting direct access to a file. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @param range the requested byte range, or {@code null} for the whole file + * @return a temporary pre-signed URL + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageFileNotFoundException if no file exists at the given index + * @throws InvalidByteRangeException if the range is malformed or out of bounds + * @throws UrlGenerationException if the URL could not be generated + * @throws StorageProviderException on any provider error + */ + PresignedUrl getPresignedUrl(String datasetId, int index, ByteRange range) + throws DatasetNotFoundException, StorageFileNotFoundException, InvalidByteRangeException, + UrlGenerationException, StorageProviderException; +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderDefinition.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderDefinition.java new file mode 100644 index 0000000..cbd4338 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderDefinition.java @@ -0,0 +1,33 @@ +package life.qbic.data_download.storage; + +import static java.util.Objects.requireNonNull; + +import java.util.Map; + +/** + * The configured definition of a storage provider, as bound from application configuration. + * + *

Each provider is identified by a unique {@code id}. The {@code type} selects the provider + * implementation and, in turn, which entries of {@code properties} are required. A disabled + * provider is not instantiated. + * + * @param id the unique provider id (the key under {@code providers:}) + * @param type the provider implementation type (e.g. {@code openbis}, {@code nfs}, {@code s3}) + * @param enabled whether the provider is active + * @param properties the type-specific configuration + */ +public record ProviderDefinition(String id, String type, boolean enabled, + Map properties) { + + public ProviderDefinition { + requireNonNull(id, "id must not be null"); + requireNonNull(type, "type must not be null"); + if (id.isBlank()) { + throw new IllegalArgumentException("id must not be blank"); + } + if (type.isBlank()) { + throw new IllegalArgumentException("type must not be blank"); + } + properties = properties == null ? Map.of() : Map.copyOf(properties); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderFactory.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderFactory.java new file mode 100644 index 0000000..62dd715 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderFactory.java @@ -0,0 +1,21 @@ +package life.qbic.data_download.storage; + +/** + * Builds a {@link StorageProvider} instance for a configured {@link ProviderDefinition}, dispatching + * on the definition's {@code type}. + * + *

A factory implementation knows the concrete provider classes of the types it supports and + * throws an exception for an unknown or unsupported type. + */ +@FunctionalInterface +public interface ProviderFactory { + + /** + * Creates a {@link StorageProvider} for the given definition. + * + * @param definition the configured provider definition + * @return the instantiated storage provider + * @throws IllegalArgumentException if the definition's type is unknown or unsupported + */ + StorageProvider create(ProviderDefinition definition); +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderRegistry.java b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderRegistry.java new file mode 100644 index 0000000..725ac50 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/ProviderRegistry.java @@ -0,0 +1,21 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.ProviderException; + +/** + * Resolves which {@link StorageProvider} handles a given dataset. + * + *

Implementations map a dataset id to a configured provider id, then return the provider + * instance for that id. + */ +public interface ProviderRegistry { + + /** + * Returns the {@link StorageProvider} responsible for the given dataset. + * + * @param datasetId the id of the dataset + * @return the storage provider for the dataset + * @throws ProviderException if no provider is configured for the dataset + */ + StorageProvider getProvider(String datasetId); +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/StorageProvider.java b/storage-provider/src/main/java/life/qbic/data_download/storage/StorageProvider.java new file mode 100644 index 0000000..24fc1dd --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/StorageProvider.java @@ -0,0 +1,58 @@ +package life.qbic.data_download.storage; + +import java.util.List; +import life.qbic.data_download.storage.exception.DatasetNotFoundException; +import life.qbic.data_download.storage.exception.StorageFileNotFoundException; +import life.qbic.data_download.storage.exception.StorageProviderException; + +/** + * Abstraction over a storage backend, exposing the files of a dataset through a lean, uniform + * contract. + * + *

Files are addressed by their index within the stable, deterministic order returned by + * {@link #listFiles(String)}. Callers can rely on that order being consistent for a given dataset + * (e.g. via a cached manifest), so an index resolved from a manifest stays valid for subsequent + * {@link #getFile(String, int)} calls. + * + *

Providers may implement additional capability interfaces (such as {@link ByteRangeProvider}) + * for behavior not every backend supports; consumers detect those via pattern matching. + */ +public interface StorageProvider { + + /** + * Lists all files of a dataset in stable, deterministic order. + * + * @param datasetId the id of the dataset + * @return the files of the dataset, in a stable order + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageProviderException on any provider error + */ + List listFiles(String datasetId) throws DatasetNotFoundException, StorageProviderException; + + /** + * Streams a file by its index within the ordered list returned by {@link #listFiles(String)}. + * The whole file is streamed from the start. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @return the file and a stream to its content + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageFileNotFoundException if no file exists at the given index + * @throws StorageProviderException on any provider error + */ + DataFile getFile(String datasetId, int index) + throws DatasetNotFoundException, StorageFileNotFoundException, StorageProviderException; + + /** + * Returns the metadata of a file by its index. + * + * @param datasetId the id of the dataset + * @param index the zero-based index of the file + * @return the file metadata + * @throws DatasetNotFoundException if the dataset does not exist + * @throws StorageFileNotFoundException if no file exists at the given index + * @throws StorageProviderException on any provider error + */ + FileInfo getFileMetadata(String datasetId, int index) + throws DatasetNotFoundException, StorageFileNotFoundException, StorageProviderException; +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/SuffixRange.java b/storage-provider/src/main/java/life/qbic/data_download/storage/SuffixRange.java new file mode 100644 index 0000000..aceaa79 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/SuffixRange.java @@ -0,0 +1,28 @@ +package life.qbic.data_download.storage; + +import life.qbic.data_download.storage.exception.InvalidByteRangeException; + +/** + * A suffix-range, {@code bytes=-suffix-length}, per RFC 9110 §14.1.2. Selects the last + * {@code length} bytes of the file; if the file is shorter than {@code length}, the entire file is + * used. + * + * @param length the number of trailing bytes to select; must be greater than zero + */ +public record SuffixRange(long length) implements ByteRange { + + public SuffixRange { + if (length <= 0) { + throw new IllegalArgumentException("suffix length must be greater than zero"); + } + } + + @Override + public ResolvedRange resolve(long fileSize) { + if (fileSize <= 0) { + throw new InvalidByteRangeException("no satisfiable range for an empty file"); + } + long start = Math.max(0, fileSize - length); + return new ResolvedRange(start, fileSize - 1); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/AuthorizationException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/AuthorizationException.java new file mode 100644 index 0000000..c9a4d84 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/AuthorizationException.java @@ -0,0 +1,13 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when the caller is not entitled to access the requested dataset. + * + *

Permanent failure: maps to a 403 for the client and must not be retried. + */ +public class AuthorizationException extends StorageProviderException { + + public AuthorizationException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/DatasetNotFoundException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/DatasetNotFoundException.java new file mode 100644 index 0000000..b5091ad --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/DatasetNotFoundException.java @@ -0,0 +1,20 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when the referenced dataset does not exist in the storage backend. + * + *

Permanent failure: maps to a 404 for the client and must not be retried. + */ +public class DatasetNotFoundException extends StorageProviderException { + + private final String datasetId; + + public DatasetNotFoundException(String datasetId) { + super("Dataset not found: " + datasetId); + this.datasetId = datasetId; + } + + public String datasetId() { + return datasetId; + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/InvalidByteRangeException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/InvalidByteRangeException.java new file mode 100644 index 0000000..6761c85 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/InvalidByteRangeException.java @@ -0,0 +1,17 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when a byte range is malformed or out of bounds. + * + *

Permanent failure: maps to a 416 for the client and must not be retried. + */ +public class InvalidByteRangeException extends StorageProviderException { + + public InvalidByteRangeException(String message) { + super(message); + } + + public InvalidByteRangeException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/NetworkException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/NetworkException.java new file mode 100644 index 0000000..90c5d52 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/NetworkException.java @@ -0,0 +1,12 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when a network-level failure prevents communication with the storage backend (timeout, + * connection reset, I/O error). Retryable. + */ +public class NetworkException extends TransientException { + + public NetworkException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderException.java new file mode 100644 index 0000000..d6089f3 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderException.java @@ -0,0 +1,16 @@ +package life.qbic.data_download.storage.exception; + +/** + * A permanent, non-retryable failure raised by the storage provider that does not fit any more + * specific category. Must not be retried. + */ +public class ProviderException extends StorageProviderException { + + public ProviderException(String message) { + super(message); + } + + public ProviderException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderUnavailableException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderUnavailableException.java new file mode 100644 index 0000000..eb22088 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/ProviderUnavailableException.java @@ -0,0 +1,16 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when the storage provider itself is temporarily unavailable (e.g. an upstream 503/502, or + * the backend is down). Retryable. + */ +public class ProviderUnavailableException extends TransientException { + + public ProviderUnavailableException(String message) { + super(message); + } + + public ProviderUnavailableException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageFileNotFoundException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageFileNotFoundException.java new file mode 100644 index 0000000..2e3444f --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageFileNotFoundException.java @@ -0,0 +1,26 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when the referenced file (by index) does not exist in the dataset. + * + *

Permanent failure: maps to a 404 for the client and must not be retried. + */ +public class StorageFileNotFoundException extends StorageProviderException { + + private final String datasetId; + private final int index; + + public StorageFileNotFoundException(String datasetId, int index) { + super("File not found in dataset " + datasetId + " at index " + index); + this.datasetId = datasetId; + this.index = index; + } + + public String datasetId() { + return datasetId; + } + + public int index() { + return index; + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageProviderException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageProviderException.java new file mode 100644 index 0000000..631793e --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/StorageProviderException.java @@ -0,0 +1,22 @@ +package life.qbic.data_download.storage.exception; + +/** + * Base class for all errors raised by a {@link life.qbic.data_download.storage.StorageProvider}. + * + *

This is an unchecked exception. Interface methods declare which of its subtypes they + * can throw in their signature for documentation purposes, but callers are not forced to handle + * them explicitly. + * + *

Only {@link TransientException} subclasses are considered retryable. All other subtypes are + * permanent and must not be retried blindly. + */ +public class StorageProviderException extends RuntimeException { + + public StorageProviderException(String message) { + super(message); + } + + public StorageProviderException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/TransientException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/TransientException.java new file mode 100644 index 0000000..ecf3ef3 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/TransientException.java @@ -0,0 +1,19 @@ +package life.qbic.data_download.storage.exception; + +/** + * Base class for errors that may succeed when retried. + * + *

This is the only branch of {@link StorageProviderException} that a retry mechanism may retry. + * Subclasses indicate transient, recoverable conditions such as network hiccups or a temporarily + * unavailable provider. Retrying a non-transient exception is a bug. + */ +public abstract class TransientException extends StorageProviderException { + + protected TransientException(String message) { + super(message); + } + + protected TransientException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/main/java/life/qbic/data_download/storage/exception/UrlGenerationException.java b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/UrlGenerationException.java new file mode 100644 index 0000000..4dbb634 --- /dev/null +++ b/storage-provider/src/main/java/life/qbic/data_download/storage/exception/UrlGenerationException.java @@ -0,0 +1,19 @@ +package life.qbic.data_download.storage.exception; + +/** + * Thrown when a pre-signed URL cannot be generated, even though the provider supports them. + * + *

This is distinct from a provider simply not supporting pre-signed URLs (which is expressed by + * not implementing {@code PresignedUrlProvider}); it signals a failure during generation that may + * or may not be transient. + */ +public class UrlGenerationException extends StorageProviderException { + + public UrlGenerationException(String message) { + super(message); + } + + public UrlGenerationException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/storage-provider/src/test/java/life/qbic/data_download/storage/ByteRangeTest.java b/storage-provider/src/test/java/life/qbic/data_download/storage/ByteRangeTest.java new file mode 100644 index 0000000..fd7726f --- /dev/null +++ b/storage-provider/src/test/java/life/qbic/data_download/storage/ByteRangeTest.java @@ -0,0 +1,131 @@ +package life.qbic.data_download.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import life.qbic.data_download.storage.exception.InvalidByteRangeException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class ByteRangeTest { + + // --- FromToRange --- + + @Test + @DisplayName("FromToRange resolves to inclusive start and end") + void fromToResolves() { + ByteRange.ResolvedRange r = new FromToRange(0, 99).resolve(1000); + assertEquals(0, r.start()); + assertEquals(99, r.end()); + assertEquals(100, r.length()); + } + + @Test + @DisplayName("FromToRange clamps an end beyond the file size") + void fromToClampsEnd() { + ByteRange.ResolvedRange r = new FromToRange(950, 2000).resolve(1000); + assertEquals(950, r.start()); + assertEquals(999, r.end()); + assertEquals(50, r.length()); + } + + @Test + @DisplayName("FromToRange throws for a start beyond the file size") + void fromToThrowsForOutOfBoundsStart() { + assertThrows(InvalidByteRangeException.class, () -> new FromToRange(1000, 2000).resolve(1000)); + } + + @Test + @DisplayName("FromToRange throws for an empty file") + void fromToThrowsForEmptyFile() { + assertThrows(InvalidByteRangeException.class, () -> new FromToRange(0, 99).resolve(0)); + } + + @Test + @DisplayName("FromToRange rejects a negative start and end before start") + void fromToRejectsInvalidBounds() { + assertThrows(IllegalArgumentException.class, () -> new FromToRange(-1, 5)); + assertThrows(IllegalArgumentException.class, () -> new FromToRange(10, 9)); + } + + // --- FromStartRange --- + + @Test + @DisplayName("FromStartRange resolves to the end of the file") + void fromStartResolves() { + ByteRange.ResolvedRange r = new FromStartRange(9500).resolve(10000); + assertEquals(9500, r.start()); + assertEquals(9999, r.end()); + assertEquals(500, r.length()); + } + + @Test + @DisplayName("FromStartRange throws for a start beyond the file size") + void fromStartThrowsForOutOfBoundsStart() { + assertThrows(InvalidByteRangeException.class, () -> new FromStartRange(10000).resolve(10000)); + } + + // --- SuffixRange --- + + @Test + @DisplayName("SuffixRange resolves to the last N bytes") + void suffixResolves() { + ByteRange.ResolvedRange r = new SuffixRange(500).resolve(10000); + assertEquals(9500, r.start()); + assertEquals(9999, r.end()); + assertEquals(500, r.length()); + } + + @Test + @DisplayName("SuffixRange uses the whole file when it is shorter than the suffix length") + void suffixUsesWholeFileWhenShorter() { + ByteRange.ResolvedRange r = new SuffixRange(2000).resolve(1000); + assertEquals(0, r.start()); + assertEquals(999, r.end()); + assertEquals(1000, r.length()); + } + + @Test + @DisplayName("SuffixRange rejects a non-positive length") + void suffixRejectsNonPositiveLength() { + assertThrows(IllegalArgumentException.class, () -> new SuffixRange(0)); + assertThrows(IllegalArgumentException.class, () -> new SuffixRange(-1)); + } + + // --- ByteRangeParser (RFC 9110 header values) --- + + @Test + @DisplayName("parser handles int-range with both bounds") + void parseIntRange() { + ByteRange range = ByteRangeParser.parse("bytes=0-499"); + assertEquals(new FromToRange(0, 499), range); + } + + @Test + @DisplayName("parser handles open-ended int-range") + void parseOpenEndedRange() { + ByteRange range = ByteRangeParser.parse("bytes=9500-"); + assertEquals(new FromStartRange(9500), range); + } + + @Test + @DisplayName("parser handles suffix-range") + void parseSuffixRange() { + ByteRange range = ByteRangeParser.parse("bytes=-500"); + assertEquals(new SuffixRange(500), range); + } + + @Test + @DisplayName("parser returns null for a blank header") + void parseBlankReturnsNull() { + assertEquals(null, ByteRangeParser.parse(null)); + assertEquals(null, ByteRangeParser.parse(" ")); + } + + @Test + @DisplayName("parser rejects a malformed header") + void parseRejectsMalformed() { + assertThrows(InvalidByteRangeException.class, () -> ByteRangeParser.parse("bytes=-")); + assertThrows(InvalidByteRangeException.class, () -> ByteRangeParser.parse("items=0-99")); + } +} \ No newline at end of file diff --git a/storage-provider/src/test/java/life/qbic/data_download/storage/ConfigurableProviderRegistryTest.java b/storage-provider/src/test/java/life/qbic/data_download/storage/ConfigurableProviderRegistryTest.java new file mode 100644 index 0000000..3999d26 --- /dev/null +++ b/storage-provider/src/test/java/life/qbic/data_download/storage/ConfigurableProviderRegistryTest.java @@ -0,0 +1,149 @@ +package life.qbic.data_download.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import life.qbic.data_download.storage.exception.ProviderException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class ConfigurableProviderRegistryTest { + + private final FakeProviderFactory factory = new FakeProviderFactory(); + + @Test + @DisplayName("registry resolves a dataset to its provider via the resolver") + void resolvesDatasetToProvider() { + ConfigurableProviderRegistry registry = newRegistry( + definitions(openbis("openbis-1")), + dataset -> Optional.of("openbis-1")); + + assertSame(factory.provider("openbis-1"), registry.getProvider("M-1")); + } + + @Test + @DisplayName("registry skips disabled providers") + void skipsDisabledProviders() { + ConfigurableProviderRegistry registry = newRegistry( + List.of(definition("openbis-1", true), definition("openbis-2", false)), + dataset -> Optional.of("openbis-2")); + + assertTrue(registry.provider("openbis-2").isEmpty()); + assertThrows(ProviderException.class, () -> registry.getProvider("M-1")); + } + + @Test + @DisplayName("registry throws ProviderException when no provider is mapped for a dataset") + void throwsWhenNoProviderMapped() { + ConfigurableProviderRegistry registry = newRegistry( + definitions(openbis("openbis-1")), + dataset -> Optional.empty()); + + assertThrows(ProviderException.class, () -> registry.getProvider("M-1")); + } + + @Test + @DisplayName("registry throws ProviderException when the resolved id is not configured") + void throwsWhenResolvedIdNotConfigured() { + ConfigurableProviderRegistry registry = newRegistry( + definitions(openbis("openbis-1")), + dataset -> Optional.of("missing")); + + assertThrows(ProviderException.class, () -> registry.getProvider("M-1")); + } + + @Test + @DisplayName("registry rejects duplicate provider ids") + void rejectsDuplicateProviderIds() { + ProviderFactory factory = new FakeProviderFactory(); + assertThrows(IllegalArgumentException.class, () -> new ConfigurableProviderRegistry( + List.of(openbis("openbis-1"), openbis("openbis-1")), + factory, dataset -> Optional.of("openbis-1"))); + } + + @Test + @DisplayName("factory is asked once per enabled provider with the full definition") + void factoryReceivesDefinition() { + new ConfigurableProviderRegistry( + definitions(openbis("openbis-1")), + factory, dataset -> Optional.of("openbis-1")); + + assertEquals(1, factory.calls); + assertEquals("openbis-1", factory.lastDefinition.id()); + assertEquals("openbis", factory.lastDefinition.type()); + } + + private ConfigurableProviderRegistry newRegistry(List definitions, + DatasetProviderResolver resolver) { + return new ConfigurableProviderRegistry(definitions, factory, resolver); + } + + private static List definitions(ProviderDefinition... defs) { + return List.of(defs); + } + + private static ProviderDefinition openbis(String id) { + return definition(id, true); + } + + private static ProviderDefinition definition(String id, boolean enabled) { + return new ProviderDefinition(id, "openbis", enabled, + Map.of("session-timeout", 3600)); + } + + /** A fake factory that records calls and returns a distinct provider per id. */ + private static final class FakeProviderFactory implements ProviderFactory { + + private final Map providers = new java.util.HashMap<>(); + private int calls; + private ProviderDefinition lastDefinition; + + FakeProviderFactory() { + } + + StorageProvider provider(String id) { + return providers.get(id); + } + + @Override + public StorageProvider create(ProviderDefinition definition) { + calls++; + lastDefinition = definition; + return providers.computeIfAbsent(definition.id(), i -> new FakeStorageProvider(i)); + } + } + + private static final class FakeStorageProvider implements StorageProvider { + + private final String id; + + FakeStorageProvider(String id) { + this.id = id; + } + + @Override + public List listFiles(String datasetId) { + return List.of(); + } + + @Override + public DataFile getFile(String datasetId, int index) { + throw new UnsupportedOperationException(); + } + + @Override + public FileInfo getFileMetadata(String datasetId, int index) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "FakeStorageProvider[" + id + "]"; + } + } +} \ No newline at end of file diff --git a/zip/pom.xml b/zip/pom.xml index df88bec..f750b18 100644 --- a/zip/pom.xml +++ b/zip/pom.xml @@ -6,7 +6,7 @@ life.qbic data-download-server - 1.0.10 + ${revision} life.qbic.data-download