diff --git a/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc b/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc index 0e6422659..a29c94e06 100644 --- a/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc +++ b/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc @@ -67,3 +67,31 @@ The preceding listing matches objects stored in your bucket in `/\{application}` │ ├── application-test.yml │ └── application.yml ``` + +[[search-paths]] +== Search paths + +Spring Cloud Config Server also supports `search-paths` for the AWS S3 backend, analogous to xref:./git-backend.adoc#placeholders-in-git-search-paths[`search-paths`] in the Git backend. You can specify a list of paths in the bucket to search for configuration files. The search paths can contain placeholders like `{application}`, `{profile}`, and `{label}`. + +The search paths support: +- *Literal paths*: Probing exact paths with supported file extensions (e.g. `properties`, `json`, `yml`, `yaml`). +- *Directory paths*: Scanning files under a specific directory. +- *Wildcards*: Patterns with `*` or `?` placeholders (e.g. `config/*`). + +The following configuration specifies search paths in the S3 bucket: + +[source,yaml] +---- +spring: + cloud: + config: + server: + awss3: + region: us-east-1 + bucket: bucket1 + search-paths: + - '{application}' + - 'config/{application}' + - 'settings/*' +---- + diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java index 1b5f757a7..340dead56 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java @@ -16,11 +16,15 @@ package org.springframework.cloud.config.server.environment; +import java.util.Collections; +import java.util.List; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cloud.config.server.support.EnvironmentRepositoryProperties; /** * @author Clay McCoy + * @author Geonwook Ham */ @ConfigurationProperties("spring.cloud.config.server.awss3") public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperties { @@ -41,13 +45,28 @@ public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperti private String bucket; /** - * Use application name as intermediate directory. Analogous to `searchPaths: - * {application}` from Git backend. + * Use application name as intermediate directory. Analogous to + * {@link #searchPaths} + * from Git backend. */ private boolean useDirectoryLayout; private int order = DEFAULT_ORDER; + /** + * List of directory paths to search for profiles in the bucket. + * Analogous to {@link #searchPaths} in Git backend. + */ + private List searchPaths = Collections.emptyList(); + + public List getSearchPaths() { + return searchPaths; + } + + public void setSearchPaths(List searchPaths) { + this.searchPaths = searchPaths; + } + public String getRegion() { return region; } diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index a1f15a2a2..0adb0701e 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -20,9 +20,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; +import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,6 +35,11 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; import org.springframework.beans.factory.config.YamlProcessor; import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; @@ -39,7 +49,9 @@ import org.springframework.core.Ordered; import org.springframework.core.env.Profiles; import org.springframework.core.io.InputStreamResource; +import org.springframework.util.AntPathMatcher; import org.springframework.util.ObjectUtils; +import org.springframework.util.PathMatcher; import org.springframework.util.StringUtils; import static org.springframework.cloud.config.server.environment.AwsS3EnvironmentRepository.PATH_SEPARATOR; @@ -48,15 +60,22 @@ * @author Clay McCoy * @author Scott Frederick * @author Daniel Aiken + * @author Geonwook Ham */ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordered, SearchPathLocator { + private final PathMatcher pathMatcher = new AntPathMatcher(); + protected static final String PATH_SEPARATOR = "/"; private static final Log LOG = LogFactory.getLog(AwsS3EnvironmentRepository.class); private static final String AWS_S3_RESOURCE_SCHEME = "s3://"; + private static final List SUPPORTED_EXTENSIONS = List.of(".properties", ".json", ".yml", ".yaml"); + + private static final List EMPTY_EXTENSION = List.of(""); + private final S3Client s3Client; private final String bucketName; @@ -67,16 +86,30 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere protected int order = Ordered.LOWEST_PRECEDENCE; + private final List searchPaths; + public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, ConfigServerProperties server) { this(s3Client, bucketName, false, server); } public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, ConfigServerProperties server) { + this(s3Client, bucketName, useApplicationAsDirectory, server, Collections.emptyList()); + } + + public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, + ConfigServerProperties server, List searchPaths) { this.s3Client = s3Client; this.bucketName = bucketName; this.serverProperties = server; this.useApplicationAsDirectory = useApplicationAsDirectory; + this.searchPaths = (searchPaths == null ? Collections.emptyList() : searchPaths); + } + + public AwsS3EnvironmentRepository(S3Client s3Client, AwsS3EnvironmentProperties properties, + ConfigServerProperties server) { + this(s3Client, properties.getBucket(), properties.isUseDirectoryLayout(), server, properties.getSearchPaths()); + this.order = properties.getOrder(); } @Override @@ -84,10 +117,6 @@ public int getOrder() { return this.order; } - public void setOrder(int order) { - this.order = order; - } - @Override public Environment findOne(String specifiedApplication, String specifiedProfiles, String specifiedLabel) { final String application = ObjectUtils.isEmpty(specifiedApplication) @@ -128,7 +157,6 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles private void addPropertySources(Environment environment, List apps, String[] profiles, List labels) { for (String label : labels) { - // If we have profiles, add property sources with those profiles for (String profile : profiles) { addPropertySourcesForApps(apps, app -> addProfileSpecificPropertySource(environment, app, profile, label)); @@ -175,7 +203,9 @@ private void addPropertySourcesForApps(List apps, Consumer addPr private void addNegatedProfilePropertySource(Environment environment, String app, String[] allProfiles, String label) { - List s3ConfigFiles = getNegatedProfileS3ConfigFileYaml(app, allProfiles, label); + List s3ConfigFiles = this.searchPaths.isEmpty() + ? getNegatedProfileS3ConfigFileYaml(app, allProfiles, label) : getS3ConfigFileWithSearchPaths(app, null, + label, key -> wrapKeyWithNegatedConfigFiles(key, app, allProfiles, label)); addPropertySource(environment, s3ConfigFiles); } @@ -200,40 +230,61 @@ private List getNegatedProfileS3ConfigFileYaml(String application, } private void addProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - List s3ConfigFiles = getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, - this::getProfileSpecificS3ConfigFileYaml); + List s3ConfigFiles = searchPaths.isEmpty() + ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, + this::getProfileSpecificS3ConfigFileYaml) + : getS3ConfigFileWithSearchPaths(app, profile, label, + key -> wrapKeyWithConfigFiles(key, app, profile, label)); addPropertySource(environment, s3ConfigFiles); } private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - List s3ConfigFiles = getS3ConfigFile(app, profile, label, - this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml); - addPropertySource(environment, s3ConfigFiles); + List s3ConfigFiles = searchPaths.isEmpty() ? getS3ConfigFile(app, profile, label, + this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml) + : Collections.emptyList(); + if (s3ConfigFiles != null) { + addPropertySource(environment, s3ConfigFiles); + } } private void addPropertySource(Environment environment, List s3ConfigFiles) { for (S3ConfigFile s3ConfigFile : s3ConfigFiles) { - final Properties config = s3ConfigFile.read(); - // This logic handles the case where the s3 file is a YAML file that is - // not profile specific (ie it does not have - in the name) - // and does not have any profile specific documents in it. In this case we do - // not want to include this - // property source we only want to include the document for the default - // profile. When we create - // the S3ConfigFile for this file we set the - // shouldIncludeWithEmptyProperties to false - // in ProfileSpecificYamlDocumentS3ConfigFile for this specific case. - if (config != null) { - if (!config.isEmpty() || s3ConfigFile.isShouldIncludeWithEmptyProperties()) { - environment.setVersion(s3ConfigFile.getVersion()); - config.putAll(serverProperties.getOverrides()); - PropertySource propertySource = new PropertySource(s3ConfigFile.getName(), config); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding property source to environment " + propertySource); - } - environment.add(propertySource); + if (s3ConfigFile == null) { + continue; + } + try { + final Properties config = s3ConfigFile.read(); + // This logic handles the case where the s3 file is a YAML file that is + // not profile specific (ie it does not have - in the name) + // and does not have any profile specific documents in it. In this case we + // do + // not want to include this + // property source we only want to include the document for the default + // profile. When we create + // the S3ConfigFile for this file we set the + // shouldIncludeWithEmptyProperties to false + // in ProfileSpecificYamlDocumentS3ConfigFile for this specific case. + if (config == null || (config.isEmpty() && !s3ConfigFile.isShouldIncludeWithEmptyProperties())) { + continue; + } + String name = s3ConfigFile.getName(); + boolean exists = environment.getPropertySources() + .stream() + .anyMatch(p -> p.getName().equals(name) && p.getSource().equals(config)); + if (exists) { + continue; + } + environment.setVersion(s3ConfigFile.getVersion()); + config.putAll(serverProperties.getOverrides()); + PropertySource propertySource = new PropertySource(name, config); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding property source to environment " + propertySource); } + environment.add(propertySource); + } + catch (Exception e) { + LOG.warn("Could not read properties from " + s3ConfigFile.getName(), e); } } } @@ -244,12 +295,202 @@ private String[] parseProfiles(String profiles) { private List getS3ConfigFile(String application, String profile, String label, JsonOrPropertiesS3ConfigFileCreator creator, YamlS3ConfigFileCreator yamlCreator) { + List configFiles = new ArrayList<>(); S3ConfigFile configFile = creator.create(application, profile, label); if (configFile != null) { - return List.of(configFile); + configFiles.add(configFile); + } + configFiles.addAll(yamlCreator.create(application, profile, label)); + return configFiles; + } + + private List getS3ConfigFileWithSearchPaths(String application, String profile, String label, + Function> keyWrapper) { + List result = new ArrayList<>(); + Set seenKeys = new LinkedHashSet<>(); + for (String template : this.searchPaths) { + String pattern = resolvePattern(template, application, profile, label); + if (!pathMatcher.isPattern(pattern)) { + boolean fileFound = probeLiteralPattern(pattern, seenKeys, result, keyWrapper); + if (!fileFound) { + scanDirectoryPattern(pattern, seenKeys, result, keyWrapper); + } + continue; + } + else if (pattern.endsWith(".*")) { + probeDotWildcardPattern(pattern, seenKeys, result, keyWrapper); + continue; + } + scanWildcardPattern(pattern, seenKeys, result, keyWrapper); + } + return result; + } + + private String resolvePattern(String template, String application, String profile, String label) { + String resolvedLabel = (label == null ? "" : label); + String resolvedProfile = (profile == null ? "" : profile); + String pattern = template.replace("{application}", application) + .replace("{profile}", resolvedProfile) + .replace("{label}", resolvedLabel); + return StringUtils.trimLeadingCharacter(pattern.replaceAll("/{2,}", "/"), '/'); + } + + private boolean probeLiteralPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + boolean fileFound = false; + List extensionsToProbe = hasSupportedExtension(pattern) ? EMPTY_EXTENSION : SUPPORTED_EXTENSIONS; + for (String ext : extensionsToProbe) { + String key = pattern + ext; + if (!seenKeys.add(key)) { + continue; + } + else if (probeKeyAndAddResult(key, result, keyWrapper)) { + fileFound = true; + } + } + return fileFound; + } + + private void probeDotWildcardPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String base = pattern.substring(0, pattern.length() - 2); + for (String ext : SUPPORTED_EXTENSIONS) { + String key = base + ext; + if (seenKeys.add(key)) { + probeKeyAndAddResult(key, result, keyWrapper); + } + } + } + + private boolean probeKeyAndAddResult(String key, List result, + Function> keyWrapper) { + try { + s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); + result.addAll(keyWrapper.apply(key)); + return true; + } + catch (S3Exception e) { + int status = e.statusCode(); + if (status != 404 && status != 403) { + if (LOG.isInfoEnabled()) { + LOG.info("Error checking S3 object key: " + key, e); + } + throw e; + } + return false; + } + } + + private void scanDirectoryPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String dirPrefix = pattern.endsWith("/") ? pattern : pattern + "/"; + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(dirPrefix) + .continuationToken(token) + .build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!hasSupportedExtension(key)) { + continue; + } + else if (seenKeys.add(key)) { + result.addAll(keyWrapper.apply(key)); + } + } + token = resp.nextContinuationToken(); + } + while (token != null); + } + + private void scanWildcardPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String prefix = extractPrefix(pattern); + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).continuationToken(token).build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!pathMatcher.match(pattern, key) || !hasSupportedExtension(key)) { + continue; + } + else if (seenKeys.add(key)) { + result.addAll(keyWrapper.apply(key)); + } + } + token = resp.nextContinuationToken(); + } + while (token != null); + } + + private boolean hasSupportedExtension(String key) { + return key.endsWith(".properties") || key.endsWith(".json") || key.endsWith(".yml") || key.endsWith(".yaml"); + } + + private List wrapKeyWithConfigFiles(String key, String application, String profile, String label) { + + if (key.endsWith(".yml") || key.endsWith(".yaml")) { + List files = new ArrayList<>(); + files.addAll(getProfileSpecificYamlFromKey(key, application, profile, label)); + files.addAll(getNonProfileSpecificYamlFromKey(key, application, profile, label)); + return files; + } + return createConfigFileFromKey(key, application, profile, label).map(Collections::singletonList) + .orElseGet(Collections::emptyList); + } + + private List getProfileSpecificYamlFromKey(String key, String application, String profile, + String label) { + + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client, + properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) + ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); + config.setShouldIncludeWithEmptyProperties(false); + return List.of(config); + } + + private List getNonProfileSpecificYamlFromKey(String key, String application, String profile, + String label) { + + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client, properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) + ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); + return List.of(config); + } + + private String extractPrefix(String pattern) { + int idx = pattern.indexOf('*'); + int q = pattern.indexOf('?'); + if (q != -1 && (idx == -1 || q < idx)) { + idx = q; + } + else if (idx <= 0) { + return ""; } - return new ArrayList<>(yamlCreator.create(application, profile, label)); + int slash = pattern.lastIndexOf('/', idx); + return (slash == -1 ? "" : pattern.substring(0, slash + 1)); + } + private Optional createConfigFileFromKey(String key, String application, String profile, + String label) { + String ext = key.substring(key.lastIndexOf('.') + 1); + if ("properties".equalsIgnoreCase(ext)) { + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); + } + else if ("json".equalsIgnoreCase(ext)) { + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); + } + else if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); + } + return Optional.empty(); } private List getNonProfileSpecificS3ConfigFileYaml(String application, String profile, @@ -329,6 +570,33 @@ private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String } } + private List wrapKeyWithNegatedConfigFiles(String key, String application, String[] allProfiles, + String label) { + if (key.endsWith(".yml") || key.endsWith(".yaml")) { + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, null, label, bucketName, + this.useApplicationAsDirectory, s3Client, properties -> { + Object onProfileValue = properties.get("spring.config.activate.on-profile"); + if (onProfileValue == null) { + onProfileValue = properties.get("spring.config.activate.onProfile"); + } + if (onProfileValue == null) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + String expression = onProfileValue.toString().trim(); + if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + List allProfilesList = Arrays.asList(allProfiles); + boolean matches = Profiles.of(expression).matches(allProfilesList::contains); + return matches ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND; + }); + config.setShouldIncludeWithEmptyProperties(false); + return List.of(config); + } + return Collections.emptyList(); + } + @Override public Locations getLocations(String application, String profiles, String label) { StringBuilder baseLocation = new StringBuilder(AWS_S3_RESOURCE_SCHEME + bucketName + PATH_SEPARATOR); @@ -343,6 +611,11 @@ public Locations getLocations(String application, String profiles, String label) return new Locations(application, profiles, label, null, new String[] { baseLocation.toString() }); } + static boolean isSimpleProfileName(String expression) { + return !expression.contains("!") && !expression.contains("&") && !expression.contains("|") + && !expression.contains("(") && !expression.contains(","); + } + interface YamlS3ConfigFileCreator { List create(String application, String profile, String label); @@ -516,10 +789,12 @@ class YamlS3ConfigFile extends S3ConfigFile { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); this.documentMatchers = documentMatchers; this.properties = read(); - } protected static boolean profileMatchesActivateProperty(String profile, Properties properties) { + if (profile == null) { + return false; + } return profile.equals(properties.get("spring.config.activate.on-profile")) || profile.equals(properties.get("spring.config.activate.onProfile")); } @@ -614,6 +889,74 @@ protected List getExtensions() { } +class S3ConfigFileFromKey extends S3ConfigFile { + + private final String key; + + private final YamlProcessor.DocumentMatcher[] documentMatchers; + + S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + S3Client s3Client) { + this(key, application, profile, label, bucketName, false, s3Client, new YamlProcessor.DocumentMatcher[] {}); + } + + S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client, YamlProcessor.DocumentMatcher... documentMatchers) { + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); + this.key = key; + this.documentMatchers = documentMatchers; + this.properties = read(); + } + + @Override + public String getName() { + return "s3:" + bucketName + "/" + key; + } + + @Override + protected String buildObjectKeyPrefix() { + return key.substring(0, key.lastIndexOf('.')); + } + + @Override + protected List getExtensions() { + return List.of(key.substring(key.lastIndexOf('.') + 1)); + } + + @Override + public Properties read() { + if (this.properties != null) { + return this.properties; + } + String ext = key.substring(key.lastIndexOf('.') + 1).toLowerCase(Locale.ROOT); + if ("properties".equals(ext)) { + Properties props = new Properties(); + try (InputStream in = getObject()) { + props.load(in); + } + catch (Exception e) { + LOG.warn("Exception thrown when reading property file", e); + throw new IllegalStateException("Cannot load environment", e); + } + return props; + } + else if ("json".equals(ext) || "yml".equals(ext) || "yaml".equals(ext)) { + final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + try (InputStream in = getObject()) { + yaml.setResources(new InputStreamResource(in)); + yaml.setDocumentMatchers(documentMatchers); + return yaml.getObject(); + } + catch (Exception e) { + LOG.warn("Could not read YAML/JSON file", e); + throw new IllegalStateException("Cannot load environment", e); + } + } + throw new IllegalStateException("Unsupported extension: " + ext); + } + +} + class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { NegatedProfileYamlDocumentS3ConfigFile(String application, String label, String bucketName, @@ -630,7 +973,7 @@ class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { // Simple positive profile names are already handled by // ProfileSpecificYamlDocumentS3ConfigFile. Only process complex or negated // expressions here to avoid adding duplicate property sources. - if (isSimpleProfileName(expression)) { + if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { return YamlProcessor.MatchStatus.NOT_FOUND; } List allProfilesList = Arrays.asList(allProfiles); @@ -639,11 +982,6 @@ class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { }); } - private static boolean isSimpleProfileName(String expression) { - return !expression.contains("!") && !expression.contains("&") && !expression.contains("|") - && !expression.contains("(") && !expression.contains(","); - } - @Override protected String buildObjectKeyPrefix() { return super.buildObjectKeyPrefix(false); diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java index 138e89ce1..2540cb277 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java @@ -23,6 +23,9 @@ import static org.springframework.cloud.config.server.environment.AwsClientBuilderConfigurer.configureClientBuilder; +/** + * @author Geonwook Ham + */ public class AwsS3EnvironmentRepositoryFactory implements EnvironmentRepositoryFactory { @@ -38,10 +41,7 @@ public AwsS3EnvironmentRepository build(AwsS3EnvironmentProperties environmentPr configureClientBuilder(clientBuilder, environmentProperties.getRegion(), environmentProperties.getEndpoint()); final S3Client client = clientBuilder.build(); - AwsS3EnvironmentRepository repository = new AwsS3EnvironmentRepository(client, - environmentProperties.getBucket(), environmentProperties.isUseDirectoryLayout(), server); - repository.setOrder(environmentProperties.getOrder()); - return repository; + return new AwsS3EnvironmentRepository(client, environmentProperties, server); } } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index 2f3a49eb5..fae01ef5a 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -21,13 +21,17 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -56,6 +60,7 @@ /** * @author Clay McCoy * @author Matej Nedić + * @author Geonwook Ham */ @Testcontainers @Tag("DockerRequired") @@ -258,8 +263,8 @@ public void negatedProfileDocumentIncludedWhenNegatedProfileIsAbsentFromMultiple String yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI()))); putFiles("application.yaml", yamlString); - // neither "default" nor "other-profile" is "my-profile", so "!my-profile" should - // match + // neither "default" nor "other-profile" is "my-profile", so "!my-profile" + // should match final Environment env = envRepo.findOne("application", "default,other-profile", null); List propertySources = env.getPropertySources(); @@ -762,6 +767,280 @@ public void getLocationsTest() { "default", "defaultlabel", null, new String[] { "s3://test/defaultlabel" })); } + @Test + public void searchPath_placeholderOnly_shouldResolveExactFile() { + List paths = List.of("{label}/{application}-{profile}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar.yml"); + } + + @Test + public void searchPath_wildcardOnly_shouldResolveAllProperties() { + List paths = List.of("{label}/common/*.properties"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/common/a.properties", "a=1\n"); + putFiles("v1/common/b.properties", "b=2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/common/a.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/b.properties"); + } + + @Test + public void searchPath_placeholderAndWildcard_shouldResolveMatchingKeys() { + List paths = List.of("{label}/{application}-{profile}.yml", "{label}/common/*.properties"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo-bar.yml", yamlContent); + putFiles("v1/common/foo.properties", "k=v\n"); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/foo.properties"); + } + + @Test + public void searchPaths_orderMatters_forPropertySourceOrder() { + List paths = List.of("{label}/common/*.properties", "{label}/{application}-{profile}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/common/foo.properties", "k=v\n"); + putFiles("v1/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/common/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo-bar.yml"); + } + + @TestFactory + public Stream searchPath_extensionPreserved() { + List exts = List.of("yml", "yaml", "properties", "json"); + return exts.stream().map(ext -> DynamicTest.dynamicTest(ext, () -> { + List paths = List.of("{label}/foo-bar." + ext); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String content = "key=v\n"; + if (ext.equals("yml") || ext.equals("yaml")) { + content = yamlContent; + } + else if (ext.equals("json")) { + content = jsonContent; + } + putFiles("v1/foo-bar." + ext, content); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar." + ext); + })); + } + + @Test + public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { + List paths = List.of("{label}/{application}/foo.*"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", true, server, paths); + putFiles("v1/foo/foo.properties", "k=v\n"); + putFiles("v1/foo/foo.json", jsonContent); + + Environment env = repo.findOne("foo", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/foo.properties"); + } + + @Test + public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws IOException { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multi = "---\na: 1\n---\nb: 2\n"; + putFiles("lab/app.yml", multi); + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(1); + @SuppressWarnings("unchecked") + Map map = (Map) env.getPropertySources().get(0).getSource(); + assertThat(map).containsEntry("a", 1).containsEntry("b", 2); + } + + @Test + public void searchPaths_deduplication_shouldOnlyAddOnce() { + List paths = List.of("{label}/foo.yml", "{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo.yml", yamlContent); + + Environment env = repo.findOne("foo", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + } + + @Test + public void searchPaths_literalStopsAtProperties() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.properties", "foo=bar\nflag=false\n"); + putFiles("v1/app.json", jsonContent); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(3); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); + } + + @Test + public void searchPaths_literalStopsAtJson() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.json", jsonContent); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.json"); + } + + @Test + public void searchPaths_literalStopsAtYaml() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.yml"); + } + + @Test + public void searchPaths_dotWildcardAutoExtProperties() { + List paths = List.of("{label}/{application}.*"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.properties", "x=1\n"); + putFiles("v1/app.json", jsonContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); + } + + @Test + public void searchPaths_literalDirectoryScan() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app/foo.properties", "a=1\n"); + putFiles("v1/app/sub/bar.yml", "bar: 2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/app/sub/bar.yml"); + } + + @Test + public void searchPaths_doubleWildcardNested() { + List paths = List.of("{label}/**/*.properties"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo/a.properties", "a=1\n"); + putFiles("v1/foo/sub/b.properties", "b=2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/a.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo/sub/b.properties"); + } + + @Test + public void searchPaths_singleCharacterWildcard_shouldMatchExactlyOneChar() { + List paths = List.of("{label}/data-?.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab/data-a.yml", "a: 1\n"); + putFiles("lab/data-b.yml", "b: 2\n"); + putFiles("lab/data-10.yml", "x: 3\n"); // should not match + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()).contains("lab/data-a.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("lab/data-b.yml"); + } + + @Test + public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { + List paths = List.of("{label}/*.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab/app.yml", yamlContent); + putFiles("lab/other.yml", yamlContent); + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(2); + } + + @Test + public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { + ConfigServerProperties serverWithDefaultLabel = new ConfigServerProperties(); + serverWithDefaultLabel.setDefaultLabel("main"); + + List paths = List.of("{label}/foo-bar.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, + serverWithDefaultLabel, paths); + + putFiles("main/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "", ""); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()).contains("main/foo-bar.yml"); + } + + @Test + public void searchPaths_multipleLabels_shouldApplyForEachLabelInReverseOrder() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab1/app.yml", yamlContent); + putFiles("lab2/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "lab1,lab2"); + assertThat(env.getPropertySources().get(0).getName()).contains("lab2/app.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("lab1/app.yml"); + } + + @Test + public void searchPaths_negatedProfile_shouldIncludeDocumentWhenProfileNotActive() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multiDocYaml = "spring:\n" + " config:\n" + " activate:\n" + " on-profile: '!my-profile'\n" + + "negated: true\n" + "---\n" + "default-key: default-value\n"; + + putFiles("v1/app.yml", multiDocYaml); + + Environment env = repo.findOne("app", "other-profile", "v1"); + + assertThat(env.getPropertySources()).hasSize(2); + boolean negatedPresent = env.getPropertySources() + .stream() + .anyMatch(ps -> Boolean.TRUE.equals(ps.getSource().get("negated"))); + assertThat(negatedPresent).isTrue(); + } + + @Test + public void searchPaths_negatedProfile_shouldExcludeDocumentWhenProfileIsActive() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multiDocYaml = "spring:\n" + " config:\n" + " activate:\n" + " on-profile: '!my-profile'\n" + + "negated: true\n" + "---\n" + "default-key: default-value\n"; + + putFiles("v1/app.yml", multiDocYaml); + + Environment env = repo.findOne("app", "my-profile", "v1"); + + assertThat(env.getPropertySources()).hasSize(1); + boolean negatedPresent = env.getPropertySources() + .stream() + .anyMatch(ps -> Boolean.TRUE.equals(ps.getSource().get("negated"))); + assertThat(negatedPresent).isFalse(); + } + private String putFiles(String fileName, String propertyContent) { toBeRemoved.add(fileName); return s3Client