Add derived_source_keep parameter for array preservation in derived source

server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java

@@ -123,6 +123,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> toType(m).nullValue
         ).acceptsNull();
 
+        private final Parameter<String> derivedSourceKeepParam = Parameter.restrictedStringParam(
+            "derived_source_keep",
+            false,
+            m -> toType(m).derivedSourceKeep.getValue(),
+            "none",
+            "arrays"
+        );
+
         private final Parameter<Float> boost = Parameter.boostParam();
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
@@ -132,11 +140,24 @@ public Builder(String name) {
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return Arrays.asList(meta, boost, docValues, indexed, nullValue, stored);
+            return Arrays.asList(meta, boost, docValues, indexed, nullValue, derivedSourceKeepParam, stored);
         }
 
         @Override
         public BooleanFieldMapper build(BuilderContext context) {
+            // Auto-enable store=true when derived_source_keep="arrays"
+            DerivedSourceKeep derivedSourceKeep = DerivedSourceKeep.fromString(this.derivedSourceKeepParam.getValue());
+            boolean storeValue = this.stored.getValue();
+
+            if (derivedSourceKeep.requiresStoredFields()) {
+                // Check if store was explicitly set to false
+                if (this.stored.isSet() && !storeValue) {
+                    throw new MapperParsingException("Cannot set derived_source_keep='arrays' with store=false for field [" + name() + "]");
+                }
+                storeValue = true;
+                this.stored.setValue(storeValue);
+            }
+
             MappedFieldType ft = new BooleanFieldType(
                 buildFullName(context),
                 indexed.getValue(),
@@ -354,6 +375,7 @@ protected FieldTypeCapabilities.Capability searchCapability() {
     private final boolean indexed;
     private final boolean hasDocValues;
     private final boolean stored;
+    private final DerivedSourceKeep derivedSourceKeep;
 
     protected BooleanFieldMapper(
         String simpleName,
@@ -367,6 +389,8 @@ protected BooleanFieldMapper(
         this.stored = builder.stored.getValue();
         this.indexed = builder.indexed.getValue();
         this.hasDocValues = builder.docValues.getValue();
+        this.derivedSourceKeep = DerivedSourceKeep.fromString(builder.derivedSourceKeepParam.getValue());
+        setDerivedFieldGenerator(derivedFieldGenerator());
     }
 
     @Override
@@ -433,7 +457,11 @@ protected String contentType() {
 
     @Override
     protected void canDeriveSourceInternal() {
-        checkStoredAndDocValuesForDerivedSource();
+        if (derivedSourceKeep.requiresStoredFields()) {
+            checkStoredForDerivedSource();
+        } else {
+            checkStoredAndDocValuesForDerivedSource();
+        }
     }
 
     /**
@@ -450,6 +478,8 @@ protected void canDeriveSourceInternal() {
      */
     @Override
     protected DerivedFieldGenerator derivedFieldGenerator() {
+        DerivedSourceKeep mode = (this.derivedSourceKeep != null) ? this.derivedSourceKeep : DerivedSourceKeep.NONE;
+
         return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) {
             @Override
             public Object convert(Object value) {
@@ -459,6 +489,6 @@ public Object convert(Object value) {
                 }
                 return val == 1;
             }
-        }, new StoredFieldFetcher(mappedFieldType, simpleName()));
+        }, new StoredFieldFetcher(mappedFieldType, simpleName()), mode);
     }
 }

server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java

@@ -232,7 +232,11 @@ private static DateFieldMapper toType(FieldMapper in) {
 
     @Override
     protected void canDeriveSourceInternal() {
-        checkStoredAndDocValuesForDerivedSource();
+        if (derivedSourceKeep.requiresStoredFields()) {
+            checkStoredForDerivedSource();
+        } else {
+            checkStoredAndDocValuesForDerivedSource();
+        }
     }
 
     /**
@@ -250,6 +254,8 @@ protected void canDeriveSourceInternal() {
      */
     @Override
     protected DerivedFieldGenerator derivedFieldGenerator() {
+        DerivedSourceKeep mode = (this.derivedSourceKeep != null) ? this.derivedSourceKeep : DerivedSourceKeep.NONE;
+
         return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) {
             @Override
             public Object convert(Object value) {
@@ -259,7 +265,7 @@ public Object convert(Object value) {
                 }
                 return fieldType().dateTimeFormatter().format(resolution.toInstant(val).atZone(ZoneOffset.UTC));
             }
-        }, new StoredFieldFetcher(mappedFieldType, simpleName()));
+        }, new StoredFieldFetcher(mappedFieldType, simpleName()), mode);
     }
 
     /**
@@ -280,6 +286,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> toType(m).skiplist
         );
 
+        private final Parameter<String> derivedSourceKeepParam = Parameter.restrictedStringParam(
+            "derived_source_keep",
+            false,
+            m -> toType(m).derivedSourceKeep.getValue(),
+            "none",
+            "arrays"
+        );
+
         private final Parameter<Float> boost = Parameter.boostParam();
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
@@ -347,7 +361,20 @@ private DateFormatter buildFormatter() {
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return Arrays.asList(index, docValues, store, skiplist, format, printFormat, locale, nullValue, ignoreMalformed, boost, meta);
+            return Arrays.asList(
+                index,
+                docValues,
+                store,
+                skiplist,
+                format,
+                printFormat,
+                locale,
+                nullValue,
+                ignoreMalformed,
+                derivedSourceKeepParam,
+                boost,
+                meta
+            );
         }
 
         private Long parseNullValue(DateFieldType fieldType) {
@@ -371,6 +398,19 @@ private Long parseNullValue(DateFieldType fieldType) {
 
         @Override
         public DateFieldMapper build(BuilderContext context) {
+            // Auto-enable store=true when derived_source_keep="arrays"
+            DerivedSourceKeep derivedSourceKeep = DerivedSourceKeep.fromString(this.derivedSourceKeepParam.getValue());
+            boolean storeValue = this.store.getValue();
+
+            if (derivedSourceKeep.requiresStoredFields()) {
+                // Check if store was explicitly set to false
+                if (this.store.isSet() && !storeValue) {
+                    throw new MapperParsingException("Cannot set derived_source_keep='arrays' with store=false for field [" + name() + "]");
+                }
+                storeValue = true;
+                this.store.setValue(storeValue);
+            }
+
             DateFieldType ft = new DateFieldType(
                 buildFullName(context),
                 index.getValue(),
@@ -768,6 +808,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
     private final Long nullValue;
     private final String nullValueAsString;
     private final Resolution resolution;
+    private final DerivedSourceKeep derivedSourceKeep;
 
     private final boolean ignoreMalformedByDefault;
     private final Version indexCreatedVersion;
@@ -794,8 +835,10 @@ private DateFieldMapper(
         this.nullValueAsString = builder.nullValue.getValue();
         this.nullValue = nullValue;
         this.resolution = resolution;
+        this.derivedSourceKeep = DerivedSourceKeep.fromString(builder.derivedSourceKeepParam.getValue());
         this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
         this.indexCreatedVersion = builder.indexCreatedVersion;
+        setDerivedFieldGenerator(derivedFieldGenerator());
     }
 
     @Override

server/src/main/java/org/opensearch/index/mapper/DerivedFieldGenerator.java

@@ -22,19 +22,58 @@ public class DerivedFieldGenerator {
 
     private final MappedFieldType mappedFieldType;
     private final FieldValueFetcher fieldValueFetcher;
+    private final DerivedSourceKeep derivedSourceKeep;
 
+    /**
+     * Creates a DerivedFieldGenerator with the specified fetchers that don't specify derived_source_keep.
+     * Defaults to DerivedSourceKeep.NONE (uses doc values when available).
+     *
+     * @param mappedFieldType the field type
+     * @param docValuesFetcher fetcher for doc values (sorted/deduplicated)
+     * @param storedFieldFetcher fetcher for stored fields (preserves order/duplicates)
+     */
     public DerivedFieldGenerator(
         MappedFieldType mappedFieldType,
         FieldValueFetcher docValuesFetcher,
         FieldValueFetcher storedFieldFetcher
+    ) {
+        this(mappedFieldType, docValuesFetcher, storedFieldFetcher, DerivedSourceKeep.NONE);
+    }
+
+    /**
+     * Creates a DerivedFieldGenerator with the specified fetchers and derived source keep mode.
+     *
+     * @param mappedFieldType the field type
+     * @param docValuesFetcher fetcher for doc values (sorted/deduplicated)
+     * @param storedFieldFetcher fetcher for stored fields (preserves order/duplicates)
+     * @param derivedSourceKeep the mode for source reconstruction (NONE or ARRAYS)
+     */
+    public DerivedFieldGenerator(
+        MappedFieldType mappedFieldType,
+        FieldValueFetcher docValuesFetcher,
+        FieldValueFetcher storedFieldFetcher,
+        DerivedSourceKeep derivedSourceKeep
     ) {
         this.mappedFieldType = mappedFieldType;
-        if (Objects.requireNonNull(getDerivedFieldPreference()) == FieldValueType.DOC_VALUES) {
-            assert docValuesFetcher != null;
-            this.fieldValueFetcher = docValuesFetcher;
-        } else {
-            assert storedFieldFetcher != null;
+        this.derivedSourceKeep = Objects.requireNonNull(derivedSourceKeep, "derivedSourceKeep cannot be null");
+
+        if (derivedSourceKeep.requiresStoredFields()) {
+            // User explicitly requested array preservation via stored fields
+            if (storedFieldFetcher == null) {
+                throw new IllegalArgumentException(
+                    "derived_source_keep='arrays' requires stored fields to be enabled for field [" + mappedFieldType.name() + "]"
+                );
+            }
             this.fieldValueFetcher = storedFieldFetcher;
+        } else {
+            // Default behavior: prefer doc values if available, otherwise use stored fields
+            if (Objects.requireNonNull(getDerivedFieldPreference()) == FieldValueType.DOC_VALUES) {
+                assert docValuesFetcher != null;
+                this.fieldValueFetcher = docValuesFetcher;
+            } else {
+                assert storedFieldFetcher != null;
+                this.fieldValueFetcher = storedFieldFetcher;
+            }
         }
     }
 
@@ -49,6 +88,13 @@ public FieldValueType getDerivedFieldPreference() {
         return FieldValueType.STORED;
     }
 
+    /**
+     * Returns the derived source keep mode for this generator.
+     */
+    public DerivedSourceKeep getDerivedSourceKeep() {
+        return derivedSourceKeep;
+    }
+
     /**
      * Generate the derived field value based on the preference of derived field and field value type
      * @param builder - builder to store the derived source filed

server/src/main/java/org/opensearch/index/mapper/DerivedSourceKeep.java

@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.PublicApi;
+
+import java.util.Locale;
+
+/**
+ * Strategy for preserving field values when reconstructing source from derived fields.
+ * Controls whether to use doc values (sorted/deduplicated) or stored fields (preserves order/duplicates).
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "3.8.0")
+public enum DerivedSourceKeep {
+    /**
+     * Use doc values (default). Values are sorted and deduplicated.
+     * Most storage-efficient but loses array order and duplicates.
+     */
+    NONE("none"),
+
+    /**
+     * Use stored fields. Preserves array order and duplicate values.
+     * Automatically enables store=true. Higher storage overhead.
+     */
+    ARRAYS("arrays");
+
+    private final String value;
+
+    DerivedSourceKeep(String value) {
+        this.value = value;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    /**
+     * Parses string value into enum. Case-insensitive.
+     * Returns NONE if value is null.
+     */
+    public static DerivedSourceKeep fromString(String value) {
+        if (value == null) {
+            return NONE;
+        }
+
+        String normalizedValue = value.toLowerCase(Locale.ROOT);
+        for (DerivedSourceKeep mode : values()) {
+            if (mode.value.equals(normalizedValue)) {
+                return mode;
+            }
+        }
+
+        throw new IllegalArgumentException("Invalid value for derived_source_keep: [" + value + "]. Valid values are: [none, arrays]");
+    }
+
+    /**
+     * Returns true if this mode requires stored fields to be enabled.
+     */
+    public boolean requiresStoredFields() {
+        return this == ARRAYS;
+    }
+
+    @Override
+    public String toString() {
+        return value;
+    }
+}