feat: Keep null variable if variable already exists during process

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -34,6 +34,7 @@
 import fr.insee.genesis.exceptions.ReviewDisabledException;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.modelefiliere.RawResponseDto;
+import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -286,7 +287,7 @@ public ResponseEntity<Object> findResponsesByInterrogationAndCollectionInstrumen
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<List<SurveyUnitModel>> getLatestByInterrogationAndCollectionInstrument(@RequestParam("interrogationId") String interrogationId,
                                                                @RequestParam("collectionInstrumentId") String collectionInstrumentId) {
-        List<SurveyUnitModel> responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId);
+        List<SurveyUnitModel> responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId);
         return ResponseEntity.ok(responses);
     }
 
@@ -300,7 +301,7 @@ public ResponseEntity<List<SurveyUnitModel>> getLatestByInterrogationAndCollecti
     public ResponseEntity<SurveyUnitSimplifiedDto> getLatestByInterrogationOneObject(@RequestParam("interrogationId") String interrogationId,
                                                                                      @RequestParam("collectionInstrumentId") String collectionInstrumentId,
                                                                                      @RequestParam("mode") Mode mode) {
-        List<SurveyUnitModel> responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId);
+        List<SurveyUnitModel> responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId);
         List<VariableModel> outputVariables = new ArrayList<>();
         List<VariableModel> outputExternalVariables = new ArrayList<>();
         RawResponseDto.QuestionnaireStateEnum questionnaireState = null;
@@ -333,7 +334,7 @@ public ResponseEntity<SurveyUnitSimplifiedDto> getLatestByInterrogationOneObject
     @Operation(summary = "Returns the response with the latest variables for a collectionInstrument, mode and " +
             "interrogation")
     @GetMapping(path = "/{collectionInstrumentId}/{mode}/{interrogationId}")
-    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<SurveyUnitSimplifiedDto> getResponseByCollectionInstrumentAndInterrogation(
             @PathVariable("collectionInstrumentId") String collectionInstrumentId,
             @PathVariable("interrogationId") String interrogationId,
@@ -365,7 +366,7 @@ public ResponseEntity<List<SurveyUnitSimplifiedDto>> getLatestForInterrogationLi
         List<SurveyUnitSimplifiedDto> results = new ArrayList<>();
         List<Mode> modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId);
         interrogationIds.forEach(interrogationId -> {
-            List<SurveyUnitModel> responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(
+            List<SurveyUnitModel> responses = surveyUnitService.findLatestByInterrogationIdAndCollectionInstrumentId(
                     interrogationId.getInterrogationId(), collectionInstrumentId
             );
             modes.forEach(mode -> {
@@ -407,6 +408,7 @@ public ResponseEntity<List<SurveyUnitSimplifiedDto>> getLatestForInterrogationLi
         return ResponseEntity.ok(results);
     }
 
+    //Kraftwerk uses this
     @Operation(summary = "Retrieve responses for a collection instrument and a list of interrogations",
             description = "Return the latest state for each variable for the given interrogationIds and a given collection instrument (formerly questionnaire).<br>" +
                     "For a given id, the endpoint returns a document by collection mode (if there is more than one)<br>" +
@@ -696,6 +698,7 @@ private static String getSuccessMessage(boolean isAnyDataSaved) {
      */
     //TODO Unused for now, reuse code for optimizations, also move it to service
     @Deprecated
+    @Hidden
     @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE",
             description = "Return the latest state for each variable for the given ids and a given questionnaire.<br>" +
                     "For a given id, the endpoint returns a document by collection mode (if there is more than one).")

src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java

@@ -1,51 +1,71 @@
 package fr.insee.genesis.domain.converter.rawdata;
 
-import fr.insee.bpm.metadata.model.Variable;
 import fr.insee.bpm.metadata.model.VariablesMap;
-import fr.insee.genesis.Constants;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
-import fr.insee.genesis.domain.model.surveyunit.VariableModel;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
 import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser;
-import fr.insee.genesis.domain.utils.GroupUtils;
-import fr.insee.genesis.domain.utils.JsonUtils;
-import lombok.RequiredArgsConstructor;
+import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 @Component
 @Slf4j
-@RequiredArgsConstructor
-public class LunaticJsonRawDataConverter {
+public class LunaticJsonRawDataConverter extends RawDataConverter {
 
     private final LunaticJsonRawDataPayloadParser payloadParser;
 
+    public LunaticJsonRawDataConverter(SurveyUnitApiPort surveyUnitApiPort,
+                                       LunaticJsonRawDataPayloadParser lunaticJsonRawDataPayloadParser) {
+        super(surveyUnitApiPort);
+        this.payloadParser = lunaticJsonRawDataPayloadParser;
+    }
+
     public List<SurveyUnitModel> convertRawData(
+            String questionnaireId,
             List<LunaticJsonRawDataModel> rawDataList,
             VariablesMap variablesMap
     ) {
-        return convertRawDataAndCollectEmptyModels(rawDataList, variablesMap, new ArrayList<>());
+        return convertRawDataAndCollectEmptyModels(
+                questionnaireId,
+                rawDataList,
+                variablesMap,
+                new ArrayList<>()
+        );
     }
 
     public List<SurveyUnitModel> convertRawDataAndCollectEmptyModels(
+            String questionnaireId,
             List<LunaticJsonRawDataModel> rawDataList,
             VariablesMap variablesMap,
             List<SurveyUnitModel> emptySurveyUnitModels
     ) {
         List<SurveyUnitModel> surveyUnitModels = new ArrayList<>();
 
+        Map<String, Map<DataState, SurveyUnitModel>> lastSurveyUnitModelsByInterrogationIdAndState =
+                getLastSurveyUnitModels(
+                        questionnaireId,
+                        rawDataList.stream().map(LunaticJsonRawDataModel::interrogationId).collect(Collectors.toList())
+                );
+
         for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) {
             for (LunaticJsonRawDataModel rawData : rawDataList) {
                 RawDataModelType rawDataModelType = getRawDataModelType(rawData);
+                SurveyUnitModel lastSurveyUnitModelForDataState = null;
+                if(lastSurveyUnitModelsByInterrogationIdAndState.containsKey(rawData.interrogationId())){
+                    lastSurveyUnitModelForDataState = lastSurveyUnitModelsByInterrogationIdAndState
+                            .get(rawData.interrogationId())
+                            .get(dataState);
+                }
 
-                SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
+                SurveyUnitModel newSurveyUnitModel = SurveyUnitModel.builder()
                         .collectionInstrumentId(rawData.questionnaireId())
                         .mode(rawData.mode())
                         .interrogationId(rawData.interrogationId())
@@ -59,25 +79,39 @@
                         .externalVariables(new ArrayList<>())
                         .build();
 
-                convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap);
+                convertCollectedVariables(
+                        rawData.data(),
+                        rawData.interrogationId(),
+                        lastSurveyUnitModelForDataState,
+                        newSurveyUnitModel,
+                        dataState,
+                        rawDataModelType,
+                        variablesMap
+                );
 
                 if (dataState == DataState.COLLECTED) {
-                    convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap);
+                    convertExternalVariables(
+                            rawData.data(),
+                            lastSurveyUnitModelForDataState,
+                            newSurveyUnitModel,
+                            rawDataModelType,
+                            variablesMap
+                    );
                 }
 
-                boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty()
-                        && surveyUnitModel.getExternalVariables().isEmpty();
+                boolean hasNoVariable = newSurveyUnitModel.getCollectedVariables().isEmpty()
+                        && newSurveyUnitModel.getExternalVariables().isEmpty();
 
                 if (hasNoVariable) {
-                    if (surveyUnitModel.getState() == DataState.COLLECTED) {
+                    if (newSurveyUnitModel.getState() == DataState.COLLECTED) {
                         log.warn("No collected or external variable for interrogation {}, raw data is ignored.",
                                 rawData.interrogationId());
                     }
-                    emptySurveyUnitModels.add(surveyUnitModel);
+                    emptySurveyUnitModels.add(newSurveyUnitModel);
                     continue;
                 }
 
-                surveyUnitModels.add(surveyUnitModel);
+                surveyUnitModels.add(newSurveyUnitModel);
             }
         }
 
@@ -89,127 +123,4 @@
                 ? RawDataModelType.FILIERE
                 : RawDataModelType.LEGACY;
     }
-
-    private void convertRawDataCollectedVariables(
-            LunaticJsonRawDataModel srcRawData,
-            SurveyUnitModel dstSurveyUnitModel,
-            DataState dataState,
-            RawDataModelType rawDataModelType,
-            VariablesMap variablesMap
-    ) {
-        Map<String, Object> dataMap = srcRawData.data();
-        if (rawDataModelType == RawDataModelType.FILIERE) {
-            dataMap = JsonUtils.asMap(dataMap.get("data"));
-        }
-
-        Map<String, Object>  collectedMap = JsonUtils.asMap(dataMap.get("COLLECTED"));
-
-        if (collectedMap == null || collectedMap.isEmpty()) {
-            if (dataState == DataState.COLLECTED) {
-                log.warn("No collected data for interrogation {}", srcRawData.interrogationId());
-            }
-            return;
-        }
-
-        String stateKey = dataState.toString();
-        List<VariableModel> destination = dstSurveyUnitModel.getCollectedVariables();
-
-        for (Map.Entry<String, Object> collectedVariable : collectedMap.entrySet()) {
-            RawResponseConverter.processCollectedVariable(
-                    collectedVariable,
-                    stateKey,
-                    variablesMap,
-                    dstSurveyUnitModel,
-                    destination
-            );
-        }
-    }
-
-
-    private static void convertRawDataExternalVariables(
-            LunaticJsonRawDataModel srcRawData,
-            SurveyUnitModel dstSurveyUnitModel,
-            RawDataModelType rawDataModelType,
-            VariablesMap variablesMap
-    ) {
-        Map<String, Object> dataMap = srcRawData.data();
-        if (rawDataModelType == RawDataModelType.FILIERE) {
-            dataMap =  JsonUtils.asMap(dataMap.get("data"));
-        }
-
-        Map<String, Object> externalMap =  JsonUtils.asMap(dataMap.get("EXTERNAL"));
-        if (externalMap != null && !externalMap.isEmpty()) {
-            convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap);
-        }
-    }
-
-    private static void convertToExternalVar(
-            SurveyUnitModel dstSurveyUnitModel,
-            VariablesMap variablesMap,
-            Map<String, Object> externalMap
-    ) {
-        for (Map.Entry<String, Object> externalVariableEntry : externalMap.entrySet()) {
-            Object valueObject = externalVariableEntry.getValue();
-
-            if (valueObject instanceof List<?>) {
-                convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables());
-                continue;
-            }
-
-            if (valueObject != null) {
-                convertOneVar(
-                        externalVariableEntry,
-                        valueObject.toString(),
-                        variablesMap,
-                        1,
-                        dstSurveyUnitModel.getExternalVariables()
-                );
-            }
-        }
-    }
-
-    private static void convertListVar(
-            Object valuesForState,
-            Map.Entry<String, Object> collectedVariable,
-            VariablesMap variablesMap,
-            List<VariableModel> destination
-    ) {
-        List<String> values = JsonUtils.asStringList(valuesForState);
-        if (!values.isEmpty()) {
-            int iteration = 1;
-            for (String value : values) {
-                if (value != null && !value.isEmpty()) {
-                    convertOneVar(collectedVariable, value, variablesMap, iteration, destination);
-                }
-                iteration++;
-            }
-        }
-    }
-
-    private static void convertOneVar(
-            Map.Entry<String, Object> variableEntry,
-            String value,
-            VariablesMap variablesMap,
-            int iteration,
-            List<VariableModel> destination
-    ) {
-        VariableModel variableModel = VariableModel.builder()
-                .varId(variableEntry.getKey())
-                .value(value)
-                .scope(getIdLoop(variablesMap, variableEntry.getKey()))
-                .iteration(iteration)
-                .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap))
-                .build();
-
-        destination.add(variableModel);
-    }
-
-    private static String getIdLoop(VariablesMap variablesMap, String variableName) {
-        Variable variable = variablesMap.getVariable(variableName);
-        if (variable == null) {
-            log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME);
-            return Constants.ROOT_GROUP_NAME;
-        }
-        return variable.getGroupName();
-    }
 }