From 9d7c4dbf020737aae84a5d1a9be8a04ecabbcd6c Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Fri, 5 Dec 2025 03:54:17 -0500 Subject: [PATCH 01/19] supress warning message on test/build execution (#384) --- package.json | 2 +- yarn.lock | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 49174880..0faada21 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "@dhis2/ui-core": "6.24.0", "@dhis2/ui-widgets": "6.24.0", "@eyeseetea/d2-api": "1.20.0", - "@eyeseetea/d2-ui-components": "2.11.0-beta.1", + "@eyeseetea/d2-ui-components": "2.12.0", "@eyeseetea/feedback-component": "0.1.3-beta.3", "@eyeseetea/xlsx-populate": "4.3.2-beta.1", "@material-ui/core": "4.12.3", diff --git a/yarn.lock b/yarn.lock index 9d92550c..11d8faa8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4642,10 +4642,10 @@ qs "6.9.7" react "^16.12.0" -"@eyeseetea/d2-ui-components@2.11.0-beta.1": - version "2.11.0-beta.1" - resolved "https://registry.yarnpkg.com/@eyeseetea/d2-ui-components/-/d2-ui-components-2.11.0-beta.1.tgz#b4b4c76285a03f4f99d2645d8c7476122da8f289" - integrity sha512-OLBphENbEUQjVe94rqKqaw9vClPLXeYSIAD0Zpx7etnY2Out2lh5DOl97muff13mKJmvN6u35BApp7DLnX0X0g== +"@eyeseetea/d2-ui-components@2.12.0": + version "2.12.0" + resolved "https://registry.yarnpkg.com/@eyeseetea/d2-ui-components/-/d2-ui-components-2.12.0.tgz#dabef61e90f42796deefe891393646fc917abb2e" + integrity sha512-lH+kuDsxU/N4MlDrBJcTBECyQF0YQkWrShu+zICzlSm4AAqq6FEqq20xrloJr1fHoB35/OmYVyry35kMwWfwdA== dependencies: "@date-io/core" "1.3.6" "@date-io/moment" "1.0.2" @@ -4659,7 +4659,6 @@ moment "2.29.4" nano-memoize "1.2.1" react-linkify "1.0.0-alpha" - rxjs-compat "6.6.3" throttle-debounce "2.1.0" "@eyeseetea/feedback-component@0.1.3-beta.3": @@ -17608,11 +17607,6 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs-compat@6.6.3: - version "6.6.3" - resolved "https://registry.npmjs.org/rxjs-compat/-/rxjs-compat-6.6.3.tgz#141405fcee11f48718d428b99c8f01826f594e5c" - integrity sha512-y+wUqq7bS2dG+7rH2fNMoxsDiJ32RQzFxZQE/JdtpnmEZmwLQrb1tCiItyHxdXJHXjmHnnzFscn3b6PEmORGKw== - rxjs-compat@6.6.7: version "6.6.7" resolved "https://registry.yarnpkg.com/rxjs-compat/-/rxjs-compat-6.6.7.tgz#6eb4ef75c0a58ea672854a701ccc8d49f41e69cb" From 62a654d526a58f115033869f61c4aadfdf87b9c5 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Wed, 11 Mar 2026 13:19:42 -0500 Subject: [PATCH 02/19] add spanish and french translations --- i18n/es.po | 6 +----- i18n/fr.po | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/i18n/es.po b/i18n/es.po index 2d105d8f..7569963a 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -211,10 +211,6 @@ msgstr "" msgid "Default" msgstr "" -#, fuzzy -msgid "Delete and import" -msgstr "Importación de datos" - #, fuzzy msgid "Import despite duplicates" msgstr "Importar datos" @@ -1385,7 +1381,7 @@ msgstr "" "continuar?" msgid "Delete and Import" -msgstr "" +msgstr "Borrar e Importar" msgid "" "All data values in the spreadsheet will be imported to the system, but any " diff --git a/i18n/fr.po b/i18n/fr.po index 7b362091..0b4860db 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -205,10 +205,6 @@ msgstr "" msgid "Default" msgstr "" -#, fuzzy -msgid "Delete and import" -msgstr "Importation de données en masse" - #, fuzzy msgid "Import despite duplicates" msgstr "Importer des données" @@ -1407,7 +1403,7 @@ msgstr "" "calcul seront enregistrées. Êtes-vous sur de vouloir continuer?" msgid "Delete and Import" -msgstr "" +msgstr "SUPPRIMER ET IMPORTER" msgid "" "All data values in the spreadsheet will be imported to the system, but any " From 1ee25c3face9b3754e1fbecea7570d7a224740c8 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Wed, 11 Mar 2026 14:23:30 -0500 Subject: [PATCH 03/19] fix duplicate entries in po files --- i18n/en.pot | 9 +++------ i18n/es.po | 18 +++++++----------- i18n/fr.po | 14 +++++++------- i18n/pt.po | 16 ++++++---------- i18n/ru.po | 16 ++++++---------- .../history/HistoryImportSummary.tsx | 2 +- 6 files changed, 30 insertions(+), 45 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 2d0dd273..56b0abcf 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2025-11-12T16:45:33.366Z\n" -"PO-Revision-Date: 2025-11-12T16:45:33.366Z\n" +"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"PO-Revision-Date: 2026-03-11T19:21:15.859Z\n" msgid "Events - Create/update" msgstr "" @@ -188,7 +188,7 @@ msgstr "" msgid "Default" msgstr "" -msgid "Delete and import" +msgid "Delete and Import" msgstr "" msgid "Import despite duplicates" @@ -1310,9 +1310,6 @@ msgid "" "existing values before importing the data?" msgstr "" -msgid "Delete and Import" -msgstr "" - msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " diff --git a/i18n/es.po b/i18n/es.po index 7569963a..be274f85 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-12T16:45:33.366Z\n" +"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -211,6 +211,9 @@ msgstr "" msgid "Default" msgstr "" +msgid "Delete and Import" +msgstr "Borrar e Importar" + #, fuzzy msgid "Import despite duplicates" msgstr "Importar datos" @@ -1287,9 +1290,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://" -"eyeseetea.com). Source code, documentation and release notes can be found at " -"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." +"com). Source code, documentation and release notes can be found at the " +"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1380,9 +1383,6 @@ msgstr "" "remplazarán por aquellos presentes en el archivo. ¿Está seguro que desea " "continuar?" -msgid "Delete and Import" -msgstr "Borrar e Importar" - msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " @@ -1465,7 +1465,3 @@ msgstr "" msgid "Settings saved" msgstr "Configuración guardada" - -#, fuzzy -#~ msgid "Pending" -#~ msgstr "Encabezamientos" diff --git a/i18n/fr.po b/i18n/fr.po index 0b4860db..38648ff4 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load App\n" -"POT-Creation-Date: 2025-11-12T16:45:33.366Z\n" +"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -205,6 +205,9 @@ msgstr "" msgid "Default" msgstr "" +msgid "Delete and Import" +msgstr "SUPPRIMER ET IMPORTER" + #, fuzzy msgid "Import despite duplicates" msgstr "Importer des données" @@ -1306,9 +1309,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://" -"eyeseetea.com). Source code, documentation and release notes can be found at " -"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." +"com). Source code, documentation and release notes can be found at the " +"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1402,9 +1405,6 @@ msgstr "" "valeurs de données seront supprimées et seules celles de la feuille de " "calcul seront enregistrées. Êtes-vous sur de vouloir continuer?" -msgid "Delete and Import" -msgstr "SUPPRIMER ET IMPORTER" - msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " diff --git a/i18n/pt.po b/i18n/pt.po index b3848033..926b8a67 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-12T16:45:33.366Z\n" +"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -213,9 +213,8 @@ msgstr "" msgid "Default" msgstr "" -#, fuzzy -msgid "Delete and import" -msgstr "Importação de dados em massa" +msgid "Delete and Import" +msgstr "" #, fuzzy msgid "Import despite duplicates" @@ -1346,9 +1345,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://" -"eyeseetea.com). Source code, documentation and release notes can be found at " -"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." +"com). Source code, documentation and release notes can be found at the " +"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1441,9 +1440,6 @@ msgstr "" "dados serão excluídos e remplazados pelos dados da planilha. Tem certeza que " "deseja continuar?" -msgid "Delete and Import" -msgstr "" - msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " diff --git a/i18n/ru.po b/i18n/ru.po index eb4bfe74..4aa1643f 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-12T16:45:33.366Z\n" +"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -214,9 +214,8 @@ msgstr "" msgid "Default" msgstr "" -#, fuzzy -msgid "Delete and import" -msgstr "Массовый импорт данных" +msgid "Delete and Import" +msgstr "" #, fuzzy msgid "Import despite duplicates" @@ -1350,9 +1349,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://" -"eyeseetea.com). Source code, documentation and release notes can be found at " -"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." +"com). Source code, documentation and release notes can be found at the " +"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1445,9 +1444,6 @@ msgstr "" "данных будут удалены, и будут сохранены только те, которые находятся в " "электронной таблице. Вы уверены?" -msgid "Delete and Import" -msgstr "" - msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " diff --git a/src/webapp/components/history/HistoryImportSummary.tsx b/src/webapp/components/history/HistoryImportSummary.tsx index c960826e..046cae75 100644 --- a/src/webapp/components/history/HistoryImportSummary.tsx +++ b/src/webapp/components/history/HistoryImportSummary.tsx @@ -88,7 +88,7 @@ function getImportStrategyLabel(entry: HistoryEntryDetails, dataFormType: Maybe< } switch (strategy) { case "IMPORT": - return dataFormType === "dataSets" ? i18n.t("Delete and import") : i18n.t("Import despite duplicates"); + return dataFormType === "dataSets" ? i18n.t("Delete and Import") : i18n.t("Import despite duplicates"); case "IGNORE": return dataFormType === "dataSets" ? i18n.t("Import only new data values") From 9371734b4b4215200b0e521b2602640d2df6d72f Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:23:15 +0800 Subject: [PATCH 04/19] chunk datavalue import --- src/data/InstanceDhisRepository.ts | 98 ++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 8b6b7b7f..7009d449 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -28,7 +28,11 @@ import { DhisInstance } from "../domain/entities/DhisInstance"; import { Locale } from "../domain/entities/Locale"; import { OrgUnit } from "../domain/entities/OrgUnit"; import { NamedRef, Ref } from "../domain/entities/ReferenceObject"; -import { SynchronizationResult } from "../domain/entities/SynchronizationResult"; +import { + SynchronizationResult, + SynchronizationStats, + SynchronizationStatus, +} from "../domain/entities/SynchronizationResult"; import { Program, TrackedEntityInstance } from "../domain/entities/TrackedEntityInstance"; import { BuilderMetadata, @@ -46,6 +50,7 @@ import { D2TrackedEntityType, DataStore, DataValueSetsGetResponse, + DataValueSetsPostResponse, Id, SelectedPick, D2SharingSchema, @@ -56,6 +61,7 @@ import { postEvents } from "./Dhis2Events"; import { getProgram, getTrackedEntityInstances, updateTrackedEntityInstances } from "./Dhis2TrackedEntityInstances"; import { Sharing } from "../domain/entities/Sharing"; import { getMetadataDetailsFromErrors } from "./Dhis2Import"; +import { Maybe } from "../types/utils"; export class InstanceDhisRepository implements InstanceRepository { private api: D2Api; @@ -458,13 +464,28 @@ export class InstanceDhisRepository implements InstanceRepository { const title = importStrategy === "DELETE" ? i18n.t("Data values - Delete") : i18n.t("Data values - Create/update"); - const { response } = await this.api.dataValues - .postSetAsync({ importStrategy }, { dataSet: dataSetId, dataValues }) - .getData(); + if (dataValues.length === 0) { + return { + title, + status: "SUCCESS", + message: i18n.t("No data values to import"), + stats: [{ imported: 0, deleted: 0, updated: 0, ignored: 0 }], + errors: [], + rawResponse: {}, + }; + } - const importSummary = await this.api.system.waitFor(response.jobType, response.id).getData(); + const chunks = _.chunk(dataValues, 3000); - if (!importSummary) { + const chunkResults = await promiseMap(chunks, async chunk => { + const { response } = await this.api.dataValues + .postSetAsync({ importStrategy }, { dataSet: dataSetId, dataValues: chunk }) + .getData(); + + return this.api.system.waitFor(response.jobType, response.id).getData(); + }); + + if (chunkResults.every(r => !r)) { return { title, status: "ERROR", @@ -475,21 +496,72 @@ export class InstanceDhisRepository implements InstanceRepository { }; } - const { status, description, conflicts, importCount } = importSummary; - const { imported, deleted, updated, ignored } = importCount; - const errors = conflicts?.map(({ object, value }) => ({ id: object, message: value, details: "" })) ?? []; + const { mergedStatus, mergedDescription, mergedImportCount, nullChunkStats, summaries } = + this.mergeChunkResults(chunks, chunkResults); + + const allConflicts = _.flatMap(summaries, s => s.conflicts ?? []); + const errors = allConflicts.map(({ object, value }) => ({ id: object, message: value, details: "" })); const errorDetails = await getMetadataDetailsFromErrors(this.api, errors); return { title, - status, - message: description, - stats: [{ imported, deleted, updated, ignored }], + status: mergedStatus, + message: mergedDescription, + stats: [mergedImportCount, ...nullChunkStats], errors: errorDetails, - rawResponse: importSummary, + rawResponse: summaries, }; } + private mergeChunkResults(chunks: AggregatedDataValue[][], chunkResults: Array) { + const emptyChunkCount = chunkResults.filter(r => !r).length; + const hasEmptySummaries = emptyChunkCount > 0; + const summaries = _.compact(chunkResults); + + const uniqueStatuses = _.uniq(summaries.map(s => s.status)); + const mergedStatus: SynchronizationStatus = + hasEmptySummaries || uniqueStatuses.length !== 1 ? "WARNING" : uniqueStatuses[0] ?? "WARNING"; + + const mergedDescription = [ + ..._.uniq(summaries.map(s => s.description).filter(Boolean)), + ...(hasEmptySummaries + ? [ + i18n.t("{{count}} chunk(s) returned no summary — import result unknown for those records.", { + count: emptyChunkCount, + }), + ] + : []), + ].join(" / "); + + const mergedImportCount = summaries.reduce( + (acc, s) => ({ + imported: acc.imported + s.importCount.imported, + deleted: acc.deleted + s.importCount.deleted, + updated: acc.updated + s.importCount.updated, + ignored: acc.ignored + s.importCount.ignored, + }), + { imported: 0, deleted: 0, updated: 0, ignored: 0 } + ); + + // Add a dedicated stat row per null chunk so the UI surfaces unknown outcomes explicitly. + // The total is set to the chunk size so the user knows how many records have an unknown outcome. + const nullChunkStats: SynchronizationStats[] = hasEmptySummaries + ? chunkResults + .map((result, i) => ({ result, chunk: chunks[i] })) + .filter(({ result }) => !result) + .map(({ chunk }) => ({ + type: i18n.t("Chunk (unknown outcome)"), + imported: 0, + deleted: 0, + updated: 0, + ignored: 0, + total: chunk?.length ?? 0, + })) + : []; + + return { mergedStatus, mergedDescription, mergedImportCount, nullChunkStats, summaries }; + } + // TODO: Review when data validation comes in private async validateAggregateImportPackage(dataValues: AggregatedDataValue[]) { const dataElements = _.uniq(dataValues.map(({ dataElement }) => dataElement)); From eaa67730be5566e15fa2bfe7ecb5e41c08229b35 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:35:08 +0800 Subject: [PATCH 05/19] clean up dependency --- src/data/InstanceDhisRepository.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 7009d449..71488513 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -61,7 +61,6 @@ import { postEvents } from "./Dhis2Events"; import { getProgram, getTrackedEntityInstances, updateTrackedEntityInstances } from "./Dhis2TrackedEntityInstances"; import { Sharing } from "../domain/entities/Sharing"; import { getMetadataDetailsFromErrors } from "./Dhis2Import"; -import { Maybe } from "../types/utils"; export class InstanceDhisRepository implements InstanceRepository { private api: D2Api; From 42fbf64765065758ff7697b107ab953df890d30b Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:37:21 +0800 Subject: [PATCH 06/19] update comment --- src/data/InstanceDhisRepository.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 71488513..6f0c852c 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -542,8 +542,7 @@ export class InstanceDhisRepository implements InstanceRepository { { imported: 0, deleted: 0, updated: 0, ignored: 0 } ); - // Add a dedicated stat row per null chunk so the UI surfaces unknown outcomes explicitly. - // The total is set to the chunk size so the user knows how many records have an unknown outcome. + // Add a dedicated stat row per null chunk with total to unknown outcomes are explicitly. const nullChunkStats: SynchronizationStats[] = hasEmptySummaries ? chunkResults .map((result, i) => ({ result, chunk: chunks[i] })) From 52767ff76607e88964266e456f20c00fdcf298f0 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:37:27 +0800 Subject: [PATCH 07/19] translations --- i18n/en.pot | 19 +++++++++++++++++-- i18n/es.po | 18 +++++++++++++++++- i18n/fr.po | 18 +++++++++++++++++- i18n/pt.po | 18 +++++++++++++++++- i18n/ru.po | 19 ++++++++++++++++++- 5 files changed, 86 insertions(+), 6 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 56b0abcf..dfa46fae 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" -"PO-Revision-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" +"PO-Revision-Date: 2026-03-18T04:23:33.926Z\n" msgid "Events - Create/update" msgstr "" @@ -38,9 +38,24 @@ msgstr "" msgid "Data values - Create/update" msgstr "" +msgid "No data values to import" +msgstr "" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/es.po b/i18n/es.po index be274f85..74a6780a 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "Valores de los datos - Borrar" msgid "Data values - Create/update" msgstr "Valores de los datos - Crear/actualizar" +#, fuzzy +msgid "No data values to import" +msgstr "valores de datos" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/fr.po b/i18n/fr.po index 38648ff4..05f98fdf 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load App\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "" msgid "Data values - Create/update" msgstr "" +#, fuzzy +msgid "No data values to import" +msgstr "valeurs de données" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/pt.po b/i18n/pt.po index 926b8a67..39ff4dea 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "Valores de dados - Eliminar" msgid "Data values - Create/update" msgstr "Valores dos dados - Criar/actualizar" +#, fuzzy +msgid "No data values to import" +msgstr "valores de dados" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/ru.po b/i18n/ru.po index 4aa1643f..ce368f6d 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -42,9 +42,26 @@ msgstr "Значения данных - Удалить" msgid "Data values - Create/update" msgstr "Значения данных - создание/обновление" +#, fuzzy +msgid "No data values to import" +msgstr "значения данных" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" +msgstr[2] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" From 19ba28b42aee55f3a6a5e1b72bbac86ab7bd04e8 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Thu, 19 Mar 2026 06:41:23 -0500 Subject: [PATCH 08/19] add tooltip translations for spanish and french --- i18n/es.po | 6 +++--- i18n/fr.po | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/i18n/es.po b/i18n/es.po index be274f85..d342f767 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1387,19 +1387,19 @@ msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " "will be deleted first, so none will be kept before doing the import." -msgstr "" +msgstr "Todos los valores de la hoja de cálculo se importarán al sistema, pero cualquier dato existente para dicha unidad organizativa y periodos se eliminará primero, por lo que no se conservará nada antes de realizar la importación." msgid "" "Import only new data values, without updating nor deleting any existing one. " "Only values in the spreadsheet that do not currently exist in the system " "will be imported" -msgstr "" +msgstr "Importar solo valores de datos nuevos, sin actualizar ni eliminar los existentes. Solo se importarán los valores de la hoja de cálculo que no existan actualmente en el sistema." msgid "" "Import new data values and also update existing ones. All data values in the " "spreadsheet will be imported to the system, but other data values present in " "the system that are not provided in the spreadsheet will be kept." -msgstr "" +msgstr "Importar nuevos valores de datos y actualizar los existentes. Todos los valores de la hoja de cálculo se importarán al sistema, pero se mantendrán los demás valores presentes en el sistema que no figuren en la hoja de cálculo." msgid "Warning: Your upload may result in the generation of duplicates" msgstr "Alerta: Su importación puede generar duplicados" diff --git a/i18n/fr.po b/i18n/fr.po index 38648ff4..fb3b21fe 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1410,18 +1410,26 @@ msgid "" "data that was existing for such organisation unit and periods in the system " "will be deleted first, so none will be kept before doing the import." msgstr "" +"Toutes les données de la feuille de calcul seront importées dans le système," +"mais les données existantes pour ces unités d'organisation et ces périodes seront d'abord supprimées" +"; aucune donnée existante ne sera donc conservée avant l'importation." msgid "" "Import only new data values, without updating nor deleting any existing one. " "Only values in the spreadsheet that do not currently exist in the system " "will be imported" msgstr "" +"Importer uniquement les nouvelles valeurs, sans mettre à jour ni supprimer les données existantes." +"Seules les valeurs de la feuille de calcul qui n'existent pas encore dans le système seront importées." msgid "" "Import new data values and also update existing ones. All data values in the " "spreadsheet will be imported to the system, but other data values present in " "the system that are not provided in the spreadsheet will be kept." msgstr "" +"Importer les nouvelles valeurs et mettre à jour les valeurs existantes." +"Toutes les données de la feuille de calcul seront importées," +"mais les autres valeurs présentes dans le système et absentes de la feuille de calcul seront conservées." msgid "Warning: Your upload may result in the generation of duplicates" msgstr "" From 48a9182b5e643369bf11557ca919819dfbd3b077 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Fri, 27 Mar 2026 19:21:34 +0800 Subject: [PATCH 09/19] add multi text support for data elements --- src/domain/entities/Template.ts | 4 ++ src/domain/usecases/ImportTemplateUseCase.ts | 46 +++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index 04c4748c..dfbee59f 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -217,6 +217,7 @@ export interface TrackerEventRowDataSource { sortBy?: string; onlyLastEvent?: boolean; dataElementProcessingRules?: DataProcessingRule[]; + multiTextDataElementDelimiter?: string; } export interface RowDataSource extends BaseDataSource { @@ -233,6 +234,7 @@ export interface RowDataSource extends BaseDataSource { longitude: ColumnRef | CellRef | ValueRef; }; geometry?: ColumnRef | CellRef | ValueRef; + multiTextDataElementDelimiter?: string; } export interface TeiRowDataSource { @@ -262,6 +264,7 @@ export interface ColumnDataSource extends BaseDataSource { categoryOption?: ColumnRef; attribute?: RowRef | CellRef; eventId?: RowRef | CellRef; + multiTextDataElementDelimiter?: string; } export interface CellDataSource extends BaseDataSource { @@ -273,6 +276,7 @@ export interface CellDataSource extends BaseDataSource { categoryOption?: CellRef | ValueRef; attribute?: CellRef | ValueRef; eventId?: CellRef | ValueRef; + multiTextDataElementDelimiter?: string; } interface DataFormRef { diff --git a/src/domain/usecases/ImportTemplateUseCase.ts b/src/domain/usecases/ImportTemplateUseCase.ts index 76fbc904..7b3c7215 100644 --- a/src/domain/usecases/ImportTemplateUseCase.ts +++ b/src/domain/usecases/ImportTemplateUseCase.ts @@ -20,6 +20,7 @@ import { TemplateTrackerProgramPackage, } from "../entities/Template"; import { ExcelReader } from "../helpers/ExcelReader"; +import { MULTI_TEXT_OPTION_DELIMITER } from "../helpers/ExcelBuilder"; import { ExcelRepository } from "../repositories/ExcelRepository"; import { InstanceRepository } from "../repositories/InstanceRepository"; import { TemplateRepository } from "../repositories/TemplateRepository"; @@ -315,6 +316,28 @@ export class ImportTemplateUseCase implements UseCase { .first(); } + private getMultiTextDataElementDelimiter(template: Template): Maybe { + if (template.type !== "custom") return undefined; + + return _(template.dataSources) + .map(dataSource => { + if (typeof dataSource === "function") { + return undefined; + } else if ( + dataSource.type !== "row" && + dataSource.type !== "rowTrackedEvent" && + dataSource.type !== "column" && + dataSource.type !== "cell" + ) { + return undefined; + } else { + return dataSource.multiTextDataElementDelimiter; + } + }) + .compact() + .first(); + } + private shouldDeleteAggregatedData(strategy: DuplicateImportStrategy): boolean { return strategy === "IMPORT"; } @@ -377,13 +400,17 @@ export class ImportTemplateUseCase implements UseCase { const trackedEntityPackage = dataPackage.type === "trackerPrograms" ? this.formatTrackedEntityInstances(dataPackage, dataForm) : {}; + const multiTextDataElementDelimiter = this.getMultiTextDataElementDelimiter(template); + return { ...dataPackage, ...trackedEntityPackage, dataEntries: dataPackage.dataEntries.map(({ dataValues, ...dataEntry }) => { return { ...dataEntry, - dataValues: _.compact(dataValues.map(value => formatDataValue(value, dataForm))), + dataValues: _.compact( + dataValues.map(value => formatDataValue(value, dataForm, multiTextDataElementDelimiter)) + ), }; }), }; @@ -725,7 +752,11 @@ function getOptionValue(originalValue: string, options?: DataOption[]): Maybe originalValue === id || originalValue === name); } -function formatDataValue(item: TemplateDataPackageDataValue, dataForm: DataForm): Maybe { +function formatDataValue( + item: TemplateDataPackageDataValue, + dataForm: DataForm, + multiTextDataElementDelimiter: Maybe +): Maybe { const dataElement = dataForm.dataElements.find(({ id }) => item.dataElement === id); const booleanValue = getBooleanValue(item); @@ -737,6 +768,17 @@ function formatDataValue(item: TemplateDataPackageDataValue, dataForm: DataForm) return booleanValue ? { ...item, value: true } : undefined; } + if (dataElement?.valueType === "MULTI_TEXT" && multiTextDataElementDelimiter && isString(item.value)) { + const optionNames = item.value.split(multiTextDataElementDelimiter).map(name => name.trim()); + const value = optionNames + .map(name => { + const option = getOptionValue(name, dataElement.options); + return option?.code ?? name; + }) + .join(MULTI_TEXT_OPTION_DELIMITER); + return { ...item, value }; + } + const optionValue = isString(item.value) ? getOptionValue(item.value, dataElement?.options) : undefined; const value = optionValue?.code ?? item.value; From f5da0373d03cbf559f5a8be34f0db70828d5d9f3 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Fri, 27 Mar 2026 22:49:05 +0800 Subject: [PATCH 10/19] add override rule --- src/domain/entities/Template.ts | 20 +++++- src/domain/helpers/DataProcessingService.ts | 79 ++++++++++++++++++--- src/domain/helpers/ExcelBuilder.ts | 37 ++++++---- 3 files changed, 113 insertions(+), 23 deletions(-) diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index dfbee59f..b3b3d9df 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -162,7 +162,7 @@ export interface Range { } type BaseDataProcessingRule = { - type: "coalesce"; + type: "coalesce" | "override"; condition: "onExport"; description?: string; }; @@ -170,11 +170,18 @@ type BaseDataProcessingRule = { export type DataProcessingRuleCoalesce = BaseDataProcessingRule & { type: "coalesce"; condition: "onExport"; - destination: ColumnRef; + destination: ColumnRef | RowRef; targetIds: Id[]; // attribute or data element id }; -type DataProcessingRule = DataProcessingRuleCoalesce; +export type DataProcessingRuleOverride = BaseDataProcessingRule & { + type: "override"; + condition: "onExport"; + destination: ColumnRef | RowRef; + target: ColumnRef | RowRef; +}; + +type DataProcessingRule = DataProcessingRuleCoalesce | DataProcessingRuleOverride; interface BaseDataSource { type: DataSourceType; @@ -235,6 +242,7 @@ export interface RowDataSource extends BaseDataSource { }; geometry?: ColumnRef | CellRef | ValueRef; multiTextDataElementDelimiter?: string; + dataElementProcessingRules?: DataProcessingRule[]; } export interface TeiRowDataSource { @@ -265,6 +273,7 @@ export interface ColumnDataSource extends BaseDataSource { attribute?: RowRef | CellRef; eventId?: RowRef | CellRef; multiTextDataElementDelimiter?: string; + dataElementProcessingRules?: DataProcessingRule[]; } export interface CellDataSource extends BaseDataSource { @@ -277,6 +286,7 @@ export interface CellDataSource extends BaseDataSource { attribute?: CellRef | ValueRef; eventId?: CellRef | ValueRef; multiTextDataElementDelimiter?: string; + dataElementProcessingRules?: DataProcessingRule[]; } interface DataFormRef { @@ -583,3 +593,7 @@ function mapFromProgramData(entry: ProgramPackageData): TemplateDataPackageData export function isDataProcessingRuleCoalesce(rule: DataProcessingRule): rule is DataProcessingRuleCoalesce { return rule.type === "coalesce"; } + +export function isDataProcessingRuleOverride(rule: DataProcessingRule): rule is DataProcessingRuleOverride { + return rule.type === "override"; +} diff --git a/src/domain/helpers/DataProcessingService.ts b/src/domain/helpers/DataProcessingService.ts index 8e0aca23..f3dea41d 100644 --- a/src/domain/helpers/DataProcessingService.ts +++ b/src/domain/helpers/DataProcessingService.ts @@ -1,4 +1,13 @@ -import { CellRef, DataProcessingRuleCoalesce, TemplateDataValue } from "../entities/Template"; +import { + CellRef, + ColumnRef, + DataProcessingRuleCoalesce, + DataProcessingRuleOverride, + isDataProcessingRuleCoalesce, + isDataProcessingRuleOverride, + RowRef, + TemplateDataValue, +} from "../entities/Template"; import { Id } from "../entities/ReferenceObject"; import _ from "lodash"; @@ -9,8 +18,23 @@ export type DataToProcess = { optionId?: TemplateDataValue["optionId"]; }; +type DataProcessingRule = DataProcessingRuleCoalesce | DataProcessingRuleOverride; + export class DataProcessingService { - static coalesceValues(props: { + static applyRules(props: { + dataDetails: DataToProcess[]; + dataProcessingRules?: DataProcessingRule[]; + }): DataToProcess[] { + const { dataDetails, dataProcessingRules } = props; + + const coalesceRules = dataProcessingRules?.filter(isDataProcessingRuleCoalesce); + const overrideRules = dataProcessingRules?.filter(isDataProcessingRuleOverride); + + const coalesced = this.coalesceValues({ dataDetails, dataProcessingRules: coalesceRules }); + return this.overrideValues({ dataDetails: coalesced, dataProcessingRules: overrideRules }); + } + + private static coalesceValues(props: { dataDetails: DataToProcess[]; dataProcessingRules?: DataProcessingRuleCoalesce[]; }): DataToProcess[] { @@ -33,18 +57,57 @@ export class DataProcessingService { if (!firstValidDetail) return undefined; return { ...firstValidDetail, - cell: this.replaceColumn(firstValidDetail.cell, rule.destination.ref), + cell: this.replaceRef(firstValidDetail.cell, rule.destination), }; }); return [...otherDataElements, ..._.compact(coalescedEntries)]; } - private static replaceColumn(cellRef: CellRef, newColumn: string): CellRef { + private static overrideValues(props: { + dataDetails: DataToProcess[]; + dataProcessingRules?: DataProcessingRuleOverride[]; + }): DataToProcess[] { + const { dataProcessingRules, dataDetails } = props; + + if (!dataProcessingRules || dataProcessingRules.length < 1) return dataDetails; + + return dataDetails.map(detail => { + const matchingRule = dataProcessingRules.find(rule => this.cellMatchesRef(detail.cell, rule.target)); + + if (!matchingRule) return detail; + + return { + ...detail, + cell: this.replaceRef(detail.cell, matchingRule.destination), + }; + }); + } + + private static cellMatchesRef(targetCell: CellRef, targetRef: ColumnRef | RowRef): boolean { + switch (targetRef.type) { + case "column": + return targetCell.ref.replace(/[0-9]+/, "") === targetRef.ref; + case "row": + return targetCell.ref.replace(/[A-Z]+/, "") === String(targetRef.ref); + } + } + + private static replaceRef(cellRef: CellRef, destinationRef: ColumnRef | RowRef): CellRef { + const colLetter = cellRef.ref.replace(/[0-9]+/, ""); const rowNumber = cellRef.ref.replace(/[A-Z]+/, ""); - return { - ...cellRef, - ref: `${newColumn}${rowNumber}`, - }; + + switch (destinationRef.type) { + case "column": + return { + ...cellRef, + ref: `${destinationRef.ref}${rowNumber}`, + }; + case "row": + return { + ...cellRef, + ref: `${colLetter}${destinationRef.ref}`, + }; + } } } diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index 38488ce1..bf8ce845 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -12,7 +12,6 @@ import { DataSource, DataSourceValue, DownloadCustomizationOptions, - isDataProcessingRuleCoalesce, RowDataSource, setDataEntrySheet, setSheet, @@ -220,12 +219,11 @@ export class ExcelBuilder { }) ); - const coalesceDataProcessRules = - dataSource.attributeDataProcessingRules?.filter(isDataProcessingRuleCoalesce); - - const attributeDetails = DataProcessingService.coalesceValues({ + const attributeDetails = DataProcessingService.applyRules({ dataDetails: _.compact(allAttributeDetails), - dataProcessingRules: coalesceDataProcessRules, + dataProcessingRules: dataSource.attributeDataProcessingRules?.filter( + rule => rule.condition === "onExport" + ), }); await Promise.all( @@ -376,12 +374,11 @@ export class ExcelBuilder { }) ); - const coalesceDataProcessRules = - dataSource.dataElementProcessingRules?.filter(isDataProcessingRuleCoalesce); - - const dataElementDetails = DataProcessingService.coalesceValues({ + const dataElementDetails = DataProcessingService.applyRules({ dataDetails: dataElementsToProcess, - dataProcessingRules: coalesceDataProcessRules, + dataProcessingRules: dataSource.dataElementProcessingRules?.filter( + rule => rule.condition === "onExport" + ), }); //TODO extract "_ dv.dataElement === dataElement && (!dv.category || dv.category === category) ) ?? {}; - if (value) await this.excelRepository.writeCell(template.id, cell, value); + if (value && dataElement) { + dataElementsToProcess.push({ cell, id: dataElement, value }); + } } + const dataElementDetails = DataProcessingService.applyRules({ + dataDetails: dataElementsToProcess, + dataProcessingRules: dataSource.dataElementProcessingRules?.filter( + rule => rule.condition === "onExport" + ), + }); + + await Promise.all( + dataElementDetails.map(({ cell, value }) => + this.excelRepository.writeCell(template.id, cell, value) + ) + ); + rowStart += 1; } } From 70e998db3b8439feaee3de5b30fdc7d58831dfd0 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Sat, 28 Mar 2026 18:38:04 +0800 Subject: [PATCH 11/19] fix single line import by doing a second pass if range isn't filled --- src/domain/helpers/ExcelBuilder.ts | 120 ++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 28 deletions(-) diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index bf8ce845..72d00753 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -530,45 +530,109 @@ export class ExcelBuilder { } } - const dataElementsToProcess: DataToProcess[] = []; - for (const cell of cells) { - const dataElementCell = await this.findRelative(template, dataSource.dataElement, cell); + const dataElementsToProcess = await this.resolveDataElementValues(template, dataSource, cells, dataValues); + + await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess); + + rowStart += 1; + } - const categoryCell = await this.findRelative(template, dataSource.categoryOption, cell); + if (dataSource.range.rowEnd !== undefined && rowStart <= dataSource.range.rowEnd) { + await this.fillUnmappedRows(template, dataSource, payload, rowStart, dataSource.range.rowEnd); + } + } + + // When a datasource range has more rows than data entries + // (e.g. dataset custom templates where each row maps to a different data element via a mapping sheet), + // the main fillRows loop only covers the first N rows. + // This method fills the remaining rows by searching ALL entries for matching data values. + private async fillUnmappedRows( + template: Template, + dataSource: RowDataSource, + payload: TemplateDataPackage, + nextRow: number, + rangeRowEnd: number + ) { + const allDataValues = payload.dataEntries.flatMap(e => e.dataValues); + const dataValueByDECOC = _.keyBy(allDataValues, dv => `${dv.dataElement}:${dv.category ?? ""}`); - const dataElement = dataElementCell - ? removeCharacters(await this.excelRepository.readCell(template.id, dataElementCell)) - : undefined; + for (let row = nextRow; row <= rangeRowEnd; row++) { + const cells = await this.excelRepository.getCellsInRange(template.id, { + ...dataSource.range, + rowStart: row, + rowEnd: row, + }); - const category = categoryCell - ? removeCharacters(await this.excelRepository.readCell(template.id, categoryCell)) - : undefined; + const dataElementsToProcess: DataToProcess[] = []; + for (const cell of cells) { + const resolved = await this.resolveCellDataElement(template, dataSource, cell); + if (!resolved) continue; - const { value } = - dataValues.find( - dv => dv.dataElement === dataElement && (!dv.category || dv.category === category) - ) ?? {}; + const { dataElement, category } = resolved; + const value = dataValueByDECOC[`${dataElement}:${category ?? ""}`]?.value; - if (value && dataElement) { + if (value !== undefined) { dataElementsToProcess.push({ cell, id: dataElement, value }); } } - const dataElementDetails = DataProcessingService.applyRules({ - dataDetails: dataElementsToProcess, - dataProcessingRules: dataSource.dataElementProcessingRules?.filter( - rule => rule.condition === "onExport" - ), - }); + await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess); + } + } - await Promise.all( - dataElementDetails.map(({ cell, value }) => - this.excelRepository.writeCell(template.id, cell, value) - ) - ); + private async resolveCellDataElement( + template: Template, + dataSource: RowDataSource, + cell: CellRef + ): Promise }>> { + const dataElementCell = await this.findRelative(template, dataSource.dataElement, cell); + if (!dataElementCell) return undefined; + + const dataElement = removeCharacters(await this.excelRepository.readCell(template.id, dataElementCell)); + if (!dataElement) return undefined; + + const categoryCell = await this.findRelative(template, dataSource.categoryOption, cell); + const category = categoryCell + ? removeCharacters(await this.excelRepository.readCell(template.id, categoryCell)) + : undefined; - rowStart += 1; - } + return { dataElement: String(dataElement), category: category ? String(category) : undefined }; + } + + private async resolveDataElementValues( + template: Template, + dataSource: RowDataSource, + cells: CellRef[], + dataValues: TemplateDataPackageData["dataValues"] + ): Promise { + const results = await promiseMap(cells, async (cell): Promise> => { + const resolved = await this.resolveCellDataElement(template, dataSource, cell); + if (!resolved) return undefined; + + const { dataElement, category } = resolved; + const { value } = + dataValues.find(dv => dv.dataElement === dataElement && (!dv.category || dv.category === category)) ?? + {}; + + return value ? { cell, id: dataElement, value } : undefined; + }); + + return _.compact(results); + } + + private async applyRulesAndWrite( + template: Template, + dataSource: RowDataSource, + dataElementsToProcess: DataToProcess[] + ): Promise { + const dataElementDetails = DataProcessingService.applyRules({ + dataDetails: dataElementsToProcess, + dataProcessingRules: dataSource.dataElementProcessingRules?.filter(rule => rule.condition === "onExport"), + }); + + await Promise.all( + dataElementDetails.map(({ cell, value }) => this.excelRepository.writeCell(template.id, cell, value)) + ); } private async findRelative(template: Template, ref?: SheetRef | ValueRef, relative?: CellRef) { From 9945a88fa5ca691868325cb96b50b67f448de7ad Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Sat, 28 Mar 2026 19:37:31 +0800 Subject: [PATCH 12/19] add handling for multi text export --- src/domain/entities/Template.ts | 14 +++ src/domain/helpers/ExcelBuilder.ts | 116 +++++++++++++++--- .../usecases/DownloadTemplateUseCase.ts | 21 +++- src/domain/usecases/ImportTemplateUseCase.ts | 17 +-- 4 files changed, 134 insertions(+), 34 deletions(-) diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index b3b3d9df..8c2621e1 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -590,6 +590,20 @@ function mapFromProgramData(entry: ProgramPackageData): TemplateDataPackageData }; } +type DataSourceWithMultiTextDelimiter = RowDataSource | TrackerEventRowDataSource | ColumnDataSource | CellDataSource; + +export function hasMultiTextDataElementDelimiter( + dataSource: DataSource +): dataSource is DataSourceWithMultiTextDelimiter { + return ( + typeof dataSource !== "function" && + (dataSource.type === "row" || + dataSource.type === "rowTrackedEvent" || + dataSource.type === "column" || + dataSource.type === "cell") + ); +} + export function isDataProcessingRuleCoalesce(rule: DataProcessingRule): rule is DataProcessingRuleCoalesce { return rule.type === "coalesce"; } diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index 72d00753..f4788471 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -20,6 +20,7 @@ import { Template, TemplateDataPackage, TemplateDataPackageData, + TemplateDataValue, templateFromDataPackage, TemplateTrackerProgramPackage, TrackerEventRowDataSource, @@ -35,6 +36,8 @@ import { ModulesRepositories } from "../repositories/ModulesRepositories"; import { Maybe } from "../../types/utils"; import { DataElementDisaggregationsMappingRepository } from "../repositories/DataElementDisaggregationsMappingRepository"; import { DataProcessingService, DataToProcess } from "./DataProcessingService"; +import { DataElement, DataForm } from "../entities/DataForm"; +import { Id } from "../entities/ReferenceObject"; const dateFormatPattern = "yyyy-MM-dd"; @@ -46,7 +49,12 @@ export class ExcelBuilder { private dataElementDisaggregationsMappingRepository: DataElementDisaggregationsMappingRepository ) {} - public async populateTemplate(template: Template, payload: DataPackage, settings: Settings): Promise { + public async populateTemplate( + template: Template, + payload: DataPackage, + settings: Settings, + dataForm?: DataForm + ): Promise { const { dataSources = [] } = template; const dataSourceValues = await this.getDataSourceValues(template, dataSources); const metadata = @@ -54,21 +62,33 @@ export class ExcelBuilder { ? await this.instanceRepository.getBuilderMetadata(payload.trackedEntityInstances) : emptyBuilderMetadata; const templatePayload = templateFromDataPackage(payload); + const dataElementById = _.keyBy(dataForm?.dataElements ?? [], de => de.id); + const multiTextLookup: MultiTextLookup = { + dataElementById, + optionByCodeByDe: _.mapValues(dataElementById, de => _.keyBy(de.options, o => o.code)), + }; for (const dataSource of dataSourceValues) { if (!dataSource.skipPopulate) { switch (dataSource.type) { case "cell": - await this.fillCells(template, dataSource, templatePayload); + await this.fillCells(template, dataSource, templatePayload, multiTextLookup); break; case "row": - await this.fillRows(template, dataSource, templatePayload); + await this.fillRows(template, dataSource, templatePayload, multiTextLookup); break; case "rowTei": await this.fillTeiRows(template, dataSource, templatePayload); break; case "rowTrackedEvent": - await this.fillTrackerEventRows(template, dataSource, templatePayload, metadata, settings); + await this.fillTrackerEventRows( + template, + dataSource, + templatePayload, + metadata, + settings, + multiTextLookup + ); break; case "rowTeiRelationship": await this.fillTrackerRelationshipRows(template, dataSource, payload); @@ -102,7 +122,12 @@ export class ExcelBuilder { }); } - private async fillCells(template: Template, dataSource: CellDataSource, payload: TemplateDataPackage) { + private async fillCells( + template: Template, + dataSource: CellDataSource, + payload: TemplateDataPackage, + multiTextLookup: MultiTextLookup + ) { const orgUnit = await this.readCellValue(template, dataSource.orgUnit); const dataElement = await this.readCellValue(template, dataSource.dataElement); const period = await this.readCellValue(template, dataSource.period); @@ -115,7 +140,13 @@ export class ExcelBuilder { .find(dv => dv.dataElement === dataElement && (!dv.category || dv.category === categoryOption)) ?? {}; if (value) { - await this.excelRepository.writeCell(template.id, dataSource.ref, value); + const writeValue = this.formatDataElementValue({ + value, + dataElementId: String(dataElement), + delimiter: dataSource.multiTextDataElementDelimiter, + multiTextLookup, + }); + await this.excelRepository.writeCell(template.id, dataSource.ref, writeValue); } } @@ -247,6 +278,21 @@ export class ExcelBuilder { return options.map(option => option.name).join(multiTextTeiDelimiter); } + private formatDataElementValue(options: { + value: TemplateDataValue["value"]; + dataElementId: Id; + delimiter: Maybe; + multiTextLookup: MultiTextLookup; + }): TemplateDataValue["value"] { + const { value, dataElementId, delimiter, multiTextLookup } = options; + const isMultiText = multiTextLookup.dataElementById[dataElementId]?.valueType === "MULTI_TEXT"; + if (!isMultiText || !delimiter) return value; + + const codes = String(value).split(MULTI_TEXT_OPTION_DELIMITER); + const optionByCode = multiTextLookup.optionByCodeByDe[dataElementId] ?? {}; + return codes.map(code => optionByCode[code]?.name ?? code).join(delimiter); + } + private async fillCell(template: Template, cellRef: CellRef, sheetRef: SheetRef, value: string | number | boolean) { const cell = await this.excelRepository.findRelativeCell(template.id, sheetRef, cellRef); @@ -294,7 +340,8 @@ export class ExcelBuilder { dataSource: TrackerEventRowDataSource, payload: TemplateDataPackage, metadata: BuilderMetadata, - settings: Settings + settings: Settings, + multiTextLookup: MultiTextLookup ) { if (payload.type !== "trackerPrograms") return; @@ -383,9 +430,17 @@ export class ExcelBuilder { //TODO extract "_ - this.excelRepository.writeCell(template.id, cell, optionId ? `_${optionId}` : value) - ) + dataElementDetails.map(({ cell, id, optionId, value }) => { + const writeValue = optionId + ? `_${optionId}` + : this.formatDataElementValue({ + value, + dataElementId: id, + delimiter: dataSource.multiTextDataElementDelimiter, + multiTextLookup, + }); + return this.excelRepository.writeCell(template.id, cell, writeValue); + }) ); rowStart += 1; @@ -479,7 +534,12 @@ export class ExcelBuilder { return dataEntriesTeisWithEvents; } - private async fillRows(template: Template, dataSource: RowDataSource, payload: TemplateDataPackage) { + private async fillRows( + template: Template, + dataSource: RowDataSource, + payload: TemplateDataPackage, + multiTextLookup: MultiTextLookup + ) { let { rowStart } = dataSource.range; for (const { id, orgUnit, period, attribute, dataValues, coordinate, geometry } of payload.dataEntries) { const cells = await this.excelRepository.getCellsInRange(template.id, { @@ -532,13 +592,20 @@ export class ExcelBuilder { const dataElementsToProcess = await this.resolveDataElementValues(template, dataSource, cells, dataValues); - await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess); + await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess, multiTextLookup); rowStart += 1; } if (dataSource.range.rowEnd !== undefined && rowStart <= dataSource.range.rowEnd) { - await this.fillUnmappedRows(template, dataSource, payload, rowStart, dataSource.range.rowEnd); + await this.fillUnmappedRows( + template, + dataSource, + payload, + rowStart, + dataSource.range.rowEnd, + multiTextLookup + ); } } @@ -551,7 +618,8 @@ export class ExcelBuilder { dataSource: RowDataSource, payload: TemplateDataPackage, nextRow: number, - rangeRowEnd: number + rangeRowEnd: number, + multiTextLookup: MultiTextLookup ) { const allDataValues = payload.dataEntries.flatMap(e => e.dataValues); const dataValueByDECOC = _.keyBy(allDataValues, dv => `${dv.dataElement}:${dv.category ?? ""}`); @@ -576,7 +644,7 @@ export class ExcelBuilder { } } - await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess); + await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess, multiTextLookup); } } @@ -623,7 +691,8 @@ export class ExcelBuilder { private async applyRulesAndWrite( template: Template, dataSource: RowDataSource, - dataElementsToProcess: DataToProcess[] + dataElementsToProcess: DataToProcess[], + multiTextLookup: MultiTextLookup ): Promise { const dataElementDetails = DataProcessingService.applyRules({ dataDetails: dataElementsToProcess, @@ -631,7 +700,15 @@ export class ExcelBuilder { }); await Promise.all( - dataElementDetails.map(({ cell, value }) => this.excelRepository.writeCell(template.id, cell, value)) + dataElementDetails.map(({ cell, id, value }) => { + const writeValue = this.formatDataElementValue({ + value, + dataElementId: id, + delimiter: dataSource.multiTextDataElementDelimiter, + multiTextLookup, + }); + return this.excelRepository.writeCell(template.id, cell, writeValue); + }) ); } @@ -679,4 +756,9 @@ export class ExcelBuilder { } } +type MultiTextLookup = { + dataElementById: Record; + optionByCodeByDe: Record>; +}; + export const MULTI_TEXT_OPTION_DELIMITER = ","; diff --git a/src/domain/usecases/DownloadTemplateUseCase.ts b/src/domain/usecases/DownloadTemplateUseCase.ts index b4f06e86..f52069e3 100644 --- a/src/domain/usecases/DownloadTemplateUseCase.ts +++ b/src/domain/usecases/DownloadTemplateUseCase.ts @@ -9,9 +9,9 @@ import { getExtensionFile, XLSX_EXTENSION } from "../../utils/files"; import { promiseMap } from "../../utils/promises"; import Settings from "../../webapp/logic/settings"; import { getGeneratedTemplateId, SheetBuilder } from "../../webapp/logic/sheetBuilder"; -import { DataFormType, dataFormTypeMap } from "../entities/DataForm"; +import { DataForm, DataFormType, dataFormTypeMap } from "../entities/DataForm"; import { Id, Ref } from "../entities/ReferenceObject"; -import { templateFromDataPackage, TemplateType } from "../entities/Template"; +import { getDataFormRef, hasMultiTextDataElementDelimiter, Template, templateFromDataPackage, TemplateType } from "../entities/Template"; import { ExcelBuilder } from "../helpers/ExcelBuilder"; import { ExcelRepository } from "../repositories/ExcelRepository"; import { InstanceRepository } from "../repositories/InstanceRepository"; @@ -215,7 +215,8 @@ export class DownloadTemplateUseCase implements UseCase { } if (dataPackage) { - await builder.populateTemplate(template, dataPackage, settings); + const dataForm = await this.getDataForm(template); + await builder.populateTemplate(template, dataPackage, settings, dataForm); } } @@ -231,6 +232,20 @@ export class DownloadTemplateUseCase implements UseCase { } } + private async getDataForm(template: Template): Promise { + if (template.type !== "custom") return undefined; + + const hasMultiText = template.dataSources?.some( + ds => hasMultiTextDataElementDelimiter(ds) && ds.multiTextDataElementDelimiter + ); + if (!hasMultiText) return undefined; + + const dataFormRef = getDataFormRef(template); + if (!dataFormRef.id) return undefined; + + return (await this.instanceRepository.getDataForms({ ids: [dataFormRef.id] }))[0]; + } + private getFirstValueOrEmpty(model: string[]): string { return _(model).first() || ""; } diff --git a/src/domain/usecases/ImportTemplateUseCase.ts b/src/domain/usecases/ImportTemplateUseCase.ts index 7b3c7215..6e433870 100644 --- a/src/domain/usecases/ImportTemplateUseCase.ts +++ b/src/domain/usecases/ImportTemplateUseCase.ts @@ -11,6 +11,7 @@ import { Either } from "../entities/Either"; import { OrgUnit } from "../entities/OrgUnit"; import { ErrorMessage, SynchronizationResult } from "../entities/SynchronizationResult"; import { + hasMultiTextDataElementDelimiter, Template, TemplateDataPackage, TemplateDataPackageData, @@ -320,20 +321,8 @@ export class ImportTemplateUseCase implements UseCase { if (template.type !== "custom") return undefined; return _(template.dataSources) - .map(dataSource => { - if (typeof dataSource === "function") { - return undefined; - } else if ( - dataSource.type !== "row" && - dataSource.type !== "rowTrackedEvent" && - dataSource.type !== "column" && - dataSource.type !== "cell" - ) { - return undefined; - } else { - return dataSource.multiTextDataElementDelimiter; - } - }) + .filter(hasMultiTextDataElementDelimiter) + .map(ds => ds.multiTextDataElementDelimiter) .compact() .first(); } From 0cd43e38c39349b3dcce2073242ab3ed3cfdbf79 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Sat, 28 Mar 2026 19:43:31 +0800 Subject: [PATCH 13/19] format --- src/domain/usecases/DownloadTemplateUseCase.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/domain/usecases/DownloadTemplateUseCase.ts b/src/domain/usecases/DownloadTemplateUseCase.ts index f52069e3..daec1835 100644 --- a/src/domain/usecases/DownloadTemplateUseCase.ts +++ b/src/domain/usecases/DownloadTemplateUseCase.ts @@ -11,7 +11,13 @@ import Settings from "../../webapp/logic/settings"; import { getGeneratedTemplateId, SheetBuilder } from "../../webapp/logic/sheetBuilder"; import { DataForm, DataFormType, dataFormTypeMap } from "../entities/DataForm"; import { Id, Ref } from "../entities/ReferenceObject"; -import { getDataFormRef, hasMultiTextDataElementDelimiter, Template, templateFromDataPackage, TemplateType } from "../entities/Template"; +import { + getDataFormRef, + hasMultiTextDataElementDelimiter, + Template, + templateFromDataPackage, + TemplateType, +} from "../entities/Template"; import { ExcelBuilder } from "../helpers/ExcelBuilder"; import { ExcelRepository } from "../repositories/ExcelRepository"; import { InstanceRepository } from "../repositories/InstanceRepository"; From 38ab9f39321303026bf95bf230e684d49e56636d Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:18:19 +0800 Subject: [PATCH 14/19] populate fixedOrgUnit and fixedPeriod when downloading empty template --- .../usecases/DownloadTemplateUseCase.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/domain/usecases/DownloadTemplateUseCase.ts b/src/domain/usecases/DownloadTemplateUseCase.ts index b4f06e86..bcc7151d 100644 --- a/src/domain/usecases/DownloadTemplateUseCase.ts +++ b/src/domain/usecases/DownloadTemplateUseCase.ts @@ -196,24 +196,24 @@ export class DownloadTemplateUseCase implements UseCase { if (theme) await builder.applyTheme(template, theme); - if (enablePopulate) { - if (template.type === "custom" && template.fixedOrgUnit) { - await this.excelRepository.writeCell( - template.id, - template.fixedOrgUnit, - dataPackage?.dataEntries[0]?.orgUnit ?? this.getFirstValueOrEmpty(orgUnits) - ); - } + if (template.type === "custom" && template.fixedOrgUnit) { + await this.excelRepository.writeCell( + template.id, + template.fixedOrgUnit, + dataPackage?.dataEntries[0]?.orgUnit ?? this.getFirstValueOrEmpty(orgUnits) + ); + } - if (template.type === "custom" && template.fixedPeriod) { - const periods = buildAllPossiblePeriods(element.periodType, populateStartDate, populateEndDate); - await this.excelRepository.writeCell( - template.id, - template.fixedPeriod, - dataPackage?.dataEntries[0]?.period ?? this.getFirstValueOrEmpty(periods) - ); - } + if (template.type === "custom" && template.fixedPeriod) { + const periods = buildAllPossiblePeriods(element.periodType, populateStartDate, populateEndDate); + await this.excelRepository.writeCell( + template.id, + template.fixedPeriod, + dataPackage?.dataEntries[0]?.period ?? this.getFirstValueOrEmpty(periods) + ); + } + if (enablePopulate) { if (dataPackage) { await builder.populateTemplate(template, dataPackage, settings); } From 0c610eb5e1171e7e5c58075e860200faff31f35b Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 1 Apr 2026 15:53:05 +0800 Subject: [PATCH 15/19] reduce chunks further because some chunks are still resulting to empty responses --- src/data/InstanceDhisRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 6f0c852c..9efadfe4 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -474,7 +474,7 @@ export class InstanceDhisRepository implements InstanceRepository { }; } - const chunks = _.chunk(dataValues, 3000); + const chunks = _.chunk(dataValues, 1000); const chunkResults = await promiseMap(chunks, async chunk => { const { response } = await this.api.dataValues From 6d911f1fbcf6424db430e2bbd36f0619b3af2479 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:12:25 +0800 Subject: [PATCH 16/19] prioritize defined name over cached values --- src/domain/helpers/ExcelBuilder.ts | 13 +++++++++--- src/domain/helpers/ExcelCellReader.ts | 30 +++++++++++++++++++++++++++ src/domain/helpers/ExcelReader.ts | 13 ++---------- 3 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 src/domain/helpers/ExcelCellReader.ts diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index f4788471..7d0c9b6c 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -36,6 +36,7 @@ import { ModulesRepositories } from "../repositories/ModulesRepositories"; import { Maybe } from "../../types/utils"; import { DataElementDisaggregationsMappingRepository } from "../repositories/DataElementDisaggregationsMappingRepository"; import { DataProcessingService, DataToProcess } from "./DataProcessingService"; +import { readCellResolvingDefinedNames } from "./ExcelCellReader"; import { DataElement, DataForm } from "../entities/DataForm"; import { Id } from "../entities/ReferenceObject"; @@ -155,7 +156,13 @@ export class ExcelBuilder { ref?: CellRef | ValueRef, options: { isFormula: boolean } = { isFormula: false } ): Promise { - return removeCharacters(await this.excelRepository.readCell(template.id, ref, { formula: options.isFormula })); + if (options.isFormula || !ref || ref.type === "value") { + return removeCharacters( + await this.excelRepository.readCell(template.id, ref, { formula: options.isFormula }) + ); + } + + return removeCharacters(await readCellResolvingDefinedNames(this.excelRepository, template.id, ref)); } private async fillTeiRows(template: Template, dataSource: TeiRowDataSource, payload: TemplateDataPackage) { @@ -656,12 +663,12 @@ export class ExcelBuilder { const dataElementCell = await this.findRelative(template, dataSource.dataElement, cell); if (!dataElementCell) return undefined; - const dataElement = removeCharacters(await this.excelRepository.readCell(template.id, dataElementCell)); + const dataElement = await readCellResolvingDefinedNames(this.excelRepository, template.id, dataElementCell); if (!dataElement) return undefined; const categoryCell = await this.findRelative(template, dataSource.categoryOption, cell); const category = categoryCell - ? removeCharacters(await this.excelRepository.readCell(template.id, categoryCell)) + ? await readCellResolvingDefinedNames(this.excelRepository, template.id, categoryCell) : undefined; return { dataElement: String(dataElement), category: category ? String(category) : undefined }; diff --git a/src/domain/helpers/ExcelCellReader.ts b/src/domain/helpers/ExcelCellReader.ts new file mode 100644 index 00000000..08aa21a4 --- /dev/null +++ b/src/domain/helpers/ExcelCellReader.ts @@ -0,0 +1,30 @@ +import { removeCharacters } from "../../utils/string"; +import { CellRef } from "../entities/Template"; +import { ExcelRepository, ExcelValue } from "../repositories/ExcelRepository"; + +/** + * Read a cell's value, resolving defined-name formulas to their IDs. + * + * Some templates store data-element / category-option IDs as Excel defined-name + * references (e.g. `=_aP6qePO2WG6`). When the workbook has been saved in Excel + * the cached text value of the formula (a human-readable label) takes precedence + * over the formula itself in a plain `readCell` call. + * + * This helper detects that case: if the cell's formula matches a defined name, + * it returns the cleaned formula (the ID) instead of the cached display value. + */ +export async function readCellResolvingDefinedNames( + excelRepository: ExcelRepository, + templateId: string, + cell: CellRef +): Promise { + const value = await excelRepository.readCell(templateId, cell); + const formula = await excelRepository.readCell(templateId, cell, { formula: true }); + + const definedNames = await excelRepository.listDefinedNames(templateId); + if (typeof formula === "string" && definedNames.includes(formula.replace(/^=/, ""))) { + return removeCharacters(formula); + } + + return value; +} diff --git a/src/domain/helpers/ExcelReader.ts b/src/domain/helpers/ExcelReader.ts index c6a28ef3..705f03a4 100644 --- a/src/domain/helpers/ExcelReader.ts +++ b/src/domain/helpers/ExcelReader.ts @@ -5,6 +5,7 @@ import moment from "moment"; import { isDefined } from "../../utils"; import { promiseMap } from "../../utils/promises"; import { removeCharacters } from "../../utils/string"; +import { readCellResolvingDefinedNames } from "./ExcelCellReader"; import { DataForm, dataFormTypeMap, DataFormFeatureType } from "../entities/DataForm"; import { buildGeometry, getGeometryFromString } from "../entities/Geometry"; import { Relationship } from "../entities/Relationship"; @@ -515,17 +516,7 @@ export class ExcelReader { const cell = await this.excelRepository.findRelativeCell(template.id, ref, relative); if (cell) { - const value = await this.excelRepository.readCell(template.id, cell); - const formula = await this.excelRepository.readCell(template.id, cell, { - formula: true, - }); - - const definedNames = await this.excelRepository.listDefinedNames(template.id); - if (typeof formula === "string" && definedNames.includes(formula.replace(/^=/, ""))) { - return removeCharacters(formula); - } - - return value; + return readCellResolvingDefinedNames(this.excelRepository, template.id, cell); } } From 582c442d87badfed6cf8dd64740f55218f549c69 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Sat, 4 Apr 2026 15:55:41 +0800 Subject: [PATCH 17/19] handle row by dataEntry and single entry by lookup --- src/domain/helpers/ExcelBuilder.ts | 105 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index 7d0c9b6c..a0c04423 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -546,6 +546,61 @@ export class ExcelBuilder { dataSource: RowDataSource, payload: TemplateDataPackage, multiTextLookup: MultiTextLookup + ) { + const isFixedOrgUnitPeriod = + template.type === "custom" && Boolean(template.fixedOrgUnit) && Boolean(template.fixedPeriod); + + if (isFixedOrgUnitPeriod) { + return this.fillRowsByKeyLookup(template, dataSource, payload, multiTextLookup); + } + + return this.fillRowsByDataEntry(template, dataSource, payload, multiTextLookup); + } + + // For templates with fixedOrgUnit/fixedPeriod: one data entry with many data values, + // each row may map to a different DE/COC. Uses exact key lookup. + private async fillRowsByKeyLookup( + template: Template, + dataSource: RowDataSource, + payload: TemplateDataPackage, + multiTextLookup: MultiTextLookup + ) { + const { rowStart, rowEnd = rowStart } = dataSource.range; + const allDataValues = payload.dataEntries.flatMap(e => e.dataValues); + const dataValueByDECOC = _.keyBy(allDataValues, dv => `${dv.dataElement}:${dv.category ?? ""}`); + + for (let row = rowStart; row <= rowEnd; row++) { + const cells = await this.excelRepository.getCellsInRange(template.id, { + ...dataSource.range, + rowStart: row, + rowEnd: row, + }); + + const dataElementsToProcess: DataToProcess[] = []; + for (const cell of cells) { + const resolved = await this.resolveCellDataElement(template, dataSource, cell); + if (!resolved) continue; + + const { dataElement, category } = resolved; + const exactKey = `${dataElement}:${category ?? ""}`; + const defaultKey = `${dataElement}:`; + const value = dataValueByDECOC[exactKey]?.value ?? dataValueByDECOC[defaultKey]?.value; + + if (value !== undefined) { + dataElementsToProcess.push({ cell, id: dataElement, value }); + } + } + + await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess, multiTextLookup); + } + } + + // For templates where each row corresponds to a different data entry (orgUnit/period/event). + private async fillRowsByDataEntry( + template: Template, + dataSource: RowDataSource, + payload: TemplateDataPackage, + multiTextLookup: MultiTextLookup ) { let { rowStart } = dataSource.range; for (const { id, orgUnit, period, attribute, dataValues, coordinate, geometry } of payload.dataEntries) { @@ -603,56 +658,6 @@ export class ExcelBuilder { rowStart += 1; } - - if (dataSource.range.rowEnd !== undefined && rowStart <= dataSource.range.rowEnd) { - await this.fillUnmappedRows( - template, - dataSource, - payload, - rowStart, - dataSource.range.rowEnd, - multiTextLookup - ); - } - } - - // When a datasource range has more rows than data entries - // (e.g. dataset custom templates where each row maps to a different data element via a mapping sheet), - // the main fillRows loop only covers the first N rows. - // This method fills the remaining rows by searching ALL entries for matching data values. - private async fillUnmappedRows( - template: Template, - dataSource: RowDataSource, - payload: TemplateDataPackage, - nextRow: number, - rangeRowEnd: number, - multiTextLookup: MultiTextLookup - ) { - const allDataValues = payload.dataEntries.flatMap(e => e.dataValues); - const dataValueByDECOC = _.keyBy(allDataValues, dv => `${dv.dataElement}:${dv.category ?? ""}`); - - for (let row = nextRow; row <= rangeRowEnd; row++) { - const cells = await this.excelRepository.getCellsInRange(template.id, { - ...dataSource.range, - rowStart: row, - rowEnd: row, - }); - - const dataElementsToProcess: DataToProcess[] = []; - for (const cell of cells) { - const resolved = await this.resolveCellDataElement(template, dataSource, cell); - if (!resolved) continue; - - const { dataElement, category } = resolved; - const value = dataValueByDECOC[`${dataElement}:${category ?? ""}`]?.value; - - if (value !== undefined) { - dataElementsToProcess.push({ cell, id: dataElement, value }); - } - } - - await this.applyRulesAndWrite(template, dataSource, dataElementsToProcess, multiTextLookup); - } } private async resolveCellDataElement( From be3dbd380897f414b98fed19d028e3669cc34b59 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Sat, 4 Apr 2026 17:33:25 +0800 Subject: [PATCH 18/19] refactor read cell helper file --- src/domain/helpers/ExcelBuilder.ts | 2 +- src/domain/helpers/ExcelReader.ts | 2 +- src/domain/helpers/{ExcelCellReader.ts => readCell.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/domain/helpers/{ExcelCellReader.ts => readCell.ts} (100%) diff --git a/src/domain/helpers/ExcelBuilder.ts b/src/domain/helpers/ExcelBuilder.ts index a0c04423..d5708e79 100644 --- a/src/domain/helpers/ExcelBuilder.ts +++ b/src/domain/helpers/ExcelBuilder.ts @@ -36,7 +36,7 @@ import { ModulesRepositories } from "../repositories/ModulesRepositories"; import { Maybe } from "../../types/utils"; import { DataElementDisaggregationsMappingRepository } from "../repositories/DataElementDisaggregationsMappingRepository"; import { DataProcessingService, DataToProcess } from "./DataProcessingService"; -import { readCellResolvingDefinedNames } from "./ExcelCellReader"; +import { readCellResolvingDefinedNames } from "./readCell"; import { DataElement, DataForm } from "../entities/DataForm"; import { Id } from "../entities/ReferenceObject"; diff --git a/src/domain/helpers/ExcelReader.ts b/src/domain/helpers/ExcelReader.ts index 705f03a4..f0ace651 100644 --- a/src/domain/helpers/ExcelReader.ts +++ b/src/domain/helpers/ExcelReader.ts @@ -5,7 +5,7 @@ import moment from "moment"; import { isDefined } from "../../utils"; import { promiseMap } from "../../utils/promises"; import { removeCharacters } from "../../utils/string"; -import { readCellResolvingDefinedNames } from "./ExcelCellReader"; +import { readCellResolvingDefinedNames } from "./readCell"; import { DataForm, dataFormTypeMap, DataFormFeatureType } from "../entities/DataForm"; import { buildGeometry, getGeometryFromString } from "../entities/Geometry"; import { Relationship } from "../entities/Relationship"; diff --git a/src/domain/helpers/ExcelCellReader.ts b/src/domain/helpers/readCell.ts similarity index 100% rename from src/domain/helpers/ExcelCellReader.ts rename to src/domain/helpers/readCell.ts From f16929b6a5f7d38dee2e7ae64fd72e4a8ebd29ee Mon Sep 17 00:00:00 2001 From: Ramon-Jimenez Date: Fri, 17 Apr 2026 12:44:06 +0200 Subject: [PATCH 19/19] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0faada21..430f5783 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "bulk-load", "description": "Bulk importing made easy", - "version": "3.32.0", + "version": "3.33.0", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".",