Feature/mars2grib era6 reencoding support

share/metkit/params.yaml

@@ -3539,6 +3539,24 @@
     - 228240
     - 228246
     - 228247
+- - levtype: sfc
+  - - 254001
+    - 254002
+    - 254003
+    - 254004
+    - 254005
+    - 254006
+    - 254007
+    - 254008
+    - 254009
+    - 254010
+    - 254011
+    - 254012
+    - 254013
+    - 254014
+    - 254015
+    - 254016
+    - 254017
 - - class: od
     levtype: al
     stream: elda

src/metkit/mars2grib/backend/concepts/AllConcepts.h

@@ -99,9 +99,11 @@
 #include "metkit/mars2grib/backend/concepts/destine/destineConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/ensemble/ensembleConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/generating-process/generatingProcessConceptDescriptor.h"
+#include "metkit/mars2grib/backend/concepts/iteration/iterationConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/level/levelConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/longrange/longrangeConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/mars/marsConceptDescriptor.h"
+#include "metkit/mars2grib/backend/concepts/model-error/modelErrorConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/nil/nilConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/origin/originConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/packing/packingConceptDescriptor.h"
@@ -168,10 +170,10 @@ using TypeList = metkit::mars2grib::backend::compile_time_registry_engine::TypeL
 /// Higher-level code should interact with concepts exclusively through
 /// registry APIs, not by iterating this list directly.
 ///
-using AllConcepts =
-    TypeList<AnalysisConcept, CompositionConcept, DataTypeConcept, DerivedConcept, DestineConcept, EnsembleConcept,
-             GeneratingProcessConcept, LevelConcept, LongrangeConcept, MarsConcept, NilConcept, OriginConcept,
-             PackingConcept, ParamConcept, PointInTimeConcept, ReferenceTimeConcept, RepresentationConcept,
-             SatelliteConcept, ShapeOfTheEarthConcept, StatisticsConcept, TablesConcept, WaveConcept>;
+using AllConcepts = TypeList<AnalysisConcept, CompositionConcept, DataTypeConcept, DerivedConcept, DestineConcept,
+                             EnsembleConcept, GeneratingProcessConcept, LevelConcept, LongrangeConcept,
+                             IterationConcept, MarsConcept, NilConcept, OriginConcept, PackingConcept, ParamConcept,
+                             PointInTimeConcept, ReferenceTimeConcept, RepresentationConcept, SatelliteConcept,
+                             ShapeOfTheEarthConcept, StatisticsConcept, TablesConcept, WaveConcept, ModelErrorConcept>;
 
 }  // namespace metkit::mars2grib::backend::concepts_::detail

src/metkit/mars2grib/backend/concepts/analysis/analysisEncoding.h

@@ -150,7 +150,7 @@ void AnalysisOp(const MarsDict_t& mars, const ParDict_t& par, const OptDict_t& o
             MARS2GRIB_LOG_CONCEPT(analysis);
 
             // Structural validation
-            validation::match_LocalDefinitionNumber_or_throw(opt, out, {36L});
+            validation::match_LocalDefinitionNumber_or_throw(opt, out, {36L, 38L, 39L});
 
             // Deductions
             long offsetToEndOf4DvarWindowVal = deductions::resolve_offsetToEndOf4DvarWindow_or_throw(mars, par, opt);

src/metkit/mars2grib/backend/concepts/concepts.md

@@ -47,7 +47,60 @@ The ordered list of variants for a concept defines its **local variant index spa
 
 ---
 
-## 3. Concept Descriptor Contract
+## 3. New Concept vs New Variant
+
+When changing the concept system, first decide whether the requested behavior is
+a new concept or a new variant of an existing concept.
+
+Use a **new concept** when the feature is an independent semantic axis that must
+be composable with other concepts. Use a **new variant** when the feature is an
+alternative realization inside an existing semantic axis and does not require
+independent composability.
+
+This distinction is often a domain decision and is usually not reliably
+deducible from the code alone. If a request does not explicitly state whether a
+new concept or a new variant is required, ask before implementing.
+
+---
+
+## 4. Level Concept Guardrail
+
+The `level` concept is one of the most constrained concepts in mars2grib.
+Although GRIB ultimately represents vertical levels through six low-level
+fixed-surface keys:
+
+* `typeOfFirstFixedSurface`
+* `scaleFactorOfFirstFixedSurface`
+* `scaledValueOfFirstFixedSurface`
+* `typeOfSecondFixedSurface`
+* `scaleFactorOfSecondFixedSurface`
+* `scaledValueOfSecondFixedSurface`
+
+mars2grib must not set these keys directly. Many combinations of these keys are
+syntactically possible but semantically meaningless for ECMWF products.
+
+Instead, the encoder must rely on the official level abstraction:
+
+* `typeOfLevel`
+* `level`, when required
+* `topLevel` / `bottomLevel`, when required
+* PV-array data, when required
+
+Each supported `typeOfLevel` corresponds to a `LevelType` variant, apart from a
+few virtual type-of-level values kept in mars2grib because they cannot be added
+to ecCodes for backward-compatibility reasons. Each variant maps to a prescribed
+configuration of the low-level fixed-surface keys.
+
+Do not implement level fixes by injecting `typeOfFirstFixedSurface`,
+`scaleFactorOfFirstFixedSurface`, `scaledValueOfFirstFixedSurface`,
+`typeOfSecondFixedSurface`, `scaleFactorOfSecondFixedSurface`, or
+`scaledValueOfSecondFixedSurface`. If a new level behavior is required, add or
+adjust the appropriate `LevelType` variant, matcher mapping, or deduction so the
+level remains encoded through `typeOfLevel` and the official level interface.
+
+---
+
+## 5. Concept Descriptor Contract
 
 Each concept is implemented as a **descriptor type** that conforms to the
 `RegisterEntryDescriptor` interface.
@@ -68,7 +121,7 @@ The descriptor contains **no runtime state** and no virtual functions.
 
 ---
 
-## 4. Capabilities
+## 6. Capabilities
 
 Concepts may expose multiple independent *capabilities*.
 
@@ -87,7 +140,7 @@ independent dispatch planes.
 
 ---
 
-## 5. The Concept Universe (`AllConcepts`)
+## 7. The Concept Universe (`AllConcepts`)
 
 All concepts known to the system are aggregated into a single ordered typelist:
 
@@ -107,7 +160,7 @@ Changing this order is a **breaking structural change**.
 
 ---
 
-## 6. Concept Identifiers
+## 8. Concept Identifiers
 
 Each concept is assigned a **stable numeric identifier** based on its position
 in `AllConcepts`.
@@ -130,7 +183,7 @@ They are used as indices into:
 
 ---
 
-## 7. Variant Index Spaces
+## 9. Variant Index Spaces
 
 Variants are indexed in two ways:
 
@@ -158,7 +211,7 @@ The global variant index is the primary key used by:
 
 ---
 
-## 8. Matching Phase
+## 10. Matching Phase
 
 Matching determines **which concepts and variants are active** for a given
 input request.
@@ -179,7 +232,7 @@ The result is an `ActiveConceptsData` structure.
 
 ---
 
-## 9. Encoding Phases
+## 11. Encoding Phases
 
 Encoding is divided into **logical stages**, such as:
 
@@ -201,7 +254,7 @@ All dispatch tables are generated **entirely at compile time**.
 
 ---
 
-## 10. Design Principles
+## 12. Design Principles
 
 The concept system is designed around the following principles:
 
@@ -217,7 +270,7 @@ Execution code performs *only iteration and invocation*.
 
 ---
 
-## 11. Adding a New Concept
+## 13. Adding a New Concept
 
 To add a new concept:
 
@@ -231,7 +284,7 @@ No registry code needs to be modified.
 
 ---
 
-## 12. Summary
+## 14. Summary
 
 Concepts are the **semantic backbone** of the mars2grib backend.
 

src/metkit/mars2grib/backend/concepts/ensemble/ensembleMatcher.h

@@ -2,6 +2,7 @@
 
 // System include
 #include <cstddef>
+#include <string>
 
 // Utils
 #include "metkit/mars2grib/backend/concepts/ensemble/ensembleEnum.h"
@@ -12,8 +13,15 @@ namespace metkit::mars2grib::backend::concepts_ {
 
 template <class MarsDict_t, class OptDict_t>
 std::size_t ensembleMatcher(const MarsDict_t& mars, const OptDict_t& opt) {
+    using metkit::mars2grib::utils::dict_traits::get_or_throw;
     using metkit::mars2grib::utils::dict_traits::has;
 
+    // Skip model-error products: in that case "number" identifies the
+    // model-error realization, not an ensemble member.
+    if (has(mars, "type") && get_or_throw<std::string>(mars, "type") == "eme") {
+        return compile_time_registry_engine::MISSING;
+    }
+
     if (has(mars, "number")) {
         return static_cast<std::size_t>(EnsembleType::Individual);
     }

src/metkit/mars2grib/backend/concepts/iteration/iterationConceptDescriptor.h

@@ -0,0 +1,160 @@
+/*
+ * (C) Copyright 2025- ECMWF and individual contributors.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+///
+/// @file IterationConcept.h
+/// @brief Compile-time registry entry for the GRIB `iteration` concept.
+///
+/// This header defines `IterationConcept`, the **compile-time descriptor**
+/// that registers the GRIB `iteration` concept into the mars2grib
+/// compile-time registry engine.
+///
+/// The descriptor provides:
+/// - The concept name
+/// - The mapping between variants and their symbolic names
+/// - The set of callbacks associated with each encoding phase
+/// - The entry-level matcher used to activate the concept
+///
+/// This file contains **no runtime logic**. All decisions are resolved
+/// at compile time through template instantiation.
+///
+/// @ingroup mars2grib_backend_concepts
+///
+#pragma once
+
+// System include
+#include <cstddef>
+
+// Registry engine
+#include "metkit/mars2grib/backend/compile-time-registry-engine/RegisterEntryDescriptor.h"
+#include "metkit/mars2grib/backend/compile-time-registry-engine/common.h"
+#include "metkit/mars2grib/utils/generalUtils.h"
+
+// Core concept includes
+#include "metkit/mars2grib/backend/concepts/iteration/iterationEncoding.h"
+#include "metkit/mars2grib/backend/concepts/iteration/iterationEnum.h"
+#include "metkit/mars2grib/backend/concepts/iteration/iterationMatcher.h"
+
+namespace metkit::mars2grib::backend::concepts_ {
+
+// Importing the compile-time registry engine namespace locally to avoid
+// excessive verbosity in template-heavy code. This is restricted to an
+// internal scope and not exposed through public headers.
+using namespace metkit::mars2grib::backend::compile_time_registry_engine;
+
+///
+/// @brief Compile-time descriptor for the `iteration` concept.
+///
+/// `IterationConcept` registers the GRIB `iteration` concept into the
+/// compile-time registry engine.
+///
+/// The descriptor defines:
+/// - The canonical concept name
+/// - The mapping from variant enum values to symbolic names
+/// - The callbacks associated with each encoding phase
+/// - The entry-level matcher used to detect applicability
+///
+/// All functions in this descriptor are `constexpr` and are evaluated
+/// entirely at compile time.
+///
+struct IterationConcept : RegisterEntryDescriptor<IterationType, IterationList> {
+
+    ///
+    /// @brief Return the canonical name of the concept.
+    ///
+    /// This name is used for:
+    /// - Registry identification
+    /// - Diagnostics and logging
+    /// - Debug and introspection facilities
+    ///
+    static constexpr std::string_view entryName() { return iterationName; }
+
+    ///
+    /// @brief Return the symbolic name of a concept variant.
+    ///
+    /// @tparam T Variant enumeration value
+    ///
+    /// @return String view representing the variant name
+    ///
+    template <IterationType T>
+    static constexpr std::string_view variantName() {
+        return iterationTypeName<T>();
+    }
+
+    ///
+    /// @brief Return the callback associated with a specific encoding phase.
+    ///
+    /// This function is queried by the registry engine to obtain the
+    /// callback implementing the `iteration` concept for a given:
+    ///
+    /// - Capability
+    /// - Encoding stage
+    /// - GRIB section
+    /// - Concept variant
+    ///
+    /// The function returns:
+    /// - A valid function pointer if the concept is applicable
+    /// - `nullptr` otherwise
+    ///
+    /// @tparam Capability Encoding capability index
+    /// @tparam Stage      Encoding stage
+    /// @tparam Sec        GRIB section
+    /// @tparam Variant    Concept variant
+    /// @tparam MarsDict_t Type of MARS dictionary
+    /// @tparam ParDict_t  Type of parameter dictionary
+    /// @tparam OptDict_t  Type of options dictionary
+    /// @tparam OutDict_t  Type of output GRIB dictionary
+    ///
+    /// @return Function pointer implementing the phase, or `nullptr`
+    ///
+    template <std::size_t Capability, std::size_t Stage, std::size_t Sec, IterationType Variant, class MarsDict_t,
+              class ParDict_t, class OptDict_t, class OutDict_t>
+    static constexpr Fn<MarsDict_t, ParDict_t, OptDict_t, OutDict_t> phaseCallbacks() {
+
+        if constexpr (Capability == 0) {
+
+            if constexpr (iterationApplicable<Stage, Sec, Variant>()) {
+                return &IterationOp<Stage, Sec, Variant, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>;
+            }
+            else {
+                return nullptr;
+            }
+        }
+        else {
+            return nullptr;
+        }
+
+        mars2gribUnreachable();
+    }
+
+    ///
+    /// @brief Variant-specific callbacks (not used for this concept).
+    ///
+    template <std::size_t Capability, IterationType Variant, class MarsDict_t, class ParDict_t, class OptDict_t,
+              class OutDict_t>
+    static constexpr Fn<MarsDict_t, ParDict_t, OptDict_t, OutDict_t> variantCallbacks() {
+        return nullptr;
+    }
+
+    ///
+    /// @brief Entry-level matcher callback.
+    ///
+    template <std::size_t Capability, class MarsDict_t, class OptDict_t>
+    static constexpr Fm<MarsDict_t, OptDict_t> entryCallbacks() {
+        if constexpr (Capability == 0) {
+            return &iterationMatcher<MarsDict_t, OptDict_t>;
+        }
+        else {
+            return nullptr;
+        }
+    }
+};
+
+}  // namespace metkit::mars2grib::backend::concepts_