diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 47573f4f1c..1346a53588 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -3287,8 +3287,8 @@ object Http4s700 { | is off, so old metrics are never archived nor deleted. | * `check_metric_retention_policy_is_respected` — flags if the oldest live | metric is older than the retention window (move job not keeping up / stopped). - | * `check_all_old_metrics_can_be_archived` — warns if old metric rows have an - | empty correlation id and so cannot be moved to the archive. + | * `check_all_old_metrics_can_be_archived` — always OK; old metric rows with no + | correlation id are now archived with a generated `ORIGINALLY_NOT_SET-` id. | * `check_archive_retention_policy_is_respected` — flags if the oldest archived | metric is older than the archive retention (cleanup not keeping up / stopped). | * `check_archive_metrics_is_fresh_enough` — flags if a backlog exists but diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 87df2ce057..60eaf4a263 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1324,20 +1324,14 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { checks += MetricsIntegrityCheckJsonV700("check_metric_retention_policy_is_respected", "OK", "The metric table is empty.") } - // Old metric rows with an empty correlation id can't be archived (the archive - // requires a UUID) and are excluded from the move job, so they sit in the metric - // table indefinitely. Surface their count so this is visible rather than looking - // like a stalled job in the oldest-within-retention check above. - val unarchivableOldMetricCount = MappedMetric.count( - By_<=(MappedMetric.date, new Date(now.getTime - retainMetricsDaysEffective * metricsOneDayInMillis)), - By(MappedMetric.correlationId, "") - ) - if (unarchivableOldMetricCount == 0) - checks += MetricsIntegrityCheckJsonV700("check_all_old_metrics_can_be_archived", "OK", - "No metric rows older than the retention window are blocked from archiving.") - else - checks += MetricsIntegrityCheckJsonV700("check_all_old_metrics_can_be_archived", "WARNING", - s"$unarchivableOldMetricCount metric row(s) older than the retention window have an empty correlation id and cannot be archived (the archive requires a UUID). They remain in the metric table and are excluded from the move job — typically legacy rows that predate correlation ids.") + // Previously: rows with an empty/null correlation id could not be archived and were + // surfaced here as a permanent backlog. As of the synthetic-id change in + // MetricsArchiveScheduler.copyRowToMetricsArchive, such rows ARE archived (with an + // "ORIGINALLY_NOT_SET-" correlation id), so there is no un-archivable category + // anymore. The check slot is retained (so consumers/dashboards keep a stable shape) + // but its condition is intentionally empty for now — it always reports OK. + checks += MetricsIntegrityCheckJsonV700("check_all_old_metrics_can_be_archived", "OK", + "All metric rows older than the retention window are archivable; rows with no correlation id are archived with a generated 'ORIGINALLY_NOT_SET-' id.") archiveOldest match { case Some(d) => @@ -1461,7 +1455,7 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { MetricsIntegrityCheckJsonV700("check_metric_retention_policy_is_respected", "OK", "Oldest metric is 85 days old, within the effective retention of 90 days (+7d grace)."), MetricsIntegrityCheckJsonV700("check_all_old_metrics_can_be_archived", "OK", - "No metric rows older than the retention window are blocked from archiving."), + "All metric rows older than the retention window are archivable; rows with no correlation id are archived with a generated 'ORIGINALLY_NOT_SET-' id."), MetricsIntegrityCheckJsonV700("check_archive_retention_policy_is_respected", "OK", "Oldest archived metric is 700 days old, within the effective archive retention of 730 days (+7d grace)."), MetricsIntegrityCheckJsonV700("check_archive_metrics_is_fresh_enough", "OK", diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index df1b02d14a..4a5375ebce 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -9,7 +9,7 @@ import code.api.util.APIUtil import code.metrics.{APIMetric, APIMetrics, MappedMetric, MetricArchive, MetricsArchiveRun} import code.util.Helper.MdcLoggable import net.liftweb.common.Full -import net.liftweb.mapper.{Ascending, By, By_<=, By_>=, MaxRows, NotBy, OrderBy} +import net.liftweb.mapper.{Ascending, By, By_<=, By_>=, MaxRows, OrderBy} import scala.concurrent.duration._ @@ -175,15 +175,13 @@ object MetricsArchiveScheduler extends MdcLoggable { // a Redis cache (stable TTL for past-dated queries), and a "which rows should I // move and delete" query must never be served from a stale snapshot. // - // We also exclude rows with an empty correlation id: the archive's correlationId - // column requires a UUID, so those rows can't be archived. Filtering them in the - // query (rather than skipping them in the loop) means the job never repeatedly - // re-scans the same un-archivable rows — which, being the oldest, would otherwise - // permanently occupy the oldest-first candidate window. Their count is surfaced - // by the /diagnostics/metrics endpoint instead. + // Note: rows with an empty/null correlation id are NOT filtered out. They used to + // be (the archive's correlationId is non-null), which left them — being the oldest — + // permanently occupying the candidate window and stalling the job. copyRowToMetricsArchive + // now assigns those rows a synthetic "ORIGINALLY_NOT_SET-" correlation id so + // they archive normally instead of accumulating forever in the live table. val candidateMetricRowsToMove: List[MappedMetric] = MappedMetric.findAll( By_<=(MappedMetric.date, someDaysAgo), - NotBy(MappedMetric.correlationId, ""), OrderBy(MappedMetric.date, Ascending), MaxRows(limit) ) @@ -209,6 +207,17 @@ object MetricsArchiveScheduler extends MdcLoggable { // Returns true when the archive copy was persisted, false otherwise. private def copyRowToMetricsArchive(i: APIMetric): Boolean = { + // Legacy rows can have an empty/null correlation id (they predate correlation ids, + // or were never authenticated via a consent / X-Request-ID). The archive's + // correlationId is non-null, so assign a traceable synthetic id rather than skipping + // — and thereby permanently accumulating — these rows. The "ORIGINALLY_NOT_SET-" + // prefix makes it obvious in the archive that the original value was absent. + val rawCorrelationId = i.getCorrelationId() + val correlationId = + if (rawCorrelationId == null || rawCorrelationId.isEmpty) + s"ORIGINALLY_NOT_SET-${generateUUID()}" + else + rawCorrelationId APIMetrics.apiMetrics.vend.saveMetricsArchive( i.getMetricId(), i.getUserId(), @@ -223,7 +232,7 @@ object MetricsArchiveScheduler extends MdcLoggable { i.getImplementedInVersion(), i.getVerb(), Some(i.getHttpCode()), - i.getCorrelationId(), + correlationId, i.getResponseBody(), i.getSourceIp(), i.getTargetIp(), diff --git a/obp-api/src/test/scala/code/scheduler/MetricsArchiveSchedulerTest.scala b/obp-api/src/test/scala/code/scheduler/MetricsArchiveSchedulerTest.scala index 15402e64d1..f36d938b02 100644 --- a/obp-api/src/test/scala/code/scheduler/MetricsArchiveSchedulerTest.scala +++ b/obp-api/src/test/scala/code/scheduler/MetricsArchiveSchedulerTest.scala @@ -105,18 +105,22 @@ class MetricsArchiveSchedulerTest extends ServerSetup { run.RowsMovedToArchive.get should equal(1) } - scenario("Old rows with an empty correlation id are left in place, never archived") { + scenario("Old rows with an empty correlation id are archived with a synthetic ORIGINALLY_NOT_SET correlation id") { val noCorr = seedMetric(daysAgo(800), "") val outcome = MetricsArchiveScheduler.runOnce() outcome shouldBe a[RunCompleted] - Then("the row stays in metric and is not in the archive") - MappedMetric.find(By(MappedMetric.id, noCorr.id.get)).isDefined should equal(true) - MetricArchive.find(By(MetricArchive.metricId, noCorr.id.get)).isDefined should equal(false) + Then("the row is moved out of metric and into the archive") + MappedMetric.find(By(MappedMetric.id, noCorr.id.get)).isDefined should equal(false) + val archived = MetricArchive.find(By(MetricArchive.metricId, noCorr.id.get)) + archived.isDefined should equal(true) - And("nothing was moved") - outcome.asInstanceOf[RunCompleted].run.RowsMovedToArchive.get should equal(0) + And("the archived copy was given a generated ORIGINALLY_NOT_SET correlation id") + archived.openOrThrowException("expected archived row").correlationId.get should startWith("ORIGINALLY_NOT_SET-") + + And("exactly one row was moved") + outcome.asInstanceOf[RunCompleted].run.RowsMovedToArchive.get should equal(1) } scenario("Outdated archive rows are deleted; recent archive rows are kept") {