From 0d280afabdece04b7fc3646b6b144d39ec6f1a50 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 12 May 2026 16:16:28 -0400 Subject: [PATCH 1/3] NIFI-15932: Implemented ability to migrate a Versioned flow's Assets and config to a new Connector --- .../main/asciidoc/administration-guide.adoc | 87 +++ .../src/main/asciidoc/developer-guide.adoc | 90 ++++ .../nifi/web/api/dto/MigrationPayloadDTO.java | 34 ++ .../nifi/web/api/dto/MigrationRequestDTO.java | 54 ++ .../dto/MigrationRequestLocalSourceDTO.java | 34 ++ .../web/api/dto/MigrationUpdateStepDTO.java | 23 + .../dto/VersionedFlowMigrationSourceDTO.java | 84 +++ .../api/entity/MigrationPayloadEntity.java | 35 ++ .../api/entity/MigrationRequestEntity.java | 35 ++ .../VersionedFlowMigrationSourcesEntity.java | 37 ++ .../http/StandardHttpResponseMapper.java | 2 + .../MigrationRequestEndpointMerger.java | 107 ++++ .../ClusteredConnectorRequestReplicator.java | 17 +- .../MigrationRequestEndpointMergerTest.java | 122 +++++ ...tandardVersionedComponentSynchronizer.java | 41 +- .../VersionedComponentStateValidator.java | 102 ++++ .../ConnectorFlowSnapshotProvider.java | 40 ++ .../connector/ConnectorMigrationManager.java | 89 ++++ .../connector/ConnectorMigrationSource.java | 80 +++ .../components/connector/ConnectorNode.java | 10 + .../connector/ConnectorRepository.java | 13 + .../FrameworkConnectorMigrationContext.java | 53 ++ .../connector/ConnectorParameterLookup.java | 7 +- .../StandardConnectorMigrationManager.java | 502 ++++++++++++++++++ .../connector/StandardConnectorNode.java | 14 +- .../StandardConnectorRepository.java | 171 ++++-- ...ardFrameworkConnectorMigrationContext.java | 135 +++++ .../StandaloneParameterContextFacade.java | 1 + .../FlowControllerFlowContextFactory.java | 8 +- ...TestStandardConnectorMigrationManager.java | 468 ++++++++++++++++ .../TestStandardConnectorRepository.java | 202 ++++--- ...ardFrameworkConnectorMigrationContext.java | 167 ++++++ .../apache/nifi/audit/ConnectorAuditor.java | 23 + .../apache/nifi/web/NiFiServiceFacade.java | 8 + .../nifi/web/StandardNiFiServiceFacade.java | 59 +- .../nifi/web/api/ConnectorResource.java | 386 ++++++++++++++ .../org/apache/nifi/web/dao/ConnectorDAO.java | 9 + .../web/dao/impl/StandardConnectorDAO.java | 51 ++ .../nifi/web/api/TestConnectorResource.java | 60 +++ .../AsymmetricFailureMigrationConnector.java | 88 +++ .../system/FailingMigrationConnector.java | 119 +++++ .../system/MigrationTargetConnector.java | 176 ++++++ .../tests/system/NonMigratingConnector.java | 59 ++ .../tests/system/AssetReadingProcessor.java | 97 ++++ ...apache.nifi.components.connector.Connector | 4 + .../org.apache.nifi.processor.Processor | 1 + .../nifi/tests/system/NiFiClientUtil.java | 129 +++++ ...nectorVersionedFlowMigrationFailureIT.java | 88 +++ ...eredConnectorVersionedFlowMigrationIT.java | 26 + ...orVersionedFlowMigrationEligibilityIT.java | 102 ++++ ...nectorVersionedFlowMigrationFailureIT.java | 122 +++++ ...onnectorVersionedFlowMigrationLocalIT.java | 261 +++++++++ ...rsionedFlowMigrationUploadedPayloadIT.java | 63 +++ .../nifi/toolkit/client/ConnectorClient.java | 13 + .../client/impl/JerseyConnectorClient.java | 75 +++ pom.xml | 2 +- 56 files changed, 4691 insertions(+), 194 deletions(-) create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index d5e4234bc39f..e28deef9f47b 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -3833,6 +3833,93 @@ The Secrets Manager delegates to Parameter Providers to retrieve secret values. |`nifi.secrets.manager.cache.duration`|The duration for which resolved secret values are cached before being refreshed from the underlying Parameter Providers. Accepts any NiFi time duration value such as `5 mins`, `30 secs`, etc. A value of `0 sec` disables caching entirely. Defaults to `5 mins`. |==== +== Migrating a Connector from a Versioned Process Group + +NiFi supports a one-time path that migrates a newly created Connector from a source Versioned Process Group. +The source Versioned Process Group is used as a reference: the Connector reads it and updates its own managed flow to mirror the source's configuration, parameters, and component state. +The source flow itself is not installed onto the Connector. +Only Connectors that explicitly opt in to migration support can use this capability. + +=== Prerequisites for a local migration + +When the source is a local Process Group on the same NiFi instance, the Process Group must satisfy all of the following requirements before NiFi will allow the migration: + +- The source Process Group must be under version control. +- The source Process Group must be `UP_TO_DATE` with the latest published version in Flow Registry. +- All processors in the source Process Group must be stopped. +- All controller services in the source Process Group must be disabled. +- All queues in the source Process Group must be empty. +- The source Process Group must not reference controller services outside of the Process Group. +- The target Connector must be newly created, stopped, and not previously migrated. +- The target Connector's active flow must be empty when migration begins. Connectors that ship a non-empty initial flow are not compatible with the migration path. + +If the source Process Group is not under version control, export it and use the uploaded payload path instead. + +=== Migration paths + +There are two ways to run the migration: + +- Local Process Group migration: + Select an eligible Process Group that already exists on the current NiFi canvas. + NiFi validates the prerequisites listed above before starting the migration. +- Uploaded payload migration: + Export a Process Group definition and upload the resulting flow snapshot payload to the Connector migration endpoint. + This path is intended for non-version-controlled sources, sources exported from another NiFi instance, or any other source that cannot satisfy the local version-control requirements. + +For uploaded payloads, NiFi does not require the source flow to be under version control. +The Connector's own migration support check determines whether the uploaded flow can be imported. + +=== Running a migration + +. Create a new Connector instance. +. If the source Process Group is local and registry-backed, choose it from the Connector migration source list. +. If the source Process Group is not eligible for the local path, export it with component state included and upload the payload to the Connector migration payload endpoint. +. Start the Connector migration request. +. Poll the request until it completes. +. Open the Connector configuration and supply any missing sensitive values or secrets. +. Verify the configuration and start the Connector. + +=== What NiFi migrates + +During migration, NiFi provides the Connector with the exported flow definition, parameter contexts, referenced assets, and any attached component state for `@Stateful` components. +The Connector performs the flow translation and updates its own managed flow to mirror the source. + +For local migrations, a Connector can also copy referenced assets from the source Process Group into the Connector's own asset namespace. +Uploaded payload migrations do not include live access to source assets, so any required asset content must be re-uploaded separately after the migration. + +Sensitive parameter values are not present in exported flow definitions. +After migration, configure the Connector with any missing secret values before starting it. + +=== Cleanup after a successful migration + +After a successful local migration, NiFi disables the source Process Group and renames it with the `(Migrated) ` prefix. +This makes it clear that the source flow has already been used as the basis for a Connector migration. + +Uploaded payloads and migration request state are kept only in memory for the lifetime of the request. +When the request succeeds, fails, or is cancelled, NiFi removes the uploaded payload associated with that request. + +=== Cluster-topology rule for component state + +When the source flow being migrated includes LOCAL component state for a stateful component (for example a `@Stateful` Processor or Controller Service), the destination cluster must have at least as many connected nodes as the source flow's exported state references. +The rule applied by the migration manager is: + +---- +size(component.localNodeStates) <= number of connected nodes in the destination cluster +---- + +This is the same rule that applies when importing a Versioned Process Group into a destination cluster. +NiFi enforces it for both the local migration path and the uploaded payload path so that LOCAL state can be unambiguously distributed across the destination cluster. + +When the migration request fails with a message such as `Cannot import flow with component state: the flow definition contains local state from N source node(s) but the destination cluster has only M connected node(s)`, take one of the following actions: + +- Reconnect any disconnected cluster nodes so that the destination cluster has at least `N` connected nodes, then re-run the migration. +- Re-export the source flow without component state (uncheck the "include component state" option) and re-upload the payload. + The Connector will start with empty LOCAL state on every destination node, which is appropriate when the LOCAL state is reproducible from the upstream system. +- Migrate into a cluster that has at least `N` connected nodes. + +The migration request is also rejected when any cluster node is in the `CONNECTING`, `DISCONNECTED`, or `DISCONNECTING` state at the time the request is submitted, because component state and asset content cannot be synchronized to a node that is not currently connected. +Reconnect the disconnected nodes and re-submit the request. + [[upgrading_nifi]] == Upgrading NiFi diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc index 1906efacc77b..1e42f013865e 100644 --- a/nifi-docs/src/main/asciidoc/developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc @@ -2697,6 +2697,96 @@ The following command is used to generate a standard binary distribution of Apac `mvn clean install -Pcontrib-check` +== Supporting migration from a Versioned Process Group + +Connectors can opt in to a one-time migration flow that uses a source Versioned Process Group as a reference for populating a freshly created Connector. +This is intended for Connectors that can translate an exported flow definition, its parameter contexts, referenced assets, and any attached component state into their own managed flow. +The source Versioned Process Group is not installed onto the Connector; instead, the Connector reads the source as input and updates its own flow to mirror the source's configuration and state. + +Migration support is optional. +Existing Connectors continue to behave as before unless they override the new default methods on `org.apache.nifi.components.connector.Connector`. + +=== Migration contract + +The Connector interface now exposes two default methods for migration: + +- `isMigrationSupported(ConnectorMigrationContext)` is a cheap, side-effect-free predicate. + It should inspect `context.getSourceFlow()` and return `true` only when the Connector can be migrated from the source structure. + Implementations should limit themselves to structural checks such as processor types, parameter names, and exported metadata. + Component state and asset content are not available at listing time and must not be relied on; per-state precondition checks belong inside the Connector's `migrate(...)` implementation. +- `migrate(ConnectorMigrationContext)` performs the actual migration. + The Connector reads the source `VersionedExternalFlow`, translates it as needed into its own representation, and applies the result to its own active `FlowContext`. + +The default implementation of `isMigrationSupported(...)` returns `false`. +The default implementation of `migrate(...)` throws `UnsupportedOperationException`, so a Connector must override both methods to participate in migration. + +=== Accessing the source flow + +`ConnectorMigrationContext` exposes the source `VersionedExternalFlow` using `getSourceFlow()`. +The source includes: + +- The exported `VersionedProcessGroup` +- Parameter contexts +- Referenced assets +- Attached `VersionedComponentState` for `@Stateful` components +- Export metadata such as flow name, registry identifiers, and version + +The same context also indicates whether the request came from a local Process Group on the same NiFi instance or from an uploaded payload using `isLocalMigration()`. + +=== Updating the Connector's flow + +When a Connector extends `AbstractConnector`, the usual pattern is to retain the `ConnectorInitializationContext` supplied in `initialize(...)` and then update the Connector's own active flow directly from `migrate(...)`. + +[source,java] +---- +@Override +public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final VersionedExternalFlow transformedFlow = transformSourceFlow(context.getSourceFlow()); + getInitializationContext().updateFlow(context.getActiveFlowContext(), transformedFlow); +} +---- + +`context.getActiveFlowContext()` always refers to the Connector's active flow context. +After `migrate(...)` returns successfully, the framework recreates the working flow context from the active flow. + +If the Connector needs to preserve state while reshaping the flow, it can read or rewrite `VersionedConfigurableExtension.componentState` on the exported components before calling `updateFlow(...)`. +When the updated flow is applied, NiFi restores LOCAL and CLUSTER state through the standard flow synchronization path. + +If `migrate(...)` fails, the framework clears LOCAL and CLUSTER state for every Processor and Controller Service currently inside the Connector's flow before reloading the initial flow. +A Connector implementer does not need to track which components it has written state to during a failed migration. + +=== Rewriting parameters and assets + +The exported source flow contains parameter contexts and any asset references that were attached to exported parameters. +Connectors are responsible for mapping those values into their own target shape. + +A Connector can copy an asset from a local source Process Group into its own asset namespace using `ConnectorMigrationContext.copyAssetFromSource(String)`. +This method is available only for local migrations. +The method throws an `IllegalArgumentException` when the supplied source asset identifier cannot be resolved and when invoked for an uploaded payload migration. +Uploaded payloads do not have access to a live source asset namespace, so asset references from uploaded payloads must be handled outside of the migration request. + +The typical flow for parameter migration is: + +. Read the source parameter values from `context.getSourceFlow().getParameterContexts()` +. Rewrite any values or parameter names that the Connector expects +. Copy any local assets that must move with the flow +. Apply the translated flow to the Connector's own active flow context with `updateFlow(...)` + +=== Sensitive values + +Sensitive parameter values and other secret material are not present in the exported source flow. +Connectors must leave those values unset during migration. +After the migration completes, the user supplies the missing values through the normal Configure step before starting the Connector. + +=== Additional expectations + +- `isMigrationSupported(...)` must not mutate flow state, parameter values, or assets. +- `migrate(...)` should either complete successfully or throw an exception. +- A Connector that supports migration must return an empty (or `null`) initial flow from `getInitialFlow()`. + Migration requires that the target Connector's active flow be empty at the time `migrate(...)` is invoked, so a non-empty initial flow is not compatible with the migration path. +- Migration is one-shot from the framework's point of view. + Once the Connector's flow has been updated by a migration, the Connector has user-installed components, and a subsequent migration request will be rejected until the Connector is discarded and a fresh one is created. + == How to contribute to Apache NiFi We are always excited to have contributions from the community - especially from new contributors! diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java new file mode 100644 index 000000000000..08a17d70c3b4 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationPayload") +public class MigrationPayloadDTO { + private String payloadId; + + @Schema(description = "The identifier of the uploaded migration payload.") + public String getPayloadId() { + return payloadId; + } + + public void setPayloadId(final String payloadId) { + this.payloadId = payloadId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java new file mode 100644 index 000000000000..9dbbce360cf8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationRequest") +public class MigrationRequestDTO extends AsynchronousRequestDTO { + private String connectorId; + private MigrationRequestLocalSourceDTO localSource; + private String payloadId; + + @Schema(description = "The identifier of the Connector receiving the migration.") + public String getConnectorId() { + return connectorId; + } + + public void setConnectorId(final String connectorId) { + this.connectorId = connectorId; + } + + @Schema(description = "The local Process Group source for the migration request.") + public MigrationRequestLocalSourceDTO getLocalSource() { + return localSource; + } + + public void setLocalSource(final MigrationRequestLocalSourceDTO localSource) { + this.localSource = localSource; + } + + @Schema(description = "The identifier of a previously uploaded migration payload.") + public String getPayloadId() { + return payloadId; + } + + public void setPayloadId(final String payloadId) { + this.payloadId = payloadId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java new file mode 100644 index 000000000000..c0434ea011c9 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationRequestLocalSource") +public class MigrationRequestLocalSourceDTO { + private String processGroupId; + + @Schema(description = "The identifier of the local source Process Group to migrate.") + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java new file mode 100644 index 000000000000..61d4d2b16ca1 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationUpdateStep") +public class MigrationUpdateStepDTO extends UpdateStepDTO { +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java new file mode 100644 index 000000000000..d38699742e51 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "versionedFlowMigrationSource") +public class VersionedFlowMigrationSourceDTO { + private String processGroupId; + private String processGroupName; + private String registryClientId; + private String bucketId; + private String flowId; + private String version; + + @Schema(description = "The identifier of the source Process Group.") + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } + + @Schema(description = "The name of the source Process Group.") + public String getProcessGroupName() { + return processGroupName; + } + + public void setProcessGroupName(final String processGroupName) { + this.processGroupName = processGroupName; + } + + @Schema(description = "The identifier of the Flow Registry client backing the source Process Group.") + public String getRegistryClientId() { + return registryClientId; + } + + public void setRegistryClientId(final String registryClientId) { + this.registryClientId = registryClientId; + } + + @Schema(description = "The Flow Registry bucket identifier for the source Process Group.") + public String getBucketId() { + return bucketId; + } + + public void setBucketId(final String bucketId) { + this.bucketId = bucketId; + } + + @Schema(description = "The Flow Registry flow identifier for the source Process Group.") + public String getFlowId() { + return flowId; + } + + public void setFlowId(final String flowId) { + this.flowId = flowId; + } + + @Schema(description = "The published version of the source Process Group.") + public String getVersion() { + return version; + } + + public void setVersion(final String version) { + this.version = version; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java new file mode 100644 index 000000000000..2a2d716dd83e --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.MigrationPayloadDTO; + +@XmlRootElement(name = "migrationPayloadEntity") +public class MigrationPayloadEntity extends Entity { + private MigrationPayloadDTO payload; + + @Schema(description = "The uploaded migration payload metadata.") + public MigrationPayloadDTO getPayload() { + return payload; + } + + public void setPayload(final MigrationPayloadDTO payload) { + this.payload = payload; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java new file mode 100644 index 000000000000..7351b4c9b44a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; + +@XmlRootElement(name = "migrationRequestEntity") +public class MigrationRequestEntity extends Entity { + private MigrationRequestDTO request; + + @Schema(description = "The migration request.") + public MigrationRequestDTO getRequest() { + return request; + } + + public void setRequest(final MigrationRequestDTO request) { + this.request = request; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java new file mode 100644 index 000000000000..6f426b2579a6 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; + +import java.util.List; + +@XmlRootElement(name = "versionedFlowMigrationSourcesEntity") +public class VersionedFlowMigrationSourcesEntity extends Entity { + private List migrationSources; + + @Schema(description = "The Versioned Process Groups that the Connector can be migrated from.") + public List getMigrationSources() { + return migrationSources; + } + + public void setMigrationSources(final List migrationSources) { + this.migrationSources = migrationSources; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java index 6c2b43a3239b..22c19c34a4a5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java @@ -63,6 +63,7 @@ import org.apache.nifi.cluster.coordination.http.endpoints.LabelsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.LatestProvenanceEventsMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ListFlowFilesEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.MigrationRequestEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarDetailsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarSummariesEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarSummaryEndpointMerger; @@ -152,6 +153,7 @@ public StandardHttpResponseMapper(final NiFiProperties nifiProperties) { endpointMergers.add(new ConnectorFlowEndpointMerger()); endpointMergers.add(new ConnectorPropertyGroupEndpointMerger()); endpointMergers.add(new ConnectorPropertyGroupNamesEndpointMerger()); + endpointMergers.add(new MigrationRequestEndpointMerger()); endpointMergers.add(new VerifyConnectorConfigStepEndpointMerger()); endpointMergers.add(new ConnectionEndpointMerger()); endpointMergers.add(new ConnectionsEndpointMerger()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java new file mode 100644 index 000000000000..e197d693d056 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; + +import java.net.URI; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class MigrationRequestEndpointMerger extends AbstractSingleEntityEndpoint { + public static final Pattern MIGRATION_REQUEST_URI_PATTERN = + Pattern.compile("/nifi-api/connectors/[a-f0-9\\-]{36}/migration-requests(/[a-f0-9\\-]{36})?"); + + @Override + protected Class getEntityClass() { + return MigrationRequestEntity.class; + } + + @Override + public boolean canHandle(final URI uri, final String method) { + return MIGRATION_REQUEST_URI_PATTERN.matcher(uri.getPath()).matches(); + } + + @Override + protected void mergeResponses(final MigrationRequestEntity clientEntity, final Map entityMap, + final Set successfulResponses, final Set problematicResponses) { + final MigrationRequestDTO clientRequest = clientEntity.getRequest(); + if (clientRequest == null) { + return; + } + + final List clientUpdateSteps = clientRequest.getUpdateSteps() == null ? List.of() : clientRequest.getUpdateSteps(); + + for (final Map.Entry nodeEntry : entityMap.entrySet()) { + final NodeIdentifier nodeId = nodeEntry.getKey(); + final MigrationRequestDTO nodeRequest = nodeEntry.getValue().getRequest(); + if (nodeRequest == null) { + continue; + } + + clientRequest.setComplete(clientRequest.isComplete() && nodeRequest.isComplete()); + + if (nodeRequest.getFailureReason() != null) { + final String nodeReason = "Node " + nodeId.getApiAddress() + ":" + nodeId.getApiPort() + ": " + nodeRequest.getFailureReason(); + final String existingReason = clientRequest.getFailureReason(); + clientRequest.setFailureReason(existingReason == null ? nodeReason : existingReason + "; " + nodeReason); + } + + final Date clientLastUpdated = clientRequest.getLastUpdated(); + final Date nodeLastUpdated = nodeRequest.getLastUpdated(); + if (nodeLastUpdated != null && (clientLastUpdated == null || nodeLastUpdated.before(clientLastUpdated))) { + clientRequest.setLastUpdated(nodeLastUpdated); + } + + clientRequest.setPercentCompleted(Math.min(clientRequest.getPercentCompleted(), nodeRequest.getPercentCompleted())); + + mergeUpdateSteps(clientUpdateSteps, nodeRequest.getUpdateSteps()); + } + + // Recompute the human-readable state to reflect any failure surfaced by a remote node so that polling clients + // see a consistent picture rather than the arbitrary client node's view. + if (clientRequest.getFailureReason() != null) { + clientRequest.setState("Failed: " + clientRequest.getFailureReason()); + } else if (clientRequest.isComplete()) { + clientRequest.setState("Complete"); + } + } + + private void mergeUpdateSteps(final List clientSteps, final List nodeSteps) { + if (clientSteps == null || nodeSteps == null) { + return; + } + + final int sharedStepCount = Math.min(clientSteps.size(), nodeSteps.size()); + for (int stepIndex = 0; stepIndex < sharedStepCount; stepIndex++) { + final MigrationUpdateStepDTO clientStep = clientSteps.get(stepIndex); + final MigrationUpdateStepDTO nodeStep = nodeSteps.get(stepIndex); + + clientStep.setComplete(clientStep.isComplete() && nodeStep.isComplete()); + if (nodeStep.getFailureReason() != null && clientStep.getFailureReason() == null) { + clientStep.setFailureReason(nodeStep.getFailureReason()); + } + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java index 9af3b62f2e8e..d438b1a0d28b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java @@ -60,17 +60,22 @@ public ConnectorState getState(final String connectorId) throws IOException { try { final NodeResponse mergedNodeResponse = asyncResponse.awaitMergedResponse(); + final Entity updatedEntity = mergedNodeResponse.getUpdatedEntity(); final Response response = mergedNodeResponse.getClientResponse(); - final int statusCode = response.getStatusInfo().getStatusCode(); - logger.debug("getState: Connector [{}] — merged response status [{}] ({})", connectorId, statusCode, response.getStatusInfo().getReasonPhrase()); - - verifyResponse(response.getStatusInfo(), connectorId); + if (response != null) { + final int statusCode = response.getStatusInfo().getStatusCode(); + logger.debug("getState: Connector [{}] — merged response status [{}] ({})", connectorId, statusCode, response.getStatusInfo().getReasonPhrase()); + verifyResponse(response.getStatusInfo(), connectorId); + } else if (mergedNodeResponse.hasThrowable()) { + throw new IOException("Failed to retrieve Connector state for " + connectorId, mergedNodeResponse.getThrowable()); + } else if (!(updatedEntity instanceof ConnectorEntity)) { + throw new IOException("Received neither response nor merged Connector entity while retrieving state for Connector " + connectorId); + } // Use the merged/updated entity if available, otherwise fall back to reading from the raw response. // The updatedEntity contains the properly merged state from all nodes, while readEntity() would // only return the state from whichever single node was selected as the "client response". final ConnectorEntity connectorEntity; - final Entity updatedEntity = mergedNodeResponse.getUpdatedEntity(); if (updatedEntity instanceof ConnectorEntity mergedConnectorEntity) { logger.debug("getState: Connector [{}] - using merged updatedEntity", connectorId); connectorEntity = mergedConnectorEntity; @@ -115,7 +120,7 @@ private void verifyResponse(final StatusType responseStatusType, final String co final String reason = responseStatusType.getReasonPhrase(); if (responseStatusType.getStatusCode() == Status.NOT_FOUND.getStatusCode()) { - throw new IllegalArgumentException("Connector with ID + " + connectorId + " does not exist"); + throw new IllegalArgumentException("Connector with ID " + connectorId + " does not exist"); } final Family responseFamily = responseStatusType.getFamily(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java new file mode 100644 index 000000000000..7492fc0e3213 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MigrationRequestEndpointMergerTest { + + private MigrationRequestEndpointMerger merger; + + @BeforeEach + public void setUp() { + merger = new MigrationRequestEndpointMerger(); + } + + @Test + public void testMergeReflectsWorstCompletionState() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(true, null, 100, new Date(2_000L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(false, null, 40, new Date(1_500L))); + + merger.mergeResponses(client, map, null, null); + + // Any node still in progress drives the merged response to incomplete. + assertFalse(client.getRequest().isComplete()); + } + + @Test + public void testMergePropagatesFailureFromAnyNode() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(true, null, 100, new Date(2_000L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(true, "rollback failed", 100, new Date(2_500L))); + + merger.mergeResponses(client, map, null, null); + + assertNotNull(client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getFailureReason().contains("rollback failed"), client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getFailureReason().contains("node-2"), client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getState().startsWith("Failed: "), client.getRequest().getState()); + } + + @Test + public void testMergePicksMinimumPercentComplete() { + final MigrationRequestEntity client = createMigrationRequest(false, null, 90, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(false, null, 60, new Date(2_500L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(false, null, 25, new Date(2_700L))); + + merger.mergeResponses(client, map, null, null); + + assertEquals(25, client.getRequest().getPercentCompleted()); + } + + @Test + public void testMergePropagatesPerStepFailures() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final MigrationRequestEntity nodeFailure = createMigrationRequest(true, null, 100, new Date(2_500L)); + nodeFailure.getRequest().getUpdateSteps().get(0).setFailureReason("step failed on node"); + nodeFailure.getRequest().getUpdateSteps().get(0).setComplete(false); + + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), nodeFailure); + + merger.mergeResponses(client, map, null, null); + + final MigrationUpdateStepDTO mergedStep = client.getRequest().getUpdateSteps().get(0); + assertFalse(mergedStep.isComplete()); + assertEquals("step failed on node", mergedStep.getFailureReason()); + } + + private MigrationRequestEntity createMigrationRequest(final boolean complete, final String failureReason, final int percentComplete, final Date lastUpdated) { + final MigrationRequestDTO request = new MigrationRequestDTO(); + request.setComplete(complete); + request.setFailureReason(failureReason); + request.setPercentCompleted(percentComplete); + request.setLastUpdated(lastUpdated); + + final MigrationUpdateStepDTO step = new MigrationUpdateStepDTO(); + step.setDescription("Migrate Versioned Flow"); + step.setComplete(complete); + request.setUpdateSteps(List.of(step)); + + final MigrationRequestEntity entity = new MigrationRequestEntity(); + entity.setRequest(request); + return entity; + } + + private NodeIdentifier nodeId(final String hostname, final int port) { + return new NodeIdentifier(hostname + "-" + port, hostname, port, hostname, port + 1, hostname, port + 2, port + 3, false); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index d5f1db9d7bee..e277fe74fecd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -63,7 +63,6 @@ import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedComponentState; -import org.apache.nifi.flow.VersionedConfigurableExtension; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedExternalFlow; @@ -4122,45 +4121,7 @@ private Map getPropertyValues(final ComponentNode componentNode) } private void validateLocalStateTopology(final VersionedProcessGroup proposed) { - final int connectedNodeCount = context.getConnectedNodeCount(); - if (connectedNodeCount <= 0) { - return; - } - - final int maxSourceNodes = findMaxLocalStateNodeCount(proposed); - if (maxSourceNodes > connectedNodeCount) { - throw new IllegalStateException( - "Cannot import flow with component state: the flow definition contains local state from %d source node(s) but the destination cluster has only %d connected node(s). " - .formatted(maxSourceNodes, connectedNodeCount) - + "Import into a cluster with at least %d node(s), or export without component state.".formatted(maxSourceNodes)); - } - } - - private int findMaxLocalStateNodeCount(final VersionedProcessGroup group) { - int max = 0; - for (final VersionedConfigurableExtension ext : getStatefulExtensions(group)) { - final VersionedComponentState state = ext.getComponentState(); - if (state != null && state.getLocalNodeStates() != null) { - max = Math.max(max, state.getLocalNodeStates().size()); - } - } - if (group.getProcessGroups() != null) { - for (final VersionedProcessGroup child : group.getProcessGroups()) { - max = Math.max(max, findMaxLocalStateNodeCount(child)); - } - } - return max; - } - - private List getStatefulExtensions(final VersionedProcessGroup group) { - final List extensions = new ArrayList<>(); - if (group.getProcessors() != null) { - extensions.addAll(group.getProcessors()); - } - if (group.getControllerServices() != null) { - extensions.addAll(group.getControllerServices()); - } - return extensions; + VersionedComponentStateValidator.validateLocalStateTopology(proposed, context.getConnectedNodeCount()); } private void restoreComponentState(final String componentId, final VersionedComponentState componentState, final ComponentNode componentNode) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java new file mode 100644 index 000000000000..4610a4acb4e2 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.synchronization; + +import org.apache.nifi.flow.VersionedComponentState; +import org.apache.nifi.flow.VersionedConfigurableExtension; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.ArrayList; +import java.util.List; + +/** + * Shared helpers that inspect the LOCAL component state contained in an exported {@link VersionedProcessGroup} + * and validate it against the destination cluster topology before importing or migrating the flow. + * + *

This is the single source of truth for the cluster-topology rule used by both the standard versioned-flow + * synchronization path and the connector migration path. Both paths must reject a flow whose LOCAL state + * contains entries for more nodes than the destination cluster currently has connected, because there is no way + * to attribute the extra source-node states to a destination node.

+ */ +public final class VersionedComponentStateValidator { + + private VersionedComponentStateValidator() { + } + + /** + * Validates that the LOCAL component state inside the given proposed {@link VersionedProcessGroup} can be + * imported into a cluster that currently has the given number of connected nodes. + * + * @param proposed the proposed flow to validate + * @param connectedNodeCount the number of nodes that are currently connected to the destination cluster + * @throws IllegalStateException when any stateful component in the proposed flow has more local-node-state entries + * than the destination cluster has connected nodes + */ + public static void validateLocalStateTopology(final VersionedProcessGroup proposed, final int connectedNodeCount) { + if (proposed == null || connectedNodeCount <= 0) { + return; + } + + final int maxSourceNodes = findMaxLocalStateNodeCount(proposed); + if (maxSourceNodes > connectedNodeCount) { + throw new IllegalStateException( + "Cannot import flow with component state: the flow definition contains local state from %d source node(s) but the destination cluster has only %d connected node(s). " + .formatted(maxSourceNodes, connectedNodeCount) + + "Import into a cluster with at least %d node(s), or export without component state.".formatted(maxSourceNodes)); + } + } + + /** + * Returns the maximum number of local-node-state entries declared by any stateful component anywhere in the + * given {@link VersionedProcessGroup} hierarchy. Returns 0 when no component declares LOCAL state. + * + * @param group the root process group to inspect + * @return the maximum local-node-state count across the group hierarchy + */ + public static int findMaxLocalStateNodeCount(final VersionedProcessGroup group) { + if (group == null) { + return 0; + } + + int max = 0; + for (final VersionedConfigurableExtension extension : getStatefulExtensions(group)) { + final VersionedComponentState state = extension.getComponentState(); + if (state != null && state.getLocalNodeStates() != null) { + max = Math.max(max, state.getLocalNodeStates().size()); + } + } + + if (group.getProcessGroups() != null) { + for (final VersionedProcessGroup child : group.getProcessGroups()) { + max = Math.max(max, findMaxLocalStateNodeCount(child)); + } + } + + return max; + } + + private static List getStatefulExtensions(final VersionedProcessGroup group) { + final List extensions = new ArrayList<>(); + if (group.getProcessors() != null) { + extensions.addAll(group.getProcessors()); + } + if (group.getControllerServices() != null) { + extensions.addAll(group.getControllerServices()); + } + return extensions; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java new file mode 100644 index 000000000000..773f0ed40f05 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; + +/** + * Supplies {@link RegisteredFlowSnapshot} instances for Process Groups that are being inspected as + * candidate sources for Connector migration. Implementations delegate to the standard flow snapshot + * facility so that the migration manager and the listing path use a single mechanism for snapshot + * acquisition rather than reimplementing flow mapping in framework-core. + */ +@FunctionalInterface +public interface ConnectorFlowSnapshotProvider { + + /** + * Returns the current flow snapshot for the Process Group with the given identifier, optionally + * including referenced controller services and exported component state. + * + * @param processGroupId the identifier of the Process Group to snapshot + * @param includeReferencedServices whether to include controller services referenced from outside the group + * @param includeComponentState whether to include LOCAL and CLUSTER component state in the snapshot + * @return the flow snapshot + */ + RegisteredFlowSnapshot getCurrentFlowSnapshotByGroupId(String processGroupId, boolean includeReferencedServices, boolean includeComponentState); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java new file mode 100644 index 000000000000..2f2fb67ebfba --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.util.List; +import java.util.function.BooleanSupplier; + +/** + * Framework service that drives the eligibility-listing path used by the migration-sources REST endpoint and the + * actual migration path used by the migration-request REST endpoint. Connector translation logic is owned by the + * extension; framework concerns (eligibility gating, source disable/rename, rollback, cluster-topology validation, + * and atomic flag persistence) live here. + */ +public interface ConnectorMigrationManager { + + /** + * Lists the Versioned Process Groups on the canvas that the given Connector can use as a migration source. + * + * @param connectorId the identifier of the target Connector + * @return the eligible source Process Groups, after framework prerequisites and the Connector's own + * {@code isMigrationSupported(...)} filter have been applied + */ + List listMigrationSources(String connectorId); + + /** + * Verifies that the given Process Group satisfies every framework prerequisite required for a local-source + * migration into the given Connector. Throws an {@link IllegalStateException} with a state-specific + * diagnostic when any prerequisite is unmet. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group + */ + void verifyEligibility(String connectorId, String processGroupId); + + /** + * Migrates the target Connector by updating the Connector's own flow to mirror the configuration, parameters, and + * component state captured in the given source flow. When {@code processGroupId} is non-null the migration is + * treated as a local-source migration (and the source Process Group is disabled and renamed on success); when + * {@code processGroupId} is null the migration is treated as an uploaded-payload migration. On any failure during + * the Connector's {@code migrate(...)} call or the framework's post-migrate work, the Connector is rolled back to + * its initial-flow state and the source Process Group is left untouched. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group for local-source migration, or {@code null} + * for an uploaded-payload migration + * @param sourceFlow the source flow whose configuration, parameters, and component state the Connector should + * mirror into its own managed flow + * @throws FlowUpdateException when the migration cannot be completed + */ + default void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow) throws FlowUpdateException { + migrateFromVersionedFlow(connectorId, processGroupId, sourceFlow, () -> false); + } + + /** + * Equivalent to {@link #migrateFromVersionedFlow(String, String, VersionedExternalFlow)} but additionally polls the + * given supplier at framework-controlled checkpoints so a caller-driven cancellation is observed promptly. The + * Connector's own {@code migrate(...)} call is not interrupted; cancellation only takes effect at the next + * framework checkpoint, after which the migration is rolled back as if it had failed. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group for local-source migration, or {@code null} + * for an uploaded-payload migration + * @param sourceFlow the source flow whose configuration, parameters, and component state the Connector should + * mirror into its own managed flow + * @param cancellationCheck a supplier that returns {@code true} when the caller has requested cancellation; the + * framework polls it at the natural boundaries between the Connector's {@code migrate(...)} + * call and the subsequent framework post-migrate work + * @throws FlowUpdateException when the migration cannot be completed + */ + void migrateFromVersionedFlow(String connectorId, String processGroupId, VersionedExternalFlow sourceFlow, + BooleanSupplier cancellationCheck) throws FlowUpdateException; +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java new file mode 100644 index 000000000000..4cf65195eb46 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +/** + * Describes a Versioned Process Group on the canvas that is eligible to be used as the source for a Connector + * migration. Returned by {@link ConnectorMigrationManager#listMigrationSources(String)} and surfaced to clients + * through the {@code GET /connectors/{id}/migration-sources} endpoint. + */ +public class ConnectorMigrationSource { + private String processGroupId; + private String processGroupName; + private String registryClientId; + private String bucketId; + private String flowId; + private String version; + + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } + + public String getProcessGroupName() { + return processGroupName; + } + + public void setProcessGroupName(final String processGroupName) { + this.processGroupName = processGroupName; + } + + public String getRegistryClientId() { + return registryClientId; + } + + public void setRegistryClientId(final String registryClientId) { + this.registryClientId = registryClientId; + } + + public String getBucketId() { + return bucketId; + } + + public void setBucketId(final String bucketId) { + this.bucketId = bucketId; + } + + public String getFlowId() { + return flowId; + } + + public void setFlowId(final String flowId) { + this.flowId = flowId; + } + + public String getVersion() { + return version; + } + + public void setVersion(final String version) { + this.version = version; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java index 08819d38857d..3b216522be09 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java @@ -23,6 +23,7 @@ import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.VersionedComponent; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; import org.apache.nifi.components.validation.ValidationState; import org.apache.nifi.components.validation.ValidationStatus; import org.apache.nifi.components.validation.ValidationTrigger; @@ -81,6 +82,15 @@ public interface ConnectorNode extends ComponentAuthorizable, VersionedComponent void loadInitialFlow() throws FlowUpdateException; + boolean isMigrationSupported(ConnectorMigrationContext context); + + /** + * Discards the working flow context, if any, and rebuilds it from the active flow context's configuration. + * The framework calls this on the success path of a Connector migration so that any subsequent configure-step + * interactions begin from the migrated flow rather than from the working state that existed before migration. + */ + void recreateWorkingFlowContext(); + /** *

* Pause triggering asynchronous validation to occur when the connector is updated. Often times, it is necessary diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java index d8d06bc79efa..fd9189d96ad4 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.Future; @@ -202,6 +203,18 @@ void inheritConfiguration(ConnectorNode connector, List assetIdentifiers); + /** * Ensures that asset binaries for the given connector are available locally by downloading * any missing or changed assets from the external {@link ConnectorConfigurationProvider}. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..61cc0917ee75 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; + +import java.util.Set; + +/** + * Framework-only extension of {@link ConnectorMigrationContext} that exposes the managers needed to + * execute migration and rollback logic. + */ +public interface FrameworkConnectorMigrationContext extends ConnectorMigrationContext { + + @Override + FrameworkFlowContext getActiveFlowContext(); + + AssetManager getSourceAssetManager(); + + ConnectorRepository getConnectorRepository(); + + StateManagerProvider getStateManagerProvider(); + + ClusterTopologyProvider getClusterTopologyProvider(); + + /** + * Returns the identifiers of assets that the running migration attempt has copied into the + * Connector asset namespace via {@link ConnectorMigrationContext#copyAssetFromSource(String)}. + * Used by the framework to scope rollback so it deletes only assets created during the attempt + * rather than every asset on the Connector. + * + * @return the identifiers of assets created during the current migration attempt + */ + Set getCopiedAssetIds(); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java index ada91a1d418f..eeca0c31e149 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java @@ -172,13 +172,14 @@ private void collectParameterValues(final VersionedParameterContext context, fin parameterValues.removeIf(param -> param.getName().equals(parameterName)); } + final List referencedAssets = versionedParameter.getReferencedAssets(); final ParameterValue.Builder builder = new ParameterValue.Builder() .name(parameterName) - .value(versionedParameter.getValue()) + .value(referencedAssets == null || referencedAssets.isEmpty() ? versionedParameter.getValue() : null) .sensitive(versionedParameter.isSensitive()); - if (assetManager != null && versionedParameter.getReferencedAssets() != null) { - for (final VersionedAsset versionedAsset : versionedParameter.getReferencedAssets()) { + if (assetManager != null && referencedAssets != null) { + for (final VersionedAsset versionedAsset : referencedAssets) { assetManager.getAsset(versionedAsset.getIdentifier()).ifPresent(builder::addReferencedAsset); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java new file mode 100644 index 000000000000..d09df0104c0c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.components.connector.components.ConnectionFacade; +import org.apache.nifi.components.connector.components.ControllerServiceFacade; +import org.apache.nifi.components.connector.components.ProcessGroupFacade; +import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.queue.QueueSize; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceState; +import org.apache.nifi.flow.ExternalControllerServiceReference; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedExternalFlowMetadata; +import org.apache.nifi.flow.synchronization.VersionedComponentStateValidator; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshotMetadata; +import org.apache.nifi.registry.flow.VersionControlInformation; +import org.apache.nifi.registry.flow.VersionedFlowState; +import org.apache.nifi.registry.flow.VersionedFlowStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BooleanSupplier; + +public class StandardConnectorMigrationManager implements ConnectorMigrationManager { + private static final Logger logger = LoggerFactory.getLogger(StandardConnectorMigrationManager.class); + + private static final String MIGRATED_SOURCE_PREFIX = "(Migrated) "; + + private final FlowController flowController; + private final ConnectorFlowSnapshotProvider snapshotProvider; + + public StandardConnectorMigrationManager(final FlowController flowController, final ConnectorFlowSnapshotProvider snapshotProvider) { + this.flowController = Objects.requireNonNull(flowController, "FlowController is required"); + this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "ConnectorFlowSnapshotProvider is required"); + } + + @Override + public List listMigrationSources(final String connectorId) { + final ConnectorNode connector = getRequiredConnector(connectorId); + final List migrationSources = new ArrayList<>(); + + for (final ProcessGroup processGroup : getCandidateSourceGroups()) { + if (isEligibleSource(connector, processGroup)) { + migrationSources.add(toMigrationSource(processGroup.getVersionControlInformation(), processGroup)); + } + } + + return migrationSources; + } + + @Override + public void verifyEligibility(final String connectorId, final String processGroupId) { + final ConnectorNode connector = getRequiredConnector(connectorId); + final ProcessGroup processGroup = getRequiredSourceProcessGroup(processGroupId); + final String ineligibilityReason = getIneligibilityReason(connector, processGroup); + if (ineligibilityReason != null) { + throw new IllegalStateException(ineligibilityReason); + } + } + + @Override + public void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow, + final BooleanSupplier cancellationCheck) throws FlowUpdateException { + final BooleanSupplier cancellation = cancellationCheck == null ? () -> false : cancellationCheck; + final ConnectorNode connector = getRequiredConnector(connectorId); + verifyConnectorCanReceiveMigration(connector); + verifyTargetIsAtInitialFlow(connector); + validateMigrationSource(sourceFlow); + + final boolean localMigration = processGroupId != null; + final ProcessGroup sourceProcessGroup = localMigration ? getRequiredSourceProcessGroup(processGroupId) : null; + final StandardFrameworkConnectorMigrationContext migrationContext = new StandardFrameworkConnectorMigrationContext( + connectorId, + sourceFlow, + localMigration, + connector.getActiveFlowContext(), + flowController.getAssetManager(), + flowController.getConnectorRepository(), + flowController.getStateManagerProvider(), + flowController + ); + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connector.getConnector().getClass(), connectorId)) { + if (!connector.getConnector().isMigrationSupported(migrationContext)) { + throw new FlowUpdateException("Connector " + connectorId + " does not support migration from the provided source flow."); + } + } + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connector.getConnector().getClass(), connectorId)) { + flowController.getConnectorRepository().syncAssetsFromProvider(connector); + connector.getConnector().migrate(migrationContext); + + if (cancellation.getAsBoolean()) { + throw new FlowUpdateException("Migration of Connector " + connectorId + " was cancelled after the Connector applied the source flow."); + } + } catch (final FlowUpdateException e) { + rollbackMigration(connector, migrationContext); + throw e; + } catch (final Exception e) { + rollbackMigration(connector, migrationContext); + throw new FlowUpdateException("Failed to migrate Connector " + connectorId + " from the provided source flow", e); + } + + try { + connector.recreateWorkingFlowContext(); + + if (cancellation.getAsBoolean()) { + throw new FlowUpdateException("Migration of Connector " + connectorId + " was cancelled before the source Process Group could be finalized."); + } + + if (sourceProcessGroup != null) { + disableAndRenameSourceProcessGroup(sourceProcessGroup); + } + } catch (final FlowUpdateException e) { + rollbackMigration(connector, migrationContext); + throw e; + } catch (final Exception e) { + rollbackMigration(connector, migrationContext); + throw new FlowUpdateException("Failed to finalize migrated configuration for Connector " + connectorId, e); + } + } + + private void rollbackMigration(final ConnectorNode connector, final StandardFrameworkConnectorMigrationContext migrationContext) { + clearConnectorComponentState(connector); + + final Set copiedAssetIds = migrationContext.getCopiedAssetIds(); + if (!copiedAssetIds.isEmpty()) { + try { + flowController.getConnectorRepository().deleteAssets(connector.getIdentifier(), copiedAssetIds); + } catch (final Exception e) { + logger.warn("Failed to delete copied assets {} for Connector {} during migration rollback", copiedAssetIds, connector.getIdentifier(), e); + } + } + + try { + connector.loadInitialFlow(); + } catch (final FlowUpdateException e) { + // Do not propagate the rollback failure: the caller's primary need is to see why the migration originally + // failed, not how the recovery attempt failed. The operator can read the rollback failure from the log. + logger.error("Failed to restore Connector {} to its initial state after migration failure; manual cleanup may be required", + connector.getIdentifier(), e); + return; + } + + flowController.getConnectorRepository().discardWorkingConfiguration(connector); + } + + private void clearConnectorComponentState(final ConnectorNode connector) { + // Migration is only permitted on a Connector whose active flow is empty, so every Processor and Controller + // Service currently inside the Connector's managed flow was created by the failed migrate(...) call. Clearing + // state for those components is safe and avoids the brittle approach of trying to map source-flow versioned + // identifiers to runtime component identifiers. + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + if (activeFlowContext == null) { + return; + } + + final ProcessGroup managedGroup = activeFlowContext.getManagedProcessGroup(); + if (managedGroup == null) { + return; + } + + for (final ProcessorNode processor : managedGroup.findAllProcessors()) { + clearComponentState(processor.getIdentifier()); + } + for (final ControllerServiceNode controllerService : managedGroup.findAllControllerServices()) { + clearComponentState(controllerService.getIdentifier()); + } + } + + private void clearComponentState(final String componentIdentifier) { + final StateManager stateManager = flowController.getStateManagerProvider().getStateManager(componentIdentifier); + if (stateManager == null) { + return; + } + + try { + stateManager.clear(Scope.LOCAL); + } catch (final Exception e) { + logger.warn("Failed to clear LOCAL state for component {} during migration rollback", componentIdentifier, e); + } + + try { + stateManager.clear(Scope.CLUSTER); + } catch (final Exception e) { + logger.warn("Failed to clear CLUSTER state for component {} during migration rollback", componentIdentifier, e); + } + } + + private ConnectorNode getRequiredConnector(final String connectorId) { + final ConnectorNode connector = flowController.getConnectorRepository().getConnector(connectorId); + if (connector == null) { + throw new IllegalArgumentException("Could not find Connector with ID " + connectorId); + } + + return connector; + } + + private ProcessGroup getRequiredSourceProcessGroup(final String processGroupId) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + final ProcessGroup processGroup = rootGroup == null ? null : rootGroup.findProcessGroup(processGroupId); + if (processGroup == null) { + throw new IllegalArgumentException("Could not find Process Group with ID " + processGroupId); + } + + return processGroup; + } + + private List getCandidateSourceGroups() { + final FlowManager flowManager = flowController.getFlowManager(); + final ProcessGroup rootGroup = flowManager.getRootGroup(); + if (rootGroup == null) { + return Collections.emptyList(); + } + + return new ArrayList<>(rootGroup.findAllProcessGroups()); + } + + private boolean isEligibleSource(final ConnectorNode connector, final ProcessGroup processGroup) { + return getIneligibilityReason(connector, processGroup) == null; + } + + private String getIneligibilityReason(final ConnectorNode connector, final ProcessGroup processGroup) { + final VersionControlInformation versionControlInformation = processGroup.getVersionControlInformation(); + if (versionControlInformation == null) { + return "Process Group " + processGroup.getIdentifier() + " is not under version control."; + } + + if (processGroup.getConnectorIdentifier().isPresent()) { + return "Process Group " + processGroup.getIdentifier() + " is managed by a Connector and cannot be used as a migration source."; + } + + final VersionedFlowStatus versionedFlowStatus = versionControlInformation.getStatus(); + final VersionedFlowState state = versionedFlowStatus == null ? null : versionedFlowStatus.getState(); + if (state != VersionedFlowState.UP_TO_DATE) { + return describeIneligibleVersionedFlowState(processGroup, state); + } + + if (hasRunningProcessors(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " has running or enabled processors."; + } + + if (hasEnabledControllerServices(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " has enabled controller services."; + } + + if (hasQueuedData(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " contains queued FlowFiles."; + } + + final VersionedExternalFlow sourceFlow = obtainSourceFlowForListing(processGroup); + if (!Optional.ofNullable(sourceFlow.getExternalControllerServices()).orElse(Collections.emptyMap()).isEmpty()) { + return "Process Group " + processGroup.getIdentifier() + " references controller services outside the Process Group."; + } + + final FrameworkConnectorMigrationContext migrationContext = new StandardFrameworkConnectorMigrationContext( + connector.getIdentifier(), + sourceFlow, + true, + connector.getActiveFlowContext(), + flowController.getAssetManager(), + flowController.getConnectorRepository(), + flowController.getStateManagerProvider(), + flowController + ); + + return connector.isMigrationSupported(migrationContext) + ? null + : "Connector " + connector.getIdentifier() + " does not support migration from Process Group " + processGroup.getIdentifier() + "."; + } + + private String describeIneligibleVersionedFlowState(final ProcessGroup processGroup, final VersionedFlowState state) { + final String prefix = "Process Group " + processGroup.getIdentifier() + " "; + if (state == null) { + return prefix + "has no published version control state available."; + } + + return switch (state) { + case STALE -> prefix + "is stale with respect to its registered flow; bring it back to the latest published version before migrating."; + case LOCALLY_MODIFIED -> prefix + "has local modifications relative to its registered flow; revert or publish them before migrating."; + case LOCALLY_MODIFIED_AND_STALE -> prefix + "is stale and has local modifications; revert them and refresh to the latest version before migrating."; + case SYNC_FAILURE -> prefix + "failed to synchronize with its flow registry; restore connectivity and refresh the version control status before migrating."; + default -> prefix + "is not up to date with its registered flow."; + }; + } + + private void verifyConnectorCanReceiveMigration(final ConnectorNode connector) { + if (connector.getCurrentState() != ConnectorState.STOPPED || connector.getDesiredState() != ConnectorState.STOPPED) { + throw new IllegalStateException("Connector " + connector.getIdentifier() + " must be stopped before it can be migrated."); + } + } + + private void verifyTargetIsAtInitialFlow(final ConnectorNode connector) { + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + if (activeFlowContext == null) { + return; + } + + final ProcessGroupFacade rootGroup = activeFlowContext.getRootGroup(); + if (rootGroup == null) { + return; + } + + if (containsUserInstalledComponents(rootGroup)) { + throw new IllegalStateException("Connector " + connector.getIdentifier() + + " already contains user-installed components in its active flow; migration is only supported for a freshly created Connector."); + } + } + + private boolean containsUserInstalledComponents(final ProcessGroupFacade processGroup) { + final Set processors = processGroup.getProcessors(); + if (processors != null && !processors.isEmpty()) { + return true; + } + + final Set controllerServices = processGroup.getControllerServices(); + if (controllerServices != null && !controllerServices.isEmpty()) { + return true; + } + + final Set connections = processGroup.getConnections(); + if (connections != null && !connections.isEmpty()) { + return true; + } + + final Set childGroups = processGroup.getProcessGroups(); + if (childGroups == null) { + return false; + } + + for (final ProcessGroupFacade childGroup : childGroups) { + if (containsUserInstalledComponents(childGroup)) { + return true; + } + } + + return false; + } + + private void validateMigrationSource(final VersionedExternalFlow sourceFlow) { + final Map externalControllerServices = + Optional.ofNullable(sourceFlow.getExternalControllerServices()).orElse(Collections.emptyMap()); + if (!externalControllerServices.isEmpty()) { + throw new IllegalStateException("Connector cannot reference services outside its managed flow."); + } + + final int connectedNodeCount = flowController.getConnectedNodeCount(); + VersionedComponentStateValidator.validateLocalStateTopology(sourceFlow.getFlowContents(), connectedNodeCount); + } + + private void disableAndRenameSourceProcessGroup(final ProcessGroup processGroup) { + disableSourceProcessors(processGroup); + disableSourceControllerServices(processGroup); + + final String currentName = processGroup.getName(); + if (currentName != null && currentName.startsWith(MIGRATED_SOURCE_PREFIX)) { + return; + } + + processGroup.setName(MIGRATED_SOURCE_PREFIX + currentName); + } + + private void disableSourceProcessors(final ProcessGroup processGroup) { + for (final ProcessorNode processor : processGroup.findAllProcessors()) { + if (processor.getDesiredState() == ScheduledState.DISABLED) { + continue; + } + + final ProcessGroup parent = processor.getProcessGroup(); + if (parent == null) { + continue; + } + + try { + parent.disableProcessor(processor); + } catch (final Exception e) { + logger.warn("Failed to disable processor {} while finalizing migrated source Process Group {}", processor.getIdentifier(), processGroup.getIdentifier(), e); + } + } + } + + private void disableSourceControllerServices(final ProcessGroup processGroup) { + final List enabledServices = new ArrayList<>(); + for (final ControllerServiceNode controllerService : processGroup.findAllControllerServices()) { + if (controllerService.getState() != ControllerServiceState.DISABLED) { + enabledServices.add(controllerService); + } + } + + if (enabledServices.isEmpty()) { + return; + } + + try { + flowController.getControllerServiceProvider().disableControllerServicesAsync(enabledServices); + } catch (final Exception e) { + logger.warn("Failed to disable controller services while finalizing migrated source Process Group {}", processGroup.getIdentifier(), e); + } + } + + private boolean hasRunningProcessors(final ProcessGroup processGroup) { + return processGroup.findAllProcessors().stream() + .anyMatch(processor -> processor.getPhysicalScheduledState() != ScheduledState.STOPPED && processor.getPhysicalScheduledState() != ScheduledState.DISABLED); + } + + private boolean hasEnabledControllerServices(final ProcessGroup processGroup) { + return processGroup.findAllControllerServices().stream() + .anyMatch(controllerService -> controllerService.getState() != ControllerServiceState.DISABLED); + } + + private boolean hasQueuedData(final ProcessGroup processGroup) { + for (final Connection connection : processGroup.findAllConnections()) { + final QueueSize queueSize = connection.getFlowFileQueue().size(); + if (queueSize.getObjectCount() > 0) { + return true; + } + } + + return false; + } + + private ConnectorMigrationSource toMigrationSource(final VersionControlInformation versionControlInformation, final ProcessGroup processGroup) { + final ConnectorMigrationSource migrationSource = new ConnectorMigrationSource(); + migrationSource.setProcessGroupId(processGroup.getIdentifier()); + migrationSource.setProcessGroupName(processGroup.getName()); + migrationSource.setRegistryClientId(versionControlInformation.getRegistryIdentifier()); + migrationSource.setBucketId(versionControlInformation.getBucketIdentifier()); + migrationSource.setFlowId(versionControlInformation.getFlowIdentifier()); + migrationSource.setVersion(versionControlInformation.getVersion()); + return migrationSource; + } + + private VersionedExternalFlow obtainSourceFlowForListing(final ProcessGroup processGroup) { + // Listing-time eligibility checks intentionally exclude component state. Materializing component state is + // expensive and is only required for the actual migration. The Connector's isMigrationSupported(...) must rely + // solely on structural information (processor types, parameter names, exported metadata) at listing time; + // per-state precondition checks belong inside the Connector's own migrate(...) implementation. + final RegisteredFlowSnapshot snapshot = snapshotProvider.getCurrentFlowSnapshotByGroupId(processGroup.getIdentifier(), true, false); + return createExternalFlow(snapshot); + } + + public VersionedExternalFlow createExternalFlow(final RegisteredFlowSnapshot flowSnapshot) { + final VersionedExternalFlowMetadata externalFlowMetadata = new VersionedExternalFlowMetadata(); + final RegisteredFlowSnapshotMetadata snapshotMetadata = flowSnapshot.getSnapshotMetadata(); + if (snapshotMetadata != null) { + externalFlowMetadata.setAuthor(snapshotMetadata.getAuthor()); + externalFlowMetadata.setBucketIdentifier(snapshotMetadata.getBucketIdentifier()); + externalFlowMetadata.setComments(snapshotMetadata.getComments()); + externalFlowMetadata.setFlowIdentifier(snapshotMetadata.getFlowIdentifier()); + externalFlowMetadata.setTimestamp(snapshotMetadata.getTimestamp()); + externalFlowMetadata.setVersion(snapshotMetadata.getVersion()); + } + + final RegisteredFlow registeredFlow = flowSnapshot.getFlow(); + if (registeredFlow == null) { + externalFlowMetadata.setFlowName(flowSnapshot.getFlowContents().getName()); + } else { + externalFlowMetadata.setFlowName(registeredFlow.getName()); + } + + final VersionedExternalFlow externalFlow = new VersionedExternalFlow(); + externalFlow.setFlowContents(flowSnapshot.getFlowContents()); + externalFlow.setExternalControllerServices(flowSnapshot.getExternalControllerServices()); + externalFlow.setParameterContexts(flowSnapshot.getParameterContexts()); + externalFlow.setParameterProviders(flowSnapshot.getParameterProviders()); + externalFlow.setMetadata(externalFlowMetadata); + return externalFlow; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index b9659ebb4860..789896db7eac 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -29,6 +29,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.components.ParameterContextFacade; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; import org.apache.nifi.components.validation.DisabledServiceValidationResult; import org.apache.nifi.components.validation.ValidationState; import org.apache.nifi.components.validation.ValidationStatus; @@ -755,6 +756,16 @@ public void loadInitialFlow() throws FlowUpdateException { recreateWorkingFlowContext(); } + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { + return getConnector().isMigrationSupported(context); + } catch (final Exception e) { + getComponentLog().warn("Failed to evaluate whether migration is supported for {}; assuming migration is not supported", context.getSourceFlow(), e); + return false; + } + } + private void stopComponents(final VersionedProcessGroup group) { for (final VersionedProcessor processor : group.getProcessors()) { if (processor.getScheduledState() == ScheduledState.RUNNING) { @@ -868,7 +879,8 @@ private boolean isBundleResolvable(final String componentType, final Bundle curr return availableBundles.size() == 1; } - private void recreateWorkingFlowContext() { + @Override + public void recreateWorkingFlowContext() { destroyWorkingContext(); workingFlowContext = flowContextFactory.createWorkingFlowContext(identifier, connectorDetails.getComponentLog(), activeFlowContext.getConfigurationContext(), activeFlowContext.getBundle()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index bdb527fecc81..13b015bb5948 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -30,8 +30,11 @@ import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedConnectorValueReference; +import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.util.BundleUtils; import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.web.api.dto.BundleDTO; @@ -42,6 +45,7 @@ import java.io.InputStream; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -53,6 +57,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; @@ -104,6 +110,7 @@ public void addConnector(final ConnectorNode connector) { @Override public void restoreConnector(final ConnectorNode connector) { connectors.put(connector.getIdentifier(), connector); + cleanUpAssets(connector); logger.debug("Successfully restored {}", connector); } @@ -113,7 +120,7 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne final ScheduledState proposedScheduledState = versionedConnector.getScheduledState(); logger.debug("syncConnector called for connector [{}]", connectorId); - // Consult the provider for external state checks and working config + // Consult the provider for external state checks and any externally managed working configuration. final ConnectorSyncDirective directive; if (configurationProvider != null) { try { @@ -158,7 +165,6 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne } // ALLOW: proceed with sync - // Look up or create the connector node ConnectorNode connector = connectors.get(connectorId); final boolean isNewConnector = connector == null; if (isNewConnector) { @@ -188,7 +194,7 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne return ConnectorSyncResult.rejected(connector); } - // Determine effective name, working config, and ScheduledState + // Determine the connector name, working configuration, and ScheduledState that should win for this sync pass. final ConnectorWorkingConfiguration providerConfig = directive.getWorkingConfiguration(); // Enrich provider-sourced SECRET_REFERENCE values with providerId before they are compared @@ -214,7 +220,6 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne // Set name locally (no provider.save()) connector.setName(effectiveName); - // Compare config and sync if changed final boolean wasRunning = currentState == ConnectorState.RUNNING; final boolean configChanged = isNewConnector || isConfigurationUpdated(connector, effectiveActiveConfig, effectiveWorkingConfig); @@ -227,8 +232,8 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne if (wasRunning) { try { stopConnector(connector); - } catch (final Exception stopEx) { - logger.error("{} also failed to stop after sync failure", connector, stopEx); + } catch (final Exception stopException) { + logger.error("{} also failed to stop after sync failure", connector, stopException); } } return ConnectorSyncResult.failed(connector); @@ -305,8 +310,6 @@ private ConnectorState waitForStableState(final ConnectorNode connector, final C return current; } - // --- Configuration comparison --- - private boolean isConfigurationUpdated(final ConnectorNode existingConnector, final List effectiveActiveConfig, final List effectiveWorkingConfig) { @@ -360,10 +363,10 @@ private boolean isConfigurationStepUpdated(final NamedStepConfiguration existing for (final Map.Entry versionedEntry : versionedProperties.entrySet()) { final String propertyName = versionedEntry.getKey(); - final VersionedConnectorValueReference versionedRef = versionedEntry.getValue(); - final ConnectorValueReference existingRef = existingProperties.get(propertyName); + final VersionedConnectorValueReference versionedValueReference = versionedEntry.getValue(); + final ConnectorValueReference existingValueReference = existingProperties.get(propertyName); - if (!valueReferencesEqual(versionedRef, existingRef)) { + if (!valueReferencesEqual(versionedValueReference, existingValueReference)) { return true; } } @@ -387,9 +390,9 @@ private boolean valueReferencesEqual(final VersionedConnectorValueReference vers return switch (existingReference) { case StringLiteralValue stringLiteral -> Objects.equals(stringLiteral.getValue(), versionedReference.getValue()); - case AssetReference assetRef -> Objects.equals(assetRef.getAssetIdentifiers(), versionedReference.getAssetIds()); - case SecretReference secretRef -> Objects.equals(secretRef.getProviderId(), versionedReference.getProviderId()) - && Objects.equals(secretRef.getSecretName(), versionedReference.getSecretName()); + case AssetReference assetReference -> Objects.equals(assetReference.getAssetIdentifiers(), versionedReference.getAssetIds()); + case SecretReference secretReference -> Objects.equals(secretReference.getProviderId(), versionedReference.getProviderId()) + && Objects.equals(secretReference.getSecretName(), versionedReference.getSecretName()); }; } @@ -429,9 +432,9 @@ private void stopConnectorAndAwait(final ConnectorNode connector) { logger.info("Stopping connector [{}] (current state: {}) and awaiting completion", connectorId, currentState); try { final Future stopFuture = stopConnector(connector); - stopFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + stopFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] stopped successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to stop", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -446,9 +449,9 @@ private void purgeConnectorAndAwait(final ConnectorNode connector) { try { logger.debug("Purging FlowFiles for connector [{}] before removal", connectorId); final Future purgeFuture = connector.purgeFlowFiles("Flow Synchronization"); - purgeFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + purgeFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] purged successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to purge FlowFiles", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -596,7 +599,16 @@ private void waitForState(final ConnectorNode connector, final Set referencedAssetIds = new HashSet<>(); collectReferencedAssetIds(connector.getActiveFlowContext(), referencedAssetIds); @@ -651,6 +667,11 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f return; } + final ProcessGroup managedProcessGroup = flowContext.getManagedProcessGroup(); + if (managedProcessGroup != null) { + collectReferencedParameterContextAssetIds(managedProcessGroup.getParameterContext(), referencedAssetIds); + } + final ConnectorConfiguration configuration = flowContext.getConfigurationContext().toConnectorConfiguration(); for (final NamedStepConfiguration namedStepConfiguration : configuration.getNamedStepConfigurations()) { final StepConfiguration stepConfiguration = namedStepConfiguration.configuration(); @@ -666,6 +687,23 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f } } + private void collectReferencedParameterContextAssetIds(final ParameterContext parameterContext, final Set referencedAssetIds) { + if (parameterContext == null) { + return; + } + + for (final Parameter parameter : parameterContext.getEffectiveParameters().values()) { + final List referencedAssets = parameter.getReferencedAssets(); + if (referencedAssets == null) { + continue; + } + + for (final Asset referencedAsset : referencedAssets) { + referencedAssetIds.add(referencedAsset.getIdentifier()); + } + } + } + @Override public void updateConnector(final ConnectorNode connector, final String name) { if (configurationProvider != null) { @@ -767,16 +805,30 @@ public List getAssets(final String connectorId) { public void deleteAssets(final String connectorId) { final List assets = assetManager.getAssets(connectorId); for (final Asset asset : assets) { - try { - if (configurationProvider != null) { - // When a provider is configured, delegate entirely to the provider, which should delete from the AssetManager and sync to the external store. - configurationProvider.deleteAsset(connectorId, asset.getIdentifier()); - } else { - assetManager.deleteAsset(asset.getIdentifier()); - } - } catch (final Exception e) { - logger.warn("Failed to delete asset [nifiUuid={}] for connector [{}]", asset.getIdentifier(), connectorId, e); + deleteAsset(connectorId, asset.getIdentifier()); + } + } + + @Override + public void deleteAssets(final String connectorId, final Collection assetIdentifiers) { + if (assetIdentifiers == null || assetIdentifiers.isEmpty()) { + return; + } + + for (final String assetIdentifier : assetIdentifiers) { + deleteAsset(connectorId, assetIdentifier); + } + } + + private void deleteAsset(final String connectorId, final String assetIdentifier) { + try { + if (configurationProvider != null) { + configurationProvider.deleteAsset(connectorId, assetIdentifier); + } else { + assetManager.deleteAsset(assetIdentifier); } + } catch (final Exception e) { + logger.warn("Failed to delete asset [nifiUuid={}] for connector [{}]", assetIdentifier, connectorId, e); } } @@ -807,12 +859,12 @@ private void syncFromProvider(final ConnectorNode connector) { return; } - final ConnectorWorkingConfiguration config = externalConfig.get(); - if (config.getName() != null) { - connector.setName(config.getName()); + final ConnectorWorkingConfiguration externalWorkingConfiguration = externalConfig.get(); + if (externalWorkingConfiguration.getName() != null) { + connector.setName(externalWorkingConfiguration.getName()); } - final List workingFlowConfiguration = config.getWorkingFlowConfiguration(); + final List workingFlowConfiguration = externalWorkingConfiguration.getWorkingFlowConfiguration(); if (workingFlowConfiguration == null) { return; @@ -838,10 +890,10 @@ private void syncFromProvider(final ConnectorNode connector) { } private ConnectorWorkingConfiguration buildWorkingConfiguration(final ConnectorNode connector) { - final ConnectorWorkingConfiguration config = new ConnectorWorkingConfiguration(); - config.setName(connector.getName()); - config.setWorkingFlowConfiguration(buildVersionedConfigurationSteps(connector.getWorkingFlowContext())); - return config; + final ConnectorWorkingConfiguration workingConfiguration = new ConnectorWorkingConfiguration(); + workingConfiguration.setName(connector.getName()); + workingConfiguration.setWorkingFlowConfiguration(buildVersionedConfigurationSteps(connector.getWorkingFlowContext())); + return workingConfiguration; } private List buildVersionedConfigurationSteps(final FrameworkFlowContext flowContext) { @@ -922,12 +974,12 @@ private VersionedConnectorValueReference toVersionedValueReference(final Connect switch (valueReference) { case StringLiteralValue stringLiteral -> versionedReference.setValue(stringLiteral.getValue()); - case AssetReference assetRef -> versionedReference.setAssetIds(assetRef.getAssetIdentifiers()); - case SecretReference secretRef -> { - versionedReference.setProviderId(secretRef.getProviderId()); - versionedReference.setProviderName(secretRef.getProviderName()); - versionedReference.setSecretName(secretRef.getSecretName()); - versionedReference.setFullyQualifiedSecretName(secretRef.getFullyQualifiedName()); + case AssetReference assetReference -> versionedReference.setAssetIds(assetReference.getAssetIdentifiers()); + case SecretReference secretReference -> { + versionedReference.setProviderId(secretReference.getProviderId()); + versionedReference.setProviderName(secretReference.getProviderName()); + versionedReference.setSecretName(secretReference.getSecretName()); + versionedReference.setFullyQualifiedSecretName(secretReference.getFullyQualifiedName()); } } @@ -939,9 +991,9 @@ private StepConfiguration toStepConfiguration(final VersionedConfigurationStep s final Map versionedProperties = step.getProperties(); if (versionedProperties != null) { for (final Map.Entry entry : versionedProperties.entrySet()) { - final VersionedConnectorValueReference versionedRef = entry.getValue(); - if (versionedRef != null) { - propertyValues.put(entry.getKey(), toConnectorValueReference(versionedRef)); + final VersionedConnectorValueReference versionedValueReference = entry.getValue(); + if (versionedValueReference != null) { + propertyValues.put(entry.getKey(), toConnectorValueReference(versionedValueReference)); } } } @@ -982,16 +1034,25 @@ private void resolveSecretReferencesFromProvider(final List properties.values().stream()) - .filter(Objects::nonNull) - .filter(ref -> ConnectorValueType.SECRET_REFERENCE.name().equals(ref.getValueType())) - .filter(ref -> ref.getProviderId() == null) - .filter(ref -> ref.getProviderName() != null) - .forEach(ref -> ref.setProviderId(findProviderIdByName(ref.getProviderName()))); + for (final VersionedConfigurationStep step : steps) { + if (step == null || step.getProperties() == null) { + continue; + } + + for (final VersionedConnectorValueReference valueReference : step.getProperties().values()) { + if (valueReference == null) { + continue; + } + if (!ConnectorValueType.SECRET_REFERENCE.name().equals(valueReference.getValueType())) { + continue; + } + if (valueReference.getProviderId() != null || valueReference.getProviderName() == null) { + continue; + } + + valueReference.setProviderId(findProviderIdByName(valueReference.getProviderName())); + } + } } /** diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..f13268b316a9 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +public class StandardFrameworkConnectorMigrationContext implements FrameworkConnectorMigrationContext { + private static final Logger logger = LoggerFactory.getLogger(StandardFrameworkConnectorMigrationContext.class); + private final String connectorId; + private final VersionedExternalFlow sourceFlow; + private final boolean localMigration; + private final FrameworkFlowContext activeFlowContext; + private final AssetManager sourceAssetManager; + private final ConnectorRepository connectorRepository; + private final StateManagerProvider stateManagerProvider; + private final ClusterTopologyProvider clusterTopologyProvider; + private final Set copiedAssetIds = Collections.synchronizedSet(new LinkedHashSet<>()); + + public StandardFrameworkConnectorMigrationContext(final String connectorId, final VersionedExternalFlow sourceFlow, final boolean localMigration, + final FrameworkFlowContext activeFlowContext, final AssetManager sourceAssetManager, final ConnectorRepository connectorRepository, + final StateManagerProvider stateManagerProvider, final ClusterTopologyProvider clusterTopologyProvider) { + this.connectorId = connectorId; + this.sourceFlow = sourceFlow; + this.localMigration = localMigration; + this.activeFlowContext = activeFlowContext; + this.sourceAssetManager = sourceAssetManager; + this.connectorRepository = connectorRepository; + this.stateManagerProvider = stateManagerProvider; + this.clusterTopologyProvider = clusterTopologyProvider; + } + + @Override + public VersionedExternalFlow getSourceFlow() { + return sourceFlow; + } + + @Override + public boolean isLocalMigration() { + return localMigration; + } + + @Override + public FrameworkFlowContext getActiveFlowContext() { + return activeFlowContext; + } + + @Override + public AssetReference copyAssetFromSource(final String sourceAssetId) { + if (sourceAssetId == null || sourceAssetId.isBlank()) { + throw new IllegalArgumentException("Source asset identifier must be specified."); + } + if (!localMigration) { + throw new IllegalArgumentException("Source assets can only be copied for migrations from a local Versioned Process Group."); + } + + final String copiedAssetId = UUID.nameUUIDFromBytes((connectorId + ":" + sourceAssetId).getBytes(StandardCharsets.UTF_8)).toString(); + final Optional existingAsset = connectorRepository.getAsset(copiedAssetId); + if (existingAsset.isPresent() && existingAsset.get().getFile().isFile() && existingAsset.get().getFile().length() > 0L) { + copiedAssetIds.add(copiedAssetId); + return new AssetReference(Set.of(copiedAssetId)); + } + + final Optional sourceAsset = sourceAssetManager.getAsset(sourceAssetId); + if (sourceAsset.isEmpty()) { + logger.warn("Connector [{}] migration could not locate source asset [{}]; the asset will not be copied and the affected parameter " + + "will be left without an asset reference for the user to re-attach after migration.", connectorId, sourceAssetId); + return new AssetReference(Set.of()); + } + + try (final InputStream sourceContents = new FileInputStream(sourceAsset.get().getFile())) { + final Asset copiedAsset = connectorRepository.storeAsset(connectorId, copiedAssetId, sourceAsset.get().getName(), sourceContents); + copiedAssetIds.add(copiedAsset.getIdentifier()); + return new AssetReference(Set.of(copiedAsset.getIdentifier())); + } catch (final IOException e) { + throw new IllegalStateException("Failed to copy source asset with identifier " + sourceAssetId, e); + } + } + + @Override + public AssetManager getSourceAssetManager() { + return sourceAssetManager; + } + + @Override + public ConnectorRepository getConnectorRepository() { + return connectorRepository; + } + + @Override + public StateManagerProvider getStateManagerProvider() { + return stateManagerProvider; + } + + @Override + public ClusterTopologyProvider getClusterTopologyProvider() { + return clusterTopologyProvider; + } + + @Override + public Set getCopiedAssetIds() { + synchronized (copiedAssetIds) { + return Set.copyOf(copiedAssetIds); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java index 2599269d3554..b4387c74503f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java @@ -109,6 +109,7 @@ private Map createParameterMap(final Collection createParameterValues(final ParameterContext context) { final List parameterValues = new ArrayList<>(); for (final Parameter parameter : context.getParameters().values()) { + final List referencedAssets = parameter.getReferencedAssets(); final ParameterValue.Builder parameterValueBuilder = new ParameterValue.Builder() .name(parameter.getDescriptor().getName()) .sensitive(parameter.getDescriptor().isSensitive()) - .value(parameter.getValue()); + .value(referencedAssets == null || referencedAssets.isEmpty() ? parameter.getValue() : null); - parameter.getReferencedAssets().forEach(parameterValueBuilder::addReferencedAsset); + if (referencedAssets != null) { + referencedAssets.forEach(parameterValueBuilder::addReferencedAsset); + } parameterValues.add(parameterValueBuilder.build()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java new file mode 100644 index 000000000000..d895dfd35066 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.connector.components.ProcessGroupFacade; +import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.controller.service.ControllerServiceState; +import org.apache.nifi.flow.ExternalControllerServiceReference; +import org.apache.nifi.flow.VersionedComponentState; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedNodeState; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; +import org.apache.nifi.registry.flow.VersionControlInformation; +import org.apache.nifi.registry.flow.VersionedFlowState; +import org.apache.nifi.registry.flow.VersionedFlowStatus; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestStandardConnectorMigrationManager { + + private static final String CONNECTOR_ID = "connector-1"; + private static final String SOURCE_GROUP_ID = "source-group"; + private static final String SOURCE_GROUP_NAME = "Source Flow"; + + @Test + public void testMigrateFromUploadedPayloadRejectsLocalStateBeyondConnectedNodeCount() { + final FlowController flowController = createFlowController(2); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(3); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("Cannot import flow"), exception.getMessage()); + + verify(connectorNode, never()).recreateWorkingFlowContext(); + } + + @Test + public void testMigrateFromLocalSourceDisablesAndRenamesProcessGroupAfterSuccess() throws Exception { + final FlowController flowController = createFlowController(1); + final ProcessGroup sourceProcessGroup = wireSourceProcessGroup(flowController, SOURCE_GROUP_ID, SOURCE_GROUP_NAME); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, SOURCE_GROUP_ID, sourceFlow); + + verify(connectorNode).recreateWorkingFlowContext(); + verify(sourceProcessGroup).setName("(Migrated) " + SOURCE_GROUP_NAME); + } + + @Test + public void testRenameIsIdempotentWhenSourceAlreadyHasMigratedPrefix() throws Exception { + final FlowController flowController = createFlowController(1); + final ProcessGroup sourceProcessGroup = wireSourceProcessGroup(flowController, SOURCE_GROUP_ID, "(Migrated) " + SOURCE_GROUP_NAME); + + final ProcessGroup sourceParent = mock(ProcessGroup.class); + final ProcessorNode runningProcessor = mock(ProcessorNode.class); + when(runningProcessor.getDesiredState()).thenReturn(ScheduledState.RUNNING); + when(runningProcessor.getProcessGroup()).thenReturn(sourceParent); + when(sourceProcessGroup.findAllProcessors()).thenReturn(List.of(runningProcessor)); + + final ControllerServiceNode enabledService = mock(ControllerServiceNode.class); + when(enabledService.getState()).thenReturn(ControllerServiceState.ENABLED); + when(sourceProcessGroup.findAllControllerServices()).thenReturn(Set.of(enabledService)); + + final ControllerServiceProvider controllerServiceProvider = mock(ControllerServiceProvider.class); + when(flowController.getControllerServiceProvider()).thenReturn(controllerServiceProvider); + + wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, SOURCE_GROUP_ID, sourceFlow); + + // The source already had the (Migrated) prefix from a previous attempt, so it must not be re-prefixed. + verify(sourceProcessGroup, never()).setName(anyString()); + + // Disable must still run unconditionally even when the rename was skipped. + verify(sourceParent).disableProcessor(runningProcessor); + verify(controllerServiceProvider).disableControllerServicesAsync(List.of(enabledService)); + } + + @Test + public void testRerunAfterFailureIsAllowed() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final Connector connector = connectorNode.getConnector(); + doThrow(new FlowUpdateException("transient")).doNothing().when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // Second attempt must succeed because the prior attempt rolled back fully. + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow); + verify(connectorNode, atLeastOnce()).recreateWorkingFlowContext(); + } + + @Test + public void testNonFlowUpdateExceptionFromMigrateStillTriggersRollback() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + final Connector connector = connectorNode.getConnector(); + doThrow(new RuntimeException("boom")).when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + verify(connectorNode).loadInitialFlow(); + verify(connectorRepository).discardWorkingConfiguration(connectorNode); + verify(connectorNode, never()).recreateWorkingFlowContext(); + } + + @Test + public void testRollbackClearsLocalAndClusterStateAndDeletesOnlyCopiedAssets() throws Exception { + final FlowController flowController = createFlowController(1); + final StateManagerProvider stateManagerProvider = flowController.getStateManagerProvider(); + final StateManager componentStateManager = mock(StateManager.class); + when(stateManagerProvider.getStateManager(anyString())).thenReturn(componentStateManager); + + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessorNode processor = mock(ProcessorNode.class); + when(processor.getIdentifier()).thenReturn("processor-runtime-id"); + final ControllerServiceNode controllerService = mock(ControllerServiceNode.class); + when(controllerService.getIdentifier()).thenReturn("service-runtime-id"); + final ProcessGroup managedGroup = mock(ProcessGroup.class); + when(managedGroup.findAllProcessors()).thenReturn(List.of(processor)); + when(managedGroup.findAllControllerServices()).thenReturn(Set.of(controllerService)); + when(connectorNode.getActiveFlowContext().getManagedProcessGroup()).thenReturn(managedGroup); + + final Connector connector = connectorNode.getConnector(); + doThrow(new FlowUpdateException("nope")).when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // Rollback clears LOCAL and CLUSTER state for each component currently in the Connector's managed flow. + verify(stateManagerProvider).getStateManager("processor-runtime-id"); + verify(stateManagerProvider).getStateManager("service-runtime-id"); + verify(componentStateManager, atLeastOnce()).clear(Scope.LOCAL); + verify(componentStateManager, atLeastOnce()).clear(Scope.CLUSTER); + + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + verify(connectorNode).loadInitialFlow(); + } + + @Test + public void testListingFiltersStaleAndLocallyModifiedSources() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + final ProcessGroup staleGroup = mockVersionedGroup("stale-group", VersionedFlowState.STALE); + final ProcessGroup locallyModifiedGroup = mockVersionedGroup("locally-modified-group", VersionedFlowState.LOCALLY_MODIFIED); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(staleGroup, locallyModifiedGroup)); + when(rootGroup.findProcessGroup("stale-group")).thenReturn(staleGroup); + when(rootGroup.findProcessGroup("locally-modified-group")).thenReturn(locallyModifiedGroup); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final List sources = migrationManager.listMigrationSources(CONNECTOR_ID); + + assertTrue(sources.isEmpty()); + + // Distinct diagnostics per VersionedFlowState are exposed via verifyEligibility. + final IllegalStateException staleException = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "stale-group")); + assertTrue(staleException.getMessage().contains("stale"), staleException.getMessage()); + + final IllegalStateException locallyModifiedException = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "locally-modified-group")); + assertTrue(locallyModifiedException.getMessage().contains("local modifications"), locallyModifiedException.getMessage()); + } + + @Test + public void testListingExcludesNonRegistryBackedProcessGroups() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + // Process group has no version control information at all, so it must not appear in listing. + final ProcessGroup nonRegistryBacked = mock(ProcessGroup.class); + when(nonRegistryBacked.getIdentifier()).thenReturn("non-registry-backed"); + when(nonRegistryBacked.getName()).thenReturn("Non Registry Backed"); + when(nonRegistryBacked.getVersionControlInformation()).thenReturn(null); + when(nonRegistryBacked.getConnectorIdentifier()).thenReturn(Optional.empty()); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(nonRegistryBacked)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final List sources = migrationManager.listMigrationSources(CONNECTOR_ID); + + assertTrue(sources.isEmpty(), "Process groups without VersionControlInformation must not appear in the migration sources listing"); + } + + @Test + public void testEligibilityRejectsNonVersionControlledSource() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup nonVersioned = mock(ProcessGroup.class); + when(nonVersioned.getIdentifier()).thenReturn("non-versioned"); + when(nonVersioned.getVersionControlInformation()).thenReturn(null); + when(nonVersioned.getConnectorIdentifier()).thenReturn(Optional.empty()); + when(rootGroup.findProcessGroup("non-versioned")).thenReturn(nonVersioned); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "non-versioned")); + assertTrue(exception.getMessage().contains("not under version control"), exception.getMessage()); + } + + @Test + public void testValidateMigrationSourceRejectsExternalControllerServices() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + sourceFlow.setExternalControllerServices(Map.of("external-service", new ExternalControllerServiceReference())); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("services outside its managed flow"), exception.getMessage()); + } + + @Test + public void testRejectMigrationWhenTargetActiveFlowAlreadyHasUserComponents() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + // Active flow has an extra processor compared to the (empty) initial flow. + final ProcessGroupFacade rootGroup = mock(ProcessGroupFacade.class); + when(rootGroup.getProcessors()).thenReturn(Set.of(mock(ProcessorFacade.class))); + when(connectorNode.getActiveFlowContext().getRootGroup()).thenReturn(rootGroup); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("user-installed components"), exception.getMessage()); + } + + @Test + public void testRollbackFailureDoesNotMaskOriginalMigrationFailure() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + final Connector connector = connectorNode.getConnector(); + + final FlowUpdateException originalFailure = new FlowUpdateException("original"); + doThrow(originalFailure).when(connector).migrate(any(ConnectorMigrationContext.class)); + doThrow(new FlowUpdateException("rollback failed")).when(connectorNode).loadInitialFlow(); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final FlowUpdateException thrown = assertThrows(FlowUpdateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // The caller must see the original migration failure, not the rollback failure. + assertEquals("original", thrown.getMessage()); + } + + @Test + public void testIsMigrationSupportedFalseDoesNotTriggerRollback() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + when(connectorNode.getConnector().isMigrationSupported(any())).thenReturn(false); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // An eligibility rejection must not exercise the rollback machinery: no flow was attempted. + verify(connectorNode, never()).loadInitialFlow(); + verify(connectorRepository, never()).discardWorkingConfiguration(connectorNode); + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + } + + @Test + public void testEachIneligibleVersionedFlowStateHasDistinctDiagnostic() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + for (final VersionedFlowState state : VersionedFlowState.values()) { + if (state == VersionedFlowState.UP_TO_DATE) { + continue; + } + final ProcessGroup processGroup = mockVersionedGroup("group-" + state, state); + when(rootGroup.findProcessGroup("group-" + state)).thenReturn(processGroup); + } + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + final String staleMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-STALE")).getMessage(); + final String locallyModifiedMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-LOCALLY_MODIFIED")).getMessage(); + final String locallyModifiedAndStaleMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-LOCALLY_MODIFIED_AND_STALE")).getMessage(); + final String syncFailureMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-SYNC_FAILURE")).getMessage(); + + assertEquals(4, Set.of(staleMessage, locallyModifiedMessage, locallyModifiedAndStaleMessage, syncFailureMessage).size(), + "Each non-UP_TO_DATE VersionedFlowState must produce a unique diagnostic message"); + } + + private StandardConnectorMigrationManager newMigrationManager(final FlowController flowController) { + // The snapshot provider is only exercised by the listing path. Tests that drive listing wire their own snapshots. + final RegisteredFlowSnapshot emptySnapshot = new RegisteredFlowSnapshot(); + emptySnapshot.setFlowContents(new VersionedProcessGroup()); + emptySnapshot.setFlow(new RegisteredFlow()); + final ConnectorFlowSnapshotProvider snapshotProvider = (groupId, includeReferencedServices, includeComponentState) -> emptySnapshot; + return new StandardConnectorMigrationManager(flowController, snapshotProvider); + } + + private FlowController createFlowController(final int connectedNodeCount) { + final FlowController flowController = mock(FlowController.class); + when(flowController.getConnectorRepository()).thenReturn(mock(ConnectorRepository.class)); + when(flowController.getFlowManager()).thenReturn(mock(FlowManager.class)); + when(flowController.getExtensionManager()).thenReturn(mock(ExtensionManager.class)); + when(flowController.getAssetManager()).thenReturn(mock(AssetManager.class)); + when(flowController.getConnectorAssetManager()).thenReturn(mock(AssetManager.class)); + when(flowController.getStateManagerProvider()).thenReturn(mock(StateManagerProvider.class)); + when(flowController.getConnectedNodeCount()).thenReturn(connectedNodeCount); + return flowController; + } + + private ConnectorNode wireFreshConnector(final FlowController flowController, final String connectorId) { + final ConnectorNode connectorNode = mock(ConnectorNode.class); + when(connectorNode.getIdentifier()).thenReturn(connectorId); + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.STOPPED); + when(connectorNode.getDesiredState()).thenReturn(ConnectorState.STOPPED); + final FrameworkFlowContext flowContext = mock(FrameworkFlowContext.class); + when(connectorNode.getActiveFlowContext()).thenReturn(flowContext); + + final Connector connector = mock(Connector.class); + when(connector.isMigrationSupported(any())).thenReturn(true); + when(connectorNode.getConnector()).thenReturn(connector); + when(flowController.getConnectorRepository().getConnector(connectorId)).thenReturn(connectorNode); + return connectorNode; + } + + private ProcessGroup wireSourceProcessGroup(final FlowController flowController, final String groupId, final String groupName) { + final FlowManager flowManager = flowController.getFlowManager(); + final ProcessGroup rootGroup = mock(ProcessGroup.class); + final ProcessGroup sourceProcessGroup = mock(ProcessGroup.class); + when(flowManager.getRootGroup()).thenReturn(rootGroup); + when(rootGroup.findProcessGroup(groupId)).thenReturn(sourceProcessGroup); + when(sourceProcessGroup.getName()).thenReturn(groupName); + return sourceProcessGroup; + } + + private ProcessGroup mockVersionedGroup(final String groupId, final VersionedFlowState state) { + final ProcessGroup processGroup = mock(ProcessGroup.class); + when(processGroup.getIdentifier()).thenReturn(groupId); + when(processGroup.getConnectorIdentifier()).thenReturn(Optional.empty()); + final VersionControlInformation versionControlInformation = mock(VersionControlInformation.class); + final VersionedFlowStatus status = mock(VersionedFlowStatus.class); + when(status.getState()).thenReturn(state); + when(versionControlInformation.getStatus()).thenReturn(status); + when(processGroup.getVersionControlInformation()).thenReturn(versionControlInformation); + return processGroup; + } + + private VersionedExternalFlow createSourceFlowWithLocalStateCount(final int localStateCount) { + final VersionedComponentState componentState = new VersionedComponentState(); + componentState.setLocalNodeStates(createNodeStates(localStateCount)); + + final VersionedProcessor processor = new VersionedProcessor(); + processor.setIdentifier("processor-1"); + processor.setName("count-1"); + processor.setComponentState(componentState); + + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setIdentifier("group-1"); + processGroup.setName("Source Flow"); + processGroup.setProcessors(Collections.singleton(processor)); + + final VersionedExternalFlow sourceFlow = new VersionedExternalFlow(); + sourceFlow.setFlowContents(processGroup); + sourceFlow.setExternalControllerServices(Collections.emptyMap()); + sourceFlow.setParameterContexts(Collections.emptyMap()); + sourceFlow.setParameterProviders(Collections.emptyMap()); + return sourceFlow; + } + + private List createNodeStates(final int localStateCount) { + final List nodeStates = new ArrayList<>(); + for (int i = 0; i < localStateCount; i++) { + nodeStates.add(new VersionedNodeState(Map.of("count", Integer.toString(i + 1)))); + } + return nodeStates; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index cba4b12f5741..215197786f4f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -66,6 +66,7 @@ import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -128,6 +129,47 @@ public void testRestoreConnector() { assertEquals(connector, repository.getConnector("connector-1", ConnectorSyncMode.SYNC_WITH_PROVIDER)); } + @Test + public void testRestoreConnectorCleansUpUnreferencedAssets() { + final AssetManager assetManager = mock(AssetManager.class); + final StandardConnectorRepository repository = createRepositoryWithProviderAndAssetManager(null, assetManager); + + final Asset referencedAsset = mock(Asset.class); + when(referencedAsset.getIdentifier()).thenReturn("referenced-asset"); + when(referencedAsset.getName()).thenReturn("referenced.txt"); + + final Asset unreferencedAsset = mock(Asset.class); + when(unreferencedAsset.getIdentifier()).thenReturn("unreferenced-asset"); + when(unreferencedAsset.getName()).thenReturn("unreferenced.txt"); + + final MutableConnectorConfigurationContext activeConfigContext = mock(MutableConnectorConfigurationContext.class); + final ConnectorConfiguration activeConfiguration = new ConnectorConfiguration(Set.of( + new NamedStepConfiguration("step1", new StepConfiguration(Map.of("property", new AssetReference(Set.of("referenced-asset"))))) + )); + when(activeConfigContext.toConnectorConfiguration()).thenReturn(activeConfiguration); + + final FrameworkFlowContext activeFlowContext = mock(FrameworkFlowContext.class); + when(activeFlowContext.getConfigurationContext()).thenReturn(activeConfigContext); + + final MutableConnectorConfigurationContext workingConfigContext = mock(MutableConnectorConfigurationContext.class); + when(workingConfigContext.toConnectorConfiguration()).thenReturn(new ConnectorConfiguration(Set.of())); + + final FrameworkFlowContext workingFlowContext = mock(FrameworkFlowContext.class); + when(workingFlowContext.getConfigurationContext()).thenReturn(workingConfigContext); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + when(connector.getActiveFlowContext()).thenReturn(activeFlowContext); + when(connector.getWorkingFlowContext()).thenReturn(workingFlowContext); + + when(assetManager.getAssets("connector-1")).thenReturn(List.of(referencedAsset, unreferencedAsset)); + + repository.restoreConnector(connector); + + verify(assetManager).deleteAsset("unreferenced-asset"); + verify(assetManager, never()).deleteAsset("referenced-asset"); + } + @Test public void testGetConnectorsReturnsNewListInstances() { final StandardConnectorRepository repository = new StandardConnectorRepository(); @@ -243,7 +285,7 @@ public void testGetConnectorWithProviderOverridesWorkingConfig() throws FlowUpda final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("External Name"); - final VersionedConfigurationStep externalStep = createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("external-value"))); + final VersionedConfigurationStep externalStep = createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("external-value"))); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); when(provider.load("connector-1")).thenReturn(Optional.of(externalConfig)); @@ -378,7 +420,7 @@ public void testConfigureConnectorSavesToProviderBeforeModifyingNode() throws Fl final List savedSteps = savedConfig.getWorkingFlowConfiguration(); assertNotNull(savedSteps); final VersionedConfigurationStep savedStep = savedSteps.stream() - .filter(s -> "step1".equals(s.getName())).findFirst().orElse(null); + .filter(versionedConfigurationStep -> "step1".equals(versionedConfigurationStep.getName())).findFirst().orElse(null); assertNotNull(savedStep); assertEquals("STRING_LITERAL", savedStep.getProperties().get("prop1").getValueType()); assertEquals("new-value", savedStep.getProperties().get("prop1").getValue()); @@ -413,7 +455,7 @@ public void testConfigureConnectorMergesPartialStepConfig() throws FlowUpdateExc repository.addConnector(connector); final VersionedConfigurationStep existingStep = createVersionedStep("step1", - Map.of("propA", createStringLiteralRef("old-A"), "propB", createStringLiteralRef("old-B"))); + Map.of("propA", createStringLiteralReference("old-A"), "propB", createStringLiteralReference("old-B"))); final ConnectorWorkingConfiguration existingConfig = new ConnectorWorkingConfiguration(); existingConfig.setName("Test Connector"); existingConfig.setWorkingFlowConfiguration(new ArrayList<>(List.of(existingStep))); @@ -431,7 +473,7 @@ public void testConfigureConnectorMergesPartialStepConfig() throws FlowUpdateExc final ConnectorWorkingConfiguration savedConfig = configCaptor.getValue(); final VersionedConfigurationStep savedStep = savedConfig.getWorkingFlowConfiguration().stream() - .filter(s -> "step1".equals(s.getName())).findFirst().orElse(null); + .filter(versionedConfigurationStep -> "step1".equals(versionedConfigurationStep.getName())).findFirst().orElse(null); assertNotNull(savedStep); final Map savedProps = savedStep.getProperties(); @@ -821,10 +863,8 @@ public void testSyncAssetsFromProviderCallsSyncAssetsThenReloads() { repository.syncAssetsFromProvider(connector); - // Step 1: provider.syncAssets() called verify(provider).syncAssets("connector-1"); - // Step 2: provider.load() called to reload updated config (may also have been called during addConnector) - verify(provider, org.mockito.Mockito.atLeastOnce()).load("connector-1"); + verify(provider, atLeastOnce()).load("connector-1"); } @Test @@ -884,11 +924,8 @@ public void testCleanUpAssetsCallsProviderDeleteForUnreferencedAssets() { repository.discardWorkingConfiguration(connector); - // Provider.deleteAsset called with NiFi UUID for unreferenced asset verify(provider).deleteAsset(connectorId, unreferencedAssetId); - // Referenced asset is not deleted verify(provider, never()).deleteAsset(connectorId, referencedAssetId); - // AssetManager.deleteAsset NOT called directly since provider handles local deletion verify(assetManager, never()).deleteAsset(anyString()); } @@ -939,10 +976,10 @@ public void testSyncConnectorStoppedWithConfigChange() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -956,9 +993,9 @@ public void testSyncConnectorStoppedNoConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); verify(connector, never()).inheritConfiguration(any(), any(), any()); @@ -972,10 +1009,10 @@ public void testSyncConnectorRunningWithConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("new-value"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -989,9 +1026,9 @@ public void testSyncConnectorDrainingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.DRAINING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1006,9 +1043,9 @@ public void testSyncConnectorPreparingForUpdateIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.PREPARING_FOR_UPDATE); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1022,9 +1059,9 @@ public void testSyncConnectorUpdatingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); } @@ -1037,10 +1074,10 @@ public void testSyncConnectorUpdateFailedRecovery() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATE_FAILED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("recovery-value"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("recovery-value"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -1054,10 +1091,10 @@ public void testSyncConnectorUpdatedState() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); } @@ -1072,10 +1109,10 @@ public void testSyncConnectorInheritConfigurationFailureWhenRunning() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector).stop(any()); @@ -1091,10 +1128,10 @@ public void testSyncConnectorInheritConfigurationFailureWhenStopped() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector, never()).stop(any()); @@ -1110,9 +1147,9 @@ public void testSyncConnectorProviderRejectsSync() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1130,9 +1167,9 @@ public void testSyncConnectorProviderThrowsException() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1149,9 +1186,9 @@ public void testSyncConnectorStartingWaitsForRunning() throws Exception { .thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); } @@ -1164,9 +1201,9 @@ public void testSyncConnectorStartingTimesOut() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STARTING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1188,9 +1225,9 @@ public void testSyncConnectorProviderReturnsRemoveStopsAndRemoves() throws Excep when(connector.stop(any())).thenReturn(CompletableFuture.completedFuture(null)); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); verify(connector).stop(any()); @@ -1209,9 +1246,9 @@ public void testSyncConnectorProviderReturnsRemoveStoppedConnector() throws Exce when(connector.getConnector()).thenReturn(mockExtension); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); verify(connector, never()).stop(any()); @@ -1224,9 +1261,9 @@ public void testSyncConnectorProviderReturnsRemoveForNonExistent() throws Except when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.remove()); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); } @@ -1237,14 +1274,14 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.reject()); final FlowManager flowManager = mock(FlowManager.class); - final StandardConnectorRepository repository = createRepositoryWithFlowManagerAndProvider(provider, flowManager); + final StandardConnectorRepository repository = createRepositoryWithFlowManager(provider, flowManager); final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(flowManager).createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean()); @@ -1262,9 +1299,9 @@ public void testSyncConnectorProviderAllowWithScheduledStateOverride() throws Ex when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); assertEquals(ScheduledState.ENABLED, result.getEffectiveScheduledState()); @@ -1277,7 +1314,7 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { final ConnectorWorkingConfiguration providerConfig = new ConnectorWorkingConfiguration(); providerConfig.setName("Provider Name"); providerConfig.setWorkingFlowConfiguration(List.of( - createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("provider-value"))))); + createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("provider-value"))))); when(provider.getSyncDirective(eq("connector-1"), any())) .thenReturn(ConnectorSyncDirective.allow(providerConfig)); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); @@ -1286,9 +1323,9 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).setName("Provider Name"); @@ -1326,10 +1363,10 @@ public void testFindProviderIdByNameReturnsNullWhenAmbiguousMatch() { // Two parameter providers share the same name; the resolution must be refused so that the // resulting SECRET_REFERENCE is surfaced as invalid in the UI rather than silently bound // to whichever provider happens to come first. - final ParameterProviderNode dup1 = mockParameterProvider("Duplicate", "id-a"); - final ParameterProviderNode dup2 = mockParameterProvider("Duplicate", "id-b"); + final ParameterProviderNode firstDuplicateProvider = mockParameterProvider("Duplicate", "id-a"); + final ParameterProviderNode secondDuplicateProvider = mockParameterProvider("Duplicate", "id-b"); final FlowManager flowManager = mock(FlowManager.class); - when(flowManager.getAllParameterProviders()).thenReturn(Set.of(dup1, dup2)); + when(flowManager.getAllParameterProviders()).thenReturn(Set.of(firstDuplicateProvider, secondDuplicateProvider)); final StandardConnectorRepository repository = createRepositoryWithFlowManager(mock(ConnectorConfigurationProvider.class), flowManager); @@ -1357,10 +1394,10 @@ public void testFindProviderIdByNameReturnsNullWhenNoProvidersRegistered() { assertNull(repository.findProviderIdByName("Anything")); } - private static ParameterProviderNode mockParameterProvider(final String name, final String id) { + private static ParameterProviderNode mockParameterProvider(final String name, final String identifier) { final ParameterProviderNode node = mock(ParameterProviderNode.class); when(node.getName()).thenReturn(name); - when(node.getIdentifier()).thenReturn(id); + when(node.getIdentifier()).thenReturn(identifier); return node; } @@ -1393,21 +1430,16 @@ private StandardConnectorRepository createRepositoryWithFlowManager(final Connec return repository; } - private StandardConnectorRepository createRepositoryWithFlowManagerAndProvider( - final ConnectorConfigurationProvider provider, final FlowManager flowManager) { - return createRepositoryWithFlowManager(provider, flowManager); - } - - private ConnectorNode createSimpleConnectorNode(final String id, final String name) { + private ConnectorNode createSimpleConnectorNode(final String identifier, final String name) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); return connector; } - private ConnectorNode createConnectorNodeWithWorkingConfig(final String id, final String name, final MutableConnectorConfigurationContext workingConfigContext) { + private ConnectorNode createConnectorNodeWithWorkingConfig(final String identifier, final String name, final MutableConnectorConfigurationContext workingConfigContext) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); final FrameworkFlowContext workingFlowContext = mock(FrameworkFlowContext.class); @@ -1417,9 +1449,9 @@ private ConnectorNode createConnectorNodeWithWorkingConfig(final String id, fina return connector; } - private ConnectorNode createConnectorNodeWithEmptyWorkingConfig(final String id, final String name) { + private ConnectorNode createConnectorNodeWithEmptyWorkingConfig(final String identifier, final String name) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); final MutableConnectorConfigurationContext workingConfigContext = mock(MutableConnectorConfigurationContext.class); @@ -1446,28 +1478,28 @@ private VersionedConfigurationStep createVersionedStep(final String name, final return step; } - private VersionedConnectorValueReference createStringLiteralRef(final String value) { + private VersionedConnectorValueReference createStringLiteralReference(final String value) { final VersionedConnectorValueReference ref = new VersionedConnectorValueReference(); ref.setValueType("STRING_LITERAL"); ref.setValue(value); return ref; } - private VersionedConnector createVersionedConnector(final String id, final String name, final ScheduledState scheduledState, + private VersionedConnector createVersionedConnector(final String identifier, final String name, final ScheduledState scheduledState, final List activeConfig) { - final VersionedConnector vc = new VersionedConnector(); - vc.setInstanceIdentifier(id); - vc.setName(name); - vc.setScheduledState(scheduledState); - vc.setActiveFlowConfiguration(activeConfig); - vc.setWorkingFlowConfiguration(activeConfig); + final VersionedConnector versionedConnector = new VersionedConnector(); + versionedConnector.setInstanceIdentifier(identifier); + versionedConnector.setName(name); + versionedConnector.setScheduledState(scheduledState); + versionedConnector.setActiveFlowConfiguration(activeConfig); + versionedConnector.setWorkingFlowConfiguration(activeConfig); final Bundle bundle = new Bundle(); bundle.setGroup("org.apache.nifi"); bundle.setArtifact("nifi-test-connector-nar"); bundle.setVersion("2.0.0"); - vc.setBundle(bundle); - vc.setType("org.apache.nifi.test.TestConnector"); - return vc; + versionedConnector.setBundle(bundle); + versionedConnector.setType("org.apache.nifi.test.TestConnector"); + return versionedConnector; } private StandardConnectorNode createRealConnectorNode(final String identifier, final Connector connector) throws FlowUpdateException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..0595d94f80fb --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestStandardFrameworkConnectorMigrationContext { + + private static final String CONNECTOR_ID = "connector-1"; + private static final String SOURCE_ASSET_ID = "source-asset-1"; + private static final String SOURCE_ASSET_NAME = "driver.jar"; + + private AssetManager sourceAssetManager; + private ConnectorRepository connectorRepository; + private StateManagerProvider stateManagerProvider; + private ClusterTopologyProvider clusterTopologyProvider; + private VersionedExternalFlow sourceFlow; + + @BeforeEach + public void setup() { + sourceAssetManager = mock(AssetManager.class); + connectorRepository = mock(ConnectorRepository.class); + stateManagerProvider = mock(StateManagerProvider.class); + clusterTopologyProvider = mock(ClusterTopologyProvider.class); + sourceFlow = mock(VersionedExternalFlow.class); + } + + @Test + public void testCopyAssetFromSourceReturnsReferenceToCopiedAsset(@TempDir final Path tempDir) throws Exception { + final File sourceFile = createAssetFile(tempDir, "source-contents"); + final Asset sourceAsset = mockAsset(SOURCE_ASSET_ID, SOURCE_ASSET_NAME, sourceFile); + when(sourceAssetManager.getAsset(SOURCE_ASSET_ID)).thenReturn(Optional.of(sourceAsset)); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.empty()); + + final File copiedFile = createAssetFile(tempDir, "copied-contents"); + when(connectorRepository.storeAsset(eq(CONNECTOR_ID), anyString(), eq(SOURCE_ASSET_NAME), any(InputStream.class))) + .thenAnswer(invocation -> mockAsset(invocation.getArgument(1), SOURCE_ASSET_NAME, copiedFile)); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertEquals(1, reference.getAssetIdentifiers().size()); + final String copiedAssetId = reference.getAssetIdentifiers().iterator().next(); + assertEquals(Set.of(copiedAssetId), context.getCopiedAssetIds()); + verify(connectorRepository).storeAsset(eq(CONNECTOR_ID), eq(copiedAssetId), eq(SOURCE_ASSET_NAME), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceReturnsEmptyReferenceWhenSourceAssetIsMissing() throws Exception { + when(sourceAssetManager.getAsset(SOURCE_ASSET_ID)).thenReturn(Optional.empty()); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.empty()); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertTrue(reference.getAssetIdentifiers().isEmpty(), + "Asset reference must be empty when the source asset cannot be located"); + assertTrue(context.getCopiedAssetIds().isEmpty(), + "Copied asset bookkeeping must remain empty when no asset is copied"); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceReusesPreviouslyCopiedAsset(@TempDir final Path tempDir) throws Exception { + final File copiedFile = createAssetFile(tempDir, "previously-copied"); + final Asset existing = mockAsset("already-copied", SOURCE_ASSET_NAME, copiedFile); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.of(existing)); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertEquals(1, reference.getAssetIdentifiers().size()); + verify(sourceAssetManager, never()).getAsset(anyString()); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceRejectsUploadedPayloadMigration() { + final StandardFrameworkConnectorMigrationContext context = createContext(false); + + final IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, + () -> context.copyAssetFromSource(SOURCE_ASSET_ID)); + assertTrue(thrown.getMessage().contains("local Versioned Process Group")); + } + + @Test + public void testCopyAssetFromSourceRejectsBlankSourceAssetIdentifier() { + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource(null)); + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource("")); + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource(" ")); + } + + private StandardFrameworkConnectorMigrationContext createContext(final boolean localMigration) { + return new StandardFrameworkConnectorMigrationContext( + CONNECTOR_ID, + sourceFlow, + localMigration, + null, + sourceAssetManager, + connectorRepository, + stateManagerProvider, + clusterTopologyProvider); + } + + private File createAssetFile(final Path tempDir, final String contents) throws IOException { + final File file = tempDir.resolve("asset-" + contents + ".bin").toFile(); + Files.writeString(file.toPath(), contents); + return file; + } + + private Asset mockAsset(final String identifier, final String name, final File file) { + final Asset asset = mock(Asset.class); + when(asset.getIdentifier()).thenReturn(identifier); + when(asset.getName()).thenReturn(name); + when(asset.getFile()).thenReturn(file); + return asset; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java index 9108e3a3b349..eae8d397509f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java @@ -372,6 +372,29 @@ public void applyConnectorUpdateAdvice(final ProceedingJoinPoint proceedingJoinP } } + @Around("within(org.apache.nifi.web.dao.ConnectorDAO+) && " + + "execution(void migrateFromVersionedFlow(java.lang.String, java.lang.String, org.apache.nifi.flow.VersionedExternalFlow, java.util.function.BooleanSupplier)) && " + + "args(connectorId, processGroupId, sourceFlow, cancellationCheck) && " + + "target(connectorDAO)") + public void migrateConnectorAdvice(final ProceedingJoinPoint proceedingJoinPoint, final String connectorId, final String processGroupId, + final Object sourceFlow, final Object cancellationCheck, final ConnectorDAO connectorDAO) throws Throwable { + final ConnectorNode connector = connectorDAO.getConnector(connectorId); + + proceedingJoinPoint.proceed(); + + if (isAuditable()) { + final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails(); + actionDetails.setName("Connector migrated from Versioned Process Group"); + actionDetails.setPreviousValue(null); + actionDetails.setValue(processGroupId == null ? "Uploaded payload" : processGroupId); + + final Action action = generateAuditRecord(connector, Operation.Configure, actionDetails); + if (action != null) { + saveAction(action, logger); + } + } + } + /** * Generates an audit record for a connector. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 11f33b91bd24..cfb38dfb8b02 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -167,6 +167,7 @@ import org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; import org.apache.nifi.web.api.entity.VersionedReportingTaskImportResponseEntity; import org.apache.nifi.web.api.request.FlowMetricsRegistry; @@ -181,6 +182,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; @@ -263,6 +265,12 @@ Set getConnectorControllerServices(String connectorId, Optional getConnectorAsset(String assetId); + VersionedFlowMigrationSourcesEntity getConnectorMigrationSources(String connectorId); + + void verifyCanMigrateConnector(String connectorId, String processGroupId); + + ConnectorEntity migrateConnector(String connectorId, String processGroupId, RegisteredFlowSnapshot flowSnapshot, BooleanSupplier cancellationCheck); + /** * Verifies that the connector is in a state where FlowFiles can be purged. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c13966b16abb..e64889a05268 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -79,6 +79,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.components.connector.Connector; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; @@ -309,6 +310,7 @@ import org.apache.nifi.web.api.dto.UserGroupDTO; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.dto.VersionedFlowDTO; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; import org.apache.nifi.web.api.dto.action.HistoryDTO; import org.apache.nifi.web.api.dto.action.HistoryQueryDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsDTO; @@ -411,6 +413,7 @@ import org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; import org.apache.nifi.web.api.entity.VersionedReportingTaskImportResponseEntity; import org.apache.nifi.web.api.request.FlowMetricsRegistry; @@ -476,6 +479,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -4028,6 +4032,59 @@ public Optional getConnectorAsset(final String assetId) { return connectorDAO.getAsset(assetId); } + @Override + public VersionedFlowMigrationSourcesEntity getConnectorMigrationSources(final String connectorId) { + final List migrationSources = connectorDAO.getMigrationSources(connectorId).stream() + .map(this::createVersionedFlowMigrationSourceDto) + .toList(); + + final VersionedFlowMigrationSourcesEntity entity = new VersionedFlowMigrationSourcesEntity(); + entity.setMigrationSources(migrationSources); + return entity; + } + + @Override + public void verifyCanMigrateConnector(final String connectorId, final String processGroupId) { + Objects.requireNonNull(processGroupId, "Process Group identifier must be specified to verify a local-source migration"); + connectorDAO.verifyCanMigrateFromVersionedFlow(connectorId, processGroupId); + } + + @Override + public ConnectorEntity migrateConnector(final String connectorId, final String processGroupId, final RegisteredFlowSnapshot flowSnapshot, + final BooleanSupplier cancellationCheck) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Revision revision = revisionManager.getRevision(connectorId); + final RevisionClaim claim = new StandardRevisionClaim(revision); + final VersionedExternalFlow externalFlow = createVersionedExternalFlow(flowSnapshot); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.migrateFromVersionedFlow(connectorId, processGroupId, externalFlow, cancellationCheck); + controllerFacade.save(); + + final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId); + final ConnectorDTO dto = dtoFactory.createConnectorDto(connectorNode); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode connectorNode = connectorDAO.getConnector(snapshot.getComponent().getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(connectorNode); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(connectorNode)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(connectorNode); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + + private VersionedFlowMigrationSourceDTO createVersionedFlowMigrationSourceDto(final ConnectorMigrationSource migrationSource) { + final VersionedFlowMigrationSourceDTO dto = new VersionedFlowMigrationSourceDTO(); + dto.setProcessGroupId(migrationSource.getProcessGroupId()); + dto.setProcessGroupName(migrationSource.getProcessGroupName()); + dto.setRegistryClientId(migrationSource.getRegistryClientId()); + dto.setBucketId(migrationSource.getBucketId()); + dto.setFlowId(migrationSource.getFlowId()); + dto.setVersion(migrationSource.getVersion()); + return dto; + } + @Override public void verifyPurgeConnectorFlowFiles(final String connectorId) { connectorDAO.verifyPurgeFlowFiles(connectorId); @@ -6117,7 +6174,7 @@ public RegisteredFlowSnapshot getCurrentFlowSnapshotByGroupId(final String proce .mapInstanceIdentifiers(false) .mapControllerServiceReferencesToVersionedId(true) .mapFlowRegistryClientId(false) - .mapAssetReferences(false) + .mapAssetReferences(true) .mapComponentState(includeComponentState) .stateManagerProvider(stateManagerProvider) .localNodeOrdinal(clusterTopologyProvider.getLocalNodeOrdinal()) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java index 087a45fdfcf4..cef4be3002aa 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.web.api; +import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -23,6 +24,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import jakarta.servlet.ServletContext; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; @@ -59,6 +62,7 @@ import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; import org.apache.nifi.stream.io.MaxLengthInputStream; import org.apache.nifi.ui.extension.UiExtension; import org.apache.nifi.ui.extension.UiExtensionMapping; @@ -80,6 +84,10 @@ import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; import org.apache.nifi.web.api.dto.DropRequestDTO; +import org.apache.nifi.web.api.dto.MigrationPayloadDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; import org.apache.nifi.web.api.dto.VerifyConnectorConfigStepRequestDTO; import org.apache.nifi.web.api.dto.search.SearchResultsDTO; import org.apache.nifi.web.api.entity.AssetEntity; @@ -93,12 +101,15 @@ import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.SearchResultsEntity; import org.apache.nifi.web.api.entity.SecretsEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; import org.apache.nifi.web.client.api.HttpResponseStatus; @@ -117,6 +128,10 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; /** @@ -128,12 +143,17 @@ public class ConnectorResource extends ApplicationResource { private static final Logger logger = LoggerFactory.getLogger(ConnectorResource.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String VERIFICATION_REQUEST_TYPE = "verification-request"; + private static final String MIGRATION_REQUEST_TYPE = "migration-request"; private static final String PURGE_REQUEST_TYPE = "purge-request"; private static final String FILENAME_HEADER = "Filename"; private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String UPLOAD_CONTENT_TYPE = "application/octet-stream"; private static final long MAX_ASSET_SIZE_BYTES = (long) DataUnit.GB.toB(1); + private static final long MIGRATION_REQUEST_TTL_MILLIS = TimeUnit.MINUTES.toMillis(1L); + private static final long MIGRATION_PAYLOAD_TTL_MILLIS = TimeUnit.MINUTES.toMillis(10L); + private static final long MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS = 30L; private NiFiServiceFacade serviceFacade; private Authorizer authorizer; @@ -143,9 +163,39 @@ public class ConnectorResource extends ApplicationResource { private final RequestManager> configVerificationRequestManager = new AsyncRequestManager<>(100, 1L, "Connector Configuration Step Verification"); + private final RequestManager migrationRequestManager = + new AsyncRequestManager<>(100, MIGRATION_REQUEST_TTL_MILLIS, "Connector Migration"); private final RequestManager purgeRequestManager = new AsyncRequestManager<>(100, 1L, "Connector FlowFile Purge"); + /** + * Uploaded migration payloads keyed by their server-assigned identifier. Each entry's age is tracked alongside + * the payload so that an upload that is never associated with a started migration request is evicted after + * {@link #MIGRATION_PAYLOAD_TTL_MILLIS}. A scheduled sweeper purges expired entries on a fixed cadence so that + * single-upload sessions do not leak payload state indefinitely. + */ + private final Map migrationPayloadsById = new ConcurrentHashMap<>(); + + private final ScheduledExecutorService migrationPayloadEvictionExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> { + final Thread thread = new Thread(runnable, "Connector Migration Payload Eviction"); + thread.setDaemon(true); + return thread; + }); + + private record MigrationPayloadEntry(RegisteredFlowSnapshot snapshot, long uploadedAtMillis) { + } + + @PostConstruct + public void startMigrationPayloadEviction() { + migrationPayloadEvictionExecutor.scheduleWithFixedDelay(this::evictExpiredMigrationPayloads, + MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS, MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + + @PreDestroy + public void stopMigrationPayloadEviction() { + migrationPayloadEvictionExecutor.shutdownNow(); + } + @Context private ServletContext servletContext; @@ -2010,6 +2060,342 @@ public Response getConnectorStatus( return generateOkResponse(entity).build(); } + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/migration-sources") + @Operation( + summary = "Lists the Versioned Process Groups that the Connector can be migrated from", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = VersionedFlowMigrationSourcesEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Read - /connectors/{uuid}") + } + ) + public Response getMigrationSources(@PathParam("id") final String connectorId) { + authorizeReadConnector(connectorId); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final VersionedFlowMigrationSourcesEntity entity = serviceFacade.getConnectorMigrationSources(connectorId); + return generateOkResponse(entity).build(); + } + + @POST + @Consumes(MediaType.APPLICATION_OCTET_STREAM) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/migration-payloads") + @Operation( + summary = "Uploads a flow snapshot payload for a later Connector migration request", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationPayloadEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid}") + } + ) + public Response createMigrationPayload( + @PathParam("id") final String connectorId, + @Parameter(description = "The migration payload snapshot", required = true) final InputStream payloadContents) throws IOException { + if (payloadContents == null) { + throw new IllegalArgumentException("Migration payload contents must be specified."); + } + + final NiFiUser currentUser = NiFiUserUtils.getNiFiUser(); + serviceFacade.authorizeAccess(lookup -> { + final Authorizable connector = lookup.getConnector(connectorId); + connector.authorize(authorizer, RequestAction.WRITE, currentUser); + }); + + if (isReplicateRequest()) { + final String uploadRequestId = UUID.randomUUID().toString(); + final UploadRequest uploadRequest = new UploadRequest.Builder() + .user(currentUser) + .filename("migration-payload.json") + .identifier(uploadRequestId) + .contents(payloadContents) + .forwardRequestHeaders(getHeaders()) + .header(CONTENT_TYPE_HEADER, UPLOAD_CONTENT_TYPE) + .header(RequestReplicationHeader.CLUSTER_ID_GENERATION_SEED.getHeader(), uploadRequestId) + .exampleRequestUri(getAbsolutePath()) + .responseClass(MigrationPayloadEntity.class) + .successfulResponseStatus(HttpResponseStatus.OK.getCode()) + .build(); + final MigrationPayloadEntity entity = uploadRequestReplicator.upload(uploadRequest); + return generateOkResponse(entity).build(); + } + + final RegisteredFlowSnapshot flowSnapshot; + try { + flowSnapshot = OBJECT_MAPPER.readValue(payloadContents, RegisteredFlowSnapshot.class); + } catch (final IOException e) { + throw new IllegalArgumentException("Deserialization of uploaded migration payload failed", e); + } + + // Bundle discovery is deferred to the asynchronous migration task in performAsyncMigration so that + // the upload thread does not block on extension-manager work. + final String payloadId = getIdGenerationSeed().orElseGet(this::generateUuid); + migrationPayloadsById.put(payloadId, new MigrationPayloadEntry(flowSnapshot, System.currentTimeMillis())); + final MigrationPayloadDTO migrationPayload = new MigrationPayloadDTO(); + migrationPayload.setPayloadId(payloadId); + + final MigrationPayloadEntity entity = new MigrationPayloadEntity(); + entity.setPayload(migrationPayload); + return generateOkResponse(entity).build(); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests") + @Operation( + summary = "Creates a Connector migration request", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid}") + } + ) + public Response createMigrationRequest(@PathParam("id") final String connectorId, final MigrationRequestEntity requestEntity) { + if (requestEntity == null || requestEntity.getRequest() == null) { + throw new IllegalArgumentException("Migration request must be specified."); + } + + final MigrationRequestDTO request = requestEntity.getRequest(); + if (!connectorId.equals(request.getConnectorId())) { + throw new IllegalArgumentException("Connector identifier in the request must match the identifier provided in the URL."); + } + + final boolean hasLocalSource = request.getLocalSource() != null && StringUtils.isNotBlank(request.getLocalSource().getProcessGroupId()); + final boolean hasPayload = StringUtils.isNotBlank(request.getPayloadId()); + if (hasLocalSource == hasPayload) { + throw new IllegalArgumentException("Migration request must specify exactly one source: either a local Process Group or an uploaded payload identifier."); + } + + if (hasPayload && !migrationPayloadsById.containsKey(request.getPayloadId())) { + throw new ResourceNotFoundException("No uploaded migration payload exists with identifier " + request.getPayloadId()); + } + + // Reject the request when not all cluster nodes are connected, mirroring the asset-upload guard, + // because component state and assets cannot be synchronized to disconnected nodes after migration. + final ClusterCoordinator clusterCoordinator = getClusterCoordinator(); + if (clusterCoordinator != null) { + final Set disconnectedNodes = clusterCoordinator.getNodeIdentifiers(NodeConnectionState.CONNECTING, NodeConnectionState.DISCONNECTED, NodeConnectionState.DISCONNECTING); + if (!disconnectedNodes.isEmpty()) { + throw new IllegalStateException("Cannot start a Connector migration because the following %s nodes are not currently connected: %s" + .formatted(disconnectedNodes.size(), disconnectedNodes)); + } + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, requestEntity); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + return withWriteLock( + serviceFacade, + requestEntity, + lookup -> { + final Authorizable connector = lookup.getConnector(connectorId); + connector.authorize(authorizer, RequestAction.WRITE, user); + }, + () -> { + if (hasLocalSource) { + serviceFacade.verifyCanMigrateConnector(connectorId, request.getLocalSource().getProcessGroupId()); + } + }, + entity -> performAsyncMigration(entity, user) + ); + } + + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests/{requestId}") + @Operation( + summary = "Gets the Connector migration request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response getMigrationRequest(@PathParam("id") final String connectorId, @PathParam("requestId") final String requestId) { + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final AsynchronousWebRequest asyncRequest = migrationRequestManager.getRequest(MIGRATION_REQUEST_TYPE, requestId, user); + return generateOkResponse(createMigrationRequestEntity(asyncRequest, connectorId, requestId)).build(); + } + + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests/{requestId}") + @Operation( + summary = "Deletes the Connector migration request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response deleteMigrationRequest(@PathParam("id") final String connectorId, @PathParam("requestId") final String requestId) { + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + if (!twoPhaseRequest || executionPhase) { + final AsynchronousWebRequest asyncRequest = + migrationRequestManager.removeRequest(MIGRATION_REQUEST_TYPE, requestId, user); + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + final MigrationRequestDTO request = asyncRequest.getRequest().getRequest(); + if (StringUtils.isNotBlank(request.getPayloadId())) { + migrationPayloadsById.remove(request.getPayloadId()); + } + + return generateOkResponse(createMigrationRequestEntity(asyncRequest, connectorId, requestId)).build(); + } + + if (isValidationPhase(httpServletRequest)) { + migrationRequestManager.getRequest(MIGRATION_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + private Response performAsyncMigration(final MigrationRequestEntity requestEntity, final NiFiUser user) { + final String requestId = generateUuid(); + final MigrationRequestDTO migrationRequest = requestEntity.getRequest(); + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Migrate Versioned Flow")); + + final AsynchronousWebRequest request = + new StandardAsynchronousWebRequest<>(requestId, requestEntity, migrationRequest.getConnectorId(), user, updateSteps); + + final Consumer> updateTask = asyncRequest -> { + final String payloadId = migrationRequest.getPayloadId(); + try { + final RegisteredFlowSnapshot flowSnapshot = getMigrationSnapshot(migrationRequest); + final String processGroupId = migrationRequest.getLocalSource() == null ? null : migrationRequest.getLocalSource().getProcessGroupId(); + final ConnectorEntity migratedConnector = serviceFacade.migrateConnector(migrationRequest.getConnectorId(), processGroupId, flowSnapshot, asyncRequest::isCancelled); + asyncRequest.markStepComplete(migratedConnector); + } catch (final Exception e) { + logger.error("Failed to migrate Connector {}", migrationRequest.getConnectorId(), e); + asyncRequest.fail("Failed to migrate Connector due to " + e); + } finally { + if (StringUtils.isNotBlank(payloadId)) { + migrationPayloadsById.remove(payloadId); + } + } + }; + + request.setCancelCallback(() -> { + final String payloadId = migrationRequest.getPayloadId(); + if (StringUtils.isNotBlank(payloadId)) { + migrationPayloadsById.remove(payloadId); + } + }); + + migrationRequestManager.submitRequest(MIGRATION_REQUEST_TYPE, requestId, request, updateTask); + final MigrationRequestEntity resultsEntity = createMigrationRequestEntity(request, migrationRequest.getConnectorId(), requestId); + return generateOkResponse(resultsEntity).build(); + } + + private RegisteredFlowSnapshot getMigrationSnapshot(final MigrationRequestDTO migrationRequest) { + final MigrationRequestLocalSourceDTO localSource = migrationRequest.getLocalSource(); + if (localSource != null && StringUtils.isNotBlank(localSource.getProcessGroupId())) { + // The local-source path uses the framework's snapshot facility, which already returns a snapshot + // whose bundles match this NiFi instance. discoverCompatibleBundles is intentionally not invoked + // here because it is reserved for snapshots that originate from external sources (uploaded payloads). + return serviceFacade.getCurrentFlowSnapshotByGroupId(localSource.getProcessGroupId(), true, true); + } + + final String payloadId = migrationRequest.getPayloadId(); + final MigrationPayloadEntry entry = migrationPayloadsById.get(payloadId); + if (entry == null) { + throw new ResourceNotFoundException("No uploaded migration payload exists with identifier " + payloadId); + } + + // Bundle discovery is performed once when the migration task picks up the uploaded payload, not on + // the upload thread, so that a slow extension manager does not block the upload-replication path. + final RegisteredFlowSnapshot flowSnapshot = entry.snapshot(); + serviceFacade.discoverCompatibleBundles(flowSnapshot.getFlowContents()); + serviceFacade.discoverCompatibleBundles(flowSnapshot.getParameterProviders()); + return flowSnapshot; + } + + private void evictExpiredMigrationPayloads() { + final long cutoffMillis = System.currentTimeMillis() - MIGRATION_PAYLOAD_TTL_MILLIS; + migrationPayloadsById.entrySet().removeIf(entry -> entry.getValue().uploadedAtMillis() < cutoffMillis); + } + + private MigrationRequestEntity createMigrationRequestEntity( + final AsynchronousWebRequest asyncRequest, + final String connectorId, + final String requestId) { + final MigrationRequestDTO requestedMigration = asyncRequest.getRequest().getRequest(); + + final MigrationRequestDTO migrationRequest = new MigrationRequestDTO(); + migrationRequest.setConnectorId(requestedMigration.getConnectorId()); + migrationRequest.setLocalSource(requestedMigration.getLocalSource()); + migrationRequest.setPayloadId(requestedMigration.getPayloadId()); + migrationRequest.setComplete(asyncRequest.isComplete()); + migrationRequest.setFailureReason(asyncRequest.getFailureReason()); + migrationRequest.setLastUpdated(asyncRequest.getLastUpdated()); + migrationRequest.setPercentCompleted(asyncRequest.getPercentComplete()); + migrationRequest.setRequestId(requestId); + migrationRequest.setState(asyncRequest.getState()); + migrationRequest.setUri(generateResourceUri("connectors", connectorId, "migration-requests", requestId)); + migrationRequest.setUpdateSteps(asyncRequest.getUpdateSteps().stream().map(updateStep -> { + final MigrationUpdateStepDTO migrationUpdateStep = new MigrationUpdateStepDTO(); + migrationUpdateStep.setDescription(updateStep.getDescription()); + migrationUpdateStep.setComplete(updateStep.isComplete()); + migrationUpdateStep.setFailureReason(updateStep.getFailureReason()); + return migrationUpdateStep; + }).toList()); + + final MigrationRequestEntity entity = new MigrationRequestEntity(); + entity.setRequest(migrationRequest); + return entity; + } + @POST @Consumes(MediaType.APPLICATION_OCTET_STREAM) @Produces(MediaType.APPLICATION_JSON) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java index acf2e7f53900..d0cbc2ef61fc 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java @@ -20,9 +20,11 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.ConnectorUpdateContext; +import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; @@ -30,6 +32,7 @@ import java.io.InputStream; import java.util.List; import java.util.Optional; +import java.util.function.BooleanSupplier; public interface ConnectorDAO { @@ -83,5 +86,11 @@ public interface ConnectorDAO { Optional getAsset(String assetId); + List getMigrationSources(String id); + + void verifyCanMigrateFromVersionedFlow(String connectorId, String processGroupId); + + void migrateFromVersionedFlow(String connectorId, String processGroupId, VersionedExternalFlow sourceFlow, BooleanSupplier cancellationCheck); + } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index 9a507e16e5ce..3b2e00520298 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -16,11 +16,15 @@ */ package org.apache.nifi.web.dao.impl; +import jakarta.annotation.PostConstruct; import org.apache.nifi.asset.Asset; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConnectorFlowSnapshotProvider; +import org.apache.nifi.components.connector.ConnectorMigrationManager; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorRepository; import org.apache.nifi.components.connector.ConnectorSyncMode; @@ -28,11 +32,14 @@ import org.apache.nifi.components.connector.ConnectorValueReference; import org.apache.nifi.components.connector.ConnectorValueType; import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StandardConnectorMigrationManager; import org.apache.nifi.components.connector.StepConfiguration; import org.apache.nifi.components.connector.StringLiteralValue; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.web.NiFiCoreException; +import org.apache.nifi.web.NiFiServiceFacade; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; @@ -43,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Repository; import java.io.IOException; @@ -54,6 +62,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.function.BooleanSupplier; import java.util.stream.Collectors; @Repository @@ -62,12 +71,31 @@ public class StandardConnectorDAO implements ConnectorDAO { private static final Logger logger = LoggerFactory.getLogger(StandardConnectorDAO.class); private FlowController flowController; + private NiFiServiceFacade serviceFacade; + private ConnectorMigrationManager connectorMigrationManager; @Autowired public void setFlowController(final FlowController flowController) { this.flowController = flowController; } + /** + * Injected with {@link Lazy @Lazy} to break the circular dependency between this DAO and + * {@link NiFiServiceFacade}. The service facade is used solely as a {@link ConnectorFlowSnapshotProvider} + * for the migration manager constructed in {@link #initialize()}; the proxy is invoked lazily at + * migration time, so the real service-facade bean is not required at DAO construction. + */ + @Autowired + public void setServiceFacade(@Lazy final NiFiServiceFacade serviceFacade) { + this.serviceFacade = serviceFacade; + } + + @PostConstruct + public void initialize() { + final ConnectorFlowSnapshotProvider snapshotProvider = serviceFacade::getCurrentFlowSnapshotByGroupId; + this.connectorMigrationManager = new StandardConnectorMigrationManager(flowController, snapshotProvider); + } + private FlowManager getFlowManager() { return flowController.getFlowManager(); } @@ -294,6 +322,29 @@ public List getAssets(final String id) { public Optional getAsset(final String assetId) { return getConnectorRepository().getAsset(assetId); } + + @Override + public List getMigrationSources(final String id) { + getConnector(id); + return connectorMigrationManager.listMigrationSources(id); + } + + @Override + public void verifyCanMigrateFromVersionedFlow(final String connectorId, final String processGroupId) { + getConnector(connectorId); + connectorMigrationManager.verifyEligibility(connectorId, processGroupId); + } + + @Override + public void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow, + final BooleanSupplier cancellationCheck) { + getConnector(connectorId); + try { + connectorMigrationManager.migrateFromVersionedFlow(connectorId, processGroupId, sourceFlow, cancellationCheck); + } catch (final Exception e) { + throw new NiFiCoreException("Failed to migrate Connector from Versioned Flow: " + e.getMessage(), e); + } + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java index 2fcac1d5dcdc..3fdf8e8f0bc3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java @@ -32,6 +32,8 @@ import org.apache.nifi.web.api.dto.AllowableValueDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterDTO; import org.apache.nifi.web.api.dto.RevisionDTO; @@ -43,10 +45,12 @@ import org.apache.nifi.web.api.entity.ConnectorRunStatusEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.SecretsEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; import org.junit.jupiter.api.AfterEach; @@ -162,6 +166,56 @@ public void testGetConnector() { verify(serviceFacade).getConnector(CONNECTOR_ID, false); } + @Test + public void testGetMigrationSources() { + final VersionedFlowMigrationSourcesEntity migrationSourcesEntity = new VersionedFlowMigrationSourcesEntity(); + when(serviceFacade.getConnectorMigrationSources(CONNECTOR_ID)).thenReturn(migrationSourcesEntity); + + try (final Response response = connectorResource.getMigrationSources(CONNECTOR_ID)) { + assertEquals(200, response.getStatus()); + assertEquals(migrationSourcesEntity, response.getEntity()); + } + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).getConnectorMigrationSources(CONNECTOR_ID); + } + + @Test + public void testCreateMigrationRequestRejectsMismatchedConnectorId() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId("different-connector"); + requestDto.setLocalSource(createLocalMigrationSource(PROCESS_GROUP_ID)); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + + @Test + public void testCreateMigrationRequestRequiresExactlyOneSource() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(CONNECTOR_ID); + requestDto.setLocalSource(createLocalMigrationSource(PROCESS_GROUP_ID)); + requestDto.setPayloadId("payload-1"); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + + @Test + public void testCreateMigrationRequestRequiresLocalSourceOrPayload() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(CONNECTOR_ID); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + @Test public void testGetConnectorClusterNodeRequestBypassesAuth() { final ConnectorResource spyResource = spy(connectorResource); @@ -878,4 +932,10 @@ public void testClearConnectorControllerServiceStateNotAuthorized() { verify(serviceFacade, never()).verifyCanClearConnectorControllerServiceState(anyString(), anyString()); verify(serviceFacade, never()).clearConnectorControllerServiceState(anyString(), anyString(), any()); } + + private MigrationRequestLocalSourceDTO createLocalMigrationSource(final String processGroupId) { + final MigrationRequestLocalSourceDTO localSource = new MigrationRequestLocalSourceDTO(); + localSource.setProcessGroupId(processGroupId); + return localSource; + } } diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java new file mode 100644 index 000000000000..e1b284d47199 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +/** + * Test connector whose {@code migrate(...)} succeeds on every cluster node except the one whose + * {@code -DnodeNumber=} JVM argument matches {@value #FAILING_NODE_NUMBER}. Used to validate that the cluster + * migration-request endpoint merger surfaces a per-node failure even when other nodes report success. + */ +public class AsymmetricFailureMigrationConnector extends AbstractConnector { + private static final String FAILING_NODE_NUMBER = "2"; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Asymmetric Migration Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + return true; + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final String currentNodeNumber = System.getProperty("nodeNumber"); + if (FAILING_NODE_NUMBER.equals(currentNodeNumber)) { + throw new FlowUpdateException("Simulated migration failure on node " + currentNodeNumber); + } + + getInitializationContext().updateFlow(context.getActiveFlowContext(), context.getSourceFlow()); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java new file mode 100644 index 000000000000..d0fbeb53f42c --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.flow.VersionedAsset; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class FailingMigrationConnector extends AbstractConnector { + private static final long FAILURE_DELAY_SECONDS = 5L; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Failing Migration Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + return true; + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + copyFirstAsset(context); + getInitializationContext().updateFlow(context.getActiveFlowContext(), context.getSourceFlow()); + try { + TimeUnit.SECONDS.sleep(FAILURE_DELAY_SECONDS); + } catch (final InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new FlowUpdateException("Interrupted while waiting to fail the migration", interruptedException); + } + + throw new FlowUpdateException("Intended migration failure for rollback testing"); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } + + private void copyFirstAsset(final ConnectorMigrationContext context) { + if (!context.isLocalMigration()) { + return; + } + + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow.getParameterContexts() == null) { + return; + } + + for (final VersionedParameterContext parameterContext : sourceFlow.getParameterContexts().values()) { + if (parameterContext.getParameters() == null) { + continue; + } + + for (final VersionedParameter parameter : parameterContext.getParameters()) { + if (parameter.getReferencedAssets() == null) { + continue; + } + + for (final VersionedAsset referencedAsset : parameter.getReferencedAssets()) { + context.copyAssetFromSource(referencedAsset.getIdentifier()); + return; + } + } + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java new file mode 100644 index 000000000000..abffdc8a372f --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.flow.VersionedAsset; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MigrationTargetConnector extends AbstractConnector { + private static final String REQUIRED_PARAMETER_NAME = "Source Topic"; + private static final String REQUIRED_PROCESSOR_TYPE = "org.apache.nifi.processors.tests.system.StatefulCountProcessor"; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Migration Target Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow == null || sourceFlow.getFlowContents() == null) { + return false; + } + + return containsRequiredParameter(sourceFlow.getParameterContexts()) && containsRequiredProcessor(sourceFlow.getFlowContents()); + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow == null) { + throw new FlowUpdateException("A source flow is required for migration"); + } + + rewriteReferencedAssets(sourceFlow, context); + getInitializationContext().updateFlow(context.getActiveFlowContext(), sourceFlow); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } + + private boolean containsRequiredParameter(final Map parameterContexts) { + if (parameterContexts == null || parameterContexts.isEmpty()) { + return false; + } + + for (final VersionedParameterContext parameterContext : parameterContexts.values()) { + final Set parameters = parameterContext.getParameters(); + if (parameters == null) { + continue; + } + + for (final VersionedParameter parameter : parameters) { + if (REQUIRED_PARAMETER_NAME.equals(parameter.getName())) { + return true; + } + } + } + + return false; + } + + private boolean containsRequiredProcessor(final VersionedProcessGroup processGroup) { + final Set processors = processGroup.getProcessors(); + if (processors != null) { + for (final VersionedProcessor processor : processors) { + if (REQUIRED_PROCESSOR_TYPE.equals(processor.getType())) { + return true; + } + } + } + + final Set childGroups = processGroup.getProcessGroups(); + if (childGroups == null) { + return false; + } + + for (final VersionedProcessGroup childGroup : childGroups) { + if (containsRequiredProcessor(childGroup)) { + return true; + } + } + + return false; + } + + private void rewriteReferencedAssets(final VersionedExternalFlow sourceFlow, final ConnectorMigrationContext context) { + final Map parameterContexts = sourceFlow.getParameterContexts(); + if (parameterContexts == null || parameterContexts.isEmpty()) { + return; + } + + for (final VersionedParameterContext parameterContext : parameterContexts.values()) { + final Set parameters = parameterContext.getParameters(); + if (parameters == null) { + continue; + } + + for (final VersionedParameter parameter : parameters) { + final List referencedAssets = parameter.getReferencedAssets(); + if (referencedAssets == null || referencedAssets.isEmpty()) { + continue; + } + + final List migratedAssets = new ArrayList<>(); + for (final VersionedAsset referencedAsset : referencedAssets) { + final AssetReference migratedReference = context.copyAssetFromSource(referencedAsset.getIdentifier()); + for (final String migratedAssetId : migratedReference.getAssetIdentifiers()) { + final VersionedAsset migratedAsset = new VersionedAsset(); + migratedAsset.setIdentifier(migratedAssetId); + migratedAsset.setName(referencedAsset.getName()); + migratedAssets.add(migratedAsset); + } + } + + parameter.setReferencedAssets(migratedAssets); + } + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java new file mode 100644 index 000000000000..6f7d80a4687e --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class NonMigratingConnector extends AbstractConnector { + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Non Migrating Flow"); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java new file mode 100644 index 000000000000..f16ed42dab19 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.tests.system; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.stream.io.StreamUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.Set; + +@Tags({"asset", "test"}) +@CapabilityDescription("Reads the contents of a file-backed asset parameter and writes the asset contents to a configured output file.") +public class AssetReadingProcessor extends AbstractProcessor { + static final PropertyDescriptor SOURCE_FILE = new PropertyDescriptor.Builder() + .name("Source File") + .description("The path of the asset file to read") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final PropertyDescriptor OUTPUT_FILE = new PropertyDescriptor.Builder() + .name("Output File") + .description("The file where the asset contents will be written") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .build(); + + static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .build(); + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of(SOURCE_FILE, OUTPUT_FILE); + } + + @Override + public Set getRelationships() { + return Set.of(REL_SUCCESS, REL_FAILURE); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + final FlowFile incomingFlowFile = session.get(); + final FlowFile flowFile = incomingFlowFile == null ? session.create() : incomingFlowFile; + + final File sourceFile = new File(context.getProperty(SOURCE_FILE).getValue()); + final File outputFile = new File(context.getProperty(OUTPUT_FILE).getValue()); + final File parentFile = outputFile.getParentFile(); + if (parentFile != null && !parentFile.exists() && !parentFile.mkdirs()) { + getLogger().error("Could not create directory {}", parentFile.getAbsolutePath()); + session.transfer(flowFile, REL_FAILURE); + return; + } + + try (final InputStream inputStream = new FileInputStream(sourceFile); + final OutputStream outputStream = new FileOutputStream(outputFile)) { + StreamUtils.copy(inputStream, outputStream); + session.transfer(flowFile, REL_SUCCESS); + } catch (final Exception e) { + getLogger().error("Failed to read asset file {} to {}", sourceFile.getAbsolutePath(), outputFile.getAbsolutePath(), e); + session.transfer(flowFile, REL_FAILURE); + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector index a2d46427433a..aef142042c3e 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -14,12 +14,16 @@ # limitations under the License. org.apache.nifi.connectors.tests.system.AssetConnector +org.apache.nifi.connectors.tests.system.AsymmetricFailureMigrationConnector org.apache.nifi.connectors.tests.system.BundleResolutionConnector org.apache.nifi.connectors.tests.system.CalculateConnector org.apache.nifi.connectors.tests.system.ComponentLifecycleConnector org.apache.nifi.connectors.tests.system.DataQueuingConnector org.apache.nifi.connectors.tests.system.GatedDataQueuingConnector +org.apache.nifi.connectors.tests.system.FailingMigrationConnector +org.apache.nifi.connectors.tests.system.MigrationTargetConnector org.apache.nifi.connectors.tests.system.NestedProcessGroupConnector +org.apache.nifi.connectors.tests.system.NonMigratingConnector org.apache.nifi.connectors.tests.system.NopConnector org.apache.nifi.connectors.tests.system.ParameterContextConnector org.apache.nifi.connectors.tests.system.SelectiveDropConnector diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index 80655da6f369..5c1de2753838 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +org.apache.nifi.processors.tests.system.AssetReadingProcessor org.apache.nifi.processors.tests.system.ClassloaderIsolationWithServiceProperty org.apache.nifi.processors.tests.system.CountEvents org.apache.nifi.processors.tests.system.CountFlowFiles diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 8960e202ce3e..f95a8e9be5c2 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -16,14 +16,19 @@ */ package org.apache.nifi.tests.system; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.nifi.cluster.coordination.node.NodeConnectionState; import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.controller.AbstractPort; import org.apache.nifi.controller.queue.LoadBalanceCompression; import org.apache.nifi.controller.queue.LoadBalanceStrategy; import org.apache.nifi.controller.queue.QueueSize; +import org.apache.nifi.flow.VersionedNodeState; +import org.apache.nifi.flow.VersionedProcessor; import org.apache.nifi.parameter.ParameterProviderConfiguration; import org.apache.nifi.provenance.search.SearchableField; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; import org.apache.nifi.scheduling.ExecutionNode; import org.apache.nifi.stream.io.StreamUtils; @@ -49,6 +54,8 @@ import org.apache.nifi.web.api.dto.FlowFileSummaryDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; import org.apache.nifi.web.api.dto.NodeDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; @@ -95,6 +102,8 @@ import org.apache.nifi.web.api.entity.FlowFileEntity; import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; import org.apache.nifi.web.api.entity.ListingRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.NodeEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; @@ -125,12 +134,14 @@ import org.apache.nifi.web.api.entity.VerifyConfigRequestEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity; import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -335,6 +346,124 @@ public ConnectorValueReferenceDTO createSecretValueReference(final String secret return valueRef; } + public VersionedFlowMigrationSourcesEntity listMigrationSources(final String connectorId) throws NiFiClientException, IOException { + return getConnectorClient().listMigrationSources(connectorId); + } + + public MigrationPayloadEntity uploadMigrationPayload(final String connectorId, final File file) throws NiFiClientException, IOException { + return getConnectorClient().uploadMigrationPayload(connectorId, file); + } + + public MigrationRequestEntity startMigrationFromLocalSource(final String connectorId, final String processGroupId) throws NiFiClientException, IOException { + final MigrationRequestLocalSourceDTO localSource = new MigrationRequestLocalSourceDTO(); + localSource.setProcessGroupId(processGroupId); + + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(connectorId); + requestDto.setLocalSource(localSource); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + return getConnectorClient().startMigration(requestEntity); + } + + public MigrationRequestEntity startMigrationFromPayload(final String connectorId, final String payloadId) throws NiFiClientException, IOException { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(connectorId); + requestDto.setPayloadId(payloadId); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + return getConnectorClient().startMigration(requestEntity); + } + + private static final long MIGRATION_WAIT_TIMEOUT_MILLIS = 60_000L; + + public MigrationRequestEntity waitForMigrationToComplete(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final long deadlineMillis = System.currentTimeMillis() + MIGRATION_WAIT_TIMEOUT_MILLIS; + int iteration = 0; + while (true) { + final MigrationRequestEntity requestEntity = getConnectorClient().getMigrationStatus(connectorId, requestId); + final MigrationRequestDTO request = requestEntity.getRequest(); + if (request.isComplete()) { + return requestEntity; + } + + if (System.currentTimeMillis() >= deadlineMillis) { + throw new IllegalStateException("Migration request " + requestId + " for Connector " + connectorId + + " did not complete within " + MIGRATION_WAIT_TIMEOUT_MILLIS + "ms; last state " + request.getState() + + " (" + request.getPercentCompleted() + "% complete)"); + } + + if (iteration++ % 30 == 0) { + logger.info("Migration request {} for Connector {} is in state {} ({}% complete)", + requestId, connectorId, request.getState(), request.getPercentCompleted()); + } + + Thread.sleep(100L); + } + } + + public MigrationRequestEntity waitForMigrationSuccess(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final MigrationRequestEntity completedRequest = waitForMigrationToComplete(connectorId, requestId); + final String failureReason = completedRequest.getRequest().getFailureReason(); + if (failureReason != null) { + throw new IllegalStateException("Migration failed: " + failureReason); + } + + return completedRequest; + } + + public MigrationRequestEntity waitForMigrationFailure(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final MigrationRequestEntity completedRequest = waitForMigrationToComplete(connectorId, requestId); + if (completedRequest.getRequest().getFailureReason() == null) { + throw new IllegalStateException("Expected migration failure but request completed successfully"); + } + + return completedRequest; + } + + public MigrationRequestEntity cancelMigration(final String connectorId, final String requestId) throws NiFiClientException, IOException { + return getConnectorClient().cancelMigration(connectorId, requestId); + } + + private static final ObjectMapper MIGRATION_PAYLOAD_OBJECT_MAPPER = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + /** + * Reads an exported flow snapshot, expands the LOCAL component state of the processor whose type ends + * with the given suffix to the requested number of node states, and writes the modified snapshot back + * to the same file. Used by clustered migration tests that need to construct a payload whose + * cluster-topology rule violation is unambiguous. + * + * @param exportFile the file containing a previously-exported {@link RegisteredFlowSnapshot} + * @param processorTypeSuffix a suffix that uniquely identifies the stateful processor inside the snapshot + * @param nodeStateValues the count values to assign to each synthetic node state, one per source node + * @throws IOException when the file cannot be read or written + * @throws IllegalStateException when no processor matches the given suffix or has no exported state + */ + public void rewriteMigrationPayloadWithNodeStates(final File exportFile, final String processorTypeSuffix, final List nodeStateValues) throws IOException { + final RegisteredFlowSnapshot flowSnapshot = MIGRATION_PAYLOAD_OBJECT_MAPPER.readValue(exportFile, RegisteredFlowSnapshot.class); + final VersionedProcessor statefulProcessor = flowSnapshot.getFlowContents().getProcessors().stream() + .filter(processor -> processor.getType().endsWith(processorTypeSuffix)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No processor in snapshot ends with type " + processorTypeSuffix)); + if (statefulProcessor.getComponentState() == null) { + throw new IllegalStateException("Processor " + statefulProcessor.getName() + " has no exported component state"); + } + + final List nodeStates = new ArrayList<>(); + for (final String value : nodeStateValues) { + nodeStates.add(new VersionedNodeState(Map.of("count", value))); + } + statefulProcessor.getComponentState().setLocalNodeStates(nodeStates); + + if (!exportFile.delete() && exportFile.exists()) { + throw new IOException("Could not delete prior export file " + exportFile.getAbsolutePath() + " before writing the modified payload"); + } + MIGRATION_PAYLOAD_OBJECT_MAPPER.writeValue(exportFile, flowSnapshot); + } + public ConfigurationStepEntity configureConnectorWithReferences(final String connectorId, final String configurationStepName, final Map propertyValues) throws NiFiClientException, IOException { final ConnectorEntity connectorEntity = getConnectorClient().getConnector(connectorId); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java new file mode 100644 index 000000000000..e11c241a7516 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ClusteredConnectorVersionedFlowMigrationFailureIT extends ConnectorVersionedFlowMigrationFailureIT { + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } + + @Test + public void testUploadedPayloadWithTooManyLocalNodeStatesFails() throws Exception { + final File outputFile = new File("target/migration/cluster-topology-output.txt"); + final File exportFile = new File("target/migration/cluster-topology-export.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("ClusterTopologySource", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + + // Inflate the LOCAL state to three node states even though the destination cluster only has two + // connected nodes; the cluster-topology rule must reject the payload. + getClientUtil().rewriteMigrationPayloadWithNodeStates(exportFile, "StatefulCountProcessor", List.of("1", "2", "3")); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connector.getId(), exportFile); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connector.getId(), payloadEntity.getPayload().getPayloadId()); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connector.getId(), requestEntity.getRequest().getRequestId()); + assertTrue(completedRequest.getRequest().getFailureReason().contains("connected node")); + + assertConnectorFresh(connector.getId()); + } + + @Test + public void testAsymmetricPerNodeMigrationFailureSurfacesInMergedResponse() throws Exception { + final File outputFile = new File("target/migration/asymmetric-failure-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("AsymmetricFailureSource", registryClient, false, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + // AsymmetricFailureMigrationConnector throws on node 2 only; node 1 succeeds. The merged response + // returned by the migration-request endpoint must surface node 2's failure, otherwise an operator polling + // the cluster would mistakenly see success. + final ConnectorEntity connector = getClientUtil().createConnector("AsymmetricFailureMigrationConnector"); + final String connectorId = connector.getId(); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceFixture.processGroup().getId()); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connectorId, requestEntity.getRequest().getRequestId()); + + final String failureReason = completedRequest.getRequest().getFailureReason(); + assertNotNull(failureReason); + assertTrue(failureReason.contains("node 2"), failureReason); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java new file mode 100644 index 000000000000..3bb23a614660 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; + +public class ClusteredConnectorVersionedFlowMigrationIT extends ConnectorVersionedFlowMigrationLocalIT { + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java new file mode 100644 index 000000000000..76a61e272da8 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ConnectorVersionedFlowMigrationEligibilityIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrationSourcesListingFiltersUnsupportedSources() throws Exception { + final FlowRegistryClientEntity registryClient = registerClient(); + final File matchingOutputFile = new File("target/migration/eligibility-matching.txt"); + final File nonMatchingOutputFile = new File("target/migration/eligibility-non-matching.txt"); + + final SourceFixture matchingFixture = createSourceFixture("EligibilityMatching", registryClient, false, matchingOutputFile, true); + final ProcessGroupEntity nonMatchingGroup = createProcessGroupWithoutRequiredParameter("EligibilityNonMatching", registryClient, nonMatchingOutputFile); + + final ConnectorEntity targetConnector = getClientUtil().createConnector("MigrationTargetConnector"); + final VersionedFlowMigrationSourcesEntity sourcesEntity = getClientUtil().listMigrationSources(targetConnector.getId()); + assertTrue(isSourceListed(sourcesEntity, matchingFixture.processGroup().getId())); + assertFalse(isSourceListed(sourcesEntity, nonMatchingGroup.getId())); + + final ConnectorEntity nonMigratingConnector = getClientUtil().createConnector("NonMigratingConnector"); + final VersionedFlowMigrationSourcesEntity nonMigratingSources = getClientUtil().listMigrationSources(nonMigratingConnector.getId()); + assertTrue(nonMigratingSources.getMigrationSources() == null || nonMigratingSources.getMigrationSources().isEmpty()); + } + + @Test + public void testLocalMigrationRejectsNonVersionControlledSource() throws Exception { + final SourceFixture sourceFixture = createSourceFixture("EligibilityLocalFailure", registerClient(), false, + new File("target/migration/eligibility-local-failure.txt"), false); + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + + assertThrows(NiFiClientException.class, () -> getClientUtil().startMigrationFromLocalSource(connector.getId(), sourceFixture.processGroup().getId())); + } + + @Test + public void testUploadedPayloadAcceptsNonVersionControlledSource() throws Exception { + final File outputFile = new File("target/migration/eligibility-uploaded.txt"); + final File exportFile = new File("target/migration/eligibility-uploaded.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("EligibilityUploaded", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + + outputFile.delete(); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connector.getId(), exportFile); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connector.getId(), payloadEntity.getPayload().getPayloadId()); + getClientUtil().waitForMigrationSuccess(connector.getId(), requestEntity.getRequest().getRequestId()); + } + + private ProcessGroupEntity createProcessGroupWithoutRequiredParameter(final String name, final FlowRegistryClientEntity registryClient, final File outputFile) throws Exception { + final ProcessGroupEntity processGroup = getClientUtil().createProcessGroup(name, "root"); + final ProcessorEntity statefulCount = getClientUtil().createProcessor("StatefulCountProcessor", processGroup.getId()); + final ProcessorEntity assetReader = getClientUtil().updateProcessorProperties( + getClientUtil().createProcessor("AssetReadingProcessor", processGroup.getId()), + Map.of( + "Source File", SAMPLE_ASSET_FILE.getAbsolutePath(), + "Output File", outputFile.getAbsolutePath())); + getClientUtil().setAutoTerminatedRelationships(assetReader, Set.of("success", "failure")); + getClientUtil().createConnection(statefulCount, assetReader, "success"); + getClientUtil().startVersionControl(processGroup, registryClient, TEST_BUCKET, name); + return processGroup; + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java new file mode 100644 index 000000000000..2d4b8049371e --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class ConnectorVersionedFlowMigrationFailureIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrationFailureRollsBackConnectorAndLeavesSourceUntouched() throws Exception { + final File outputFile = new File("target/migration/failure-local-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("FailureSource", registryClient, true, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + final String sourceGroupId = sourceFixture.processGroup().getId(); + final String originalName = getNifiClient().getProcessGroupClient().getProcessGroup(sourceGroupId).getComponent().getName(); + + final ConnectorEntity connector = getClientUtil().createConnector("FailingMigrationConnector"); + final String connectorId = connector.getId(); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceGroupId); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connectorId, requestEntity.getRequest().getRequestId()); + assertNotNull(completedRequest.getRequest().getFailureReason()); + + assertConnectorFresh(connectorId); + assertSourceUntouched(sourceFixture, originalName); + } + + @Test + public void testCorruptPayloadUploadRejectedAndConnectorRemainsFresh() throws Exception { + final File corruptPayload = new File("target/migration/corrupt-payload.json"); + corruptPayload.getParentFile().mkdirs(); + Files.writeString(corruptPayload.toPath(), "{ this is not valid json"); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + assertThrows(NiFiClientException.class, () -> getClientUtil().uploadMigrationPayload(connector.getId(), corruptPayload)); + + assertConnectorFresh(connector.getId()); + } + + @Test + public void testMigrationInterruptedByRestartLeavesConnectorFresh() throws Exception { + final File outputFile = new File("target/migration/failure-restart-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("FailureRestartSource", registryClient, false, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + final String sourceGroupId = sourceFixture.processGroup().getId(); + final String originalName = getNifiClient().getProcessGroupClient().getProcessGroup(sourceGroupId).getComponent().getName(); + + final ConnectorEntity connector = getClientUtil().createConnector("FailingMigrationConnector"); + final String connectorId = connector.getId(); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceGroupId); + + Thread.sleep(1000L); + getNiFiInstance().stop(); + getNiFiInstance().start(true); + setupClient(); + + waitFor(() -> { + try { + getNifiClient().getConnectorClient().getConnector(connectorId); + return true; + } catch (final Exception e) { + return false; + } + }); + + assertThrows(NiFiClientException.class, () -> getNifiClient().getConnectorClient().getMigrationStatus(connectorId, requestEntity.getRequest().getRequestId())); + + waitFor(() -> isConnectorFresh(connectorId)); + assertConnectorFresh(connectorId); + assertSourceUntouched(sourceFixture, originalName); + } + + private boolean isConnectorFresh(final String connectorId) { + try { + final ConnectorEntity connectorEntity = getNifiClient().getConnectorClient().getConnector(connectorId); + final String managedGroupId = connectorEntity.getComponent().getManagedProcessGroupId(); + final ProcessGroupFlowEntity flowEntity = getNifiClient().getConnectorClient().getFlow(connectorId, managedGroupId); + if (flowEntity.getProcessGroupFlow().getFlow().getProcessors() != null && !flowEntity.getProcessGroupFlow().getFlow().getProcessors().isEmpty()) { + return false; + } + if (flowEntity.getProcessGroupFlow().getFlow().getConnections() != null && !flowEntity.getProcessGroupFlow().getFlow().getConnections().isEmpty()) { + return false; + } + + return getNifiClient().getConnectorClient().getAssets(connectorId).getAssets() == null + || getNifiClient().getConnectorClient().getAssets(connectorId).getAssets().isEmpty(); + } catch (final Exception e) { + return false; + } + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java new file mode 100644 index 000000000000..9703f16a6475 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.ConnectorClient; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.ComponentStateEntity; +import org.apache.nifi.web.api.entity.ConnectionEntity; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.ControllerServiceEntity; +import org.apache.nifi.web.api.entity.ControllerServicesEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ConnectorVersionedFlowMigrationLocalIT extends NiFiSystemIT { + protected static final String TEST_BUCKET = "test-flows"; + private static final String ASSET_PARAMETER_NAME = "Asset File"; + private static final String SOURCE_TOPIC_PARAMETER_NAME = "Source Topic"; + protected static final File SAMPLE_ASSET_FILE = new File("src/test/resources/sample-assets/helloworld.txt"); + + @Test + public void testMigrateConnectorFromLocalVersionedFlow() throws Exception { + final File outputFile = new File("target/migration/local-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("LocalMigrationSource", registryClient, true, outputFile, true); + + prepareSourceForMigration(sourceFixture, outputFile); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final String connectorId = connector.getId(); + final VersionedFlowMigrationSourcesEntity sourcesEntity = getClientUtil().listMigrationSources(connectorId); + assertTrue(isSourceListed(sourcesEntity, sourceFixture.processGroup().getId())); + + outputFile.delete(); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceFixture.processGroup().getId()); + final String requestId = requestEntity.getRequest().getRequestId(); + getClientUtil().waitForMigrationSuccess(connectorId, requestId); + + final ProcessGroupEntity migratedSourceGroup = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + assertTrue(migratedSourceGroup.getComponent().getName().startsWith("(Migrated) ")); + + // The source process group's components must all be disabled after a successful migration so + // they cannot be inadvertently restarted, regardless of whether the migration also renamed the group. + final ProcessGroupFlowEntity migratedSourceFlow = + getNifiClient().getFlowClient().getProcessGroup(sourceFixture.processGroup().getId()); + for (final ProcessorEntity sourceProcessor : migratedSourceFlow.getProcessGroupFlow().getFlow().getProcessors()) { + assertEquals("DISABLED", sourceProcessor.getComponent().getState(), + "Migrated source processor " + sourceProcessor.getComponent().getName() + " must be DISABLED after successful migration"); + } + + final ConnectorClient connectorClient = getNifiClient().getConnectorClient(); + final AssetsEntity connectorAssets = connectorClient.getAssets(connectorId); + assertNotNull(connectorAssets.getAssets()); + assertFalse(connectorAssets.getAssets().isEmpty()); + + final ConnectorEntity migratedConnector = connectorClient.getConnector(connectorId); + final String managedGroupId = migratedConnector.getComponent().getManagedProcessGroupId(); + final String migratedCountProcessorId = getProcessorId(connectorId, managedGroupId, "StatefulCountProcessor"); + assertMigratedProcessorState(connectorId, migratedCountProcessorId); + + getClientUtil().startConnector(connectorId); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + assertEquals(Files.readString(SAMPLE_ASSET_FILE.toPath()).trim(), Files.readString(outputFile.toPath()).trim()); + } + + protected SourceFixture createSourceFixture(final String name, final FlowRegistryClientEntity registryClient, final boolean includeAsset, final File outputFile, + final boolean versionControlled) throws Exception { + final ProcessGroupEntity processGroup = getClientUtil().createProcessGroup(name, "root"); + + final ParameterContextEntity assetContext = getClientUtil().createParameterContext(name + "-asset", Map.of()); + if (includeAsset) { + final AssetEntity assetEntity = getNifiClient().getParamContextClient().createAsset(assetContext.getId(), SAMPLE_ASSET_FILE.getName(), SAMPLE_ASSET_FILE); + final ParameterContextUpdateRequestEntity requestEntity = getClientUtil().updateParameterAssetReferences( + assetContext, Map.of(ASSET_PARAMETER_NAME, List.of(assetEntity.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(assetContext.getId(), requestEntity.getRequest().getRequestId()); + } + + final ParameterContextEntity sourceContext = getClientUtil().createParameterContext( + name + "-source", + Map.of(SOURCE_TOPIC_PARAMETER_NAME, "orders"), + List.of(assetContext.getId()), + null); + getClientUtil().setParameterContext(processGroup.getId(), sourceContext); + + final ProcessorEntity statefulCount = getClientUtil().createProcessor("StatefulCountProcessor", processGroup.getId()); + final ProcessorEntity assetReader = getClientUtil().updateProcessorProperties(getClientUtil().createProcessor("AssetReadingProcessor", processGroup.getId()), Map.of( + "Source File", includeAsset ? "#{" + ASSET_PARAMETER_NAME + "}" : SAMPLE_ASSET_FILE.getAbsolutePath(), + "Output File", outputFile.getAbsolutePath())); + getClientUtil().setAutoTerminatedRelationships(assetReader, Set.of("success", "failure")); + + final ConnectionEntity connection = getClientUtil().createConnection(statefulCount, assetReader, "success"); + + if (versionControlled) { + assertNotNull(getClientUtil().startVersionControl(processGroup, registryClient, TEST_BUCKET, name)); + } + + return new SourceFixture(processGroup, connection); + } + + protected String getProcessorId(final String connectorId, final String groupId, final String processorTypeSuffix) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity flowEntity = getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + for (final ProcessorEntity processor : flowEntity.getProcessGroupFlow().getFlow().getProcessors()) { + if (processor.getComponent().getType().endsWith(processorTypeSuffix)) { + return processor.getId(); + } + } + + throw new IllegalStateException("Could not find processor ending with type " + processorTypeSuffix); + } + + protected boolean isSourceListed(final VersionedFlowMigrationSourcesEntity sourcesEntity, final String processGroupId) { + if (sourcesEntity.getMigrationSources() == null) { + return false; + } + + for (final VersionedFlowMigrationSourceDTO migrationSource : sourcesEntity.getMigrationSources()) { + if (processGroupId.equals(migrationSource.getProcessGroupId())) { + return true; + } + } + + return false; + } + + protected void assertMigratedProcessorState(final String connectorId, final String processorId) throws Exception { + if (!getNiFiInstance().isClustered()) { + final ComponentStateEntity stateEntity = getNifiClient().getConnectorClient().getProcessorState(connectorId, processorId); + assertStatePresent(stateEntity); + return; + } + + for (int nodeIndex = 1; nodeIndex <= getNumberOfNodes(); nodeIndex++) { + switchClientToNode(nodeIndex); + final ComponentStateEntity stateEntity = getNifiClient().getConnectorClient(DO_NOT_REPLICATE).getProcessorState(connectorId, processorId); + assertStatePresent(stateEntity); + assertNotNull(stateEntity.getComponentState().getClusterState()); + assertFalse(stateEntity.getComponentState().getClusterState().getState().isEmpty()); + } + } + + private void assertStatePresent(final ComponentStateEntity stateEntity) { + assertNotNull(stateEntity.getComponentState()); + assertNotNull(stateEntity.getComponentState().getLocalState()); + assertFalse(stateEntity.getComponentState().getLocalState().getState().isEmpty()); + } + + protected void prepareSourceForMigration(final SourceFixture sourceFixture, final File outputFile) throws Exception { + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + getClientUtil().assertFlowUpToDate(sourceFixture.processGroup().getId()); + } + + protected void assertConnectorFresh(final String connectorId) throws NiFiClientException, IOException { + final ConnectorClient connectorClient = getNifiClient().getConnectorClient(); + final ConnectorEntity connectorEntity = connectorClient.getConnector(connectorId); + + final String managedGroupId = connectorEntity.getComponent().getManagedProcessGroupId(); + final ProcessGroupFlowEntity flowEntity = connectorClient.getFlow(connectorId, managedGroupId); + assertTrue(flowEntity.getProcessGroupFlow().getFlow().getProcessors() == null + || flowEntity.getProcessGroupFlow().getFlow().getProcessors().isEmpty()); + assertTrue(flowEntity.getProcessGroupFlow().getFlow().getConnections() == null + || flowEntity.getProcessGroupFlow().getFlow().getConnections().isEmpty()); + + final AssetsEntity assetsEntity = connectorClient.getAssets(connectorId); + assertTrue(assetsEntity.getAssets() == null || assetsEntity.getAssets().isEmpty()); + } + + /** + * Asserts that the source process group has been left in the same state it was in before a failed + * migration attempt. The source must keep its original name, its original version-control state + * (registry coordinates and UP_TO_DATE state), its originally-applied parameter context, and must + * not have been disabled by the migration framework. + */ + protected void assertSourceUntouched(final SourceFixture sourceFixture, final String expectedName) throws NiFiClientException, IOException { + final ProcessGroupEntity processGroupEntity = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + assertEquals(expectedName, processGroupEntity.getComponent().getName()); + + // Version control coordinates and state must be unchanged. + assertNotNull(processGroupEntity.getComponent().getVersionControlInformation(), + "Source process group must remain under version control after a failed migration attempt"); + getClientUtil().assertFlowUpToDate(sourceFixture.processGroup().getId()); + + // Parameter context binding must be unchanged from the original setup. + final ProcessGroupEntity originalProcessGroupEntity = sourceFixture.processGroup(); + if (originalProcessGroupEntity.getComponent().getParameterContext() != null) { + assertNotNull(processGroupEntity.getComponent().getParameterContext(), + "Source process group must retain its parameter context after a failed migration attempt"); + assertEquals(originalProcessGroupEntity.getComponent().getParameterContext().getId(), + processGroupEntity.getComponent().getParameterContext().getId()); + } + + // Processors and controller services must NOT have been disabled by the migration framework. + // prepareSourceForMigration leaves processors STOPPED (started, output produced, then stopped), + // and any controller services in their original ENABLED state. A failed migration must leave + // those scheduled/enabled states unchanged. + final ProcessGroupFlowEntity flowEntity = + getNifiClient().getFlowClient().getProcessGroup(sourceFixture.processGroup().getId()); + for (final ProcessorEntity sourceProcessor : flowEntity.getProcessGroupFlow().getFlow().getProcessors()) { + final String state = sourceProcessor.getComponent().getState(); + assertNotEquals("DISABLED", state, + "Source processor " + sourceProcessor.getComponent().getName() + " must not be DISABLED after a failed migration attempt"); + assertEquals("STOPPED", state, + "Source processor " + sourceProcessor.getComponent().getName() + " must remain STOPPED after a failed migration attempt"); + } + final ControllerServicesEntity controllerServicesEntity = + getNifiClient().getFlowClient().getControllerServices(sourceFixture.processGroup().getId()); + if (controllerServicesEntity.getControllerServices() != null) { + for (final ControllerServiceEntity sourceService : controllerServicesEntity.getControllerServices()) { + final String state = sourceService.getComponent().getState(); + assertNotEquals("DISABLED", state, + "Source controller service " + sourceService.getComponent().getName() + " must not be DISABLED after a failed migration attempt"); + } + } + } + + protected record SourceFixture(ProcessGroupEntity processGroup, ConnectionEntity connection) { + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java new file mode 100644 index 000000000000..37d9500e1b9f --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConnectorVersionedFlowMigrationUploadedPayloadIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrateConnectorFromUploadedPayload() throws Exception { + final File outputFile = new File("target/migration/uploaded-output.txt"); + final File exportFile = new File("target/migration/uploaded-source.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("UploadedMigrationSource", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + final ProcessGroupEntity sourceProcessGroup = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + getNifiClient().getProcessGroupClient().deleteProcessGroup(sourceProcessGroup); + + outputFile.delete(); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final String connectorId = connector.getId(); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connectorId, exportFile); + assertNotNull(payloadEntity.getPayload()); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connectorId, payloadEntity.getPayload().getPayloadId()); + getClientUtil().waitForMigrationSuccess(connectorId, requestEntity.getRequest().getRequestId()); + + getClientUtil().startConnector(connectorId); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + assertEquals(Files.readString(SAMPLE_ASSET_FILE.toPath()).trim(), Files.readString(outputFile.toPath()).trim()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java index 05f9ed173c3d..73f05e9b8e5a 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java @@ -24,9 +24,12 @@ import org.apache.nifi.web.api.entity.ConnectorEntity; import org.apache.nifi.web.api.entity.ConnectorPropertyAllowableValuesEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import java.io.File; import java.io.IOException; @@ -259,6 +262,16 @@ VerifyConnectorConfigStepRequestEntity getConfigStepVerificationRequest(String c VerifyConnectorConfigStepRequestEntity deleteConfigStepVerificationRequest(String connectorId, String configurationStepName, String requestId) throws NiFiClientException, IOException; + VersionedFlowMigrationSourcesEntity listMigrationSources(String connectorId) throws NiFiClientException, IOException; + + MigrationPayloadEntity uploadMigrationPayload(String connectorId, File file) throws NiFiClientException, IOException; + + MigrationRequestEntity startMigration(MigrationRequestEntity requestEntity) throws NiFiClientException, IOException; + + MigrationRequestEntity getMigrationStatus(String connectorId, String requestId) throws NiFiClientException, IOException; + + MigrationRequestEntity cancelMigration(String connectorId, String requestId) throws NiFiClientException, IOException; + /** * Applies an update to a connector. * diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java index 7b648e7183e6..1ff058ec5851 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java @@ -34,9 +34,12 @@ import org.apache.nifi.web.api.entity.ConnectorPropertyAllowableValuesEntity; import org.apache.nifi.web.api.entity.ConnectorRunStatusEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import java.io.File; import java.io.FileInputStream; @@ -402,6 +405,78 @@ public VerifyConnectorConfigStepRequestEntity deleteConfigStepVerificationReques }); } + @Override + public VersionedFlowMigrationSourcesEntity listMigrationSources(final String connectorId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + + return executeAction("Error retrieving connector migration sources", () -> { + final WebTarget target = connectorTarget + .path("/migration-sources") + .resolveTemplate("id", connectorId); + return getRequestBuilder(target).get(VersionedFlowMigrationSourcesEntity.class); + }); + } + + @Override + public MigrationPayloadEntity uploadMigrationPayload(final String connectorId, final File file) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(file, "Migration payload file required"); + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath()); + } + + try (final InputStream payloadInputStream = new FileInputStream(file)) { + return executeAction("Error uploading connector migration payload", () -> { + final WebTarget target = connectorTarget + .path("/migration-payloads") + .resolveTemplate("id", connectorId); + return getRequestBuilder(target).post(Entity.entity(payloadInputStream, MediaType.APPLICATION_OCTET_STREAM_TYPE), MigrationPayloadEntity.class); + }); + } + } + + @Override + public MigrationRequestEntity startMigration(final MigrationRequestEntity requestEntity) throws NiFiClientException, IOException { + Objects.requireNonNull(requestEntity, "Migration request entity required"); + Objects.requireNonNull(requestEntity.getRequest(), "Migration request required"); + Objects.requireNonNull(requestEntity.getRequest().getConnectorId(), "Connector ID required"); + + return executeAction("Error creating connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests") + .resolveTemplate("id", requestEntity.getRequest().getConnectorId()); + return getRequestBuilder(target).post(Entity.entity(requestEntity, MediaType.APPLICATION_JSON_TYPE), MigrationRequestEntity.class); + }); + } + + @Override + public MigrationRequestEntity getMigrationStatus(final String connectorId, final String requestId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(requestId, "Migration request ID required"); + + return executeAction("Error retrieving connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + return getRequestBuilder(target).get(MigrationRequestEntity.class); + }); + } + + @Override + public MigrationRequestEntity cancelMigration(final String connectorId, final String requestId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(requestId, "Migration request ID required"); + + return executeAction("Error deleting connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + return getRequestBuilder(target).delete(MigrationRequestEntity.class); + }); + } + @Override public ConnectorEntity applyUpdate(final ConnectorEntity connectorEntity) throws NiFiClientException, IOException { if (connectorEntity == null) { diff --git a/pom.xml b/pom.xml index af98cd0c526d..87eadc2567cc 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ v24.14.1 - 2.8.0 + 2.9.0-SNAPSHOT 2.3.0 From 0ce1b2908b3dc0dace6a79a3ad81b275f920df74 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 19 May 2026 13:31:00 -0400 Subject: [PATCH 2/3] NIFI-15932: Refactored the Connector Migration logic into a separate MigratableConnector interface --- .../src/main/asciidoc/developer-guide.adoc | 11 +- .../EligibilityConnectorMigrationContext.java | 89 ++++++++++++ .../StandardConnectorMigrationManager.java | 17 ++- .../connector/StandardConnectorNode.java | 7 +- ...ardFrameworkConnectorMigrationContext.java | 2 +- ...tEligibilityConnectorMigrationContext.java | 64 +++++++++ ...TestStandardConnectorMigrationManager.java | 128 +++++++++++++++++- ...ardFrameworkConnectorMigrationContext.java | 2 +- .../AsymmetricFailureMigrationConnector.java | 3 +- .../system/FailingMigrationConnector.java | 3 +- .../system/MigrationTargetConnector.java | 3 +- 11 files changed, 305 insertions(+), 24 deletions(-) create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc index 1e42f013865e..1a87a8375509 100644 --- a/nifi-docs/src/main/asciidoc/developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc @@ -2704,22 +2704,21 @@ This is intended for Connectors that can translate an exported flow definition, The source Versioned Process Group is not installed onto the Connector; instead, the Connector reads the source as input and updates its own flow to mirror the source's configuration and state. Migration support is optional. -Existing Connectors continue to behave as before unless they override the new default methods on `org.apache.nifi.components.connector.Connector`. +Connectors opt in by implementing the optional capability interface `org.apache.nifi.components.connector.migration.MigratableConnector`. +A Connector that does not implement `MigratableConnector` is never offered as a migration target and continues to behave as before. === Migration contract -The Connector interface now exposes two default methods for migration: +`MigratableConnector` is a separate interface that the Connector class must implement (typically alongside extending `AbstractConnector`). It exposes two methods: - `isMigrationSupported(ConnectorMigrationContext)` is a cheap, side-effect-free predicate. It should inspect `context.getSourceFlow()` and return `true` only when the Connector can be migrated from the source structure. Implementations should limit themselves to structural checks such as processor types, parameter names, and exported metadata. Component state and asset content are not available at listing time and must not be relied on; per-state precondition checks belong inside the Connector's `migrate(...)` implementation. + This method must not call `ConnectorMigrationContext.copyAssetFromSource(...)`; the framework wraps the eligibility-time context so any such attempt is rejected with `IllegalStateException` and the Connector is filtered out of the listing. - `migrate(ConnectorMigrationContext)` performs the actual migration. The Connector reads the source `VersionedExternalFlow`, translates it as needed into its own representation, and applies the result to its own active `FlowContext`. -The default implementation of `isMigrationSupported(...)` returns `false`. -The default implementation of `migrate(...)` throws `UnsupportedOperationException`, so a Connector must override both methods to participate in migration. - === Accessing the source flow `ConnectorMigrationContext` exposes the source `VersionedExternalFlow` using `getSourceFlow()`. @@ -2762,7 +2761,7 @@ Connectors are responsible for mapping those values into their own target shape. A Connector can copy an asset from a local source Process Group into its own asset namespace using `ConnectorMigrationContext.copyAssetFromSource(String)`. This method is available only for local migrations. -The method throws an `IllegalArgumentException` when the supplied source asset identifier cannot be resolved and when invoked for an uploaded payload migration. +It throws `IllegalArgumentException` when the supplied source asset identifier is null or blank, and `IllegalStateException` when invoked for an uploaded-payload migration. Uploaded payloads do not have access to a live source asset namespace, so asset references from uploaded payloads must be handled outside of the migration request. The typical flow for parameter migration is: diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java new file mode 100644 index 000000000000..e0f445c894fc --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.util.Objects; +import java.util.Set; + +/** + * Delegating {@link FrameworkConnectorMigrationContext} used on the listing/eligibility path. The + * {@link org.apache.nifi.components.connector.migration.MigratableConnector#isMigrationSupported(org.apache.nifi.components.connector.migration.ConnectorMigrationContext)} + * contract requires the call to be side-effect-free; in particular it must not invoke + * {@link #copyAssetFromSource(String)}. This wrapper enforces that contract by forwarding every other accessor + * to the delegate while throwing {@link IllegalStateException} from {@code copyAssetFromSource(String)}, even + * when the underlying context would otherwise allow the copy. Implementations that ignore the JavaDoc and try + * to copy assets during eligibility evaluation are caught here rather than silently mutating the Connector's + * asset namespace during a read-only listing. + */ +final class EligibilityConnectorMigrationContext implements FrameworkConnectorMigrationContext { + + private final FrameworkConnectorMigrationContext delegate; + + EligibilityConnectorMigrationContext(final FrameworkConnectorMigrationContext delegate) { + this.delegate = Objects.requireNonNull(delegate, "Delegate FrameworkConnectorMigrationContext is required"); + } + + @Override + public VersionedExternalFlow getSourceFlow() { + return delegate.getSourceFlow(); + } + + @Override + public boolean isLocalMigration() { + return delegate.isLocalMigration(); + } + + @Override + public FrameworkFlowContext getActiveFlowContext() { + return delegate.getActiveFlowContext(); + } + + @Override + public AssetReference copyAssetFromSource(final String sourceAssetId) { + throw new IllegalStateException("copyAssetFromSource(...) must not be invoked from isMigrationSupported(...); the listing path is read-only."); + } + + @Override + public AssetManager getSourceAssetManager() { + return delegate.getSourceAssetManager(); + } + + @Override + public ConnectorRepository getConnectorRepository() { + return delegate.getConnectorRepository(); + } + + @Override + public StateManagerProvider getStateManagerProvider() { + return delegate.getStateManagerProvider(); + } + + @Override + public ClusterTopologyProvider getClusterTopologyProvider() { + return delegate.getClusterTopologyProvider(); + } + + @Override + public Set getCopiedAssetIds() { + return delegate.getCopiedAssetIds(); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java index d09df0104c0c..e7d923a76d6e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java @@ -20,6 +20,7 @@ import org.apache.nifi.components.connector.components.ControllerServiceFacade; import org.apache.nifi.components.connector.components.ProcessGroupFacade; import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.connectable.Connection; @@ -113,15 +114,20 @@ public void migrateFromVersionedFlow(final String connectorId, final String proc flowController ); - try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connector.getConnector().getClass(), connectorId)) { - if (!connector.getConnector().isMigrationSupported(migrationContext)) { + final Connector rawConnector = connector.getConnector(); + if (!(rawConnector instanceof final MigratableConnector migratableConnector)) { + throw new FlowUpdateException("Connector " + connectorId + " does not support migration from the provided source flow."); + } + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), rawConnector.getClass(), connectorId)) { + if (!migratableConnector.isMigrationSupported(migrationContext)) { throw new FlowUpdateException("Connector " + connectorId + " does not support migration from the provided source flow."); } } - try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connector.getConnector().getClass(), connectorId)) { + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), rawConnector.getClass(), connectorId)) { flowController.getConnectorRepository().syncAssetsFromProvider(connector); - connector.getConnector().migrate(migrationContext); + migratableConnector.migrate(migrationContext); if (cancellation.getAsBoolean()) { throw new FlowUpdateException("Migration of Connector " + connectorId + " was cancelled after the Connector applied the source flow."); @@ -286,7 +292,7 @@ private String getIneligibilityReason(final ConnectorNode connector, final Proce return "Process Group " + processGroup.getIdentifier() + " references controller services outside the Process Group."; } - final FrameworkConnectorMigrationContext migrationContext = new StandardFrameworkConnectorMigrationContext( + final StandardFrameworkConnectorMigrationContext underlyingContext = new StandardFrameworkConnectorMigrationContext( connector.getIdentifier(), sourceFlow, true, @@ -296,6 +302,7 @@ private String getIneligibilityReason(final ConnectorNode connector, final Proce flowController.getStateManagerProvider(), flowController ); + final FrameworkConnectorMigrationContext migrationContext = new EligibilityConnectorMigrationContext(underlyingContext); return connector.isMigrationSupported(migrationContext) ? null diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index 789896db7eac..c61231041324 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -30,6 +30,7 @@ import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.components.ParameterContextFacade; import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.components.validation.DisabledServiceValidationResult; import org.apache.nifi.components.validation.ValidationState; import org.apache.nifi.components.validation.ValidationStatus; @@ -759,7 +760,11 @@ public void loadInitialFlow() throws FlowUpdateException { @Override public boolean isMigrationSupported(final ConnectorMigrationContext context) { try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { - return getConnector().isMigrationSupported(context); + final Connector connector = getConnector(); + if (!(connector instanceof final MigratableConnector migratableConnector)) { + return false; + } + return migratableConnector.isMigrationSupported(context); } catch (final Exception e) { getComponentLog().warn("Failed to evaluate whether migration is supported for {}; assuming migration is not supported", context.getSourceFlow(), e); return false; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java index f13268b316a9..59a5e1a3e28c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java @@ -80,7 +80,7 @@ public AssetReference copyAssetFromSource(final String sourceAssetId) { throw new IllegalArgumentException("Source asset identifier must be specified."); } if (!localMigration) { - throw new IllegalArgumentException("Source assets can only be copied for migrations from a local Versioned Process Group."); + throw new IllegalStateException("Source assets can only be copied for migrations from a local Versioned Process Group."); } final String copiedAssetId = UUID.nameUUIDFromBytes((connectorId + ":" + sourceAssetId).getBytes(StandardCharsets.UTF_8)).toString(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java new file mode 100644 index 000000000000..e467926c95fe --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +public class TestEligibilityConnectorMigrationContext { + + private static final String CONNECTOR_ID = "connector-1"; + + @Test + public void testEligibilityContextRejectsCopyAssetFromSource() { + final VersionedExternalFlow sourceFlow = mock(VersionedExternalFlow.class); + final FrameworkFlowContext activeFlowContext = mock(FrameworkFlowContext.class); + final AssetManager sourceAssetManager = mock(AssetManager.class); + final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); + final StateManagerProvider stateManagerProvider = mock(StateManagerProvider.class); + final ClusterTopologyProvider clusterTopologyProvider = mock(ClusterTopologyProvider.class); + + final StandardFrameworkConnectorMigrationContext underlying = new StandardFrameworkConnectorMigrationContext( + CONNECTOR_ID, sourceFlow, true, activeFlowContext, sourceAssetManager, connectorRepository, + stateManagerProvider, clusterTopologyProvider); + final EligibilityConnectorMigrationContext eligibilityContext = new EligibilityConnectorMigrationContext(underlying); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> eligibilityContext.copyAssetFromSource("any-asset-id")); + assertTrue(thrown.getMessage().contains("copyAssetFromSource")); + assertTrue(thrown.getMessage().contains("isMigrationSupported")); + + // Every other accessor must return whatever the underlying context returns. + assertSame(sourceFlow, eligibilityContext.getSourceFlow()); + assertTrue(eligibilityContext.isLocalMigration()); + assertSame(activeFlowContext, eligibilityContext.getActiveFlowContext()); + assertSame(sourceAssetManager, eligibilityContext.getSourceAssetManager()); + assertSame(connectorRepository, eligibilityContext.getConnectorRepository()); + assertSame(stateManagerProvider, eligibilityContext.getStateManagerProvider()); + assertSame(clusterTopologyProvider, eligibilityContext.getClusterTopologyProvider()); + assertEquals(underlying.getCopiedAssetIds(), eligibilityContext.getCopiedAssetIds()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java index d895dfd35066..3fc77bb4632e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java @@ -20,6 +20,7 @@ import org.apache.nifi.components.connector.components.ProcessGroupFacade; import org.apache.nifi.components.connector.components.ProcessorFacade; import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; @@ -45,6 +46,7 @@ import org.apache.nifi.registry.flow.VersionedFlowStatus; import org.junit.jupiter.api.Test; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -64,6 +66,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; public class TestStandardConnectorMigrationManager { @@ -139,7 +142,7 @@ public void testRerunAfterFailureIsAllowed() throws Exception { final FlowController flowController = createFlowController(1); final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); - final Connector connector = connectorNode.getConnector(); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); doThrow(new FlowUpdateException("transient")).doNothing().when(connector).migrate(any(ConnectorMigrationContext.class)); final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); @@ -157,7 +160,7 @@ public void testNonFlowUpdateExceptionFromMigrateStillTriggersRollback() throws final FlowController flowController = createFlowController(1); final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); - final Connector connector = connectorNode.getConnector(); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); doThrow(new RuntimeException("boom")).when(connector).migrate(any(ConnectorMigrationContext.class)); final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); @@ -189,7 +192,7 @@ public void testRollbackClearsLocalAndClusterStateAndDeletesOnlyCopiedAssets() t when(managedGroup.findAllControllerServices()).thenReturn(Set.of(controllerService)); when(connectorNode.getActiveFlowContext().getManagedProcessGroup()).thenReturn(managedGroup); - final Connector connector = connectorNode.getConnector(); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); doThrow(new FlowUpdateException("nope")).when(connector).migrate(any(ConnectorMigrationContext.class)); final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); @@ -313,7 +316,7 @@ public void testRejectMigrationWhenTargetActiveFlowAlreadyHasUserComponents() { public void testRollbackFailureDoesNotMaskOriginalMigrationFailure() throws Exception { final FlowController flowController = createFlowController(1); final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); - final Connector connector = connectorNode.getConnector(); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); final FlowUpdateException originalFailure = new FlowUpdateException("original"); doThrow(originalFailure).when(connector).migrate(any(ConnectorMigrationContext.class)); @@ -334,7 +337,7 @@ public void testIsMigrationSupportedFalseDoesNotTriggerRollback() throws Excepti final FlowController flowController = createFlowController(1); final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); - when(connectorNode.getConnector().isMigrationSupported(any())).thenReturn(false); + when(((MigratableConnector) connectorNode.getConnector()).isMigrationSupported(any())).thenReturn(false); final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); @@ -378,6 +381,88 @@ public void testEachIneligibleVersionedFlowStateHasDistinctDiagnostic() { "Each non-UP_TO_DATE VersionedFlowState must produce a unique diagnostic message"); } + @Test + public void testListMigrationSourcesReturnsEmptyForNonMigratableConnector() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID, false); + stubFrameworkMigrationGating(connectorNode); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup eligibleGroup = mockEligibleVersionedGroup("eligible-group"); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(eligibleGroup)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + assertTrue(migrationManager.listMigrationSources(CONNECTOR_ID).isEmpty(), + "A Connector that does not implement MigratableConnector must never appear in the listing"); + } + + @Test + public void testMigrateRejectsConnectorThatDoesNotImplementMigratableConnector() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID, false); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final FlowUpdateException exception = assertThrows(FlowUpdateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("does not support migration"), exception.getMessage()); + + // Rejection happens before any flow manipulation is attempted, so rollback must not be exercised. + verify(connectorNode, never()).loadInitialFlow(); + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + } + + @Test + public void testMigrateRejectsRunningConnector() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.RUNNING); + when(connectorNode.getDesiredState()).thenReturn(ConnectorState.RUNNING); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("must be stopped before it can be migrated"), exception.getMessage()); + } + + @Test + public void testListMigrationSourcesPropagatesEligibilityWrapper() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + stubFrameworkMigrationGating(connectorNode); + + final MigratableConnector migratableConnector = (MigratableConnector) connectorNode.getConnector(); + when(migratableConnector.isMigrationSupported(any())).thenAnswer(invocation -> { + final ConnectorMigrationContext context = invocation.getArgument(0); + context.copyAssetFromSource("any-id"); + return true; + }); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup eligibleGroup = mockEligibleVersionedGroup("eligible-group"); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(eligibleGroup)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + assertTrue(migrationManager.listMigrationSources(CONNECTOR_ID).isEmpty(), + "A Connector that calls copyAssetFromSource(...) from isMigrationSupported(...) must be filtered out by the eligibility wrapper"); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + private ProcessGroup mockEligibleVersionedGroup(final String groupId) { + final ProcessGroup processGroup = mockVersionedGroup(groupId, VersionedFlowState.UP_TO_DATE); + when(processGroup.getName()).thenReturn(groupId); + return processGroup; + } + private StandardConnectorMigrationManager newMigrationManager(final FlowController flowController) { // The snapshot provider is only exercised by the listing path. Tests that drive listing wire their own snapshots. final RegisteredFlowSnapshot emptySnapshot = new RegisteredFlowSnapshot(); @@ -400,6 +485,10 @@ private FlowController createFlowController(final int connectedNodeCount) { } private ConnectorNode wireFreshConnector(final FlowController flowController, final String connectorId) { + return wireFreshConnector(flowController, connectorId, true); + } + + private ConnectorNode wireFreshConnector(final FlowController flowController, final String connectorId, final boolean migratable) { final ConnectorNode connectorNode = mock(ConnectorNode.class); when(connectorNode.getIdentifier()).thenReturn(connectorId); when(connectorNode.getCurrentState()).thenReturn(ConnectorState.STOPPED); @@ -407,13 +496,38 @@ private ConnectorNode wireFreshConnector(final FlowController flowController, fi final FrameworkFlowContext flowContext = mock(FrameworkFlowContext.class); when(connectorNode.getActiveFlowContext()).thenReturn(flowContext); - final Connector connector = mock(Connector.class); - when(connector.isMigrationSupported(any())).thenReturn(true); + final Connector connector; + if (migratable) { + connector = mock(Connector.class, withSettings().extraInterfaces(MigratableConnector.class)); + when(((MigratableConnector) connector).isMigrationSupported(any())).thenReturn(true); + } else { + connector = mock(Connector.class); + } when(connectorNode.getConnector()).thenReturn(connector); when(flowController.getConnectorRepository().getConnector(connectorId)).thenReturn(connectorNode); return connectorNode; } + /** + * Mirrors {@link StandardConnectorNode#isMigrationSupported} on the mock {@link ConnectorNode}: returns false + * unless the underlying {@link Connector} implements {@link MigratableConnector}, and swallows any exception + * thrown by the underlying call by returning false. Tests that exercise the listing/eligibility path must call + * this so the mock reflects the framework-side gating rather than Mockito's default {@code false}. + */ + private void stubFrameworkMigrationGating(final ConnectorNode connectorNode) { + when(connectorNode.isMigrationSupported(any())).thenAnswer(invocation -> { + final Connector connector = connectorNode.getConnector(); + if (!(connector instanceof final MigratableConnector migratableConnector)) { + return false; + } + try { + return migratableConnector.isMigrationSupported(invocation.getArgument(0)); + } catch (final Exception ignored) { + return false; + } + }); + } + private ProcessGroup wireSourceProcessGroup(final FlowController flowController, final String groupId, final String groupName) { final FlowManager flowManager = flowController.getFlowManager(); final ProcessGroup rootGroup = mock(ProcessGroup.class); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java index 0595d94f80fb..9133b5454298 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java @@ -125,7 +125,7 @@ public void testCopyAssetFromSourceReusesPreviouslyCopiedAsset(@TempDir final Pa public void testCopyAssetFromSourceRejectsUploadedPayloadMigration() { final StandardFrameworkConnectorMigrationContext context = createContext(false); - final IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, + final IllegalStateException thrown = assertThrows(IllegalStateException.class, () -> context.copyAssetFromSource(SOURCE_ASSET_ID)); assertTrue(thrown.getMessage().contains("local Versioned Process Group")); } diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java index e1b284d47199..eaae7033a06e 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java @@ -22,6 +22,7 @@ import org.apache.nifi.components.connector.FlowUpdateException; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.flow.VersionedProcessGroup; @@ -35,7 +36,7 @@ * {@code -DnodeNumber=} JVM argument matches {@value #FAILING_NODE_NUMBER}. Used to validate that the cluster * migration-request endpoint merger surfaces a per-node failure even when other nodes report success. */ -public class AsymmetricFailureMigrationConnector extends AbstractConnector { +public class AsymmetricFailureMigrationConnector extends AbstractConnector implements MigratableConnector { private static final String FAILING_NODE_NUMBER = "2"; @Override diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java index d0fbeb53f42c..fb3bd1c833bd 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java @@ -22,6 +22,7 @@ import org.apache.nifi.components.connector.FlowUpdateException; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.flow.VersionedParameter; @@ -34,7 +35,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -public class FailingMigrationConnector extends AbstractConnector { +public class FailingMigrationConnector extends AbstractConnector implements MigratableConnector { private static final long FAILURE_DELAY_SECONDS = 5L; @Override diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java index abffdc8a372f..7f8a30972d41 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java @@ -23,6 +23,7 @@ import org.apache.nifi.components.connector.FlowUpdateException; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.flow.VersionedParameter; @@ -37,7 +38,7 @@ import java.util.Map; import java.util.Set; -public class MigrationTargetConnector extends AbstractConnector { +public class MigrationTargetConnector extends AbstractConnector implements MigratableConnector { private static final String REQUIRED_PARAMETER_NAME = "Source Topic"; private static final String REQUIRED_PROCESSOR_TYPE = "org.apache.nifi.processors.tests.system.StatefulCountProcessor"; From fec0abab2b538db1909fe6137d8b190a1eae637e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 28 May 2026 11:18:09 -0400 Subject: [PATCH 3/3] NIFI-15932: Add MIGRATE allowable action and align with nifi-api 2.9.0-SNAPSHOT Adds the MIGRATE allowable action to StandardConnectorNode, returning a reasonNotAllowed when the connector is not stopped or does not implement MigratableConnector. Wires MIGRATE through the ConnectorActionName union in the frontend type definitions. Updates StandardConnectorMigrationManager, StandardNiFiServiceFacade, and StandardConnectorDAO to pass ConnectorSyncMode.LOCAL_ONLY when looking up connectors via ConnectorRepository, matching the convention introduced on main when the sync-mode parameter was added. Adds stub overrides of Connector.getActiveFlow(FlowContext) to every concrete Connector implementation in the source tree (mock connectors, framework-core test connectors, system-test connectors, KafkaToS3, and DummyConnector), as required by the abstract method added in nifi-api 2.9.0-SNAPSHOT. Converts between ScheduledState and VersionedConnectorState at the boundary between the framework-side ConnectorSyncResult / ConnectorSync Directive / ConnectorConfigurationProvider (still ScheduledState) and VersionedConnector (now VersionedConnectorState). --- .../connectors/AllowableValuesConnector.java | 5 ++++ .../connectors/CronScheduleConnector.java | 5 ++++ .../nifi/mock/connectors/GenerateAndLog.java | 5 ++++ .../connectors/MissingBundleConnector.java | 5 ++++ .../nifi/connectors/kafkas3/KafkaToS3.java | 5 ++++ .../mapping/VersionedComponentFlowMapper.java | 6 ++-- .../components/connector/GhostConnector.java | 5 ++++ .../StandardConnectorMigrationManager.java | 2 +- .../connector/StandardConnectorNode.java | 19 ++++++++++++ .../StandardConnectorRepository.java | 15 +++++++++- .../VersionedDataflowMapper.java | 3 +- .../connector/BlockingConnector.java | 5 ++++ .../DynamicAllowableValuesConnector.java | 5 ++++ .../connector/DynamicFlowConnector.java | 5 ++++ .../connector/MissingBundleConnector.java | 5 ++++ .../OnPropertyModifiedConnector.java | 5 ++++ .../connector/ParameterConnector.java | 5 ++++ .../connector/SleepingConnector.java | 5 ++++ ...TestStandardConnectorMigrationManager.java | 2 +- .../connector/TestStandardConnectorNode.java | 30 +++++++++++++++++++ .../TestStandardConnectorRepository.java | 28 +++++++++++++---- .../nifi/controller/flow/NopConnector.java | 5 ++++ .../VersionedFlowSynchronizerTest.java | 11 +++---- .../org/apache/nifi/nar/DummyConnector.java | 5 ++++ .../nifi/web/StandardNiFiServiceFacade.java | 4 +-- .../web/dao/impl/StandardConnectorDAO.java | 6 ++-- .../frontend/libs/shared/src/types/index.ts | 1 + .../tests/system/AssetConnector.java | 5 ++++ .../AsymmetricFailureMigrationConnector.java | 5 ++++ .../system/BundleResolutionConnector.java | 5 ++++ .../tests/system/CalculateConnector.java | 5 ++++ .../system/ComponentLifecycleConnector.java | 5 ++++ .../tests/system/DataQueuingConnector.java | 5 ++++ .../system/FailingMigrationConnector.java | 5 ++++ .../system/GatedDataQueuingConnector.java | 5 ++++ .../system/MigrationTargetConnector.java | 5 ++++ .../system/NestedProcessGroupConnector.java | 5 ++++ .../tests/system/NonMigratingConnector.java | 5 ++++ .../connectors/tests/system/NopConnector.java | 5 ++++ .../system/ParameterContextConnector.java | 5 ++++ .../tests/system/SelectiveDropConnector.java | 5 ++++ 41 files changed, 250 insertions(+), 22 deletions(-) diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java index a263482eda28..d8172b7a2bbe 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java index 1e257c245c6d..75628f7ae574 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java @@ -64,6 +64,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Cron_Schedule_Connector.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(SCHEDULE_STEP); diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java index 96b9fa63c38e..40a6a5faa690 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java @@ -35,6 +35,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java index d49674ffd038..1dad60bc72f0 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java @@ -88,6 +88,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java index bb964a8181ba..71708cdbd103 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java @@ -112,6 +112,11 @@ public VersionedExternalFlow getInitialFlow() { return KafkaToS3FlowBuilder.loadInitialFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException { final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index 216e4fd76428..0d96b2eec9c9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -1158,14 +1158,14 @@ private Map mapPropertyValues(final St return versionedProperties; } - private org.apache.nifi.flow.ScheduledState mapConnectorState(final ConnectorState connectorState) { + private org.apache.nifi.flow.VersionedConnectorState mapConnectorState(final ConnectorState connectorState) { if (connectorState == null) { return null; } return switch (connectorState) { - case RUNNING, STARTING -> org.apache.nifi.flow.ScheduledState.RUNNING; - default -> org.apache.nifi.flow.ScheduledState.ENABLED; + case RUNNING, STARTING -> org.apache.nifi.flow.VersionedConnectorState.RUNNING; + default -> org.apache.nifi.flow.VersionedConnectorState.ENABLED; }; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java index 0508821e9d1f..c5c43314668b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java index e7d923a76d6e..6fe34369f93b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java @@ -227,7 +227,7 @@ private void clearComponentState(final String componentIdentifier) { } private ConnectorNode getRequiredConnector(final String connectorId) { - final ConnectorNode connector = flowController.getConnectorRepository().getConnector(connectorId); + final ConnectorNode connector = flowController.getConnectorRepository().getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); if (connector == null) { throw new IllegalArgumentException("Could not find Connector with ID " + connectorId); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index c61231041324..e4815b78a6ed 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -1304,11 +1304,30 @@ public List getAvailableActions() { actions.add(createDrainFlowFilesAction(stopped, dataQueued)); actions.add(createCancelDrainFlowFilesAction(currentState == ConnectorState.DRAINING)); actions.add(createApplyUpdatesAction(currentState)); + actions.add(createMigrateAction(stopped)); actions.add(createDeleteAction(stopped, dataQueued)); return actions; } + private ConnectorAction createMigrateAction(final boolean stopped) { + final boolean allowed; + final String reason; + + if (!stopped) { + allowed = false; + reason = "Connector must be stopped"; + } else if (!(getConnector() instanceof MigratableConnector)) { + allowed = false; + reason = "Connector does not support migration from a Versioned flow"; + } else { + allowed = true; + reason = null; + } + + return new StandardConnectorAction("MIGRATE", "Migrate a Versioned flow's assets and configuration into this Connector", allowed, reason); + } + private boolean isStopped() { final ConnectorState currentState = getCurrentState(); if (currentState == ConnectorState.STOPPED) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index 13b015bb5948..1171febf54be 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -29,6 +29,7 @@ import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; @@ -117,7 +118,7 @@ public void restoreConnector(final ConnectorNode connector) { @Override public ConnectorSyncResult syncConnector(final VersionedConnector versionedConnector) { final String connectorId = versionedConnector.getInstanceIdentifier(); - final ScheduledState proposedScheduledState = versionedConnector.getScheduledState(); + final ScheduledState proposedScheduledState = toScheduledState(versionedConnector.getScheduledState()); logger.debug("syncConnector called for connector [{}]", connectorId); // Consult the provider for external state checks and any externally managed working configuration. @@ -1092,4 +1093,16 @@ String findProviderIdByName(final String providerName) { logger.debug("Resolved parameter provider name [{}] to id [{}]", providerName, resolvedId); return resolvedId; } + + private static ScheduledState toScheduledState(final VersionedConnectorState versionedConnectorState) { + if (versionedConnectorState == null) { + return null; + } + + return switch (versionedConnectorState) { + case RUNNING -> ScheduledState.RUNNING; + case DISABLED -> ScheduledState.DISABLED; + case ENABLED, TROUBLESHOOTING -> ScheduledState.ENABLED; + }; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index 7e74a7f2389b..ed0e9a78230e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -29,6 +29,7 @@ import org.apache.nifi.controller.flow.VersionedFlowEncodingVersion; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; @@ -101,7 +102,7 @@ private List mapConnectors() { for (final ConnectorNode connectorNode : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode); if (flowController.isStartAfterInitialization(connectorNode)) { - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); } connectors.add(versionedConnector); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java index 75c35154d693..e59f15da0ce2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java @@ -49,6 +49,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java index 404a78b9727f..73d5cd965256 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java @@ -79,6 +79,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java index bf8148214393..a4d35d575414 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java @@ -130,6 +130,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-duplicate-log-flow.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + public boolean isInitialized() { return initialized; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java index c21b55dfbbaf..55377dd15380 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java @@ -96,6 +96,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java index bf5f242a939e..a2e59992aff8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java @@ -60,6 +60,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/on-property-modified-tracker.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(CONFIG_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java index bbc071aca0eb..78b7b4b48198 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java @@ -76,6 +76,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-and-log-with-parameter.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(TEXT_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java index 6818b5b059cc..ef4f7dc0d680 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java @@ -44,6 +44,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java index 3fc77bb4632e..6c6e381e603e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java @@ -504,7 +504,7 @@ private ConnectorNode wireFreshConnector(final FlowController flowController, fi connector = mock(Connector.class); } when(connectorNode.getConnector()).thenReturn(connector); - when(flowController.getConnectorRepository().getConnector(connectorId)).thenReturn(connectorNode); + when(flowController.getConnectorRepository().getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY)).thenReturn(connectorNode); return connectorNode; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java index 00e136a66358..820576fdc350 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java @@ -881,6 +881,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -947,6 +952,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -997,6 +1007,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1068,6 +1083,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1124,6 +1144,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1168,6 +1193,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index 215197786f4f..affa9ed5fcb9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -29,6 +29,7 @@ import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedConnectorValueReference; @@ -783,7 +784,7 @@ public void testGetConnectorTriggersOnConfigurationStepConfiguredWhenValuesChang trackingConnector.reset(); final VersionedConfigurationStep externalStep = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("updated-value"))); + Map.of("prop1", createStringLiteralReference("updated-value"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); @@ -808,7 +809,7 @@ public void testGetConnectorDoesNotTriggerOnConfigurationStepConfiguredWhenValue trackingConnector.reset(); final VersionedConfigurationStep externalStep = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("same-value"))); + Map.of("prop1", createStringLiteralReference("same-value"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); @@ -832,9 +833,9 @@ public void testGetConnectorContinuesWhenOneStepFailsToReplace() throws FlowUpda repository.addConnector(connector); final VersionedConfigurationStep stepOne = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("value1"))); + Map.of("prop1", createStringLiteralReference("value1"))); final VersionedConfigurationStep stepTwo = createVersionedStep("step2", - Map.of("prop2", createStringLiteralRef("value2"))); + Map.of("prop2", createStringLiteralReference("value2"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(stepOne, stepTwo)); @@ -1490,7 +1491,7 @@ private VersionedConnector createVersionedConnector(final String identifier, fin final VersionedConnector versionedConnector = new VersionedConnector(); versionedConnector.setInstanceIdentifier(identifier); versionedConnector.setName(name); - versionedConnector.setScheduledState(scheduledState); + versionedConnector.setScheduledState(toVersionedConnectorState(scheduledState)); versionedConnector.setActiveFlowConfiguration(activeConfig); versionedConnector.setWorkingFlowConfiguration(activeConfig); final Bundle bundle = new Bundle(); @@ -1502,6 +1503,18 @@ private VersionedConnector createVersionedConnector(final String identifier, fin return versionedConnector; } + private static VersionedConnectorState toVersionedConnectorState(final ScheduledState scheduledState) { + if (scheduledState == null) { + return null; + } + + return switch (scheduledState) { + case RUNNING -> VersionedConnectorState.RUNNING; + case DISABLED -> VersionedConnectorState.DISABLED; + case ENABLED -> VersionedConnectorState.ENABLED; + }; + } + private StandardConnectorNode createRealConnectorNode(final String identifier, final Connector connector) throws FlowUpdateException { final ExtensionManager extensionManager = mock(ExtensionManager.class); final AssetManager assetManager = mock(AssetManager.class); @@ -1600,6 +1613,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java index c2dc1ab49a89..c37168fb1387 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java @@ -56,6 +56,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { if (!initialized) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java index 2684627afc3b..0216f078486b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java @@ -33,6 +33,7 @@ import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedProcessGroup; @@ -306,7 +307,7 @@ void testSyncInheritConnectorDelegatesSyncToConnectorRepository() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); @@ -334,7 +335,7 @@ void testSyncInheritConnectorNotStartedWhenEnabled() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); @@ -363,7 +364,7 @@ void testSyncOrphanConnectorIsRemoved() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -402,7 +403,7 @@ void testSyncOrphanConnectorNotRemovedWhenInProposedFlow() { versionedConnector.setName("Test Connector"); versionedConnector.setType("org.apache.nifi.connectors.TestConnector"); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) @@ -432,7 +433,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java index 2a2b21cde1b3..84afd7dd15ac 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java @@ -51,6 +51,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void start(final FlowContext flowContext) throws FlowUpdateException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index e64889a05268..eb2b610aa628 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -4061,13 +4061,13 @@ public ConnectorEntity migrateConnector(final String connectorId, final String p connectorDAO.migrateFromVersionedFlow(connectorId, processGroupId, externalFlow, cancellationCheck); controllerFacade.save(); - final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId); + final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); final ConnectorDTO dto = dtoFactory.createConnectorDto(connectorNode); final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); return new StandardRevisionUpdate<>(dto, lastMod); }); - final ConnectorNode connectorNode = connectorDAO.getConnector(snapshot.getComponent().getId()); + final ConnectorNode connectorNode = connectorDAO.getConnector(snapshot.getComponent().getId(), ConnectorSyncMode.LOCAL_ONLY); final PermissionsDTO permissions = dtoFactory.createPermissionsDto(connectorNode); final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(connectorNode)); final ConnectorStatusDTO statusDto = createConnectorStatusDto(connectorNode); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index 3b2e00520298..e68f1755895b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -325,20 +325,20 @@ public Optional getAsset(final String assetId) { @Override public List getMigrationSources(final String id) { - getConnector(id); + requireConnector(id, ConnectorSyncMode.LOCAL_ONLY); return connectorMigrationManager.listMigrationSources(id); } @Override public void verifyCanMigrateFromVersionedFlow(final String connectorId, final String processGroupId) { - getConnector(connectorId); + requireConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); connectorMigrationManager.verifyEligibility(connectorId, processGroupId); } @Override public void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow, final BooleanSupplier cancellationCheck) { - getConnector(connectorId); + requireConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); try { connectorMigrationManager.migrateFromVersionedFlow(connectorId, processGroupId, sourceFlow, cancellationCheck); } catch (final Exception e) { diff --git a/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts b/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts index b8e82ac3f617..6a8f5098ec71 100644 --- a/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts +++ b/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts @@ -234,6 +234,7 @@ export type ConnectorActionName = | 'DRAIN_FLOWFILES' | 'CANCEL_DRAIN_FLOWFILES' | 'APPLY_UPDATES' + | 'MIGRATE' | 'DELETE'; export interface ConnectorAction { diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java index 2e927bb038a7..0654f5f7655d 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java @@ -63,6 +63,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { // No-op: this connector does not manipulate the flow. diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java index eaae7033a06e..014b6b8eff0d 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java @@ -54,6 +54,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public boolean isMigrationSupported(final ConnectorMigrationContext context) { return true; diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java index f15d668ef51f..aa3a65953af8 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java @@ -110,6 +110,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedExternalFlow createFlowWithBundleScenarios() { final VersionedProcessGroup group = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Bundle Resolution Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java index bb767219154d..cf93d4695452 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java @@ -172,6 +172,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(CALCULATION_STEP); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java index 81c6af171b13..c9fa2df56997 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java @@ -67,6 +67,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedProcessGroup createRootGroup() { final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Component Lifecycle Root"); rootGroup.setPosition(new Position(0, 0)); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java index 1c50941c8371..970427dc60da 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java @@ -63,6 +63,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java index fb3bd1c833bd..c6b3eb81f41c 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java @@ -53,6 +53,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public boolean isMigrationSupported(final ConnectorMigrationContext context) { return true; diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java index 8c7f680d86ba..555f18caf161 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java @@ -94,6 +94,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java index 7f8a30972d41..c09d3db487a6 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java @@ -57,6 +57,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public boolean isMigrationSupported(final ConnectorMigrationContext context) { final VersionedExternalFlow sourceFlow = context.getSourceFlow(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java index e3903a38f1de..86d692826e4e 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java @@ -58,6 +58,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java index 6f7d80a4687e..fa65f3997dbe 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java @@ -39,6 +39,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext workingContext) { } diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java index 3d28240ca256..b07edfddfe07 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java @@ -100,6 +100,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(new ConfigVerificationResult.Builder() diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java index 9d160a11f1c2..f1ca456b1744 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java @@ -134,6 +134,11 @@ public VersionedExternalFlow getInitialFlow() { return createEmptyFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedExternalFlow createEmptyFlow() { final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(ROOT_GROUP_ID, "Parameter Context Test Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java index 441f4bc3d47f..cdd6e49befc5 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java @@ -83,6 +83,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of();