diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java index a263482eda28..d8172b7a2bbe 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java index 1e257c245c6d..75628f7ae574 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java @@ -64,6 +64,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Cron_Schedule_Connector.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(SCHEDULE_STEP); diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java index 96b9fa63c38e..40a6a5faa690 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java @@ -35,6 +35,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java index d49674ffd038..1dad60bc72f0 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java @@ -88,6 +88,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java index bb964a8181ba..71708cdbd103 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java @@ -112,6 +112,11 @@ public VersionedExternalFlow getInitialFlow() { return KafkaToS3FlowBuilder.loadInitialFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException { final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext()); diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index d5e4234bc39f..e28deef9f47b 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -3833,6 +3833,93 @@ The Secrets Manager delegates to Parameter Providers to retrieve secret values. |`nifi.secrets.manager.cache.duration`|The duration for which resolved secret values are cached before being refreshed from the underlying Parameter Providers. Accepts any NiFi time duration value such as `5 mins`, `30 secs`, etc. A value of `0 sec` disables caching entirely. Defaults to `5 mins`. |==== +== Migrating a Connector from a Versioned Process Group + +NiFi supports a one-time path that migrates a newly created Connector from a source Versioned Process Group. +The source Versioned Process Group is used as a reference: the Connector reads it and updates its own managed flow to mirror the source's configuration, parameters, and component state. +The source flow itself is not installed onto the Connector. +Only Connectors that explicitly opt in to migration support can use this capability. + +=== Prerequisites for a local migration + +When the source is a local Process Group on the same NiFi instance, the Process Group must satisfy all of the following requirements before NiFi will allow the migration: + +- The source Process Group must be under version control. +- The source Process Group must be `UP_TO_DATE` with the latest published version in Flow Registry. +- All processors in the source Process Group must be stopped. +- All controller services in the source Process Group must be disabled. +- All queues in the source Process Group must be empty. +- The source Process Group must not reference controller services outside of the Process Group. +- The target Connector must be newly created, stopped, and not previously migrated. +- The target Connector's active flow must be empty when migration begins. Connectors that ship a non-empty initial flow are not compatible with the migration path. + +If the source Process Group is not under version control, export it and use the uploaded payload path instead. + +=== Migration paths + +There are two ways to run the migration: + +- Local Process Group migration: + Select an eligible Process Group that already exists on the current NiFi canvas. + NiFi validates the prerequisites listed above before starting the migration. +- Uploaded payload migration: + Export a Process Group definition and upload the resulting flow snapshot payload to the Connector migration endpoint. + This path is intended for non-version-controlled sources, sources exported from another NiFi instance, or any other source that cannot satisfy the local version-control requirements. + +For uploaded payloads, NiFi does not require the source flow to be under version control. +The Connector's own migration support check determines whether the uploaded flow can be imported. + +=== Running a migration + +. Create a new Connector instance. +. If the source Process Group is local and registry-backed, choose it from the Connector migration source list. +. If the source Process Group is not eligible for the local path, export it with component state included and upload the payload to the Connector migration payload endpoint. +. Start the Connector migration request. +. Poll the request until it completes. +. Open the Connector configuration and supply any missing sensitive values or secrets. +. Verify the configuration and start the Connector. + +=== What NiFi migrates + +During migration, NiFi provides the Connector with the exported flow definition, parameter contexts, referenced assets, and any attached component state for `@Stateful` components. +The Connector performs the flow translation and updates its own managed flow to mirror the source. + +For local migrations, a Connector can also copy referenced assets from the source Process Group into the Connector's own asset namespace. +Uploaded payload migrations do not include live access to source assets, so any required asset content must be re-uploaded separately after the migration. + +Sensitive parameter values are not present in exported flow definitions. +After migration, configure the Connector with any missing secret values before starting it. + +=== Cleanup after a successful migration + +After a successful local migration, NiFi disables the source Process Group and renames it with the `(Migrated) ` prefix. +This makes it clear that the source flow has already been used as the basis for a Connector migration. + +Uploaded payloads and migration request state are kept only in memory for the lifetime of the request. +When the request succeeds, fails, or is cancelled, NiFi removes the uploaded payload associated with that request. + +=== Cluster-topology rule for component state + +When the source flow being migrated includes LOCAL component state for a stateful component (for example a `@Stateful` Processor or Controller Service), the destination cluster must have at least as many connected nodes as the source flow's exported state references. +The rule applied by the migration manager is: + +---- +size(component.localNodeStates) <= number of connected nodes in the destination cluster +---- + +This is the same rule that applies when importing a Versioned Process Group into a destination cluster. +NiFi enforces it for both the local migration path and the uploaded payload path so that LOCAL state can be unambiguously distributed across the destination cluster. + +When the migration request fails with a message such as `Cannot import flow with component state: the flow definition contains local state from N source node(s) but the destination cluster has only M connected node(s)`, take one of the following actions: + +- Reconnect any disconnected cluster nodes so that the destination cluster has at least `N` connected nodes, then re-run the migration. +- Re-export the source flow without component state (uncheck the "include component state" option) and re-upload the payload. + The Connector will start with empty LOCAL state on every destination node, which is appropriate when the LOCAL state is reproducible from the upstream system. +- Migrate into a cluster that has at least `N` connected nodes. + +The migration request is also rejected when any cluster node is in the `CONNECTING`, `DISCONNECTED`, or `DISCONNECTING` state at the time the request is submitted, because component state and asset content cannot be synchronized to a node that is not currently connected. +Reconnect the disconnected nodes and re-submit the request. + [[upgrading_nifi]] == Upgrading NiFi diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc index 1906efacc77b..1a87a8375509 100644 --- a/nifi-docs/src/main/asciidoc/developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc @@ -2697,6 +2697,95 @@ The following command is used to generate a standard binary distribution of Apac `mvn clean install -Pcontrib-check` +== Supporting migration from a Versioned Process Group + +Connectors can opt in to a one-time migration flow that uses a source Versioned Process Group as a reference for populating a freshly created Connector. +This is intended for Connectors that can translate an exported flow definition, its parameter contexts, referenced assets, and any attached component state into their own managed flow. +The source Versioned Process Group is not installed onto the Connector; instead, the Connector reads the source as input and updates its own flow to mirror the source's configuration and state. + +Migration support is optional. +Connectors opt in by implementing the optional capability interface `org.apache.nifi.components.connector.migration.MigratableConnector`. +A Connector that does not implement `MigratableConnector` is never offered as a migration target and continues to behave as before. + +=== Migration contract + +`MigratableConnector` is a separate interface that the Connector class must implement (typically alongside extending `AbstractConnector`). It exposes two methods: + +- `isMigrationSupported(ConnectorMigrationContext)` is a cheap, side-effect-free predicate. + It should inspect `context.getSourceFlow()` and return `true` only when the Connector can be migrated from the source structure. + Implementations should limit themselves to structural checks such as processor types, parameter names, and exported metadata. + Component state and asset content are not available at listing time and must not be relied on; per-state precondition checks belong inside the Connector's `migrate(...)` implementation. + This method must not call `ConnectorMigrationContext.copyAssetFromSource(...)`; the framework wraps the eligibility-time context so any such attempt is rejected with `IllegalStateException` and the Connector is filtered out of the listing. +- `migrate(ConnectorMigrationContext)` performs the actual migration. + The Connector reads the source `VersionedExternalFlow`, translates it as needed into its own representation, and applies the result to its own active `FlowContext`. + +=== Accessing the source flow + +`ConnectorMigrationContext` exposes the source `VersionedExternalFlow` using `getSourceFlow()`. +The source includes: + +- The exported `VersionedProcessGroup` +- Parameter contexts +- Referenced assets +- Attached `VersionedComponentState` for `@Stateful` components +- Export metadata such as flow name, registry identifiers, and version + +The same context also indicates whether the request came from a local Process Group on the same NiFi instance or from an uploaded payload using `isLocalMigration()`. + +=== Updating the Connector's flow + +When a Connector extends `AbstractConnector`, the usual pattern is to retain the `ConnectorInitializationContext` supplied in `initialize(...)` and then update the Connector's own active flow directly from `migrate(...)`. + +[source,java] +---- +@Override +public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final VersionedExternalFlow transformedFlow = transformSourceFlow(context.getSourceFlow()); + getInitializationContext().updateFlow(context.getActiveFlowContext(), transformedFlow); +} +---- + +`context.getActiveFlowContext()` always refers to the Connector's active flow context. +After `migrate(...)` returns successfully, the framework recreates the working flow context from the active flow. + +If the Connector needs to preserve state while reshaping the flow, it can read or rewrite `VersionedConfigurableExtension.componentState` on the exported components before calling `updateFlow(...)`. +When the updated flow is applied, NiFi restores LOCAL and CLUSTER state through the standard flow synchronization path. + +If `migrate(...)` fails, the framework clears LOCAL and CLUSTER state for every Processor and Controller Service currently inside the Connector's flow before reloading the initial flow. +A Connector implementer does not need to track which components it has written state to during a failed migration. + +=== Rewriting parameters and assets + +The exported source flow contains parameter contexts and any asset references that were attached to exported parameters. +Connectors are responsible for mapping those values into their own target shape. + +A Connector can copy an asset from a local source Process Group into its own asset namespace using `ConnectorMigrationContext.copyAssetFromSource(String)`. +This method is available only for local migrations. +It throws `IllegalArgumentException` when the supplied source asset identifier is null or blank, and `IllegalStateException` when invoked for an uploaded-payload migration. +Uploaded payloads do not have access to a live source asset namespace, so asset references from uploaded payloads must be handled outside of the migration request. + +The typical flow for parameter migration is: + +. Read the source parameter values from `context.getSourceFlow().getParameterContexts()` +. Rewrite any values or parameter names that the Connector expects +. Copy any local assets that must move with the flow +. Apply the translated flow to the Connector's own active flow context with `updateFlow(...)` + +=== Sensitive values + +Sensitive parameter values and other secret material are not present in the exported source flow. +Connectors must leave those values unset during migration. +After the migration completes, the user supplies the missing values through the normal Configure step before starting the Connector. + +=== Additional expectations + +- `isMigrationSupported(...)` must not mutate flow state, parameter values, or assets. +- `migrate(...)` should either complete successfully or throw an exception. +- A Connector that supports migration must return an empty (or `null`) initial flow from `getInitialFlow()`. + Migration requires that the target Connector's active flow be empty at the time `migrate(...)` is invoked, so a non-empty initial flow is not compatible with the migration path. +- Migration is one-shot from the framework's point of view. + Once the Connector's flow has been updated by a migration, the Connector has user-installed components, and a subsequent migration request will be rejected until the Connector is discarded and a fresh one is created. + == How to contribute to Apache NiFi We are always excited to have contributions from the community - especially from new contributors! diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java new file mode 100644 index 000000000000..08a17d70c3b4 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationPayloadDTO.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationPayload") +public class MigrationPayloadDTO { + private String payloadId; + + @Schema(description = "The identifier of the uploaded migration payload.") + public String getPayloadId() { + return payloadId; + } + + public void setPayloadId(final String payloadId) { + this.payloadId = payloadId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java new file mode 100644 index 000000000000..9dbbce360cf8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestDTO.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationRequest") +public class MigrationRequestDTO extends AsynchronousRequestDTO { + private String connectorId; + private MigrationRequestLocalSourceDTO localSource; + private String payloadId; + + @Schema(description = "The identifier of the Connector receiving the migration.") + public String getConnectorId() { + return connectorId; + } + + public void setConnectorId(final String connectorId) { + this.connectorId = connectorId; + } + + @Schema(description = "The local Process Group source for the migration request.") + public MigrationRequestLocalSourceDTO getLocalSource() { + return localSource; + } + + public void setLocalSource(final MigrationRequestLocalSourceDTO localSource) { + this.localSource = localSource; + } + + @Schema(description = "The identifier of a previously uploaded migration payload.") + public String getPayloadId() { + return payloadId; + } + + public void setPayloadId(final String payloadId) { + this.payloadId = payloadId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java new file mode 100644 index 000000000000..c0434ea011c9 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationRequestLocalSourceDTO.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationRequestLocalSource") +public class MigrationRequestLocalSourceDTO { + private String processGroupId; + + @Schema(description = "The identifier of the local source Process Group to migrate.") + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java new file mode 100644 index 000000000000..61d4d2b16ca1 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/MigrationUpdateStepDTO.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "migrationUpdateStep") +public class MigrationUpdateStepDTO extends UpdateStepDTO { +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java new file mode 100644 index 000000000000..d38699742e51 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/VersionedFlowMigrationSourceDTO.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "versionedFlowMigrationSource") +public class VersionedFlowMigrationSourceDTO { + private String processGroupId; + private String processGroupName; + private String registryClientId; + private String bucketId; + private String flowId; + private String version; + + @Schema(description = "The identifier of the source Process Group.") + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } + + @Schema(description = "The name of the source Process Group.") + public String getProcessGroupName() { + return processGroupName; + } + + public void setProcessGroupName(final String processGroupName) { + this.processGroupName = processGroupName; + } + + @Schema(description = "The identifier of the Flow Registry client backing the source Process Group.") + public String getRegistryClientId() { + return registryClientId; + } + + public void setRegistryClientId(final String registryClientId) { + this.registryClientId = registryClientId; + } + + @Schema(description = "The Flow Registry bucket identifier for the source Process Group.") + public String getBucketId() { + return bucketId; + } + + public void setBucketId(final String bucketId) { + this.bucketId = bucketId; + } + + @Schema(description = "The Flow Registry flow identifier for the source Process Group.") + public String getFlowId() { + return flowId; + } + + public void setFlowId(final String flowId) { + this.flowId = flowId; + } + + @Schema(description = "The published version of the source Process Group.") + public String getVersion() { + return version; + } + + public void setVersion(final String version) { + this.version = version; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java new file mode 100644 index 000000000000..2a2d716dd83e --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationPayloadEntity.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.MigrationPayloadDTO; + +@XmlRootElement(name = "migrationPayloadEntity") +public class MigrationPayloadEntity extends Entity { + private MigrationPayloadDTO payload; + + @Schema(description = "The uploaded migration payload metadata.") + public MigrationPayloadDTO getPayload() { + return payload; + } + + public void setPayload(final MigrationPayloadDTO payload) { + this.payload = payload; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java new file mode 100644 index 000000000000..7351b4c9b44a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/MigrationRequestEntity.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; + +@XmlRootElement(name = "migrationRequestEntity") +public class MigrationRequestEntity extends Entity { + private MigrationRequestDTO request; + + @Schema(description = "The migration request.") + public MigrationRequestDTO getRequest() { + return request; + } + + public void setRequest(final MigrationRequestDTO request) { + this.request = request; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java new file mode 100644 index 000000000000..6f426b2579a6 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/VersionedFlowMigrationSourcesEntity.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; + +import java.util.List; + +@XmlRootElement(name = "versionedFlowMigrationSourcesEntity") +public class VersionedFlowMigrationSourcesEntity extends Entity { + private List migrationSources; + + @Schema(description = "The Versioned Process Groups that the Connector can be migrated from.") + public List getMigrationSources() { + return migrationSources; + } + + public void setMigrationSources(final List migrationSources) { + this.migrationSources = migrationSources; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java index 6c2b43a3239b..22c19c34a4a5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java @@ -63,6 +63,7 @@ import org.apache.nifi.cluster.coordination.http.endpoints.LabelsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.LatestProvenanceEventsMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ListFlowFilesEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.MigrationRequestEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarDetailsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarSummariesEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.NarSummaryEndpointMerger; @@ -152,6 +153,7 @@ public StandardHttpResponseMapper(final NiFiProperties nifiProperties) { endpointMergers.add(new ConnectorFlowEndpointMerger()); endpointMergers.add(new ConnectorPropertyGroupEndpointMerger()); endpointMergers.add(new ConnectorPropertyGroupNamesEndpointMerger()); + endpointMergers.add(new MigrationRequestEndpointMerger()); endpointMergers.add(new VerifyConnectorConfigStepEndpointMerger()); endpointMergers.add(new ConnectionEndpointMerger()); endpointMergers.add(new ConnectionsEndpointMerger()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java new file mode 100644 index 000000000000..e197d693d056 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMerger.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; + +import java.net.URI; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class MigrationRequestEndpointMerger extends AbstractSingleEntityEndpoint { + public static final Pattern MIGRATION_REQUEST_URI_PATTERN = + Pattern.compile("/nifi-api/connectors/[a-f0-9\\-]{36}/migration-requests(/[a-f0-9\\-]{36})?"); + + @Override + protected Class getEntityClass() { + return MigrationRequestEntity.class; + } + + @Override + public boolean canHandle(final URI uri, final String method) { + return MIGRATION_REQUEST_URI_PATTERN.matcher(uri.getPath()).matches(); + } + + @Override + protected void mergeResponses(final MigrationRequestEntity clientEntity, final Map entityMap, + final Set successfulResponses, final Set problematicResponses) { + final MigrationRequestDTO clientRequest = clientEntity.getRequest(); + if (clientRequest == null) { + return; + } + + final List clientUpdateSteps = clientRequest.getUpdateSteps() == null ? List.of() : clientRequest.getUpdateSteps(); + + for (final Map.Entry nodeEntry : entityMap.entrySet()) { + final NodeIdentifier nodeId = nodeEntry.getKey(); + final MigrationRequestDTO nodeRequest = nodeEntry.getValue().getRequest(); + if (nodeRequest == null) { + continue; + } + + clientRequest.setComplete(clientRequest.isComplete() && nodeRequest.isComplete()); + + if (nodeRequest.getFailureReason() != null) { + final String nodeReason = "Node " + nodeId.getApiAddress() + ":" + nodeId.getApiPort() + ": " + nodeRequest.getFailureReason(); + final String existingReason = clientRequest.getFailureReason(); + clientRequest.setFailureReason(existingReason == null ? nodeReason : existingReason + "; " + nodeReason); + } + + final Date clientLastUpdated = clientRequest.getLastUpdated(); + final Date nodeLastUpdated = nodeRequest.getLastUpdated(); + if (nodeLastUpdated != null && (clientLastUpdated == null || nodeLastUpdated.before(clientLastUpdated))) { + clientRequest.setLastUpdated(nodeLastUpdated); + } + + clientRequest.setPercentCompleted(Math.min(clientRequest.getPercentCompleted(), nodeRequest.getPercentCompleted())); + + mergeUpdateSteps(clientUpdateSteps, nodeRequest.getUpdateSteps()); + } + + // Recompute the human-readable state to reflect any failure surfaced by a remote node so that polling clients + // see a consistent picture rather than the arbitrary client node's view. + if (clientRequest.getFailureReason() != null) { + clientRequest.setState("Failed: " + clientRequest.getFailureReason()); + } else if (clientRequest.isComplete()) { + clientRequest.setState("Complete"); + } + } + + private void mergeUpdateSteps(final List clientSteps, final List nodeSteps) { + if (clientSteps == null || nodeSteps == null) { + return; + } + + final int sharedStepCount = Math.min(clientSteps.size(), nodeSteps.size()); + for (int stepIndex = 0; stepIndex < sharedStepCount; stepIndex++) { + final MigrationUpdateStepDTO clientStep = clientSteps.get(stepIndex); + final MigrationUpdateStepDTO nodeStep = nodeSteps.get(stepIndex); + + clientStep.setComplete(clientStep.isComplete() && nodeStep.isComplete()); + if (nodeStep.getFailureReason() != null && clientStep.getFailureReason() == null) { + clientStep.setFailureReason(nodeStep.getFailureReason()); + } + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java index 9af3b62f2e8e..d438b1a0d28b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java @@ -60,17 +60,22 @@ public ConnectorState getState(final String connectorId) throws IOException { try { final NodeResponse mergedNodeResponse = asyncResponse.awaitMergedResponse(); + final Entity updatedEntity = mergedNodeResponse.getUpdatedEntity(); final Response response = mergedNodeResponse.getClientResponse(); - final int statusCode = response.getStatusInfo().getStatusCode(); - logger.debug("getState: Connector [{}] — merged response status [{}] ({})", connectorId, statusCode, response.getStatusInfo().getReasonPhrase()); - - verifyResponse(response.getStatusInfo(), connectorId); + if (response != null) { + final int statusCode = response.getStatusInfo().getStatusCode(); + logger.debug("getState: Connector [{}] — merged response status [{}] ({})", connectorId, statusCode, response.getStatusInfo().getReasonPhrase()); + verifyResponse(response.getStatusInfo(), connectorId); + } else if (mergedNodeResponse.hasThrowable()) { + throw new IOException("Failed to retrieve Connector state for " + connectorId, mergedNodeResponse.getThrowable()); + } else if (!(updatedEntity instanceof ConnectorEntity)) { + throw new IOException("Received neither response nor merged Connector entity while retrieving state for Connector " + connectorId); + } // Use the merged/updated entity if available, otherwise fall back to reading from the raw response. // The updatedEntity contains the properly merged state from all nodes, while readEntity() would // only return the state from whichever single node was selected as the "client response". final ConnectorEntity connectorEntity; - final Entity updatedEntity = mergedNodeResponse.getUpdatedEntity(); if (updatedEntity instanceof ConnectorEntity mergedConnectorEntity) { logger.debug("getState: Connector [{}] - using merged updatedEntity", connectorId); connectorEntity = mergedConnectorEntity; @@ -115,7 +120,7 @@ private void verifyResponse(final StatusType responseStatusType, final String co final String reason = responseStatusType.getReasonPhrase(); if (responseStatusType.getStatusCode() == Status.NOT_FOUND.getStatusCode()) { - throw new IllegalArgumentException("Connector with ID + " + connectorId + " does not exist"); + throw new IllegalArgumentException("Connector with ID " + connectorId + " does not exist"); } final Family responseFamily = responseStatusType.getFamily(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java new file mode 100644 index 000000000000..7492fc0e3213 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/MigrationRequestEndpointMergerTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MigrationRequestEndpointMergerTest { + + private MigrationRequestEndpointMerger merger; + + @BeforeEach + public void setUp() { + merger = new MigrationRequestEndpointMerger(); + } + + @Test + public void testMergeReflectsWorstCompletionState() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(true, null, 100, new Date(2_000L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(false, null, 40, new Date(1_500L))); + + merger.mergeResponses(client, map, null, null); + + // Any node still in progress drives the merged response to incomplete. + assertFalse(client.getRequest().isComplete()); + } + + @Test + public void testMergePropagatesFailureFromAnyNode() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(true, null, 100, new Date(2_000L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(true, "rollback failed", 100, new Date(2_500L))); + + merger.mergeResponses(client, map, null, null); + + assertNotNull(client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getFailureReason().contains("rollback failed"), client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getFailureReason().contains("node-2"), client.getRequest().getFailureReason()); + assertTrue(client.getRequest().getState().startsWith("Failed: "), client.getRequest().getState()); + } + + @Test + public void testMergePicksMinimumPercentComplete() { + final MigrationRequestEntity client = createMigrationRequest(false, null, 90, new Date(2_000L)); + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), createMigrationRequest(false, null, 60, new Date(2_500L))); + map.put(nodeId("node-2", 8081), createMigrationRequest(false, null, 25, new Date(2_700L))); + + merger.mergeResponses(client, map, null, null); + + assertEquals(25, client.getRequest().getPercentCompleted()); + } + + @Test + public void testMergePropagatesPerStepFailures() { + final MigrationRequestEntity client = createMigrationRequest(true, null, 100, new Date(2_000L)); + final MigrationRequestEntity nodeFailure = createMigrationRequest(true, null, 100, new Date(2_500L)); + nodeFailure.getRequest().getUpdateSteps().get(0).setFailureReason("step failed on node"); + nodeFailure.getRequest().getUpdateSteps().get(0).setComplete(false); + + final Map map = new LinkedHashMap<>(); + map.put(nodeId("node-1", 8080), nodeFailure); + + merger.mergeResponses(client, map, null, null); + + final MigrationUpdateStepDTO mergedStep = client.getRequest().getUpdateSteps().get(0); + assertFalse(mergedStep.isComplete()); + assertEquals("step failed on node", mergedStep.getFailureReason()); + } + + private MigrationRequestEntity createMigrationRequest(final boolean complete, final String failureReason, final int percentComplete, final Date lastUpdated) { + final MigrationRequestDTO request = new MigrationRequestDTO(); + request.setComplete(complete); + request.setFailureReason(failureReason); + request.setPercentCompleted(percentComplete); + request.setLastUpdated(lastUpdated); + + final MigrationUpdateStepDTO step = new MigrationUpdateStepDTO(); + step.setDescription("Migrate Versioned Flow"); + step.setComplete(complete); + request.setUpdateSteps(List.of(step)); + + final MigrationRequestEntity entity = new MigrationRequestEntity(); + entity.setRequest(request); + return entity; + } + + private NodeIdentifier nodeId(final String hostname, final int port) { + return new NodeIdentifier(hostname + "-" + port, hostname, port, hostname, port + 1, hostname, port + 2, port + 3, false); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index d5f1db9d7bee..e277fe74fecd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -63,7 +63,6 @@ import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedComponentState; -import org.apache.nifi.flow.VersionedConfigurableExtension; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedExternalFlow; @@ -4122,45 +4121,7 @@ private Map getPropertyValues(final ComponentNode componentNode) } private void validateLocalStateTopology(final VersionedProcessGroup proposed) { - final int connectedNodeCount = context.getConnectedNodeCount(); - if (connectedNodeCount <= 0) { - return; - } - - final int maxSourceNodes = findMaxLocalStateNodeCount(proposed); - if (maxSourceNodes > connectedNodeCount) { - throw new IllegalStateException( - "Cannot import flow with component state: the flow definition contains local state from %d source node(s) but the destination cluster has only %d connected node(s). " - .formatted(maxSourceNodes, connectedNodeCount) - + "Import into a cluster with at least %d node(s), or export without component state.".formatted(maxSourceNodes)); - } - } - - private int findMaxLocalStateNodeCount(final VersionedProcessGroup group) { - int max = 0; - for (final VersionedConfigurableExtension ext : getStatefulExtensions(group)) { - final VersionedComponentState state = ext.getComponentState(); - if (state != null && state.getLocalNodeStates() != null) { - max = Math.max(max, state.getLocalNodeStates().size()); - } - } - if (group.getProcessGroups() != null) { - for (final VersionedProcessGroup child : group.getProcessGroups()) { - max = Math.max(max, findMaxLocalStateNodeCount(child)); - } - } - return max; - } - - private List getStatefulExtensions(final VersionedProcessGroup group) { - final List extensions = new ArrayList<>(); - if (group.getProcessors() != null) { - extensions.addAll(group.getProcessors()); - } - if (group.getControllerServices() != null) { - extensions.addAll(group.getControllerServices()); - } - return extensions; + VersionedComponentStateValidator.validateLocalStateTopology(proposed, context.getConnectedNodeCount()); } private void restoreComponentState(final String componentId, final VersionedComponentState componentState, final ComponentNode componentNode) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java new file mode 100644 index 000000000000..4610a4acb4e2 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentStateValidator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.synchronization; + +import org.apache.nifi.flow.VersionedComponentState; +import org.apache.nifi.flow.VersionedConfigurableExtension; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.ArrayList; +import java.util.List; + +/** + * Shared helpers that inspect the LOCAL component state contained in an exported {@link VersionedProcessGroup} + * and validate it against the destination cluster topology before importing or migrating the flow. + * + *

This is the single source of truth for the cluster-topology rule used by both the standard versioned-flow + * synchronization path and the connector migration path. Both paths must reject a flow whose LOCAL state + * contains entries for more nodes than the destination cluster currently has connected, because there is no way + * to attribute the extra source-node states to a destination node.

+ */ +public final class VersionedComponentStateValidator { + + private VersionedComponentStateValidator() { + } + + /** + * Validates that the LOCAL component state inside the given proposed {@link VersionedProcessGroup} can be + * imported into a cluster that currently has the given number of connected nodes. + * + * @param proposed the proposed flow to validate + * @param connectedNodeCount the number of nodes that are currently connected to the destination cluster + * @throws IllegalStateException when any stateful component in the proposed flow has more local-node-state entries + * than the destination cluster has connected nodes + */ + public static void validateLocalStateTopology(final VersionedProcessGroup proposed, final int connectedNodeCount) { + if (proposed == null || connectedNodeCount <= 0) { + return; + } + + final int maxSourceNodes = findMaxLocalStateNodeCount(proposed); + if (maxSourceNodes > connectedNodeCount) { + throw new IllegalStateException( + "Cannot import flow with component state: the flow definition contains local state from %d source node(s) but the destination cluster has only %d connected node(s). " + .formatted(maxSourceNodes, connectedNodeCount) + + "Import into a cluster with at least %d node(s), or export without component state.".formatted(maxSourceNodes)); + } + } + + /** + * Returns the maximum number of local-node-state entries declared by any stateful component anywhere in the + * given {@link VersionedProcessGroup} hierarchy. Returns 0 when no component declares LOCAL state. + * + * @param group the root process group to inspect + * @return the maximum local-node-state count across the group hierarchy + */ + public static int findMaxLocalStateNodeCount(final VersionedProcessGroup group) { + if (group == null) { + return 0; + } + + int max = 0; + for (final VersionedConfigurableExtension extension : getStatefulExtensions(group)) { + final VersionedComponentState state = extension.getComponentState(); + if (state != null && state.getLocalNodeStates() != null) { + max = Math.max(max, state.getLocalNodeStates().size()); + } + } + + if (group.getProcessGroups() != null) { + for (final VersionedProcessGroup child : group.getProcessGroups()) { + max = Math.max(max, findMaxLocalStateNodeCount(child)); + } + } + + return max; + } + + private static List getStatefulExtensions(final VersionedProcessGroup group) { + final List extensions = new ArrayList<>(); + if (group.getProcessors() != null) { + extensions.addAll(group.getProcessors()); + } + if (group.getControllerServices() != null) { + extensions.addAll(group.getControllerServices()); + } + return extensions; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index 216e4fd76428..0d96b2eec9c9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -1158,14 +1158,14 @@ private Map mapPropertyValues(final St return versionedProperties; } - private org.apache.nifi.flow.ScheduledState mapConnectorState(final ConnectorState connectorState) { + private org.apache.nifi.flow.VersionedConnectorState mapConnectorState(final ConnectorState connectorState) { if (connectorState == null) { return null; } return switch (connectorState) { - case RUNNING, STARTING -> org.apache.nifi.flow.ScheduledState.RUNNING; - default -> org.apache.nifi.flow.ScheduledState.ENABLED; + case RUNNING, STARTING -> org.apache.nifi.flow.VersionedConnectorState.RUNNING; + default -> org.apache.nifi.flow.VersionedConnectorState.ENABLED; }; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java new file mode 100644 index 000000000000..773f0ed40f05 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorFlowSnapshotProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; + +/** + * Supplies {@link RegisteredFlowSnapshot} instances for Process Groups that are being inspected as + * candidate sources for Connector migration. Implementations delegate to the standard flow snapshot + * facility so that the migration manager and the listing path use a single mechanism for snapshot + * acquisition rather than reimplementing flow mapping in framework-core. + */ +@FunctionalInterface +public interface ConnectorFlowSnapshotProvider { + + /** + * Returns the current flow snapshot for the Process Group with the given identifier, optionally + * including referenced controller services and exported component state. + * + * @param processGroupId the identifier of the Process Group to snapshot + * @param includeReferencedServices whether to include controller services referenced from outside the group + * @param includeComponentState whether to include LOCAL and CLUSTER component state in the snapshot + * @return the flow snapshot + */ + RegisteredFlowSnapshot getCurrentFlowSnapshotByGroupId(String processGroupId, boolean includeReferencedServices, boolean includeComponentState); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java new file mode 100644 index 000000000000..2f2fb67ebfba --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationManager.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.util.List; +import java.util.function.BooleanSupplier; + +/** + * Framework service that drives the eligibility-listing path used by the migration-sources REST endpoint and the + * actual migration path used by the migration-request REST endpoint. Connector translation logic is owned by the + * extension; framework concerns (eligibility gating, source disable/rename, rollback, cluster-topology validation, + * and atomic flag persistence) live here. + */ +public interface ConnectorMigrationManager { + + /** + * Lists the Versioned Process Groups on the canvas that the given Connector can use as a migration source. + * + * @param connectorId the identifier of the target Connector + * @return the eligible source Process Groups, after framework prerequisites and the Connector's own + * {@code isMigrationSupported(...)} filter have been applied + */ + List listMigrationSources(String connectorId); + + /** + * Verifies that the given Process Group satisfies every framework prerequisite required for a local-source + * migration into the given Connector. Throws an {@link IllegalStateException} with a state-specific + * diagnostic when any prerequisite is unmet. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group + */ + void verifyEligibility(String connectorId, String processGroupId); + + /** + * Migrates the target Connector by updating the Connector's own flow to mirror the configuration, parameters, and + * component state captured in the given source flow. When {@code processGroupId} is non-null the migration is + * treated as a local-source migration (and the source Process Group is disabled and renamed on success); when + * {@code processGroupId} is null the migration is treated as an uploaded-payload migration. On any failure during + * the Connector's {@code migrate(...)} call or the framework's post-migrate work, the Connector is rolled back to + * its initial-flow state and the source Process Group is left untouched. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group for local-source migration, or {@code null} + * for an uploaded-payload migration + * @param sourceFlow the source flow whose configuration, parameters, and component state the Connector should + * mirror into its own managed flow + * @throws FlowUpdateException when the migration cannot be completed + */ + default void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow) throws FlowUpdateException { + migrateFromVersionedFlow(connectorId, processGroupId, sourceFlow, () -> false); + } + + /** + * Equivalent to {@link #migrateFromVersionedFlow(String, String, VersionedExternalFlow)} but additionally polls the + * given supplier at framework-controlled checkpoints so a caller-driven cancellation is observed promptly. The + * Connector's own {@code migrate(...)} call is not interrupted; cancellation only takes effect at the next + * framework checkpoint, after which the migration is rolled back as if it had failed. + * + * @param connectorId the identifier of the target Connector + * @param processGroupId the identifier of the source Process Group for local-source migration, or {@code null} + * for an uploaded-payload migration + * @param sourceFlow the source flow whose configuration, parameters, and component state the Connector should + * mirror into its own managed flow + * @param cancellationCheck a supplier that returns {@code true} when the caller has requested cancellation; the + * framework polls it at the natural boundaries between the Connector's {@code migrate(...)} + * call and the subsequent framework post-migrate work + * @throws FlowUpdateException when the migration cannot be completed + */ + void migrateFromVersionedFlow(String connectorId, String processGroupId, VersionedExternalFlow sourceFlow, + BooleanSupplier cancellationCheck) throws FlowUpdateException; +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java new file mode 100644 index 000000000000..4cf65195eb46 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorMigrationSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +/** + * Describes a Versioned Process Group on the canvas that is eligible to be used as the source for a Connector + * migration. Returned by {@link ConnectorMigrationManager#listMigrationSources(String)} and surfaced to clients + * through the {@code GET /connectors/{id}/migration-sources} endpoint. + */ +public class ConnectorMigrationSource { + private String processGroupId; + private String processGroupName; + private String registryClientId; + private String bucketId; + private String flowId; + private String version; + + public String getProcessGroupId() { + return processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } + + public String getProcessGroupName() { + return processGroupName; + } + + public void setProcessGroupName(final String processGroupName) { + this.processGroupName = processGroupName; + } + + public String getRegistryClientId() { + return registryClientId; + } + + public void setRegistryClientId(final String registryClientId) { + this.registryClientId = registryClientId; + } + + public String getBucketId() { + return bucketId; + } + + public void setBucketId(final String bucketId) { + this.bucketId = bucketId; + } + + public String getFlowId() { + return flowId; + } + + public void setFlowId(final String flowId) { + this.flowId = flowId; + } + + public String getVersion() { + return version; + } + + public void setVersion(final String version) { + this.version = version; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java index 08819d38857d..3b216522be09 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java @@ -23,6 +23,7 @@ import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.VersionedComponent; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; import org.apache.nifi.components.validation.ValidationState; import org.apache.nifi.components.validation.ValidationStatus; import org.apache.nifi.components.validation.ValidationTrigger; @@ -81,6 +82,15 @@ public interface ConnectorNode extends ComponentAuthorizable, VersionedComponent void loadInitialFlow() throws FlowUpdateException; + boolean isMigrationSupported(ConnectorMigrationContext context); + + /** + * Discards the working flow context, if any, and rebuilds it from the active flow context's configuration. + * The framework calls this on the success path of a Connector migration so that any subsequent configure-step + * interactions begin from the migrated flow rather than from the working state that existed before migration. + */ + void recreateWorkingFlowContext(); + /** *

* Pause triggering asynchronous validation to occur when the connector is updated. Often times, it is necessary diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java index d8d06bc79efa..fd9189d96ad4 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.Future; @@ -202,6 +203,18 @@ void inheritConfiguration(ConnectorNode connector, List assetIdentifiers); + /** * Ensures that asset binaries for the given connector are available locally by downloading * any missing or changed assets from the external {@link ConnectorConfigurationProvider}. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..61cc0917ee75 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/FrameworkConnectorMigrationContext.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; + +import java.util.Set; + +/** + * Framework-only extension of {@link ConnectorMigrationContext} that exposes the managers needed to + * execute migration and rollback logic. + */ +public interface FrameworkConnectorMigrationContext extends ConnectorMigrationContext { + + @Override + FrameworkFlowContext getActiveFlowContext(); + + AssetManager getSourceAssetManager(); + + ConnectorRepository getConnectorRepository(); + + StateManagerProvider getStateManagerProvider(); + + ClusterTopologyProvider getClusterTopologyProvider(); + + /** + * Returns the identifiers of assets that the running migration attempt has copied into the + * Connector asset namespace via {@link ConnectorMigrationContext#copyAssetFromSource(String)}. + * Used by the framework to scope rollback so it deletes only assets created during the attempt + * rather than every asset on the Connector. + * + * @return the identifiers of assets created during the current migration attempt + */ + Set getCopiedAssetIds(); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java index ada91a1d418f..eeca0c31e149 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorParameterLookup.java @@ -172,13 +172,14 @@ private void collectParameterValues(final VersionedParameterContext context, fin parameterValues.removeIf(param -> param.getName().equals(parameterName)); } + final List referencedAssets = versionedParameter.getReferencedAssets(); final ParameterValue.Builder builder = new ParameterValue.Builder() .name(parameterName) - .value(versionedParameter.getValue()) + .value(referencedAssets == null || referencedAssets.isEmpty() ? versionedParameter.getValue() : null) .sensitive(versionedParameter.isSensitive()); - if (assetManager != null && versionedParameter.getReferencedAssets() != null) { - for (final VersionedAsset versionedAsset : versionedParameter.getReferencedAssets()) { + if (assetManager != null && referencedAssets != null) { + for (final VersionedAsset versionedAsset : referencedAssets) { assetManager.getAsset(versionedAsset.getIdentifier()).ifPresent(builder::addReferencedAsset); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java new file mode 100644 index 000000000000..e0f445c894fc --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/EligibilityConnectorMigrationContext.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.util.Objects; +import java.util.Set; + +/** + * Delegating {@link FrameworkConnectorMigrationContext} used on the listing/eligibility path. The + * {@link org.apache.nifi.components.connector.migration.MigratableConnector#isMigrationSupported(org.apache.nifi.components.connector.migration.ConnectorMigrationContext)} + * contract requires the call to be side-effect-free; in particular it must not invoke + * {@link #copyAssetFromSource(String)}. This wrapper enforces that contract by forwarding every other accessor + * to the delegate while throwing {@link IllegalStateException} from {@code copyAssetFromSource(String)}, even + * when the underlying context would otherwise allow the copy. Implementations that ignore the JavaDoc and try + * to copy assets during eligibility evaluation are caught here rather than silently mutating the Connector's + * asset namespace during a read-only listing. + */ +final class EligibilityConnectorMigrationContext implements FrameworkConnectorMigrationContext { + + private final FrameworkConnectorMigrationContext delegate; + + EligibilityConnectorMigrationContext(final FrameworkConnectorMigrationContext delegate) { + this.delegate = Objects.requireNonNull(delegate, "Delegate FrameworkConnectorMigrationContext is required"); + } + + @Override + public VersionedExternalFlow getSourceFlow() { + return delegate.getSourceFlow(); + } + + @Override + public boolean isLocalMigration() { + return delegate.isLocalMigration(); + } + + @Override + public FrameworkFlowContext getActiveFlowContext() { + return delegate.getActiveFlowContext(); + } + + @Override + public AssetReference copyAssetFromSource(final String sourceAssetId) { + throw new IllegalStateException("copyAssetFromSource(...) must not be invoked from isMigrationSupported(...); the listing path is read-only."); + } + + @Override + public AssetManager getSourceAssetManager() { + return delegate.getSourceAssetManager(); + } + + @Override + public ConnectorRepository getConnectorRepository() { + return delegate.getConnectorRepository(); + } + + @Override + public StateManagerProvider getStateManagerProvider() { + return delegate.getStateManagerProvider(); + } + + @Override + public ClusterTopologyProvider getClusterTopologyProvider() { + return delegate.getClusterTopologyProvider(); + } + + @Override + public Set getCopiedAssetIds() { + return delegate.getCopiedAssetIds(); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java index 0508821e9d1f..c5c43314668b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java new file mode 100644 index 000000000000..6fe34369f93b --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorMigrationManager.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.components.connector.components.ConnectionFacade; +import org.apache.nifi.components.connector.components.ControllerServiceFacade; +import org.apache.nifi.components.connector.components.ProcessGroupFacade; +import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.connector.migration.MigratableConnector; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.queue.QueueSize; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceState; +import org.apache.nifi.flow.ExternalControllerServiceReference; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedExternalFlowMetadata; +import org.apache.nifi.flow.synchronization.VersionedComponentStateValidator; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshotMetadata; +import org.apache.nifi.registry.flow.VersionControlInformation; +import org.apache.nifi.registry.flow.VersionedFlowState; +import org.apache.nifi.registry.flow.VersionedFlowStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BooleanSupplier; + +public class StandardConnectorMigrationManager implements ConnectorMigrationManager { + private static final Logger logger = LoggerFactory.getLogger(StandardConnectorMigrationManager.class); + + private static final String MIGRATED_SOURCE_PREFIX = "(Migrated) "; + + private final FlowController flowController; + private final ConnectorFlowSnapshotProvider snapshotProvider; + + public StandardConnectorMigrationManager(final FlowController flowController, final ConnectorFlowSnapshotProvider snapshotProvider) { + this.flowController = Objects.requireNonNull(flowController, "FlowController is required"); + this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "ConnectorFlowSnapshotProvider is required"); + } + + @Override + public List listMigrationSources(final String connectorId) { + final ConnectorNode connector = getRequiredConnector(connectorId); + final List migrationSources = new ArrayList<>(); + + for (final ProcessGroup processGroup : getCandidateSourceGroups()) { + if (isEligibleSource(connector, processGroup)) { + migrationSources.add(toMigrationSource(processGroup.getVersionControlInformation(), processGroup)); + } + } + + return migrationSources; + } + + @Override + public void verifyEligibility(final String connectorId, final String processGroupId) { + final ConnectorNode connector = getRequiredConnector(connectorId); + final ProcessGroup processGroup = getRequiredSourceProcessGroup(processGroupId); + final String ineligibilityReason = getIneligibilityReason(connector, processGroup); + if (ineligibilityReason != null) { + throw new IllegalStateException(ineligibilityReason); + } + } + + @Override + public void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow, + final BooleanSupplier cancellationCheck) throws FlowUpdateException { + final BooleanSupplier cancellation = cancellationCheck == null ? () -> false : cancellationCheck; + final ConnectorNode connector = getRequiredConnector(connectorId); + verifyConnectorCanReceiveMigration(connector); + verifyTargetIsAtInitialFlow(connector); + validateMigrationSource(sourceFlow); + + final boolean localMigration = processGroupId != null; + final ProcessGroup sourceProcessGroup = localMigration ? getRequiredSourceProcessGroup(processGroupId) : null; + final StandardFrameworkConnectorMigrationContext migrationContext = new StandardFrameworkConnectorMigrationContext( + connectorId, + sourceFlow, + localMigration, + connector.getActiveFlowContext(), + flowController.getAssetManager(), + flowController.getConnectorRepository(), + flowController.getStateManagerProvider(), + flowController + ); + + final Connector rawConnector = connector.getConnector(); + if (!(rawConnector instanceof final MigratableConnector migratableConnector)) { + throw new FlowUpdateException("Connector " + connectorId + " does not support migration from the provided source flow."); + } + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), rawConnector.getClass(), connectorId)) { + if (!migratableConnector.isMigrationSupported(migrationContext)) { + throw new FlowUpdateException("Connector " + connectorId + " does not support migration from the provided source flow."); + } + } + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), rawConnector.getClass(), connectorId)) { + flowController.getConnectorRepository().syncAssetsFromProvider(connector); + migratableConnector.migrate(migrationContext); + + if (cancellation.getAsBoolean()) { + throw new FlowUpdateException("Migration of Connector " + connectorId + " was cancelled after the Connector applied the source flow."); + } + } catch (final FlowUpdateException e) { + rollbackMigration(connector, migrationContext); + throw e; + } catch (final Exception e) { + rollbackMigration(connector, migrationContext); + throw new FlowUpdateException("Failed to migrate Connector " + connectorId + " from the provided source flow", e); + } + + try { + connector.recreateWorkingFlowContext(); + + if (cancellation.getAsBoolean()) { + throw new FlowUpdateException("Migration of Connector " + connectorId + " was cancelled before the source Process Group could be finalized."); + } + + if (sourceProcessGroup != null) { + disableAndRenameSourceProcessGroup(sourceProcessGroup); + } + } catch (final FlowUpdateException e) { + rollbackMigration(connector, migrationContext); + throw e; + } catch (final Exception e) { + rollbackMigration(connector, migrationContext); + throw new FlowUpdateException("Failed to finalize migrated configuration for Connector " + connectorId, e); + } + } + + private void rollbackMigration(final ConnectorNode connector, final StandardFrameworkConnectorMigrationContext migrationContext) { + clearConnectorComponentState(connector); + + final Set copiedAssetIds = migrationContext.getCopiedAssetIds(); + if (!copiedAssetIds.isEmpty()) { + try { + flowController.getConnectorRepository().deleteAssets(connector.getIdentifier(), copiedAssetIds); + } catch (final Exception e) { + logger.warn("Failed to delete copied assets {} for Connector {} during migration rollback", copiedAssetIds, connector.getIdentifier(), e); + } + } + + try { + connector.loadInitialFlow(); + } catch (final FlowUpdateException e) { + // Do not propagate the rollback failure: the caller's primary need is to see why the migration originally + // failed, not how the recovery attempt failed. The operator can read the rollback failure from the log. + logger.error("Failed to restore Connector {} to its initial state after migration failure; manual cleanup may be required", + connector.getIdentifier(), e); + return; + } + + flowController.getConnectorRepository().discardWorkingConfiguration(connector); + } + + private void clearConnectorComponentState(final ConnectorNode connector) { + // Migration is only permitted on a Connector whose active flow is empty, so every Processor and Controller + // Service currently inside the Connector's managed flow was created by the failed migrate(...) call. Clearing + // state for those components is safe and avoids the brittle approach of trying to map source-flow versioned + // identifiers to runtime component identifiers. + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + if (activeFlowContext == null) { + return; + } + + final ProcessGroup managedGroup = activeFlowContext.getManagedProcessGroup(); + if (managedGroup == null) { + return; + } + + for (final ProcessorNode processor : managedGroup.findAllProcessors()) { + clearComponentState(processor.getIdentifier()); + } + for (final ControllerServiceNode controllerService : managedGroup.findAllControllerServices()) { + clearComponentState(controllerService.getIdentifier()); + } + } + + private void clearComponentState(final String componentIdentifier) { + final StateManager stateManager = flowController.getStateManagerProvider().getStateManager(componentIdentifier); + if (stateManager == null) { + return; + } + + try { + stateManager.clear(Scope.LOCAL); + } catch (final Exception e) { + logger.warn("Failed to clear LOCAL state for component {} during migration rollback", componentIdentifier, e); + } + + try { + stateManager.clear(Scope.CLUSTER); + } catch (final Exception e) { + logger.warn("Failed to clear CLUSTER state for component {} during migration rollback", componentIdentifier, e); + } + } + + private ConnectorNode getRequiredConnector(final String connectorId) { + final ConnectorNode connector = flowController.getConnectorRepository().getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + if (connector == null) { + throw new IllegalArgumentException("Could not find Connector with ID " + connectorId); + } + + return connector; + } + + private ProcessGroup getRequiredSourceProcessGroup(final String processGroupId) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + final ProcessGroup processGroup = rootGroup == null ? null : rootGroup.findProcessGroup(processGroupId); + if (processGroup == null) { + throw new IllegalArgumentException("Could not find Process Group with ID " + processGroupId); + } + + return processGroup; + } + + private List getCandidateSourceGroups() { + final FlowManager flowManager = flowController.getFlowManager(); + final ProcessGroup rootGroup = flowManager.getRootGroup(); + if (rootGroup == null) { + return Collections.emptyList(); + } + + return new ArrayList<>(rootGroup.findAllProcessGroups()); + } + + private boolean isEligibleSource(final ConnectorNode connector, final ProcessGroup processGroup) { + return getIneligibilityReason(connector, processGroup) == null; + } + + private String getIneligibilityReason(final ConnectorNode connector, final ProcessGroup processGroup) { + final VersionControlInformation versionControlInformation = processGroup.getVersionControlInformation(); + if (versionControlInformation == null) { + return "Process Group " + processGroup.getIdentifier() + " is not under version control."; + } + + if (processGroup.getConnectorIdentifier().isPresent()) { + return "Process Group " + processGroup.getIdentifier() + " is managed by a Connector and cannot be used as a migration source."; + } + + final VersionedFlowStatus versionedFlowStatus = versionControlInformation.getStatus(); + final VersionedFlowState state = versionedFlowStatus == null ? null : versionedFlowStatus.getState(); + if (state != VersionedFlowState.UP_TO_DATE) { + return describeIneligibleVersionedFlowState(processGroup, state); + } + + if (hasRunningProcessors(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " has running or enabled processors."; + } + + if (hasEnabledControllerServices(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " has enabled controller services."; + } + + if (hasQueuedData(processGroup)) { + return "Process Group " + processGroup.getIdentifier() + " contains queued FlowFiles."; + } + + final VersionedExternalFlow sourceFlow = obtainSourceFlowForListing(processGroup); + if (!Optional.ofNullable(sourceFlow.getExternalControllerServices()).orElse(Collections.emptyMap()).isEmpty()) { + return "Process Group " + processGroup.getIdentifier() + " references controller services outside the Process Group."; + } + + final StandardFrameworkConnectorMigrationContext underlyingContext = new StandardFrameworkConnectorMigrationContext( + connector.getIdentifier(), + sourceFlow, + true, + connector.getActiveFlowContext(), + flowController.getAssetManager(), + flowController.getConnectorRepository(), + flowController.getStateManagerProvider(), + flowController + ); + final FrameworkConnectorMigrationContext migrationContext = new EligibilityConnectorMigrationContext(underlyingContext); + + return connector.isMigrationSupported(migrationContext) + ? null + : "Connector " + connector.getIdentifier() + " does not support migration from Process Group " + processGroup.getIdentifier() + "."; + } + + private String describeIneligibleVersionedFlowState(final ProcessGroup processGroup, final VersionedFlowState state) { + final String prefix = "Process Group " + processGroup.getIdentifier() + " "; + if (state == null) { + return prefix + "has no published version control state available."; + } + + return switch (state) { + case STALE -> prefix + "is stale with respect to its registered flow; bring it back to the latest published version before migrating."; + case LOCALLY_MODIFIED -> prefix + "has local modifications relative to its registered flow; revert or publish them before migrating."; + case LOCALLY_MODIFIED_AND_STALE -> prefix + "is stale and has local modifications; revert them and refresh to the latest version before migrating."; + case SYNC_FAILURE -> prefix + "failed to synchronize with its flow registry; restore connectivity and refresh the version control status before migrating."; + default -> prefix + "is not up to date with its registered flow."; + }; + } + + private void verifyConnectorCanReceiveMigration(final ConnectorNode connector) { + if (connector.getCurrentState() != ConnectorState.STOPPED || connector.getDesiredState() != ConnectorState.STOPPED) { + throw new IllegalStateException("Connector " + connector.getIdentifier() + " must be stopped before it can be migrated."); + } + } + + private void verifyTargetIsAtInitialFlow(final ConnectorNode connector) { + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + if (activeFlowContext == null) { + return; + } + + final ProcessGroupFacade rootGroup = activeFlowContext.getRootGroup(); + if (rootGroup == null) { + return; + } + + if (containsUserInstalledComponents(rootGroup)) { + throw new IllegalStateException("Connector " + connector.getIdentifier() + + " already contains user-installed components in its active flow; migration is only supported for a freshly created Connector."); + } + } + + private boolean containsUserInstalledComponents(final ProcessGroupFacade processGroup) { + final Set processors = processGroup.getProcessors(); + if (processors != null && !processors.isEmpty()) { + return true; + } + + final Set controllerServices = processGroup.getControllerServices(); + if (controllerServices != null && !controllerServices.isEmpty()) { + return true; + } + + final Set connections = processGroup.getConnections(); + if (connections != null && !connections.isEmpty()) { + return true; + } + + final Set childGroups = processGroup.getProcessGroups(); + if (childGroups == null) { + return false; + } + + for (final ProcessGroupFacade childGroup : childGroups) { + if (containsUserInstalledComponents(childGroup)) { + return true; + } + } + + return false; + } + + private void validateMigrationSource(final VersionedExternalFlow sourceFlow) { + final Map externalControllerServices = + Optional.ofNullable(sourceFlow.getExternalControllerServices()).orElse(Collections.emptyMap()); + if (!externalControllerServices.isEmpty()) { + throw new IllegalStateException("Connector cannot reference services outside its managed flow."); + } + + final int connectedNodeCount = flowController.getConnectedNodeCount(); + VersionedComponentStateValidator.validateLocalStateTopology(sourceFlow.getFlowContents(), connectedNodeCount); + } + + private void disableAndRenameSourceProcessGroup(final ProcessGroup processGroup) { + disableSourceProcessors(processGroup); + disableSourceControllerServices(processGroup); + + final String currentName = processGroup.getName(); + if (currentName != null && currentName.startsWith(MIGRATED_SOURCE_PREFIX)) { + return; + } + + processGroup.setName(MIGRATED_SOURCE_PREFIX + currentName); + } + + private void disableSourceProcessors(final ProcessGroup processGroup) { + for (final ProcessorNode processor : processGroup.findAllProcessors()) { + if (processor.getDesiredState() == ScheduledState.DISABLED) { + continue; + } + + final ProcessGroup parent = processor.getProcessGroup(); + if (parent == null) { + continue; + } + + try { + parent.disableProcessor(processor); + } catch (final Exception e) { + logger.warn("Failed to disable processor {} while finalizing migrated source Process Group {}", processor.getIdentifier(), processGroup.getIdentifier(), e); + } + } + } + + private void disableSourceControllerServices(final ProcessGroup processGroup) { + final List enabledServices = new ArrayList<>(); + for (final ControllerServiceNode controllerService : processGroup.findAllControllerServices()) { + if (controllerService.getState() != ControllerServiceState.DISABLED) { + enabledServices.add(controllerService); + } + } + + if (enabledServices.isEmpty()) { + return; + } + + try { + flowController.getControllerServiceProvider().disableControllerServicesAsync(enabledServices); + } catch (final Exception e) { + logger.warn("Failed to disable controller services while finalizing migrated source Process Group {}", processGroup.getIdentifier(), e); + } + } + + private boolean hasRunningProcessors(final ProcessGroup processGroup) { + return processGroup.findAllProcessors().stream() + .anyMatch(processor -> processor.getPhysicalScheduledState() != ScheduledState.STOPPED && processor.getPhysicalScheduledState() != ScheduledState.DISABLED); + } + + private boolean hasEnabledControllerServices(final ProcessGroup processGroup) { + return processGroup.findAllControllerServices().stream() + .anyMatch(controllerService -> controllerService.getState() != ControllerServiceState.DISABLED); + } + + private boolean hasQueuedData(final ProcessGroup processGroup) { + for (final Connection connection : processGroup.findAllConnections()) { + final QueueSize queueSize = connection.getFlowFileQueue().size(); + if (queueSize.getObjectCount() > 0) { + return true; + } + } + + return false; + } + + private ConnectorMigrationSource toMigrationSource(final VersionControlInformation versionControlInformation, final ProcessGroup processGroup) { + final ConnectorMigrationSource migrationSource = new ConnectorMigrationSource(); + migrationSource.setProcessGroupId(processGroup.getIdentifier()); + migrationSource.setProcessGroupName(processGroup.getName()); + migrationSource.setRegistryClientId(versionControlInformation.getRegistryIdentifier()); + migrationSource.setBucketId(versionControlInformation.getBucketIdentifier()); + migrationSource.setFlowId(versionControlInformation.getFlowIdentifier()); + migrationSource.setVersion(versionControlInformation.getVersion()); + return migrationSource; + } + + private VersionedExternalFlow obtainSourceFlowForListing(final ProcessGroup processGroup) { + // Listing-time eligibility checks intentionally exclude component state. Materializing component state is + // expensive and is only required for the actual migration. The Connector's isMigrationSupported(...) must rely + // solely on structural information (processor types, parameter names, exported metadata) at listing time; + // per-state precondition checks belong inside the Connector's own migrate(...) implementation. + final RegisteredFlowSnapshot snapshot = snapshotProvider.getCurrentFlowSnapshotByGroupId(processGroup.getIdentifier(), true, false); + return createExternalFlow(snapshot); + } + + public VersionedExternalFlow createExternalFlow(final RegisteredFlowSnapshot flowSnapshot) { + final VersionedExternalFlowMetadata externalFlowMetadata = new VersionedExternalFlowMetadata(); + final RegisteredFlowSnapshotMetadata snapshotMetadata = flowSnapshot.getSnapshotMetadata(); + if (snapshotMetadata != null) { + externalFlowMetadata.setAuthor(snapshotMetadata.getAuthor()); + externalFlowMetadata.setBucketIdentifier(snapshotMetadata.getBucketIdentifier()); + externalFlowMetadata.setComments(snapshotMetadata.getComments()); + externalFlowMetadata.setFlowIdentifier(snapshotMetadata.getFlowIdentifier()); + externalFlowMetadata.setTimestamp(snapshotMetadata.getTimestamp()); + externalFlowMetadata.setVersion(snapshotMetadata.getVersion()); + } + + final RegisteredFlow registeredFlow = flowSnapshot.getFlow(); + if (registeredFlow == null) { + externalFlowMetadata.setFlowName(flowSnapshot.getFlowContents().getName()); + } else { + externalFlowMetadata.setFlowName(registeredFlow.getName()); + } + + final VersionedExternalFlow externalFlow = new VersionedExternalFlow(); + externalFlow.setFlowContents(flowSnapshot.getFlowContents()); + externalFlow.setExternalControllerServices(flowSnapshot.getExternalControllerServices()); + externalFlow.setParameterContexts(flowSnapshot.getParameterContexts()); + externalFlow.setParameterProviders(flowSnapshot.getParameterProviders()); + externalFlow.setMetadata(externalFlowMetadata); + return externalFlow; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index b9659ebb4860..e4815b78a6ed 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -29,6 +29,8 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.components.ParameterContextFacade; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; import org.apache.nifi.components.validation.DisabledServiceValidationResult; import org.apache.nifi.components.validation.ValidationState; import org.apache.nifi.components.validation.ValidationStatus; @@ -755,6 +757,20 @@ public void loadInitialFlow() throws FlowUpdateException { recreateWorkingFlowContext(); } + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { + final Connector connector = getConnector(); + if (!(connector instanceof final MigratableConnector migratableConnector)) { + return false; + } + return migratableConnector.isMigrationSupported(context); + } catch (final Exception e) { + getComponentLog().warn("Failed to evaluate whether migration is supported for {}; assuming migration is not supported", context.getSourceFlow(), e); + return false; + } + } + private void stopComponents(final VersionedProcessGroup group) { for (final VersionedProcessor processor : group.getProcessors()) { if (processor.getScheduledState() == ScheduledState.RUNNING) { @@ -868,7 +884,8 @@ private boolean isBundleResolvable(final String componentType, final Bundle curr return availableBundles.size() == 1; } - private void recreateWorkingFlowContext() { + @Override + public void recreateWorkingFlowContext() { destroyWorkingContext(); workingFlowContext = flowContextFactory.createWorkingFlowContext(identifier, connectorDetails.getComponentLog(), activeFlowContext.getConfigurationContext(), activeFlowContext.getBundle()); @@ -1287,11 +1304,30 @@ public List getAvailableActions() { actions.add(createDrainFlowFilesAction(stopped, dataQueued)); actions.add(createCancelDrainFlowFilesAction(currentState == ConnectorState.DRAINING)); actions.add(createApplyUpdatesAction(currentState)); + actions.add(createMigrateAction(stopped)); actions.add(createDeleteAction(stopped, dataQueued)); return actions; } + private ConnectorAction createMigrateAction(final boolean stopped) { + final boolean allowed; + final String reason; + + if (!stopped) { + allowed = false; + reason = "Connector must be stopped"; + } else if (!(getConnector() instanceof MigratableConnector)) { + allowed = false; + reason = "Connector does not support migration from a Versioned flow"; + } else { + allowed = true; + reason = null; + } + + return new StandardConnectorAction("MIGRATE", "Migrate a Versioned flow's assets and configuration into this Connector", allowed, reason); + } + private boolean isStopped() { final ConnectorState currentState = getCurrentState(); if (currentState == ConnectorState.STOPPED) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index bdb527fecc81..1171febf54be 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -29,9 +29,13 @@ import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; +import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.util.BundleUtils; import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.web.api.dto.BundleDTO; @@ -42,6 +46,7 @@ import java.io.InputStream; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -53,6 +58,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; @@ -104,16 +111,17 @@ public void addConnector(final ConnectorNode connector) { @Override public void restoreConnector(final ConnectorNode connector) { connectors.put(connector.getIdentifier(), connector); + cleanUpAssets(connector); logger.debug("Successfully restored {}", connector); } @Override public ConnectorSyncResult syncConnector(final VersionedConnector versionedConnector) { final String connectorId = versionedConnector.getInstanceIdentifier(); - final ScheduledState proposedScheduledState = versionedConnector.getScheduledState(); + final ScheduledState proposedScheduledState = toScheduledState(versionedConnector.getScheduledState()); logger.debug("syncConnector called for connector [{}]", connectorId); - // Consult the provider for external state checks and working config + // Consult the provider for external state checks and any externally managed working configuration. final ConnectorSyncDirective directive; if (configurationProvider != null) { try { @@ -158,7 +166,6 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne } // ALLOW: proceed with sync - // Look up or create the connector node ConnectorNode connector = connectors.get(connectorId); final boolean isNewConnector = connector == null; if (isNewConnector) { @@ -188,7 +195,7 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne return ConnectorSyncResult.rejected(connector); } - // Determine effective name, working config, and ScheduledState + // Determine the connector name, working configuration, and ScheduledState that should win for this sync pass. final ConnectorWorkingConfiguration providerConfig = directive.getWorkingConfiguration(); // Enrich provider-sourced SECRET_REFERENCE values with providerId before they are compared @@ -214,7 +221,6 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne // Set name locally (no provider.save()) connector.setName(effectiveName); - // Compare config and sync if changed final boolean wasRunning = currentState == ConnectorState.RUNNING; final boolean configChanged = isNewConnector || isConfigurationUpdated(connector, effectiveActiveConfig, effectiveWorkingConfig); @@ -227,8 +233,8 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne if (wasRunning) { try { stopConnector(connector); - } catch (final Exception stopEx) { - logger.error("{} also failed to stop after sync failure", connector, stopEx); + } catch (final Exception stopException) { + logger.error("{} also failed to stop after sync failure", connector, stopException); } } return ConnectorSyncResult.failed(connector); @@ -305,8 +311,6 @@ private ConnectorState waitForStableState(final ConnectorNode connector, final C return current; } - // --- Configuration comparison --- - private boolean isConfigurationUpdated(final ConnectorNode existingConnector, final List effectiveActiveConfig, final List effectiveWorkingConfig) { @@ -360,10 +364,10 @@ private boolean isConfigurationStepUpdated(final NamedStepConfiguration existing for (final Map.Entry versionedEntry : versionedProperties.entrySet()) { final String propertyName = versionedEntry.getKey(); - final VersionedConnectorValueReference versionedRef = versionedEntry.getValue(); - final ConnectorValueReference existingRef = existingProperties.get(propertyName); + final VersionedConnectorValueReference versionedValueReference = versionedEntry.getValue(); + final ConnectorValueReference existingValueReference = existingProperties.get(propertyName); - if (!valueReferencesEqual(versionedRef, existingRef)) { + if (!valueReferencesEqual(versionedValueReference, existingValueReference)) { return true; } } @@ -387,9 +391,9 @@ private boolean valueReferencesEqual(final VersionedConnectorValueReference vers return switch (existingReference) { case StringLiteralValue stringLiteral -> Objects.equals(stringLiteral.getValue(), versionedReference.getValue()); - case AssetReference assetRef -> Objects.equals(assetRef.getAssetIdentifiers(), versionedReference.getAssetIds()); - case SecretReference secretRef -> Objects.equals(secretRef.getProviderId(), versionedReference.getProviderId()) - && Objects.equals(secretRef.getSecretName(), versionedReference.getSecretName()); + case AssetReference assetReference -> Objects.equals(assetReference.getAssetIdentifiers(), versionedReference.getAssetIds()); + case SecretReference secretReference -> Objects.equals(secretReference.getProviderId(), versionedReference.getProviderId()) + && Objects.equals(secretReference.getSecretName(), versionedReference.getSecretName()); }; } @@ -429,9 +433,9 @@ private void stopConnectorAndAwait(final ConnectorNode connector) { logger.info("Stopping connector [{}] (current state: {}) and awaiting completion", connectorId, currentState); try { final Future stopFuture = stopConnector(connector); - stopFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + stopFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] stopped successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to stop", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -446,9 +450,9 @@ private void purgeConnectorAndAwait(final ConnectorNode connector) { try { logger.debug("Purging FlowFiles for connector [{}] before removal", connectorId); final Future purgeFuture = connector.purgeFlowFiles("Flow Synchronization"); - purgeFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + purgeFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] purged successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to purge FlowFiles", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -596,7 +600,16 @@ private void waitForState(final ConnectorNode connector, final Set referencedAssetIds = new HashSet<>(); collectReferencedAssetIds(connector.getActiveFlowContext(), referencedAssetIds); @@ -651,6 +668,11 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f return; } + final ProcessGroup managedProcessGroup = flowContext.getManagedProcessGroup(); + if (managedProcessGroup != null) { + collectReferencedParameterContextAssetIds(managedProcessGroup.getParameterContext(), referencedAssetIds); + } + final ConnectorConfiguration configuration = flowContext.getConfigurationContext().toConnectorConfiguration(); for (final NamedStepConfiguration namedStepConfiguration : configuration.getNamedStepConfigurations()) { final StepConfiguration stepConfiguration = namedStepConfiguration.configuration(); @@ -666,6 +688,23 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f } } + private void collectReferencedParameterContextAssetIds(final ParameterContext parameterContext, final Set referencedAssetIds) { + if (parameterContext == null) { + return; + } + + for (final Parameter parameter : parameterContext.getEffectiveParameters().values()) { + final List referencedAssets = parameter.getReferencedAssets(); + if (referencedAssets == null) { + continue; + } + + for (final Asset referencedAsset : referencedAssets) { + referencedAssetIds.add(referencedAsset.getIdentifier()); + } + } + } + @Override public void updateConnector(final ConnectorNode connector, final String name) { if (configurationProvider != null) { @@ -767,16 +806,30 @@ public List getAssets(final String connectorId) { public void deleteAssets(final String connectorId) { final List assets = assetManager.getAssets(connectorId); for (final Asset asset : assets) { - try { - if (configurationProvider != null) { - // When a provider is configured, delegate entirely to the provider, which should delete from the AssetManager and sync to the external store. - configurationProvider.deleteAsset(connectorId, asset.getIdentifier()); - } else { - assetManager.deleteAsset(asset.getIdentifier()); - } - } catch (final Exception e) { - logger.warn("Failed to delete asset [nifiUuid={}] for connector [{}]", asset.getIdentifier(), connectorId, e); + deleteAsset(connectorId, asset.getIdentifier()); + } + } + + @Override + public void deleteAssets(final String connectorId, final Collection assetIdentifiers) { + if (assetIdentifiers == null || assetIdentifiers.isEmpty()) { + return; + } + + for (final String assetIdentifier : assetIdentifiers) { + deleteAsset(connectorId, assetIdentifier); + } + } + + private void deleteAsset(final String connectorId, final String assetIdentifier) { + try { + if (configurationProvider != null) { + configurationProvider.deleteAsset(connectorId, assetIdentifier); + } else { + assetManager.deleteAsset(assetIdentifier); } + } catch (final Exception e) { + logger.warn("Failed to delete asset [nifiUuid={}] for connector [{}]", assetIdentifier, connectorId, e); } } @@ -807,12 +860,12 @@ private void syncFromProvider(final ConnectorNode connector) { return; } - final ConnectorWorkingConfiguration config = externalConfig.get(); - if (config.getName() != null) { - connector.setName(config.getName()); + final ConnectorWorkingConfiguration externalWorkingConfiguration = externalConfig.get(); + if (externalWorkingConfiguration.getName() != null) { + connector.setName(externalWorkingConfiguration.getName()); } - final List workingFlowConfiguration = config.getWorkingFlowConfiguration(); + final List workingFlowConfiguration = externalWorkingConfiguration.getWorkingFlowConfiguration(); if (workingFlowConfiguration == null) { return; @@ -838,10 +891,10 @@ private void syncFromProvider(final ConnectorNode connector) { } private ConnectorWorkingConfiguration buildWorkingConfiguration(final ConnectorNode connector) { - final ConnectorWorkingConfiguration config = new ConnectorWorkingConfiguration(); - config.setName(connector.getName()); - config.setWorkingFlowConfiguration(buildVersionedConfigurationSteps(connector.getWorkingFlowContext())); - return config; + final ConnectorWorkingConfiguration workingConfiguration = new ConnectorWorkingConfiguration(); + workingConfiguration.setName(connector.getName()); + workingConfiguration.setWorkingFlowConfiguration(buildVersionedConfigurationSteps(connector.getWorkingFlowContext())); + return workingConfiguration; } private List buildVersionedConfigurationSteps(final FrameworkFlowContext flowContext) { @@ -922,12 +975,12 @@ private VersionedConnectorValueReference toVersionedValueReference(final Connect switch (valueReference) { case StringLiteralValue stringLiteral -> versionedReference.setValue(stringLiteral.getValue()); - case AssetReference assetRef -> versionedReference.setAssetIds(assetRef.getAssetIdentifiers()); - case SecretReference secretRef -> { - versionedReference.setProviderId(secretRef.getProviderId()); - versionedReference.setProviderName(secretRef.getProviderName()); - versionedReference.setSecretName(secretRef.getSecretName()); - versionedReference.setFullyQualifiedSecretName(secretRef.getFullyQualifiedName()); + case AssetReference assetReference -> versionedReference.setAssetIds(assetReference.getAssetIdentifiers()); + case SecretReference secretReference -> { + versionedReference.setProviderId(secretReference.getProviderId()); + versionedReference.setProviderName(secretReference.getProviderName()); + versionedReference.setSecretName(secretReference.getSecretName()); + versionedReference.setFullyQualifiedSecretName(secretReference.getFullyQualifiedName()); } } @@ -939,9 +992,9 @@ private StepConfiguration toStepConfiguration(final VersionedConfigurationStep s final Map versionedProperties = step.getProperties(); if (versionedProperties != null) { for (final Map.Entry entry : versionedProperties.entrySet()) { - final VersionedConnectorValueReference versionedRef = entry.getValue(); - if (versionedRef != null) { - propertyValues.put(entry.getKey(), toConnectorValueReference(versionedRef)); + final VersionedConnectorValueReference versionedValueReference = entry.getValue(); + if (versionedValueReference != null) { + propertyValues.put(entry.getKey(), toConnectorValueReference(versionedValueReference)); } } } @@ -982,16 +1035,25 @@ private void resolveSecretReferencesFromProvider(final List properties.values().stream()) - .filter(Objects::nonNull) - .filter(ref -> ConnectorValueType.SECRET_REFERENCE.name().equals(ref.getValueType())) - .filter(ref -> ref.getProviderId() == null) - .filter(ref -> ref.getProviderName() != null) - .forEach(ref -> ref.setProviderId(findProviderIdByName(ref.getProviderName()))); + for (final VersionedConfigurationStep step : steps) { + if (step == null || step.getProperties() == null) { + continue; + } + + for (final VersionedConnectorValueReference valueReference : step.getProperties().values()) { + if (valueReference == null) { + continue; + } + if (!ConnectorValueType.SECRET_REFERENCE.name().equals(valueReference.getValueType())) { + continue; + } + if (valueReference.getProviderId() != null || valueReference.getProviderName() == null) { + continue; + } + + valueReference.setProviderId(findProviderIdByName(valueReference.getProviderName())); + } + } } /** @@ -1031,4 +1093,16 @@ String findProviderIdByName(final String providerName) { logger.debug("Resolved parameter provider name [{}] to id [{}]", providerName, resolvedId); return resolvedId; } + + private static ScheduledState toScheduledState(final VersionedConnectorState versionedConnectorState) { + if (versionedConnectorState == null) { + return null; + } + + return switch (versionedConnectorState) { + case RUNNING -> ScheduledState.RUNNING; + case DISABLED -> ScheduledState.DISABLED; + case ENABLED, TROUBLESHOOTING -> ScheduledState.ENABLED; + }; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..59a5e1a3e28c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFrameworkConnectorMigrationContext.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +public class StandardFrameworkConnectorMigrationContext implements FrameworkConnectorMigrationContext { + private static final Logger logger = LoggerFactory.getLogger(StandardFrameworkConnectorMigrationContext.class); + private final String connectorId; + private final VersionedExternalFlow sourceFlow; + private final boolean localMigration; + private final FrameworkFlowContext activeFlowContext; + private final AssetManager sourceAssetManager; + private final ConnectorRepository connectorRepository; + private final StateManagerProvider stateManagerProvider; + private final ClusterTopologyProvider clusterTopologyProvider; + private final Set copiedAssetIds = Collections.synchronizedSet(new LinkedHashSet<>()); + + public StandardFrameworkConnectorMigrationContext(final String connectorId, final VersionedExternalFlow sourceFlow, final boolean localMigration, + final FrameworkFlowContext activeFlowContext, final AssetManager sourceAssetManager, final ConnectorRepository connectorRepository, + final StateManagerProvider stateManagerProvider, final ClusterTopologyProvider clusterTopologyProvider) { + this.connectorId = connectorId; + this.sourceFlow = sourceFlow; + this.localMigration = localMigration; + this.activeFlowContext = activeFlowContext; + this.sourceAssetManager = sourceAssetManager; + this.connectorRepository = connectorRepository; + this.stateManagerProvider = stateManagerProvider; + this.clusterTopologyProvider = clusterTopologyProvider; + } + + @Override + public VersionedExternalFlow getSourceFlow() { + return sourceFlow; + } + + @Override + public boolean isLocalMigration() { + return localMigration; + } + + @Override + public FrameworkFlowContext getActiveFlowContext() { + return activeFlowContext; + } + + @Override + public AssetReference copyAssetFromSource(final String sourceAssetId) { + if (sourceAssetId == null || sourceAssetId.isBlank()) { + throw new IllegalArgumentException("Source asset identifier must be specified."); + } + if (!localMigration) { + throw new IllegalStateException("Source assets can only be copied for migrations from a local Versioned Process Group."); + } + + final String copiedAssetId = UUID.nameUUIDFromBytes((connectorId + ":" + sourceAssetId).getBytes(StandardCharsets.UTF_8)).toString(); + final Optional existingAsset = connectorRepository.getAsset(copiedAssetId); + if (existingAsset.isPresent() && existingAsset.get().getFile().isFile() && existingAsset.get().getFile().length() > 0L) { + copiedAssetIds.add(copiedAssetId); + return new AssetReference(Set.of(copiedAssetId)); + } + + final Optional sourceAsset = sourceAssetManager.getAsset(sourceAssetId); + if (sourceAsset.isEmpty()) { + logger.warn("Connector [{}] migration could not locate source asset [{}]; the asset will not be copied and the affected parameter " + + "will be left without an asset reference for the user to re-attach after migration.", connectorId, sourceAssetId); + return new AssetReference(Set.of()); + } + + try (final InputStream sourceContents = new FileInputStream(sourceAsset.get().getFile())) { + final Asset copiedAsset = connectorRepository.storeAsset(connectorId, copiedAssetId, sourceAsset.get().getName(), sourceContents); + copiedAssetIds.add(copiedAsset.getIdentifier()); + return new AssetReference(Set.of(copiedAsset.getIdentifier())); + } catch (final IOException e) { + throw new IllegalStateException("Failed to copy source asset with identifier " + sourceAssetId, e); + } + } + + @Override + public AssetManager getSourceAssetManager() { + return sourceAssetManager; + } + + @Override + public ConnectorRepository getConnectorRepository() { + return connectorRepository; + } + + @Override + public StateManagerProvider getStateManagerProvider() { + return stateManagerProvider; + } + + @Override + public ClusterTopologyProvider getClusterTopologyProvider() { + return clusterTopologyProvider; + } + + @Override + public Set getCopiedAssetIds() { + synchronized (copiedAssetIds) { + return Set.copyOf(copiedAssetIds); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java index 2599269d3554..b4387c74503f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneParameterContextFacade.java @@ -109,6 +109,7 @@ private Map createParameterMap(final Collection createParameterValues(final ParameterContext context) { final List parameterValues = new ArrayList<>(); for (final Parameter parameter : context.getParameters().values()) { + final List referencedAssets = parameter.getReferencedAssets(); final ParameterValue.Builder parameterValueBuilder = new ParameterValue.Builder() .name(parameter.getDescriptor().getName()) .sensitive(parameter.getDescriptor().isSensitive()) - .value(parameter.getValue()); + .value(referencedAssets == null || referencedAssets.isEmpty() ? parameter.getValue() : null); - parameter.getReferencedAssets().forEach(parameterValueBuilder::addReferencedAsset); + if (referencedAssets != null) { + referencedAssets.forEach(parameterValueBuilder::addReferencedAsset); + } parameterValues.add(parameterValueBuilder.build()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index 7e74a7f2389b..ed0e9a78230e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -29,6 +29,7 @@ import org.apache.nifi.controller.flow.VersionedFlowEncodingVersion; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; @@ -101,7 +102,7 @@ private List mapConnectors() { for (final ConnectorNode connectorNode : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode); if (flowController.isStartAfterInitialization(connectorNode)) { - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); } connectors.add(versionedConnector); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java index 75c35154d693..e59f15da0ce2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java @@ -49,6 +49,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java index 404a78b9727f..73d5cd965256 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java @@ -79,6 +79,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java index bf8148214393..a4d35d575414 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java @@ -130,6 +130,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-duplicate-log-flow.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + public boolean isInitialized() { return initialized; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java index c21b55dfbbaf..55377dd15380 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java @@ -96,6 +96,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java index bf5f242a939e..a2e59992aff8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java @@ -60,6 +60,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/on-property-modified-tracker.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(CONFIG_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java index bbc071aca0eb..78b7b4b48198 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java @@ -76,6 +76,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-and-log-with-parameter.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(TEXT_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java index 6818b5b059cc..ef4f7dc0d680 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java @@ -44,6 +44,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java new file mode 100644 index 000000000000..e467926c95fe --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestEligibilityConnectorMigrationContext.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +public class TestEligibilityConnectorMigrationContext { + + private static final String CONNECTOR_ID = "connector-1"; + + @Test + public void testEligibilityContextRejectsCopyAssetFromSource() { + final VersionedExternalFlow sourceFlow = mock(VersionedExternalFlow.class); + final FrameworkFlowContext activeFlowContext = mock(FrameworkFlowContext.class); + final AssetManager sourceAssetManager = mock(AssetManager.class); + final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); + final StateManagerProvider stateManagerProvider = mock(StateManagerProvider.class); + final ClusterTopologyProvider clusterTopologyProvider = mock(ClusterTopologyProvider.class); + + final StandardFrameworkConnectorMigrationContext underlying = new StandardFrameworkConnectorMigrationContext( + CONNECTOR_ID, sourceFlow, true, activeFlowContext, sourceAssetManager, connectorRepository, + stateManagerProvider, clusterTopologyProvider); + final EligibilityConnectorMigrationContext eligibilityContext = new EligibilityConnectorMigrationContext(underlying); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> eligibilityContext.copyAssetFromSource("any-asset-id")); + assertTrue(thrown.getMessage().contains("copyAssetFromSource")); + assertTrue(thrown.getMessage().contains("isMigrationSupported")); + + // Every other accessor must return whatever the underlying context returns. + assertSame(sourceFlow, eligibilityContext.getSourceFlow()); + assertTrue(eligibilityContext.isLocalMigration()); + assertSame(activeFlowContext, eligibilityContext.getActiveFlowContext()); + assertSame(sourceAssetManager, eligibilityContext.getSourceAssetManager()); + assertSame(connectorRepository, eligibilityContext.getConnectorRepository()); + assertSame(stateManagerProvider, eligibilityContext.getStateManagerProvider()); + assertSame(clusterTopologyProvider, eligibilityContext.getClusterTopologyProvider()); + assertEquals(underlying.getCopiedAssetIds(), eligibilityContext.getCopiedAssetIds()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java new file mode 100644 index 000000000000..6c6e381e603e --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorMigrationManager.java @@ -0,0 +1,582 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.connector.components.ProcessGroupFacade; +import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.controller.service.ControllerServiceState; +import org.apache.nifi.flow.ExternalControllerServiceReference; +import org.apache.nifi.flow.VersionedComponentState; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedNodeState; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; +import org.apache.nifi.registry.flow.VersionControlInformation; +import org.apache.nifi.registry.flow.VersionedFlowState; +import org.apache.nifi.registry.flow.VersionedFlowStatus; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +public class TestStandardConnectorMigrationManager { + + private static final String CONNECTOR_ID = "connector-1"; + private static final String SOURCE_GROUP_ID = "source-group"; + private static final String SOURCE_GROUP_NAME = "Source Flow"; + + @Test + public void testMigrateFromUploadedPayloadRejectsLocalStateBeyondConnectedNodeCount() { + final FlowController flowController = createFlowController(2); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(3); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("Cannot import flow"), exception.getMessage()); + + verify(connectorNode, never()).recreateWorkingFlowContext(); + } + + @Test + public void testMigrateFromLocalSourceDisablesAndRenamesProcessGroupAfterSuccess() throws Exception { + final FlowController flowController = createFlowController(1); + final ProcessGroup sourceProcessGroup = wireSourceProcessGroup(flowController, SOURCE_GROUP_ID, SOURCE_GROUP_NAME); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, SOURCE_GROUP_ID, sourceFlow); + + verify(connectorNode).recreateWorkingFlowContext(); + verify(sourceProcessGroup).setName("(Migrated) " + SOURCE_GROUP_NAME); + } + + @Test + public void testRenameIsIdempotentWhenSourceAlreadyHasMigratedPrefix() throws Exception { + final FlowController flowController = createFlowController(1); + final ProcessGroup sourceProcessGroup = wireSourceProcessGroup(flowController, SOURCE_GROUP_ID, "(Migrated) " + SOURCE_GROUP_NAME); + + final ProcessGroup sourceParent = mock(ProcessGroup.class); + final ProcessorNode runningProcessor = mock(ProcessorNode.class); + when(runningProcessor.getDesiredState()).thenReturn(ScheduledState.RUNNING); + when(runningProcessor.getProcessGroup()).thenReturn(sourceParent); + when(sourceProcessGroup.findAllProcessors()).thenReturn(List.of(runningProcessor)); + + final ControllerServiceNode enabledService = mock(ControllerServiceNode.class); + when(enabledService.getState()).thenReturn(ControllerServiceState.ENABLED); + when(sourceProcessGroup.findAllControllerServices()).thenReturn(Set.of(enabledService)); + + final ControllerServiceProvider controllerServiceProvider = mock(ControllerServiceProvider.class); + when(flowController.getControllerServiceProvider()).thenReturn(controllerServiceProvider); + + wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, SOURCE_GROUP_ID, sourceFlow); + + // The source already had the (Migrated) prefix from a previous attempt, so it must not be re-prefixed. + verify(sourceProcessGroup, never()).setName(anyString()); + + // Disable must still run unconditionally even when the rename was skipped. + verify(sourceParent).disableProcessor(runningProcessor); + verify(controllerServiceProvider).disableControllerServicesAsync(List.of(enabledService)); + } + + @Test + public void testRerunAfterFailureIsAllowed() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); + doThrow(new FlowUpdateException("transient")).doNothing().when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // Second attempt must succeed because the prior attempt rolled back fully. + migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow); + verify(connectorNode, atLeastOnce()).recreateWorkingFlowContext(); + } + + @Test + public void testNonFlowUpdateExceptionFromMigrateStillTriggersRollback() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); + doThrow(new RuntimeException("boom")).when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + verify(connectorNode).loadInitialFlow(); + verify(connectorRepository).discardWorkingConfiguration(connectorNode); + verify(connectorNode, never()).recreateWorkingFlowContext(); + } + + @Test + public void testRollbackClearsLocalAndClusterStateAndDeletesOnlyCopiedAssets() throws Exception { + final FlowController flowController = createFlowController(1); + final StateManagerProvider stateManagerProvider = flowController.getStateManagerProvider(); + final StateManager componentStateManager = mock(StateManager.class); + when(stateManagerProvider.getStateManager(anyString())).thenReturn(componentStateManager); + + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessorNode processor = mock(ProcessorNode.class); + when(processor.getIdentifier()).thenReturn("processor-runtime-id"); + final ControllerServiceNode controllerService = mock(ControllerServiceNode.class); + when(controllerService.getIdentifier()).thenReturn("service-runtime-id"); + final ProcessGroup managedGroup = mock(ProcessGroup.class); + when(managedGroup.findAllProcessors()).thenReturn(List.of(processor)); + when(managedGroup.findAllControllerServices()).thenReturn(Set.of(controllerService)); + when(connectorNode.getActiveFlowContext().getManagedProcessGroup()).thenReturn(managedGroup); + + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); + doThrow(new FlowUpdateException("nope")).when(connector).migrate(any(ConnectorMigrationContext.class)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // Rollback clears LOCAL and CLUSTER state for each component currently in the Connector's managed flow. + verify(stateManagerProvider).getStateManager("processor-runtime-id"); + verify(stateManagerProvider).getStateManager("service-runtime-id"); + verify(componentStateManager, atLeastOnce()).clear(Scope.LOCAL); + verify(componentStateManager, atLeastOnce()).clear(Scope.CLUSTER); + + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + verify(connectorNode).loadInitialFlow(); + } + + @Test + public void testListingFiltersStaleAndLocallyModifiedSources() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + final ProcessGroup staleGroup = mockVersionedGroup("stale-group", VersionedFlowState.STALE); + final ProcessGroup locallyModifiedGroup = mockVersionedGroup("locally-modified-group", VersionedFlowState.LOCALLY_MODIFIED); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(staleGroup, locallyModifiedGroup)); + when(rootGroup.findProcessGroup("stale-group")).thenReturn(staleGroup); + when(rootGroup.findProcessGroup("locally-modified-group")).thenReturn(locallyModifiedGroup); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final List sources = migrationManager.listMigrationSources(CONNECTOR_ID); + + assertTrue(sources.isEmpty()); + + // Distinct diagnostics per VersionedFlowState are exposed via verifyEligibility. + final IllegalStateException staleException = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "stale-group")); + assertTrue(staleException.getMessage().contains("stale"), staleException.getMessage()); + + final IllegalStateException locallyModifiedException = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "locally-modified-group")); + assertTrue(locallyModifiedException.getMessage().contains("local modifications"), locallyModifiedException.getMessage()); + } + + @Test + public void testListingExcludesNonRegistryBackedProcessGroups() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + // Process group has no version control information at all, so it must not appear in listing. + final ProcessGroup nonRegistryBacked = mock(ProcessGroup.class); + when(nonRegistryBacked.getIdentifier()).thenReturn("non-registry-backed"); + when(nonRegistryBacked.getName()).thenReturn("Non Registry Backed"); + when(nonRegistryBacked.getVersionControlInformation()).thenReturn(null); + when(nonRegistryBacked.getConnectorIdentifier()).thenReturn(Optional.empty()); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(nonRegistryBacked)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final List sources = migrationManager.listMigrationSources(CONNECTOR_ID); + + assertTrue(sources.isEmpty(), "Process groups without VersionControlInformation must not appear in the migration sources listing"); + } + + @Test + public void testEligibilityRejectsNonVersionControlledSource() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup nonVersioned = mock(ProcessGroup.class); + when(nonVersioned.getIdentifier()).thenReturn("non-versioned"); + when(nonVersioned.getVersionControlInformation()).thenReturn(null); + when(nonVersioned.getConnectorIdentifier()).thenReturn(Optional.empty()); + when(rootGroup.findProcessGroup("non-versioned")).thenReturn(nonVersioned); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "non-versioned")); + assertTrue(exception.getMessage().contains("not under version control"), exception.getMessage()); + } + + @Test + public void testValidateMigrationSourceRejectsExternalControllerServices() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + sourceFlow.setExternalControllerServices(Map.of("external-service", new ExternalControllerServiceReference())); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("services outside its managed flow"), exception.getMessage()); + } + + @Test + public void testRejectMigrationWhenTargetActiveFlowAlreadyHasUserComponents() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + // Active flow has an extra processor compared to the (empty) initial flow. + final ProcessGroupFacade rootGroup = mock(ProcessGroupFacade.class); + when(rootGroup.getProcessors()).thenReturn(Set.of(mock(ProcessorFacade.class))); + when(connectorNode.getActiveFlowContext().getRootGroup()).thenReturn(rootGroup); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("user-installed components"), exception.getMessage()); + } + + @Test + public void testRollbackFailureDoesNotMaskOriginalMigrationFailure() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + final MigratableConnector connector = (MigratableConnector) connectorNode.getConnector(); + + final FlowUpdateException originalFailure = new FlowUpdateException("original"); + doThrow(originalFailure).when(connector).migrate(any(ConnectorMigrationContext.class)); + doThrow(new FlowUpdateException("rollback failed")).when(connectorNode).loadInitialFlow(); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final FlowUpdateException thrown = assertThrows(FlowUpdateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // The caller must see the original migration failure, not the rollback failure. + assertEquals("original", thrown.getMessage()); + } + + @Test + public void testIsMigrationSupportedFalseDoesNotTriggerRollback() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + when(((MigratableConnector) connectorNode.getConnector()).isMigrationSupported(any())).thenReturn(false); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + assertThrows(FlowUpdateException.class, () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + + // An eligibility rejection must not exercise the rollback machinery: no flow was attempted. + verify(connectorNode, never()).loadInitialFlow(); + verify(connectorRepository, never()).discardWorkingConfiguration(connectorNode); + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + } + + @Test + public void testEachIneligibleVersionedFlowStateHasDistinctDiagnostic() { + final FlowController flowController = createFlowController(1); + wireFreshConnector(flowController, CONNECTOR_ID); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + + for (final VersionedFlowState state : VersionedFlowState.values()) { + if (state == VersionedFlowState.UP_TO_DATE) { + continue; + } + final ProcessGroup processGroup = mockVersionedGroup("group-" + state, state); + when(rootGroup.findProcessGroup("group-" + state)).thenReturn(processGroup); + } + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + final String staleMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-STALE")).getMessage(); + final String locallyModifiedMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-LOCALLY_MODIFIED")).getMessage(); + final String locallyModifiedAndStaleMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-LOCALLY_MODIFIED_AND_STALE")).getMessage(); + final String syncFailureMessage = assertThrows(IllegalStateException.class, + () -> migrationManager.verifyEligibility(CONNECTOR_ID, "group-SYNC_FAILURE")).getMessage(); + + assertEquals(4, Set.of(staleMessage, locallyModifiedMessage, locallyModifiedAndStaleMessage, syncFailureMessage).size(), + "Each non-UP_TO_DATE VersionedFlowState must produce a unique diagnostic message"); + } + + @Test + public void testListMigrationSourcesReturnsEmptyForNonMigratableConnector() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID, false); + stubFrameworkMigrationGating(connectorNode); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup eligibleGroup = mockEligibleVersionedGroup("eligible-group"); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(eligibleGroup)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + assertTrue(migrationManager.listMigrationSources(CONNECTOR_ID).isEmpty(), + "A Connector that does not implement MigratableConnector must never appear in the listing"); + } + + @Test + public void testMigrateRejectsConnectorThatDoesNotImplementMigratableConnector() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID, false); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final FlowUpdateException exception = assertThrows(FlowUpdateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("does not support migration"), exception.getMessage()); + + // Rejection happens before any flow manipulation is attempted, so rollback must not be exercised. + verify(connectorNode, never()).loadInitialFlow(); + verify(connectorRepository, never()).deleteAssets(eq(CONNECTOR_ID), any()); + } + + @Test + public void testMigrateRejectsRunningConnector() { + final FlowController flowController = createFlowController(1); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.RUNNING); + when(connectorNode.getDesiredState()).thenReturn(ConnectorState.RUNNING); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + final VersionedExternalFlow sourceFlow = createSourceFlowWithLocalStateCount(1); + + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> migrationManager.migrateFromVersionedFlow(CONNECTOR_ID, null, sourceFlow)); + assertTrue(exception.getMessage().contains("must be stopped before it can be migrated"), exception.getMessage()); + } + + @Test + public void testListMigrationSourcesPropagatesEligibilityWrapper() throws Exception { + final FlowController flowController = createFlowController(1); + final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); + final ConnectorNode connectorNode = wireFreshConnector(flowController, CONNECTOR_ID); + stubFrameworkMigrationGating(connectorNode); + + final MigratableConnector migratableConnector = (MigratableConnector) connectorNode.getConnector(); + when(migratableConnector.isMigrationSupported(any())).thenAnswer(invocation -> { + final ConnectorMigrationContext context = invocation.getArgument(0); + context.copyAssetFromSource("any-id"); + return true; + }); + + final ProcessGroup rootGroup = mock(ProcessGroup.class); + when(flowController.getFlowManager().getRootGroup()).thenReturn(rootGroup); + final ProcessGroup eligibleGroup = mockEligibleVersionedGroup("eligible-group"); + when(rootGroup.findAllProcessGroups()).thenReturn(List.of(eligibleGroup)); + + final StandardConnectorMigrationManager migrationManager = newMigrationManager(flowController); + + assertTrue(migrationManager.listMigrationSources(CONNECTOR_ID).isEmpty(), + "A Connector that calls copyAssetFromSource(...) from isMigrationSupported(...) must be filtered out by the eligibility wrapper"); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + private ProcessGroup mockEligibleVersionedGroup(final String groupId) { + final ProcessGroup processGroup = mockVersionedGroup(groupId, VersionedFlowState.UP_TO_DATE); + when(processGroup.getName()).thenReturn(groupId); + return processGroup; + } + + private StandardConnectorMigrationManager newMigrationManager(final FlowController flowController) { + // The snapshot provider is only exercised by the listing path. Tests that drive listing wire their own snapshots. + final RegisteredFlowSnapshot emptySnapshot = new RegisteredFlowSnapshot(); + emptySnapshot.setFlowContents(new VersionedProcessGroup()); + emptySnapshot.setFlow(new RegisteredFlow()); + final ConnectorFlowSnapshotProvider snapshotProvider = (groupId, includeReferencedServices, includeComponentState) -> emptySnapshot; + return new StandardConnectorMigrationManager(flowController, snapshotProvider); + } + + private FlowController createFlowController(final int connectedNodeCount) { + final FlowController flowController = mock(FlowController.class); + when(flowController.getConnectorRepository()).thenReturn(mock(ConnectorRepository.class)); + when(flowController.getFlowManager()).thenReturn(mock(FlowManager.class)); + when(flowController.getExtensionManager()).thenReturn(mock(ExtensionManager.class)); + when(flowController.getAssetManager()).thenReturn(mock(AssetManager.class)); + when(flowController.getConnectorAssetManager()).thenReturn(mock(AssetManager.class)); + when(flowController.getStateManagerProvider()).thenReturn(mock(StateManagerProvider.class)); + when(flowController.getConnectedNodeCount()).thenReturn(connectedNodeCount); + return flowController; + } + + private ConnectorNode wireFreshConnector(final FlowController flowController, final String connectorId) { + return wireFreshConnector(flowController, connectorId, true); + } + + private ConnectorNode wireFreshConnector(final FlowController flowController, final String connectorId, final boolean migratable) { + final ConnectorNode connectorNode = mock(ConnectorNode.class); + when(connectorNode.getIdentifier()).thenReturn(connectorId); + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.STOPPED); + when(connectorNode.getDesiredState()).thenReturn(ConnectorState.STOPPED); + final FrameworkFlowContext flowContext = mock(FrameworkFlowContext.class); + when(connectorNode.getActiveFlowContext()).thenReturn(flowContext); + + final Connector connector; + if (migratable) { + connector = mock(Connector.class, withSettings().extraInterfaces(MigratableConnector.class)); + when(((MigratableConnector) connector).isMigrationSupported(any())).thenReturn(true); + } else { + connector = mock(Connector.class); + } + when(connectorNode.getConnector()).thenReturn(connector); + when(flowController.getConnectorRepository().getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY)).thenReturn(connectorNode); + return connectorNode; + } + + /** + * Mirrors {@link StandardConnectorNode#isMigrationSupported} on the mock {@link ConnectorNode}: returns false + * unless the underlying {@link Connector} implements {@link MigratableConnector}, and swallows any exception + * thrown by the underlying call by returning false. Tests that exercise the listing/eligibility path must call + * this so the mock reflects the framework-side gating rather than Mockito's default {@code false}. + */ + private void stubFrameworkMigrationGating(final ConnectorNode connectorNode) { + when(connectorNode.isMigrationSupported(any())).thenAnswer(invocation -> { + final Connector connector = connectorNode.getConnector(); + if (!(connector instanceof final MigratableConnector migratableConnector)) { + return false; + } + try { + return migratableConnector.isMigrationSupported(invocation.getArgument(0)); + } catch (final Exception ignored) { + return false; + } + }); + } + + private ProcessGroup wireSourceProcessGroup(final FlowController flowController, final String groupId, final String groupName) { + final FlowManager flowManager = flowController.getFlowManager(); + final ProcessGroup rootGroup = mock(ProcessGroup.class); + final ProcessGroup sourceProcessGroup = mock(ProcessGroup.class); + when(flowManager.getRootGroup()).thenReturn(rootGroup); + when(rootGroup.findProcessGroup(groupId)).thenReturn(sourceProcessGroup); + when(sourceProcessGroup.getName()).thenReturn(groupName); + return sourceProcessGroup; + } + + private ProcessGroup mockVersionedGroup(final String groupId, final VersionedFlowState state) { + final ProcessGroup processGroup = mock(ProcessGroup.class); + when(processGroup.getIdentifier()).thenReturn(groupId); + when(processGroup.getConnectorIdentifier()).thenReturn(Optional.empty()); + final VersionControlInformation versionControlInformation = mock(VersionControlInformation.class); + final VersionedFlowStatus status = mock(VersionedFlowStatus.class); + when(status.getState()).thenReturn(state); + when(versionControlInformation.getStatus()).thenReturn(status); + when(processGroup.getVersionControlInformation()).thenReturn(versionControlInformation); + return processGroup; + } + + private VersionedExternalFlow createSourceFlowWithLocalStateCount(final int localStateCount) { + final VersionedComponentState componentState = new VersionedComponentState(); + componentState.setLocalNodeStates(createNodeStates(localStateCount)); + + final VersionedProcessor processor = new VersionedProcessor(); + processor.setIdentifier("processor-1"); + processor.setName("count-1"); + processor.setComponentState(componentState); + + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setIdentifier("group-1"); + processGroup.setName("Source Flow"); + processGroup.setProcessors(Collections.singleton(processor)); + + final VersionedExternalFlow sourceFlow = new VersionedExternalFlow(); + sourceFlow.setFlowContents(processGroup); + sourceFlow.setExternalControllerServices(Collections.emptyMap()); + sourceFlow.setParameterContexts(Collections.emptyMap()); + sourceFlow.setParameterProviders(Collections.emptyMap()); + return sourceFlow; + } + + private List createNodeStates(final int localStateCount) { + final List nodeStates = new ArrayList<>(); + for (int i = 0; i < localStateCount; i++) { + nodeStates.add(new VersionedNodeState(Map.of("count", Integer.toString(i + 1)))); + } + return nodeStates; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java index 00e136a66358..820576fdc350 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java @@ -881,6 +881,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -947,6 +952,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -997,6 +1007,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1068,6 +1083,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1124,6 +1144,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1168,6 +1193,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index cba4b12f5741..affa9ed5fcb9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -29,6 +29,7 @@ import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedConnectorValueReference; @@ -66,6 +67,7 @@ import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -128,6 +130,47 @@ public void testRestoreConnector() { assertEquals(connector, repository.getConnector("connector-1", ConnectorSyncMode.SYNC_WITH_PROVIDER)); } + @Test + public void testRestoreConnectorCleansUpUnreferencedAssets() { + final AssetManager assetManager = mock(AssetManager.class); + final StandardConnectorRepository repository = createRepositoryWithProviderAndAssetManager(null, assetManager); + + final Asset referencedAsset = mock(Asset.class); + when(referencedAsset.getIdentifier()).thenReturn("referenced-asset"); + when(referencedAsset.getName()).thenReturn("referenced.txt"); + + final Asset unreferencedAsset = mock(Asset.class); + when(unreferencedAsset.getIdentifier()).thenReturn("unreferenced-asset"); + when(unreferencedAsset.getName()).thenReturn("unreferenced.txt"); + + final MutableConnectorConfigurationContext activeConfigContext = mock(MutableConnectorConfigurationContext.class); + final ConnectorConfiguration activeConfiguration = new ConnectorConfiguration(Set.of( + new NamedStepConfiguration("step1", new StepConfiguration(Map.of("property", new AssetReference(Set.of("referenced-asset"))))) + )); + when(activeConfigContext.toConnectorConfiguration()).thenReturn(activeConfiguration); + + final FrameworkFlowContext activeFlowContext = mock(FrameworkFlowContext.class); + when(activeFlowContext.getConfigurationContext()).thenReturn(activeConfigContext); + + final MutableConnectorConfigurationContext workingConfigContext = mock(MutableConnectorConfigurationContext.class); + when(workingConfigContext.toConnectorConfiguration()).thenReturn(new ConnectorConfiguration(Set.of())); + + final FrameworkFlowContext workingFlowContext = mock(FrameworkFlowContext.class); + when(workingFlowContext.getConfigurationContext()).thenReturn(workingConfigContext); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + when(connector.getActiveFlowContext()).thenReturn(activeFlowContext); + when(connector.getWorkingFlowContext()).thenReturn(workingFlowContext); + + when(assetManager.getAssets("connector-1")).thenReturn(List.of(referencedAsset, unreferencedAsset)); + + repository.restoreConnector(connector); + + verify(assetManager).deleteAsset("unreferenced-asset"); + verify(assetManager, never()).deleteAsset("referenced-asset"); + } + @Test public void testGetConnectorsReturnsNewListInstances() { final StandardConnectorRepository repository = new StandardConnectorRepository(); @@ -243,7 +286,7 @@ public void testGetConnectorWithProviderOverridesWorkingConfig() throws FlowUpda final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("External Name"); - final VersionedConfigurationStep externalStep = createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("external-value"))); + final VersionedConfigurationStep externalStep = createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("external-value"))); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); when(provider.load("connector-1")).thenReturn(Optional.of(externalConfig)); @@ -378,7 +421,7 @@ public void testConfigureConnectorSavesToProviderBeforeModifyingNode() throws Fl final List savedSteps = savedConfig.getWorkingFlowConfiguration(); assertNotNull(savedSteps); final VersionedConfigurationStep savedStep = savedSteps.stream() - .filter(s -> "step1".equals(s.getName())).findFirst().orElse(null); + .filter(versionedConfigurationStep -> "step1".equals(versionedConfigurationStep.getName())).findFirst().orElse(null); assertNotNull(savedStep); assertEquals("STRING_LITERAL", savedStep.getProperties().get("prop1").getValueType()); assertEquals("new-value", savedStep.getProperties().get("prop1").getValue()); @@ -413,7 +456,7 @@ public void testConfigureConnectorMergesPartialStepConfig() throws FlowUpdateExc repository.addConnector(connector); final VersionedConfigurationStep existingStep = createVersionedStep("step1", - Map.of("propA", createStringLiteralRef("old-A"), "propB", createStringLiteralRef("old-B"))); + Map.of("propA", createStringLiteralReference("old-A"), "propB", createStringLiteralReference("old-B"))); final ConnectorWorkingConfiguration existingConfig = new ConnectorWorkingConfiguration(); existingConfig.setName("Test Connector"); existingConfig.setWorkingFlowConfiguration(new ArrayList<>(List.of(existingStep))); @@ -431,7 +474,7 @@ public void testConfigureConnectorMergesPartialStepConfig() throws FlowUpdateExc final ConnectorWorkingConfiguration savedConfig = configCaptor.getValue(); final VersionedConfigurationStep savedStep = savedConfig.getWorkingFlowConfiguration().stream() - .filter(s -> "step1".equals(s.getName())).findFirst().orElse(null); + .filter(versionedConfigurationStep -> "step1".equals(versionedConfigurationStep.getName())).findFirst().orElse(null); assertNotNull(savedStep); final Map savedProps = savedStep.getProperties(); @@ -741,7 +784,7 @@ public void testGetConnectorTriggersOnConfigurationStepConfiguredWhenValuesChang trackingConnector.reset(); final VersionedConfigurationStep externalStep = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("updated-value"))); + Map.of("prop1", createStringLiteralReference("updated-value"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); @@ -766,7 +809,7 @@ public void testGetConnectorDoesNotTriggerOnConfigurationStepConfiguredWhenValue trackingConnector.reset(); final VersionedConfigurationStep externalStep = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("same-value"))); + Map.of("prop1", createStringLiteralReference("same-value"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(externalStep)); @@ -790,9 +833,9 @@ public void testGetConnectorContinuesWhenOneStepFailsToReplace() throws FlowUpda repository.addConnector(connector); final VersionedConfigurationStep stepOne = createVersionedStep("step1", - Map.of("prop1", createStringLiteralRef("value1"))); + Map.of("prop1", createStringLiteralReference("value1"))); final VersionedConfigurationStep stepTwo = createVersionedStep("step2", - Map.of("prop2", createStringLiteralRef("value2"))); + Map.of("prop2", createStringLiteralReference("value2"))); final ConnectorWorkingConfiguration externalConfig = new ConnectorWorkingConfiguration(); externalConfig.setName("connector-1"); externalConfig.setWorkingFlowConfiguration(List.of(stepOne, stepTwo)); @@ -821,10 +864,8 @@ public void testSyncAssetsFromProviderCallsSyncAssetsThenReloads() { repository.syncAssetsFromProvider(connector); - // Step 1: provider.syncAssets() called verify(provider).syncAssets("connector-1"); - // Step 2: provider.load() called to reload updated config (may also have been called during addConnector) - verify(provider, org.mockito.Mockito.atLeastOnce()).load("connector-1"); + verify(provider, atLeastOnce()).load("connector-1"); } @Test @@ -884,11 +925,8 @@ public void testCleanUpAssetsCallsProviderDeleteForUnreferencedAssets() { repository.discardWorkingConfiguration(connector); - // Provider.deleteAsset called with NiFi UUID for unreferenced asset verify(provider).deleteAsset(connectorId, unreferencedAssetId); - // Referenced asset is not deleted verify(provider, never()).deleteAsset(connectorId, referencedAssetId); - // AssetManager.deleteAsset NOT called directly since provider handles local deletion verify(assetManager, never()).deleteAsset(anyString()); } @@ -939,10 +977,10 @@ public void testSyncConnectorStoppedWithConfigChange() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -956,9 +994,9 @@ public void testSyncConnectorStoppedNoConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); verify(connector, never()).inheritConfiguration(any(), any(), any()); @@ -972,10 +1010,10 @@ public void testSyncConnectorRunningWithConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("new-value"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -989,9 +1027,9 @@ public void testSyncConnectorDrainingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.DRAINING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1006,9 +1044,9 @@ public void testSyncConnectorPreparingForUpdateIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.PREPARING_FOR_UPDATE); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1022,9 +1060,9 @@ public void testSyncConnectorUpdatingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); } @@ -1037,10 +1075,10 @@ public void testSyncConnectorUpdateFailedRecovery() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATE_FAILED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("recovery-value"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("recovery-value"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).inheritConfiguration(any(), any(), any()); @@ -1054,10 +1092,10 @@ public void testSyncConnectorUpdatedState() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); } @@ -1072,10 +1110,10 @@ public void testSyncConnectorInheritConfigurationFailureWhenRunning() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector).stop(any()); @@ -1091,10 +1129,10 @@ public void testSyncConnectorInheritConfigurationFailureWhenStopped() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, - List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("value1"))))); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector, never()).stop(any()); @@ -1110,9 +1148,9 @@ public void testSyncConnectorProviderRejectsSync() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1130,9 +1168,9 @@ public void testSyncConnectorProviderThrowsException() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1149,9 +1187,9 @@ public void testSyncConnectorStartingWaitsForRunning() throws Exception { .thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); } @@ -1164,9 +1202,9 @@ public void testSyncConnectorStartingTimesOut() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STARTING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); @@ -1188,9 +1226,9 @@ public void testSyncConnectorProviderReturnsRemoveStopsAndRemoves() throws Excep when(connector.stop(any())).thenReturn(CompletableFuture.completedFuture(null)); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); verify(connector).stop(any()); @@ -1209,9 +1247,9 @@ public void testSyncConnectorProviderReturnsRemoveStoppedConnector() throws Exce when(connector.getConnector()).thenReturn(mockExtension); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); verify(connector, never()).stop(any()); @@ -1224,9 +1262,9 @@ public void testSyncConnectorProviderReturnsRemoveForNonExistent() throws Except when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.remove()); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REMOVED, result.getOutcome()); } @@ -1237,14 +1275,14 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.reject()); final FlowManager flowManager = mock(FlowManager.class); - final StandardConnectorRepository repository = createRepositoryWithFlowManagerAndProvider(provider, flowManager); + final StandardConnectorRepository repository = createRepositoryWithFlowManager(provider, flowManager); final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.REJECTED, result.getOutcome()); verify(flowManager).createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean()); @@ -1262,9 +1300,9 @@ public void testSyncConnectorProviderAllowWithScheduledStateOverride() throws Ex when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); assertEquals(ScheduledState.ENABLED, result.getEffectiveScheduledState()); @@ -1277,7 +1315,7 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { final ConnectorWorkingConfiguration providerConfig = new ConnectorWorkingConfiguration(); providerConfig.setName("Provider Name"); providerConfig.setWorkingFlowConfiguration(List.of( - createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("provider-value"))))); + createVersionedStep("step1", Map.of("prop1", createStringLiteralReference("provider-value"))))); when(provider.getSyncDirective(eq("connector-1"), any())) .thenReturn(ConnectorSyncDirective.allow(providerConfig)); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); @@ -1286,9 +1324,9 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versionedConnector = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); - final ConnectorSyncResult result = repository.syncConnector(versioned); + final ConnectorSyncResult result = repository.syncConnector(versionedConnector); assertEquals(ConnectorSyncResult.Outcome.SYNCED, result.getOutcome()); verify(connector).setName("Provider Name"); @@ -1326,10 +1364,10 @@ public void testFindProviderIdByNameReturnsNullWhenAmbiguousMatch() { // Two parameter providers share the same name; the resolution must be refused so that the // resulting SECRET_REFERENCE is surfaced as invalid in the UI rather than silently bound // to whichever provider happens to come first. - final ParameterProviderNode dup1 = mockParameterProvider("Duplicate", "id-a"); - final ParameterProviderNode dup2 = mockParameterProvider("Duplicate", "id-b"); + final ParameterProviderNode firstDuplicateProvider = mockParameterProvider("Duplicate", "id-a"); + final ParameterProviderNode secondDuplicateProvider = mockParameterProvider("Duplicate", "id-b"); final FlowManager flowManager = mock(FlowManager.class); - when(flowManager.getAllParameterProviders()).thenReturn(Set.of(dup1, dup2)); + when(flowManager.getAllParameterProviders()).thenReturn(Set.of(firstDuplicateProvider, secondDuplicateProvider)); final StandardConnectorRepository repository = createRepositoryWithFlowManager(mock(ConnectorConfigurationProvider.class), flowManager); @@ -1357,10 +1395,10 @@ public void testFindProviderIdByNameReturnsNullWhenNoProvidersRegistered() { assertNull(repository.findProviderIdByName("Anything")); } - private static ParameterProviderNode mockParameterProvider(final String name, final String id) { + private static ParameterProviderNode mockParameterProvider(final String name, final String identifier) { final ParameterProviderNode node = mock(ParameterProviderNode.class); when(node.getName()).thenReturn(name); - when(node.getIdentifier()).thenReturn(id); + when(node.getIdentifier()).thenReturn(identifier); return node; } @@ -1393,21 +1431,16 @@ private StandardConnectorRepository createRepositoryWithFlowManager(final Connec return repository; } - private StandardConnectorRepository createRepositoryWithFlowManagerAndProvider( - final ConnectorConfigurationProvider provider, final FlowManager flowManager) { - return createRepositoryWithFlowManager(provider, flowManager); - } - - private ConnectorNode createSimpleConnectorNode(final String id, final String name) { + private ConnectorNode createSimpleConnectorNode(final String identifier, final String name) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); return connector; } - private ConnectorNode createConnectorNodeWithWorkingConfig(final String id, final String name, final MutableConnectorConfigurationContext workingConfigContext) { + private ConnectorNode createConnectorNodeWithWorkingConfig(final String identifier, final String name, final MutableConnectorConfigurationContext workingConfigContext) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); final FrameworkFlowContext workingFlowContext = mock(FrameworkFlowContext.class); @@ -1417,9 +1450,9 @@ private ConnectorNode createConnectorNodeWithWorkingConfig(final String id, fina return connector; } - private ConnectorNode createConnectorNodeWithEmptyWorkingConfig(final String id, final String name) { + private ConnectorNode createConnectorNodeWithEmptyWorkingConfig(final String identifier, final String name) { final ConnectorNode connector = mock(ConnectorNode.class); - when(connector.getIdentifier()).thenReturn(id); + when(connector.getIdentifier()).thenReturn(identifier); when(connector.getName()).thenReturn(name); final MutableConnectorConfigurationContext workingConfigContext = mock(MutableConnectorConfigurationContext.class); @@ -1446,28 +1479,40 @@ private VersionedConfigurationStep createVersionedStep(final String name, final return step; } - private VersionedConnectorValueReference createStringLiteralRef(final String value) { + private VersionedConnectorValueReference createStringLiteralReference(final String value) { final VersionedConnectorValueReference ref = new VersionedConnectorValueReference(); ref.setValueType("STRING_LITERAL"); ref.setValue(value); return ref; } - private VersionedConnector createVersionedConnector(final String id, final String name, final ScheduledState scheduledState, + private VersionedConnector createVersionedConnector(final String identifier, final String name, final ScheduledState scheduledState, final List activeConfig) { - final VersionedConnector vc = new VersionedConnector(); - vc.setInstanceIdentifier(id); - vc.setName(name); - vc.setScheduledState(scheduledState); - vc.setActiveFlowConfiguration(activeConfig); - vc.setWorkingFlowConfiguration(activeConfig); + final VersionedConnector versionedConnector = new VersionedConnector(); + versionedConnector.setInstanceIdentifier(identifier); + versionedConnector.setName(name); + versionedConnector.setScheduledState(toVersionedConnectorState(scheduledState)); + versionedConnector.setActiveFlowConfiguration(activeConfig); + versionedConnector.setWorkingFlowConfiguration(activeConfig); final Bundle bundle = new Bundle(); bundle.setGroup("org.apache.nifi"); bundle.setArtifact("nifi-test-connector-nar"); bundle.setVersion("2.0.0"); - vc.setBundle(bundle); - vc.setType("org.apache.nifi.test.TestConnector"); - return vc; + versionedConnector.setBundle(bundle); + versionedConnector.setType("org.apache.nifi.test.TestConnector"); + return versionedConnector; + } + + private static VersionedConnectorState toVersionedConnectorState(final ScheduledState scheduledState) { + if (scheduledState == null) { + return null; + } + + return switch (scheduledState) { + case RUNNING -> VersionedConnectorState.RUNNING; + case DISABLED -> VersionedConnectorState.DISABLED; + case ENABLED -> VersionedConnectorState.ENABLED; + }; } private StandardConnectorNode createRealConnectorNode(final String identifier, final Connector connector) throws FlowUpdateException { @@ -1568,6 +1613,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java new file mode 100644 index 000000000000..9133b5454298 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardFrameworkConnectorMigrationContext.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.state.StateManagerProvider; +import org.apache.nifi.controller.ClusterTopologyProvider; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestStandardFrameworkConnectorMigrationContext { + + private static final String CONNECTOR_ID = "connector-1"; + private static final String SOURCE_ASSET_ID = "source-asset-1"; + private static final String SOURCE_ASSET_NAME = "driver.jar"; + + private AssetManager sourceAssetManager; + private ConnectorRepository connectorRepository; + private StateManagerProvider stateManagerProvider; + private ClusterTopologyProvider clusterTopologyProvider; + private VersionedExternalFlow sourceFlow; + + @BeforeEach + public void setup() { + sourceAssetManager = mock(AssetManager.class); + connectorRepository = mock(ConnectorRepository.class); + stateManagerProvider = mock(StateManagerProvider.class); + clusterTopologyProvider = mock(ClusterTopologyProvider.class); + sourceFlow = mock(VersionedExternalFlow.class); + } + + @Test + public void testCopyAssetFromSourceReturnsReferenceToCopiedAsset(@TempDir final Path tempDir) throws Exception { + final File sourceFile = createAssetFile(tempDir, "source-contents"); + final Asset sourceAsset = mockAsset(SOURCE_ASSET_ID, SOURCE_ASSET_NAME, sourceFile); + when(sourceAssetManager.getAsset(SOURCE_ASSET_ID)).thenReturn(Optional.of(sourceAsset)); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.empty()); + + final File copiedFile = createAssetFile(tempDir, "copied-contents"); + when(connectorRepository.storeAsset(eq(CONNECTOR_ID), anyString(), eq(SOURCE_ASSET_NAME), any(InputStream.class))) + .thenAnswer(invocation -> mockAsset(invocation.getArgument(1), SOURCE_ASSET_NAME, copiedFile)); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertEquals(1, reference.getAssetIdentifiers().size()); + final String copiedAssetId = reference.getAssetIdentifiers().iterator().next(); + assertEquals(Set.of(copiedAssetId), context.getCopiedAssetIds()); + verify(connectorRepository).storeAsset(eq(CONNECTOR_ID), eq(copiedAssetId), eq(SOURCE_ASSET_NAME), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceReturnsEmptyReferenceWhenSourceAssetIsMissing() throws Exception { + when(sourceAssetManager.getAsset(SOURCE_ASSET_ID)).thenReturn(Optional.empty()); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.empty()); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertTrue(reference.getAssetIdentifiers().isEmpty(), + "Asset reference must be empty when the source asset cannot be located"); + assertTrue(context.getCopiedAssetIds().isEmpty(), + "Copied asset bookkeeping must remain empty when no asset is copied"); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceReusesPreviouslyCopiedAsset(@TempDir final Path tempDir) throws Exception { + final File copiedFile = createAssetFile(tempDir, "previously-copied"); + final Asset existing = mockAsset("already-copied", SOURCE_ASSET_NAME, copiedFile); + when(connectorRepository.getAsset(anyString())).thenReturn(Optional.of(existing)); + + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + final AssetReference reference = context.copyAssetFromSource(SOURCE_ASSET_ID); + + assertNotNull(reference); + assertEquals(1, reference.getAssetIdentifiers().size()); + verify(sourceAssetManager, never()).getAsset(anyString()); + verify(connectorRepository, never()).storeAsset(anyString(), anyString(), anyString(), any(InputStream.class)); + } + + @Test + public void testCopyAssetFromSourceRejectsUploadedPayloadMigration() { + final StandardFrameworkConnectorMigrationContext context = createContext(false); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> context.copyAssetFromSource(SOURCE_ASSET_ID)); + assertTrue(thrown.getMessage().contains("local Versioned Process Group")); + } + + @Test + public void testCopyAssetFromSourceRejectsBlankSourceAssetIdentifier() { + final StandardFrameworkConnectorMigrationContext context = createContext(true); + + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource(null)); + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource("")); + assertThrows(IllegalArgumentException.class, () -> context.copyAssetFromSource(" ")); + } + + private StandardFrameworkConnectorMigrationContext createContext(final boolean localMigration) { + return new StandardFrameworkConnectorMigrationContext( + CONNECTOR_ID, + sourceFlow, + localMigration, + null, + sourceAssetManager, + connectorRepository, + stateManagerProvider, + clusterTopologyProvider); + } + + private File createAssetFile(final Path tempDir, final String contents) throws IOException { + final File file = tempDir.resolve("asset-" + contents + ".bin").toFile(); + Files.writeString(file.toPath(), contents); + return file; + } + + private Asset mockAsset(final String identifier, final String name, final File file) { + final Asset asset = mock(Asset.class); + when(asset.getIdentifier()).thenReturn(identifier); + when(asset.getName()).thenReturn(name); + when(asset.getFile()).thenReturn(file); + return asset; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java index c2dc1ab49a89..c37168fb1387 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java @@ -56,6 +56,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { if (!initialized) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java index 2684627afc3b..0216f078486b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java @@ -33,6 +33,7 @@ import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnector; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedProcessGroup; @@ -306,7 +307,7 @@ void testSyncInheritConnectorDelegatesSyncToConnectorRepository() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); @@ -334,7 +335,7 @@ void testSyncInheritConnectorNotStartedWhenEnabled() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); @@ -363,7 +364,7 @@ void testSyncOrphanConnectorIsRemoved() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -402,7 +403,7 @@ void testSyncOrphanConnectorNotRemovedWhenInProposedFlow() { versionedConnector.setName("Test Connector"); versionedConnector.setType("org.apache.nifi.connectors.TestConnector"); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) @@ -432,7 +433,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java index 2a2b21cde1b3..84afd7dd15ac 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java @@ -51,6 +51,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void start(final FlowContext flowContext) throws FlowUpdateException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java index 9108e3a3b349..eae8d397509f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ConnectorAuditor.java @@ -372,6 +372,29 @@ public void applyConnectorUpdateAdvice(final ProceedingJoinPoint proceedingJoinP } } + @Around("within(org.apache.nifi.web.dao.ConnectorDAO+) && " + + "execution(void migrateFromVersionedFlow(java.lang.String, java.lang.String, org.apache.nifi.flow.VersionedExternalFlow, java.util.function.BooleanSupplier)) && " + + "args(connectorId, processGroupId, sourceFlow, cancellationCheck) && " + + "target(connectorDAO)") + public void migrateConnectorAdvice(final ProceedingJoinPoint proceedingJoinPoint, final String connectorId, final String processGroupId, + final Object sourceFlow, final Object cancellationCheck, final ConnectorDAO connectorDAO) throws Throwable { + final ConnectorNode connector = connectorDAO.getConnector(connectorId); + + proceedingJoinPoint.proceed(); + + if (isAuditable()) { + final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails(); + actionDetails.setName("Connector migrated from Versioned Process Group"); + actionDetails.setPreviousValue(null); + actionDetails.setValue(processGroupId == null ? "Uploaded payload" : processGroupId); + + final Action action = generateAuditRecord(connector, Operation.Configure, actionDetails); + if (action != null) { + saveAction(action, logger); + } + } + } + /** * Generates an audit record for a connector. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 11f33b91bd24..cfb38dfb8b02 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -167,6 +167,7 @@ import org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; import org.apache.nifi.web.api.entity.VersionedReportingTaskImportResponseEntity; import org.apache.nifi.web.api.request.FlowMetricsRegistry; @@ -181,6 +182,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; @@ -263,6 +265,12 @@ Set getConnectorControllerServices(String connectorId, Optional getConnectorAsset(String assetId); + VersionedFlowMigrationSourcesEntity getConnectorMigrationSources(String connectorId); + + void verifyCanMigrateConnector(String connectorId, String processGroupId); + + ConnectorEntity migrateConnector(String connectorId, String processGroupId, RegisteredFlowSnapshot flowSnapshot, BooleanSupplier cancellationCheck); + /** * Verifies that the connector is in a state where FlowFiles can be purged. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c13966b16abb..eb2b610aa628 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -79,6 +79,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.components.connector.Connector; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; @@ -309,6 +310,7 @@ import org.apache.nifi.web.api.dto.UserGroupDTO; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.dto.VersionedFlowDTO; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; import org.apache.nifi.web.api.dto.action.HistoryDTO; import org.apache.nifi.web.api.dto.action.HistoryQueryDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsDTO; @@ -411,6 +413,7 @@ import org.apache.nifi.web.api.entity.VersionControlComponentMappingEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; import org.apache.nifi.web.api.entity.VersionedReportingTaskImportResponseEntity; import org.apache.nifi.web.api.request.FlowMetricsRegistry; @@ -476,6 +479,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -4028,6 +4032,59 @@ public Optional getConnectorAsset(final String assetId) { return connectorDAO.getAsset(assetId); } + @Override + public VersionedFlowMigrationSourcesEntity getConnectorMigrationSources(final String connectorId) { + final List migrationSources = connectorDAO.getMigrationSources(connectorId).stream() + .map(this::createVersionedFlowMigrationSourceDto) + .toList(); + + final VersionedFlowMigrationSourcesEntity entity = new VersionedFlowMigrationSourcesEntity(); + entity.setMigrationSources(migrationSources); + return entity; + } + + @Override + public void verifyCanMigrateConnector(final String connectorId, final String processGroupId) { + Objects.requireNonNull(processGroupId, "Process Group identifier must be specified to verify a local-source migration"); + connectorDAO.verifyCanMigrateFromVersionedFlow(connectorId, processGroupId); + } + + @Override + public ConnectorEntity migrateConnector(final String connectorId, final String processGroupId, final RegisteredFlowSnapshot flowSnapshot, + final BooleanSupplier cancellationCheck) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Revision revision = revisionManager.getRevision(connectorId); + final RevisionClaim claim = new StandardRevisionClaim(revision); + final VersionedExternalFlow externalFlow = createVersionedExternalFlow(flowSnapshot); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.migrateFromVersionedFlow(connectorId, processGroupId, externalFlow, cancellationCheck); + controllerFacade.save(); + + final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + final ConnectorDTO dto = dtoFactory.createConnectorDto(connectorNode); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode connectorNode = connectorDAO.getConnector(snapshot.getComponent().getId(), ConnectorSyncMode.LOCAL_ONLY); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(connectorNode); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(connectorNode)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(connectorNode); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + + private VersionedFlowMigrationSourceDTO createVersionedFlowMigrationSourceDto(final ConnectorMigrationSource migrationSource) { + final VersionedFlowMigrationSourceDTO dto = new VersionedFlowMigrationSourceDTO(); + dto.setProcessGroupId(migrationSource.getProcessGroupId()); + dto.setProcessGroupName(migrationSource.getProcessGroupName()); + dto.setRegistryClientId(migrationSource.getRegistryClientId()); + dto.setBucketId(migrationSource.getBucketId()); + dto.setFlowId(migrationSource.getFlowId()); + dto.setVersion(migrationSource.getVersion()); + return dto; + } + @Override public void verifyPurgeConnectorFlowFiles(final String connectorId) { connectorDAO.verifyPurgeFlowFiles(connectorId); @@ -6117,7 +6174,7 @@ public RegisteredFlowSnapshot getCurrentFlowSnapshotByGroupId(final String proce .mapInstanceIdentifiers(false) .mapControllerServiceReferencesToVersionedId(true) .mapFlowRegistryClientId(false) - .mapAssetReferences(false) + .mapAssetReferences(true) .mapComponentState(includeComponentState) .stateManagerProvider(stateManagerProvider) .localNodeOrdinal(clusterTopologyProvider.getLocalNodeOrdinal()) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java index 087a45fdfcf4..cef4be3002aa 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.web.api; +import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -23,6 +24,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import jakarta.servlet.ServletContext; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; @@ -59,6 +62,7 @@ import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; import org.apache.nifi.stream.io.MaxLengthInputStream; import org.apache.nifi.ui.extension.UiExtension; import org.apache.nifi.ui.extension.UiExtensionMapping; @@ -80,6 +84,10 @@ import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; import org.apache.nifi.web.api.dto.DropRequestDTO; +import org.apache.nifi.web.api.dto.MigrationPayloadDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; +import org.apache.nifi.web.api.dto.MigrationUpdateStepDTO; import org.apache.nifi.web.api.dto.VerifyConnectorConfigStepRequestDTO; import org.apache.nifi.web.api.dto.search.SearchResultsDTO; import org.apache.nifi.web.api.entity.AssetEntity; @@ -93,12 +101,15 @@ import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.SearchResultsEntity; import org.apache.nifi.web.api.entity.SecretsEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; import org.apache.nifi.web.client.api.HttpResponseStatus; @@ -117,6 +128,10 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; /** @@ -128,12 +143,17 @@ public class ConnectorResource extends ApplicationResource { private static final Logger logger = LoggerFactory.getLogger(ConnectorResource.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String VERIFICATION_REQUEST_TYPE = "verification-request"; + private static final String MIGRATION_REQUEST_TYPE = "migration-request"; private static final String PURGE_REQUEST_TYPE = "purge-request"; private static final String FILENAME_HEADER = "Filename"; private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String UPLOAD_CONTENT_TYPE = "application/octet-stream"; private static final long MAX_ASSET_SIZE_BYTES = (long) DataUnit.GB.toB(1); + private static final long MIGRATION_REQUEST_TTL_MILLIS = TimeUnit.MINUTES.toMillis(1L); + private static final long MIGRATION_PAYLOAD_TTL_MILLIS = TimeUnit.MINUTES.toMillis(10L); + private static final long MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS = 30L; private NiFiServiceFacade serviceFacade; private Authorizer authorizer; @@ -143,9 +163,39 @@ public class ConnectorResource extends ApplicationResource { private final RequestManager> configVerificationRequestManager = new AsyncRequestManager<>(100, 1L, "Connector Configuration Step Verification"); + private final RequestManager migrationRequestManager = + new AsyncRequestManager<>(100, MIGRATION_REQUEST_TTL_MILLIS, "Connector Migration"); private final RequestManager purgeRequestManager = new AsyncRequestManager<>(100, 1L, "Connector FlowFile Purge"); + /** + * Uploaded migration payloads keyed by their server-assigned identifier. Each entry's age is tracked alongside + * the payload so that an upload that is never associated with a started migration request is evicted after + * {@link #MIGRATION_PAYLOAD_TTL_MILLIS}. A scheduled sweeper purges expired entries on a fixed cadence so that + * single-upload sessions do not leak payload state indefinitely. + */ + private final Map migrationPayloadsById = new ConcurrentHashMap<>(); + + private final ScheduledExecutorService migrationPayloadEvictionExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> { + final Thread thread = new Thread(runnable, "Connector Migration Payload Eviction"); + thread.setDaemon(true); + return thread; + }); + + private record MigrationPayloadEntry(RegisteredFlowSnapshot snapshot, long uploadedAtMillis) { + } + + @PostConstruct + public void startMigrationPayloadEviction() { + migrationPayloadEvictionExecutor.scheduleWithFixedDelay(this::evictExpiredMigrationPayloads, + MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS, MIGRATION_PAYLOAD_SWEEP_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + + @PreDestroy + public void stopMigrationPayloadEviction() { + migrationPayloadEvictionExecutor.shutdownNow(); + } + @Context private ServletContext servletContext; @@ -2010,6 +2060,342 @@ public Response getConnectorStatus( return generateOkResponse(entity).build(); } + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/migration-sources") + @Operation( + summary = "Lists the Versioned Process Groups that the Connector can be migrated from", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = VersionedFlowMigrationSourcesEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Read - /connectors/{uuid}") + } + ) + public Response getMigrationSources(@PathParam("id") final String connectorId) { + authorizeReadConnector(connectorId); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final VersionedFlowMigrationSourcesEntity entity = serviceFacade.getConnectorMigrationSources(connectorId); + return generateOkResponse(entity).build(); + } + + @POST + @Consumes(MediaType.APPLICATION_OCTET_STREAM) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/migration-payloads") + @Operation( + summary = "Uploads a flow snapshot payload for a later Connector migration request", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationPayloadEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid}") + } + ) + public Response createMigrationPayload( + @PathParam("id") final String connectorId, + @Parameter(description = "The migration payload snapshot", required = true) final InputStream payloadContents) throws IOException { + if (payloadContents == null) { + throw new IllegalArgumentException("Migration payload contents must be specified."); + } + + final NiFiUser currentUser = NiFiUserUtils.getNiFiUser(); + serviceFacade.authorizeAccess(lookup -> { + final Authorizable connector = lookup.getConnector(connectorId); + connector.authorize(authorizer, RequestAction.WRITE, currentUser); + }); + + if (isReplicateRequest()) { + final String uploadRequestId = UUID.randomUUID().toString(); + final UploadRequest uploadRequest = new UploadRequest.Builder() + .user(currentUser) + .filename("migration-payload.json") + .identifier(uploadRequestId) + .contents(payloadContents) + .forwardRequestHeaders(getHeaders()) + .header(CONTENT_TYPE_HEADER, UPLOAD_CONTENT_TYPE) + .header(RequestReplicationHeader.CLUSTER_ID_GENERATION_SEED.getHeader(), uploadRequestId) + .exampleRequestUri(getAbsolutePath()) + .responseClass(MigrationPayloadEntity.class) + .successfulResponseStatus(HttpResponseStatus.OK.getCode()) + .build(); + final MigrationPayloadEntity entity = uploadRequestReplicator.upload(uploadRequest); + return generateOkResponse(entity).build(); + } + + final RegisteredFlowSnapshot flowSnapshot; + try { + flowSnapshot = OBJECT_MAPPER.readValue(payloadContents, RegisteredFlowSnapshot.class); + } catch (final IOException e) { + throw new IllegalArgumentException("Deserialization of uploaded migration payload failed", e); + } + + // Bundle discovery is deferred to the asynchronous migration task in performAsyncMigration so that + // the upload thread does not block on extension-manager work. + final String payloadId = getIdGenerationSeed().orElseGet(this::generateUuid); + migrationPayloadsById.put(payloadId, new MigrationPayloadEntry(flowSnapshot, System.currentTimeMillis())); + final MigrationPayloadDTO migrationPayload = new MigrationPayloadDTO(); + migrationPayload.setPayloadId(payloadId); + + final MigrationPayloadEntity entity = new MigrationPayloadEntity(); + entity.setPayload(migrationPayload); + return generateOkResponse(entity).build(); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests") + @Operation( + summary = "Creates a Connector migration request", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid}") + } + ) + public Response createMigrationRequest(@PathParam("id") final String connectorId, final MigrationRequestEntity requestEntity) { + if (requestEntity == null || requestEntity.getRequest() == null) { + throw new IllegalArgumentException("Migration request must be specified."); + } + + final MigrationRequestDTO request = requestEntity.getRequest(); + if (!connectorId.equals(request.getConnectorId())) { + throw new IllegalArgumentException("Connector identifier in the request must match the identifier provided in the URL."); + } + + final boolean hasLocalSource = request.getLocalSource() != null && StringUtils.isNotBlank(request.getLocalSource().getProcessGroupId()); + final boolean hasPayload = StringUtils.isNotBlank(request.getPayloadId()); + if (hasLocalSource == hasPayload) { + throw new IllegalArgumentException("Migration request must specify exactly one source: either a local Process Group or an uploaded payload identifier."); + } + + if (hasPayload && !migrationPayloadsById.containsKey(request.getPayloadId())) { + throw new ResourceNotFoundException("No uploaded migration payload exists with identifier " + request.getPayloadId()); + } + + // Reject the request when not all cluster nodes are connected, mirroring the asset-upload guard, + // because component state and assets cannot be synchronized to disconnected nodes after migration. + final ClusterCoordinator clusterCoordinator = getClusterCoordinator(); + if (clusterCoordinator != null) { + final Set disconnectedNodes = clusterCoordinator.getNodeIdentifiers(NodeConnectionState.CONNECTING, NodeConnectionState.DISCONNECTED, NodeConnectionState.DISCONNECTING); + if (!disconnectedNodes.isEmpty()) { + throw new IllegalStateException("Cannot start a Connector migration because the following %s nodes are not currently connected: %s" + .formatted(disconnectedNodes.size(), disconnectedNodes)); + } + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, requestEntity); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + return withWriteLock( + serviceFacade, + requestEntity, + lookup -> { + final Authorizable connector = lookup.getConnector(connectorId); + connector.authorize(authorizer, RequestAction.WRITE, user); + }, + () -> { + if (hasLocalSource) { + serviceFacade.verifyCanMigrateConnector(connectorId, request.getLocalSource().getProcessGroupId()); + } + }, + entity -> performAsyncMigration(entity, user) + ); + } + + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests/{requestId}") + @Operation( + summary = "Gets the Connector migration request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response getMigrationRequest(@PathParam("id") final String connectorId, @PathParam("requestId") final String requestId) { + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final AsynchronousWebRequest asyncRequest = migrationRequestManager.getRequest(MIGRATION_REQUEST_TYPE, requestId, user); + return generateOkResponse(createMigrationRequestEntity(asyncRequest, connectorId, requestId)).build(); + } + + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/migration-requests/{requestId}") + @Operation( + summary = "Deletes the Connector migration request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MigrationRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response deleteMigrationRequest(@PathParam("id") final String connectorId, @PathParam("requestId") final String requestId) { + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + if (!twoPhaseRequest || executionPhase) { + final AsynchronousWebRequest asyncRequest = + migrationRequestManager.removeRequest(MIGRATION_REQUEST_TYPE, requestId, user); + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + final MigrationRequestDTO request = asyncRequest.getRequest().getRequest(); + if (StringUtils.isNotBlank(request.getPayloadId())) { + migrationPayloadsById.remove(request.getPayloadId()); + } + + return generateOkResponse(createMigrationRequestEntity(asyncRequest, connectorId, requestId)).build(); + } + + if (isValidationPhase(httpServletRequest)) { + migrationRequestManager.getRequest(MIGRATION_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + private Response performAsyncMigration(final MigrationRequestEntity requestEntity, final NiFiUser user) { + final String requestId = generateUuid(); + final MigrationRequestDTO migrationRequest = requestEntity.getRequest(); + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Migrate Versioned Flow")); + + final AsynchronousWebRequest request = + new StandardAsynchronousWebRequest<>(requestId, requestEntity, migrationRequest.getConnectorId(), user, updateSteps); + + final Consumer> updateTask = asyncRequest -> { + final String payloadId = migrationRequest.getPayloadId(); + try { + final RegisteredFlowSnapshot flowSnapshot = getMigrationSnapshot(migrationRequest); + final String processGroupId = migrationRequest.getLocalSource() == null ? null : migrationRequest.getLocalSource().getProcessGroupId(); + final ConnectorEntity migratedConnector = serviceFacade.migrateConnector(migrationRequest.getConnectorId(), processGroupId, flowSnapshot, asyncRequest::isCancelled); + asyncRequest.markStepComplete(migratedConnector); + } catch (final Exception e) { + logger.error("Failed to migrate Connector {}", migrationRequest.getConnectorId(), e); + asyncRequest.fail("Failed to migrate Connector due to " + e); + } finally { + if (StringUtils.isNotBlank(payloadId)) { + migrationPayloadsById.remove(payloadId); + } + } + }; + + request.setCancelCallback(() -> { + final String payloadId = migrationRequest.getPayloadId(); + if (StringUtils.isNotBlank(payloadId)) { + migrationPayloadsById.remove(payloadId); + } + }); + + migrationRequestManager.submitRequest(MIGRATION_REQUEST_TYPE, requestId, request, updateTask); + final MigrationRequestEntity resultsEntity = createMigrationRequestEntity(request, migrationRequest.getConnectorId(), requestId); + return generateOkResponse(resultsEntity).build(); + } + + private RegisteredFlowSnapshot getMigrationSnapshot(final MigrationRequestDTO migrationRequest) { + final MigrationRequestLocalSourceDTO localSource = migrationRequest.getLocalSource(); + if (localSource != null && StringUtils.isNotBlank(localSource.getProcessGroupId())) { + // The local-source path uses the framework's snapshot facility, which already returns a snapshot + // whose bundles match this NiFi instance. discoverCompatibleBundles is intentionally not invoked + // here because it is reserved for snapshots that originate from external sources (uploaded payloads). + return serviceFacade.getCurrentFlowSnapshotByGroupId(localSource.getProcessGroupId(), true, true); + } + + final String payloadId = migrationRequest.getPayloadId(); + final MigrationPayloadEntry entry = migrationPayloadsById.get(payloadId); + if (entry == null) { + throw new ResourceNotFoundException("No uploaded migration payload exists with identifier " + payloadId); + } + + // Bundle discovery is performed once when the migration task picks up the uploaded payload, not on + // the upload thread, so that a slow extension manager does not block the upload-replication path. + final RegisteredFlowSnapshot flowSnapshot = entry.snapshot(); + serviceFacade.discoverCompatibleBundles(flowSnapshot.getFlowContents()); + serviceFacade.discoverCompatibleBundles(flowSnapshot.getParameterProviders()); + return flowSnapshot; + } + + private void evictExpiredMigrationPayloads() { + final long cutoffMillis = System.currentTimeMillis() - MIGRATION_PAYLOAD_TTL_MILLIS; + migrationPayloadsById.entrySet().removeIf(entry -> entry.getValue().uploadedAtMillis() < cutoffMillis); + } + + private MigrationRequestEntity createMigrationRequestEntity( + final AsynchronousWebRequest asyncRequest, + final String connectorId, + final String requestId) { + final MigrationRequestDTO requestedMigration = asyncRequest.getRequest().getRequest(); + + final MigrationRequestDTO migrationRequest = new MigrationRequestDTO(); + migrationRequest.setConnectorId(requestedMigration.getConnectorId()); + migrationRequest.setLocalSource(requestedMigration.getLocalSource()); + migrationRequest.setPayloadId(requestedMigration.getPayloadId()); + migrationRequest.setComplete(asyncRequest.isComplete()); + migrationRequest.setFailureReason(asyncRequest.getFailureReason()); + migrationRequest.setLastUpdated(asyncRequest.getLastUpdated()); + migrationRequest.setPercentCompleted(asyncRequest.getPercentComplete()); + migrationRequest.setRequestId(requestId); + migrationRequest.setState(asyncRequest.getState()); + migrationRequest.setUri(generateResourceUri("connectors", connectorId, "migration-requests", requestId)); + migrationRequest.setUpdateSteps(asyncRequest.getUpdateSteps().stream().map(updateStep -> { + final MigrationUpdateStepDTO migrationUpdateStep = new MigrationUpdateStepDTO(); + migrationUpdateStep.setDescription(updateStep.getDescription()); + migrationUpdateStep.setComplete(updateStep.isComplete()); + migrationUpdateStep.setFailureReason(updateStep.getFailureReason()); + return migrationUpdateStep; + }).toList()); + + final MigrationRequestEntity entity = new MigrationRequestEntity(); + entity.setRequest(migrationRequest); + return entity; + } + @POST @Consumes(MediaType.APPLICATION_OCTET_STREAM) @Produces(MediaType.APPLICATION_JSON) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java index acf2e7f53900..d0cbc2ef61fc 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java @@ -20,9 +20,11 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.ConnectorUpdateContext; +import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; @@ -30,6 +32,7 @@ import java.io.InputStream; import java.util.List; import java.util.Optional; +import java.util.function.BooleanSupplier; public interface ConnectorDAO { @@ -83,5 +86,11 @@ public interface ConnectorDAO { Optional getAsset(String assetId); + List getMigrationSources(String id); + + void verifyCanMigrateFromVersionedFlow(String connectorId, String processGroupId); + + void migrateFromVersionedFlow(String connectorId, String processGroupId, VersionedExternalFlow sourceFlow, BooleanSupplier cancellationCheck); + } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index 9a507e16e5ce..e68f1755895b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -16,11 +16,15 @@ */ package org.apache.nifi.web.dao.impl; +import jakarta.annotation.PostConstruct; import org.apache.nifi.asset.Asset; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConnectorFlowSnapshotProvider; +import org.apache.nifi.components.connector.ConnectorMigrationManager; +import org.apache.nifi.components.connector.ConnectorMigrationSource; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorRepository; import org.apache.nifi.components.connector.ConnectorSyncMode; @@ -28,11 +32,14 @@ import org.apache.nifi.components.connector.ConnectorValueReference; import org.apache.nifi.components.connector.ConnectorValueType; import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StandardConnectorMigrationManager; import org.apache.nifi.components.connector.StepConfiguration; import org.apache.nifi.components.connector.StringLiteralValue; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.flow.VersionedExternalFlow; import org.apache.nifi.web.NiFiCoreException; +import org.apache.nifi.web.NiFiServiceFacade; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; @@ -43,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Repository; import java.io.IOException; @@ -54,6 +62,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.function.BooleanSupplier; import java.util.stream.Collectors; @Repository @@ -62,12 +71,31 @@ public class StandardConnectorDAO implements ConnectorDAO { private static final Logger logger = LoggerFactory.getLogger(StandardConnectorDAO.class); private FlowController flowController; + private NiFiServiceFacade serviceFacade; + private ConnectorMigrationManager connectorMigrationManager; @Autowired public void setFlowController(final FlowController flowController) { this.flowController = flowController; } + /** + * Injected with {@link Lazy @Lazy} to break the circular dependency between this DAO and + * {@link NiFiServiceFacade}. The service facade is used solely as a {@link ConnectorFlowSnapshotProvider} + * for the migration manager constructed in {@link #initialize()}; the proxy is invoked lazily at + * migration time, so the real service-facade bean is not required at DAO construction. + */ + @Autowired + public void setServiceFacade(@Lazy final NiFiServiceFacade serviceFacade) { + this.serviceFacade = serviceFacade; + } + + @PostConstruct + public void initialize() { + final ConnectorFlowSnapshotProvider snapshotProvider = serviceFacade::getCurrentFlowSnapshotByGroupId; + this.connectorMigrationManager = new StandardConnectorMigrationManager(flowController, snapshotProvider); + } + private FlowManager getFlowManager() { return flowController.getFlowManager(); } @@ -294,6 +322,29 @@ public List getAssets(final String id) { public Optional getAsset(final String assetId) { return getConnectorRepository().getAsset(assetId); } + + @Override + public List getMigrationSources(final String id) { + requireConnector(id, ConnectorSyncMode.LOCAL_ONLY); + return connectorMigrationManager.listMigrationSources(id); + } + + @Override + public void verifyCanMigrateFromVersionedFlow(final String connectorId, final String processGroupId) { + requireConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + connectorMigrationManager.verifyEligibility(connectorId, processGroupId); + } + + @Override + public void migrateFromVersionedFlow(final String connectorId, final String processGroupId, final VersionedExternalFlow sourceFlow, + final BooleanSupplier cancellationCheck) { + requireConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + try { + connectorMigrationManager.migrateFromVersionedFlow(connectorId, processGroupId, sourceFlow, cancellationCheck); + } catch (final Exception e) { + throw new NiFiCoreException("Failed to migrate Connector from Versioned Flow: " + e.getMessage(), e); + } + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java index 2fcac1d5dcdc..3fdf8e8f0bc3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java @@ -32,6 +32,8 @@ import org.apache.nifi.web.api.dto.AllowableValueDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterDTO; import org.apache.nifi.web.api.dto.RevisionDTO; @@ -43,10 +45,12 @@ import org.apache.nifi.web.api.entity.ConnectorRunStatusEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.SecretsEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; import org.junit.jupiter.api.AfterEach; @@ -162,6 +166,56 @@ public void testGetConnector() { verify(serviceFacade).getConnector(CONNECTOR_ID, false); } + @Test + public void testGetMigrationSources() { + final VersionedFlowMigrationSourcesEntity migrationSourcesEntity = new VersionedFlowMigrationSourcesEntity(); + when(serviceFacade.getConnectorMigrationSources(CONNECTOR_ID)).thenReturn(migrationSourcesEntity); + + try (final Response response = connectorResource.getMigrationSources(CONNECTOR_ID)) { + assertEquals(200, response.getStatus()); + assertEquals(migrationSourcesEntity, response.getEntity()); + } + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).getConnectorMigrationSources(CONNECTOR_ID); + } + + @Test + public void testCreateMigrationRequestRejectsMismatchedConnectorId() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId("different-connector"); + requestDto.setLocalSource(createLocalMigrationSource(PROCESS_GROUP_ID)); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + + @Test + public void testCreateMigrationRequestRequiresExactlyOneSource() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(CONNECTOR_ID); + requestDto.setLocalSource(createLocalMigrationSource(PROCESS_GROUP_ID)); + requestDto.setPayloadId("payload-1"); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + + @Test + public void testCreateMigrationRequestRequiresLocalSourceOrPayload() { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(CONNECTOR_ID); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + + assertThrows(IllegalArgumentException.class, () -> connectorResource.createMigrationRequest(CONNECTOR_ID, requestEntity)); + } + @Test public void testGetConnectorClusterNodeRequestBypassesAuth() { final ConnectorResource spyResource = spy(connectorResource); @@ -878,4 +932,10 @@ public void testClearConnectorControllerServiceStateNotAuthorized() { verify(serviceFacade, never()).verifyCanClearConnectorControllerServiceState(anyString(), anyString()); verify(serviceFacade, never()).clearConnectorControllerServiceState(anyString(), anyString(), any()); } + + private MigrationRequestLocalSourceDTO createLocalMigrationSource(final String processGroupId) { + final MigrationRequestLocalSourceDTO localSource = new MigrationRequestLocalSourceDTO(); + localSource.setProcessGroupId(processGroupId); + return localSource; + } } diff --git a/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts b/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts index b8e82ac3f617..6a8f5098ec71 100644 --- a/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts +++ b/nifi-frontend/src/main/frontend/libs/shared/src/types/index.ts @@ -234,6 +234,7 @@ export type ConnectorActionName = | 'DRAIN_FLOWFILES' | 'CANCEL_DRAIN_FLOWFILES' | 'APPLY_UPDATES' + | 'MIGRATE' | 'DELETE'; export interface ConnectorAction { diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java index 2e927bb038a7..0654f5f7655d 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java @@ -63,6 +63,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { // No-op: this connector does not manipulate the flow. diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java new file mode 100644 index 000000000000..014b6b8eff0d --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AsymmetricFailureMigrationConnector.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +/** + * Test connector whose {@code migrate(...)} succeeds on every cluster node except the one whose + * {@code -DnodeNumber=} JVM argument matches {@value #FAILING_NODE_NUMBER}. Used to validate that the cluster + * migration-request endpoint merger surfaces a per-node failure even when other nodes report success. + */ +public class AsymmetricFailureMigrationConnector extends AbstractConnector implements MigratableConnector { + private static final String FAILING_NODE_NUMBER = "2"; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Asymmetric Migration Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + return true; + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final String currentNodeNumber = System.getProperty("nodeNumber"); + if (FAILING_NODE_NUMBER.equals(currentNodeNumber)) { + throw new FlowUpdateException("Simulated migration failure on node " + currentNodeNumber); + } + + getInitializationContext().updateFlow(context.getActiveFlowContext(), context.getSourceFlow()); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java index f15d668ef51f..aa3a65953af8 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java @@ -110,6 +110,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedExternalFlow createFlowWithBundleScenarios() { final VersionedProcessGroup group = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Bundle Resolution Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java index bb767219154d..cf93d4695452 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java @@ -172,6 +172,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(CALCULATION_STEP); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java index 81c6af171b13..c9fa2df56997 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java @@ -67,6 +67,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedProcessGroup createRootGroup() { final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Component Lifecycle Root"); rootGroup.setPosition(new Position(0, 0)); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java index 1c50941c8371..970427dc60da 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java @@ -63,6 +63,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java new file mode 100644 index 000000000000..c6b3eb81f41c --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/FailingMigrationConnector.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; +import org.apache.nifi.flow.VersionedAsset; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class FailingMigrationConnector extends AbstractConnector implements MigratableConnector { + private static final long FAILURE_DELAY_SECONDS = 5L; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Failing Migration Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + return true; + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + copyFirstAsset(context); + getInitializationContext().updateFlow(context.getActiveFlowContext(), context.getSourceFlow()); + try { + TimeUnit.SECONDS.sleep(FAILURE_DELAY_SECONDS); + } catch (final InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new FlowUpdateException("Interrupted while waiting to fail the migration", interruptedException); + } + + throw new FlowUpdateException("Intended migration failure for rollback testing"); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } + + private void copyFirstAsset(final ConnectorMigrationContext context) { + if (!context.isLocalMigration()) { + return; + } + + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow.getParameterContexts() == null) { + return; + } + + for (final VersionedParameterContext parameterContext : sourceFlow.getParameterContexts().values()) { + if (parameterContext.getParameters() == null) { + continue; + } + + for (final VersionedParameter parameter : parameterContext.getParameters()) { + if (parameter.getReferencedAssets() == null) { + continue; + } + + for (final VersionedAsset referencedAsset : parameter.getReferencedAssets()) { + context.copyAssetFromSource(referencedAsset.getIdentifier()); + return; + } + } + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java index 8c7f680d86ba..555f18caf161 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java @@ -94,6 +94,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java new file mode 100644 index 000000000000..c09d3db487a6 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigrationTargetConnector.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.migration.ConnectorMigrationContext; +import org.apache.nifi.components.connector.migration.MigratableConnector; +import org.apache.nifi.flow.VersionedAsset; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MigrationTargetConnector extends AbstractConnector implements MigratableConnector { + private static final String REQUIRED_PARAMETER_NAME = "Source Topic"; + private static final String REQUIRED_PROCESSOR_TYPE = "org.apache.nifi.processors.tests.system.StatefulCountProcessor"; + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Migration Target Flow"); + processGroup.setProcessors(new HashSet<>()); + processGroup.setConnections(new HashSet<>()); + processGroup.setProcessGroups(new HashSet<>()); + processGroup.setControllerServices(new HashSet<>()); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + public boolean isMigrationSupported(final ConnectorMigrationContext context) { + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow == null || sourceFlow.getFlowContents() == null) { + return false; + } + + return containsRequiredParameter(sourceFlow.getParameterContexts()) && containsRequiredProcessor(sourceFlow.getFlowContents()); + } + + @Override + public void migrate(final ConnectorMigrationContext context) throws FlowUpdateException { + final VersionedExternalFlow sourceFlow = context.getSourceFlow(); + if (sourceFlow == null) { + throw new FlowUpdateException("A source flow is required for migration"); + } + + rewriteReferencedAssets(sourceFlow, context); + getInitializationContext().updateFlow(context.getActiveFlowContext(), sourceFlow); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } + + private boolean containsRequiredParameter(final Map parameterContexts) { + if (parameterContexts == null || parameterContexts.isEmpty()) { + return false; + } + + for (final VersionedParameterContext parameterContext : parameterContexts.values()) { + final Set parameters = parameterContext.getParameters(); + if (parameters == null) { + continue; + } + + for (final VersionedParameter parameter : parameters) { + if (REQUIRED_PARAMETER_NAME.equals(parameter.getName())) { + return true; + } + } + } + + return false; + } + + private boolean containsRequiredProcessor(final VersionedProcessGroup processGroup) { + final Set processors = processGroup.getProcessors(); + if (processors != null) { + for (final VersionedProcessor processor : processors) { + if (REQUIRED_PROCESSOR_TYPE.equals(processor.getType())) { + return true; + } + } + } + + final Set childGroups = processGroup.getProcessGroups(); + if (childGroups == null) { + return false; + } + + for (final VersionedProcessGroup childGroup : childGroups) { + if (containsRequiredProcessor(childGroup)) { + return true; + } + } + + return false; + } + + private void rewriteReferencedAssets(final VersionedExternalFlow sourceFlow, final ConnectorMigrationContext context) { + final Map parameterContexts = sourceFlow.getParameterContexts(); + if (parameterContexts == null || parameterContexts.isEmpty()) { + return; + } + + for (final VersionedParameterContext parameterContext : parameterContexts.values()) { + final Set parameters = parameterContext.getParameters(); + if (parameters == null) { + continue; + } + + for (final VersionedParameter parameter : parameters) { + final List referencedAssets = parameter.getReferencedAssets(); + if (referencedAssets == null || referencedAssets.isEmpty()) { + continue; + } + + final List migratedAssets = new ArrayList<>(); + for (final VersionedAsset referencedAsset : referencedAssets) { + final AssetReference migratedReference = context.copyAssetFromSource(referencedAsset.getIdentifier()); + for (final String migratedAssetId : migratedReference.getAssetIdentifiers()) { + final VersionedAsset migratedAsset = new VersionedAsset(); + migratedAsset.setIdentifier(migratedAssetId); + migratedAsset.setName(referencedAsset.getName()); + migratedAssets.add(migratedAsset); + } + } + + parameter.setReferencedAssets(migratedAssets); + } + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java index e3903a38f1de..86d692826e4e 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java @@ -58,6 +58,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java new file mode 100644 index 000000000000..fa65f3997dbe --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NonMigratingConnector.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class NonMigratingConnector extends AbstractConnector { + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup processGroup = new VersionedProcessGroup(); + processGroup.setName("Non Migrating Flow"); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(processGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java index 3d28240ca256..b07edfddfe07 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java @@ -100,6 +100,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(new ConfigVerificationResult.Builder() diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java index 9d160a11f1c2..f1ca456b1744 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java @@ -134,6 +134,11 @@ public VersionedExternalFlow getInitialFlow() { return createEmptyFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + private VersionedExternalFlow createEmptyFlow() { final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(ROOT_GROUP_ID, "Parameter Context Test Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java index 441f4bc3d47f..cdd6e49befc5 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java @@ -83,6 +83,11 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java new file mode 100644 index 000000000000..f16ed42dab19 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/AssetReadingProcessor.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.tests.system; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.stream.io.StreamUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.Set; + +@Tags({"asset", "test"}) +@CapabilityDescription("Reads the contents of a file-backed asset parameter and writes the asset contents to a configured output file.") +public class AssetReadingProcessor extends AbstractProcessor { + static final PropertyDescriptor SOURCE_FILE = new PropertyDescriptor.Builder() + .name("Source File") + .description("The path of the asset file to read") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final PropertyDescriptor OUTPUT_FILE = new PropertyDescriptor.Builder() + .name("Output File") + .description("The file where the asset contents will be written") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .build(); + + static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .build(); + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of(SOURCE_FILE, OUTPUT_FILE); + } + + @Override + public Set getRelationships() { + return Set.of(REL_SUCCESS, REL_FAILURE); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + final FlowFile incomingFlowFile = session.get(); + final FlowFile flowFile = incomingFlowFile == null ? session.create() : incomingFlowFile; + + final File sourceFile = new File(context.getProperty(SOURCE_FILE).getValue()); + final File outputFile = new File(context.getProperty(OUTPUT_FILE).getValue()); + final File parentFile = outputFile.getParentFile(); + if (parentFile != null && !parentFile.exists() && !parentFile.mkdirs()) { + getLogger().error("Could not create directory {}", parentFile.getAbsolutePath()); + session.transfer(flowFile, REL_FAILURE); + return; + } + + try (final InputStream inputStream = new FileInputStream(sourceFile); + final OutputStream outputStream = new FileOutputStream(outputFile)) { + StreamUtils.copy(inputStream, outputStream); + session.transfer(flowFile, REL_SUCCESS); + } catch (final Exception e) { + getLogger().error("Failed to read asset file {} to {}", sourceFile.getAbsolutePath(), outputFile.getAbsolutePath(), e); + session.transfer(flowFile, REL_FAILURE); + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector index a2d46427433a..aef142042c3e 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -14,12 +14,16 @@ # limitations under the License. org.apache.nifi.connectors.tests.system.AssetConnector +org.apache.nifi.connectors.tests.system.AsymmetricFailureMigrationConnector org.apache.nifi.connectors.tests.system.BundleResolutionConnector org.apache.nifi.connectors.tests.system.CalculateConnector org.apache.nifi.connectors.tests.system.ComponentLifecycleConnector org.apache.nifi.connectors.tests.system.DataQueuingConnector org.apache.nifi.connectors.tests.system.GatedDataQueuingConnector +org.apache.nifi.connectors.tests.system.FailingMigrationConnector +org.apache.nifi.connectors.tests.system.MigrationTargetConnector org.apache.nifi.connectors.tests.system.NestedProcessGroupConnector +org.apache.nifi.connectors.tests.system.NonMigratingConnector org.apache.nifi.connectors.tests.system.NopConnector org.apache.nifi.connectors.tests.system.ParameterContextConnector org.apache.nifi.connectors.tests.system.SelectiveDropConnector diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index 80655da6f369..5c1de2753838 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +org.apache.nifi.processors.tests.system.AssetReadingProcessor org.apache.nifi.processors.tests.system.ClassloaderIsolationWithServiceProperty org.apache.nifi.processors.tests.system.CountEvents org.apache.nifi.processors.tests.system.CountFlowFiles diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 8960e202ce3e..f95a8e9be5c2 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -16,14 +16,19 @@ */ package org.apache.nifi.tests.system; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.nifi.cluster.coordination.node.NodeConnectionState; import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.controller.AbstractPort; import org.apache.nifi.controller.queue.LoadBalanceCompression; import org.apache.nifi.controller.queue.LoadBalanceStrategy; import org.apache.nifi.controller.queue.QueueSize; +import org.apache.nifi.flow.VersionedNodeState; +import org.apache.nifi.flow.VersionedProcessor; import org.apache.nifi.parameter.ParameterProviderConfiguration; import org.apache.nifi.provenance.search.SearchableField; +import org.apache.nifi.registry.flow.RegisteredFlowSnapshot; import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; import org.apache.nifi.scheduling.ExecutionNode; import org.apache.nifi.stream.io.StreamUtils; @@ -49,6 +54,8 @@ import org.apache.nifi.web.api.dto.FlowFileSummaryDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; +import org.apache.nifi.web.api.dto.MigrationRequestDTO; +import org.apache.nifi.web.api.dto.MigrationRequestLocalSourceDTO; import org.apache.nifi.web.api.dto.NodeDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; @@ -95,6 +102,8 @@ import org.apache.nifi.web.api.entity.FlowFileEntity; import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; import org.apache.nifi.web.api.entity.ListingRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.NodeEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; @@ -125,12 +134,14 @@ import org.apache.nifi.web.api.entity.VerifyConfigRequestEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity; import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -335,6 +346,124 @@ public ConnectorValueReferenceDTO createSecretValueReference(final String secret return valueRef; } + public VersionedFlowMigrationSourcesEntity listMigrationSources(final String connectorId) throws NiFiClientException, IOException { + return getConnectorClient().listMigrationSources(connectorId); + } + + public MigrationPayloadEntity uploadMigrationPayload(final String connectorId, final File file) throws NiFiClientException, IOException { + return getConnectorClient().uploadMigrationPayload(connectorId, file); + } + + public MigrationRequestEntity startMigrationFromLocalSource(final String connectorId, final String processGroupId) throws NiFiClientException, IOException { + final MigrationRequestLocalSourceDTO localSource = new MigrationRequestLocalSourceDTO(); + localSource.setProcessGroupId(processGroupId); + + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(connectorId); + requestDto.setLocalSource(localSource); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + return getConnectorClient().startMigration(requestEntity); + } + + public MigrationRequestEntity startMigrationFromPayload(final String connectorId, final String payloadId) throws NiFiClientException, IOException { + final MigrationRequestDTO requestDto = new MigrationRequestDTO(); + requestDto.setConnectorId(connectorId); + requestDto.setPayloadId(payloadId); + + final MigrationRequestEntity requestEntity = new MigrationRequestEntity(); + requestEntity.setRequest(requestDto); + return getConnectorClient().startMigration(requestEntity); + } + + private static final long MIGRATION_WAIT_TIMEOUT_MILLIS = 60_000L; + + public MigrationRequestEntity waitForMigrationToComplete(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final long deadlineMillis = System.currentTimeMillis() + MIGRATION_WAIT_TIMEOUT_MILLIS; + int iteration = 0; + while (true) { + final MigrationRequestEntity requestEntity = getConnectorClient().getMigrationStatus(connectorId, requestId); + final MigrationRequestDTO request = requestEntity.getRequest(); + if (request.isComplete()) { + return requestEntity; + } + + if (System.currentTimeMillis() >= deadlineMillis) { + throw new IllegalStateException("Migration request " + requestId + " for Connector " + connectorId + + " did not complete within " + MIGRATION_WAIT_TIMEOUT_MILLIS + "ms; last state " + request.getState() + + " (" + request.getPercentCompleted() + "% complete)"); + } + + if (iteration++ % 30 == 0) { + logger.info("Migration request {} for Connector {} is in state {} ({}% complete)", + requestId, connectorId, request.getState(), request.getPercentCompleted()); + } + + Thread.sleep(100L); + } + } + + public MigrationRequestEntity waitForMigrationSuccess(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final MigrationRequestEntity completedRequest = waitForMigrationToComplete(connectorId, requestId); + final String failureReason = completedRequest.getRequest().getFailureReason(); + if (failureReason != null) { + throw new IllegalStateException("Migration failed: " + failureReason); + } + + return completedRequest; + } + + public MigrationRequestEntity waitForMigrationFailure(final String connectorId, final String requestId) throws NiFiClientException, IOException, InterruptedException { + final MigrationRequestEntity completedRequest = waitForMigrationToComplete(connectorId, requestId); + if (completedRequest.getRequest().getFailureReason() == null) { + throw new IllegalStateException("Expected migration failure but request completed successfully"); + } + + return completedRequest; + } + + public MigrationRequestEntity cancelMigration(final String connectorId, final String requestId) throws NiFiClientException, IOException { + return getConnectorClient().cancelMigration(connectorId, requestId); + } + + private static final ObjectMapper MIGRATION_PAYLOAD_OBJECT_MAPPER = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + /** + * Reads an exported flow snapshot, expands the LOCAL component state of the processor whose type ends + * with the given suffix to the requested number of node states, and writes the modified snapshot back + * to the same file. Used by clustered migration tests that need to construct a payload whose + * cluster-topology rule violation is unambiguous. + * + * @param exportFile the file containing a previously-exported {@link RegisteredFlowSnapshot} + * @param processorTypeSuffix a suffix that uniquely identifies the stateful processor inside the snapshot + * @param nodeStateValues the count values to assign to each synthetic node state, one per source node + * @throws IOException when the file cannot be read or written + * @throws IllegalStateException when no processor matches the given suffix or has no exported state + */ + public void rewriteMigrationPayloadWithNodeStates(final File exportFile, final String processorTypeSuffix, final List nodeStateValues) throws IOException { + final RegisteredFlowSnapshot flowSnapshot = MIGRATION_PAYLOAD_OBJECT_MAPPER.readValue(exportFile, RegisteredFlowSnapshot.class); + final VersionedProcessor statefulProcessor = flowSnapshot.getFlowContents().getProcessors().stream() + .filter(processor -> processor.getType().endsWith(processorTypeSuffix)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No processor in snapshot ends with type " + processorTypeSuffix)); + if (statefulProcessor.getComponentState() == null) { + throw new IllegalStateException("Processor " + statefulProcessor.getName() + " has no exported component state"); + } + + final List nodeStates = new ArrayList<>(); + for (final String value : nodeStateValues) { + nodeStates.add(new VersionedNodeState(Map.of("count", value))); + } + statefulProcessor.getComponentState().setLocalNodeStates(nodeStates); + + if (!exportFile.delete() && exportFile.exists()) { + throw new IOException("Could not delete prior export file " + exportFile.getAbsolutePath() + " before writing the modified payload"); + } + MIGRATION_PAYLOAD_OBJECT_MAPPER.writeValue(exportFile, flowSnapshot); + } + public ConfigurationStepEntity configureConnectorWithReferences(final String connectorId, final String configurationStepName, final Map propertyValues) throws NiFiClientException, IOException { final ConnectorEntity connectorEntity = getConnectorClient().getConnector(connectorId); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java new file mode 100644 index 000000000000..e11c241a7516 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationFailureIT.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ClusteredConnectorVersionedFlowMigrationFailureIT extends ConnectorVersionedFlowMigrationFailureIT { + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } + + @Test + public void testUploadedPayloadWithTooManyLocalNodeStatesFails() throws Exception { + final File outputFile = new File("target/migration/cluster-topology-output.txt"); + final File exportFile = new File("target/migration/cluster-topology-export.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("ClusterTopologySource", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + + // Inflate the LOCAL state to three node states even though the destination cluster only has two + // connected nodes; the cluster-topology rule must reject the payload. + getClientUtil().rewriteMigrationPayloadWithNodeStates(exportFile, "StatefulCountProcessor", List.of("1", "2", "3")); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connector.getId(), exportFile); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connector.getId(), payloadEntity.getPayload().getPayloadId()); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connector.getId(), requestEntity.getRequest().getRequestId()); + assertTrue(completedRequest.getRequest().getFailureReason().contains("connected node")); + + assertConnectorFresh(connector.getId()); + } + + @Test + public void testAsymmetricPerNodeMigrationFailureSurfacesInMergedResponse() throws Exception { + final File outputFile = new File("target/migration/asymmetric-failure-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("AsymmetricFailureSource", registryClient, false, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + // AsymmetricFailureMigrationConnector throws on node 2 only; node 1 succeeds. The merged response + // returned by the migration-request endpoint must surface node 2's failure, otherwise an operator polling + // the cluster would mistakenly see success. + final ConnectorEntity connector = getClientUtil().createConnector("AsymmetricFailureMigrationConnector"); + final String connectorId = connector.getId(); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceFixture.processGroup().getId()); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connectorId, requestEntity.getRequest().getRequestId()); + + final String failureReason = completedRequest.getRequest().getFailureReason(); + assertNotNull(failureReason); + assertTrue(failureReason.contains("node 2"), failureReason); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java new file mode 100644 index 000000000000..3bb23a614660 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorVersionedFlowMigrationIT.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; + +public class ClusteredConnectorVersionedFlowMigrationIT extends ConnectorVersionedFlowMigrationLocalIT { + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java new file mode 100644 index 000000000000..76a61e272da8 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationEligibilityIT.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ConnectorVersionedFlowMigrationEligibilityIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrationSourcesListingFiltersUnsupportedSources() throws Exception { + final FlowRegistryClientEntity registryClient = registerClient(); + final File matchingOutputFile = new File("target/migration/eligibility-matching.txt"); + final File nonMatchingOutputFile = new File("target/migration/eligibility-non-matching.txt"); + + final SourceFixture matchingFixture = createSourceFixture("EligibilityMatching", registryClient, false, matchingOutputFile, true); + final ProcessGroupEntity nonMatchingGroup = createProcessGroupWithoutRequiredParameter("EligibilityNonMatching", registryClient, nonMatchingOutputFile); + + final ConnectorEntity targetConnector = getClientUtil().createConnector("MigrationTargetConnector"); + final VersionedFlowMigrationSourcesEntity sourcesEntity = getClientUtil().listMigrationSources(targetConnector.getId()); + assertTrue(isSourceListed(sourcesEntity, matchingFixture.processGroup().getId())); + assertFalse(isSourceListed(sourcesEntity, nonMatchingGroup.getId())); + + final ConnectorEntity nonMigratingConnector = getClientUtil().createConnector("NonMigratingConnector"); + final VersionedFlowMigrationSourcesEntity nonMigratingSources = getClientUtil().listMigrationSources(nonMigratingConnector.getId()); + assertTrue(nonMigratingSources.getMigrationSources() == null || nonMigratingSources.getMigrationSources().isEmpty()); + } + + @Test + public void testLocalMigrationRejectsNonVersionControlledSource() throws Exception { + final SourceFixture sourceFixture = createSourceFixture("EligibilityLocalFailure", registerClient(), false, + new File("target/migration/eligibility-local-failure.txt"), false); + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + + assertThrows(NiFiClientException.class, () -> getClientUtil().startMigrationFromLocalSource(connector.getId(), sourceFixture.processGroup().getId())); + } + + @Test + public void testUploadedPayloadAcceptsNonVersionControlledSource() throws Exception { + final File outputFile = new File("target/migration/eligibility-uploaded.txt"); + final File exportFile = new File("target/migration/eligibility-uploaded.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("EligibilityUploaded", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + + outputFile.delete(); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connector.getId(), exportFile); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connector.getId(), payloadEntity.getPayload().getPayloadId()); + getClientUtil().waitForMigrationSuccess(connector.getId(), requestEntity.getRequest().getRequestId()); + } + + private ProcessGroupEntity createProcessGroupWithoutRequiredParameter(final String name, final FlowRegistryClientEntity registryClient, final File outputFile) throws Exception { + final ProcessGroupEntity processGroup = getClientUtil().createProcessGroup(name, "root"); + final ProcessorEntity statefulCount = getClientUtil().createProcessor("StatefulCountProcessor", processGroup.getId()); + final ProcessorEntity assetReader = getClientUtil().updateProcessorProperties( + getClientUtil().createProcessor("AssetReadingProcessor", processGroup.getId()), + Map.of( + "Source File", SAMPLE_ASSET_FILE.getAbsolutePath(), + "Output File", outputFile.getAbsolutePath())); + getClientUtil().setAutoTerminatedRelationships(assetReader, Set.of("success", "failure")); + getClientUtil().createConnection(statefulCount, assetReader, "success"); + getClientUtil().startVersionControl(processGroup, registryClient, TEST_BUCKET, name); + return processGroup; + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java new file mode 100644 index 000000000000..2d4b8049371e --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationFailureIT.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class ConnectorVersionedFlowMigrationFailureIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrationFailureRollsBackConnectorAndLeavesSourceUntouched() throws Exception { + final File outputFile = new File("target/migration/failure-local-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("FailureSource", registryClient, true, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + final String sourceGroupId = sourceFixture.processGroup().getId(); + final String originalName = getNifiClient().getProcessGroupClient().getProcessGroup(sourceGroupId).getComponent().getName(); + + final ConnectorEntity connector = getClientUtil().createConnector("FailingMigrationConnector"); + final String connectorId = connector.getId(); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceGroupId); + final MigrationRequestEntity completedRequest = getClientUtil().waitForMigrationFailure(connectorId, requestEntity.getRequest().getRequestId()); + assertNotNull(completedRequest.getRequest().getFailureReason()); + + assertConnectorFresh(connectorId); + assertSourceUntouched(sourceFixture, originalName); + } + + @Test + public void testCorruptPayloadUploadRejectedAndConnectorRemainsFresh() throws Exception { + final File corruptPayload = new File("target/migration/corrupt-payload.json"); + corruptPayload.getParentFile().mkdirs(); + Files.writeString(corruptPayload.toPath(), "{ this is not valid json"); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + assertThrows(NiFiClientException.class, () -> getClientUtil().uploadMigrationPayload(connector.getId(), corruptPayload)); + + assertConnectorFresh(connector.getId()); + } + + @Test + public void testMigrationInterruptedByRestartLeavesConnectorFresh() throws Exception { + final File outputFile = new File("target/migration/failure-restart-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("FailureRestartSource", registryClient, false, outputFile, true); + prepareSourceForMigration(sourceFixture, outputFile); + + final String sourceGroupId = sourceFixture.processGroup().getId(); + final String originalName = getNifiClient().getProcessGroupClient().getProcessGroup(sourceGroupId).getComponent().getName(); + + final ConnectorEntity connector = getClientUtil().createConnector("FailingMigrationConnector"); + final String connectorId = connector.getId(); + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceGroupId); + + Thread.sleep(1000L); + getNiFiInstance().stop(); + getNiFiInstance().start(true); + setupClient(); + + waitFor(() -> { + try { + getNifiClient().getConnectorClient().getConnector(connectorId); + return true; + } catch (final Exception e) { + return false; + } + }); + + assertThrows(NiFiClientException.class, () -> getNifiClient().getConnectorClient().getMigrationStatus(connectorId, requestEntity.getRequest().getRequestId())); + + waitFor(() -> isConnectorFresh(connectorId)); + assertConnectorFresh(connectorId); + assertSourceUntouched(sourceFixture, originalName); + } + + private boolean isConnectorFresh(final String connectorId) { + try { + final ConnectorEntity connectorEntity = getNifiClient().getConnectorClient().getConnector(connectorId); + final String managedGroupId = connectorEntity.getComponent().getManagedProcessGroupId(); + final ProcessGroupFlowEntity flowEntity = getNifiClient().getConnectorClient().getFlow(connectorId, managedGroupId); + if (flowEntity.getProcessGroupFlow().getFlow().getProcessors() != null && !flowEntity.getProcessGroupFlow().getFlow().getProcessors().isEmpty()) { + return false; + } + if (flowEntity.getProcessGroupFlow().getFlow().getConnections() != null && !flowEntity.getProcessGroupFlow().getFlow().getConnections().isEmpty()) { + return false; + } + + return getNifiClient().getConnectorClient().getAssets(connectorId).getAssets() == null + || getNifiClient().getConnectorClient().getAssets(connectorId).getAssets().isEmpty(); + } catch (final Exception e) { + return false; + } + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java new file mode 100644 index 000000000000..9703f16a6475 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationLocalIT.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.ConnectorClient; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.VersionedFlowMigrationSourceDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.ComponentStateEntity; +import org.apache.nifi.web.api.entity.ConnectionEntity; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.ControllerServiceEntity; +import org.apache.nifi.web.api.entity.ControllerServicesEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ConnectorVersionedFlowMigrationLocalIT extends NiFiSystemIT { + protected static final String TEST_BUCKET = "test-flows"; + private static final String ASSET_PARAMETER_NAME = "Asset File"; + private static final String SOURCE_TOPIC_PARAMETER_NAME = "Source Topic"; + protected static final File SAMPLE_ASSET_FILE = new File("src/test/resources/sample-assets/helloworld.txt"); + + @Test + public void testMigrateConnectorFromLocalVersionedFlow() throws Exception { + final File outputFile = new File("target/migration/local-output.txt"); + outputFile.delete(); + + final FlowRegistryClientEntity registryClient = registerClient(); + final SourceFixture sourceFixture = createSourceFixture("LocalMigrationSource", registryClient, true, outputFile, true); + + prepareSourceForMigration(sourceFixture, outputFile); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final String connectorId = connector.getId(); + final VersionedFlowMigrationSourcesEntity sourcesEntity = getClientUtil().listMigrationSources(connectorId); + assertTrue(isSourceListed(sourcesEntity, sourceFixture.processGroup().getId())); + + outputFile.delete(); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromLocalSource(connectorId, sourceFixture.processGroup().getId()); + final String requestId = requestEntity.getRequest().getRequestId(); + getClientUtil().waitForMigrationSuccess(connectorId, requestId); + + final ProcessGroupEntity migratedSourceGroup = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + assertTrue(migratedSourceGroup.getComponent().getName().startsWith("(Migrated) ")); + + // The source process group's components must all be disabled after a successful migration so + // they cannot be inadvertently restarted, regardless of whether the migration also renamed the group. + final ProcessGroupFlowEntity migratedSourceFlow = + getNifiClient().getFlowClient().getProcessGroup(sourceFixture.processGroup().getId()); + for (final ProcessorEntity sourceProcessor : migratedSourceFlow.getProcessGroupFlow().getFlow().getProcessors()) { + assertEquals("DISABLED", sourceProcessor.getComponent().getState(), + "Migrated source processor " + sourceProcessor.getComponent().getName() + " must be DISABLED after successful migration"); + } + + final ConnectorClient connectorClient = getNifiClient().getConnectorClient(); + final AssetsEntity connectorAssets = connectorClient.getAssets(connectorId); + assertNotNull(connectorAssets.getAssets()); + assertFalse(connectorAssets.getAssets().isEmpty()); + + final ConnectorEntity migratedConnector = connectorClient.getConnector(connectorId); + final String managedGroupId = migratedConnector.getComponent().getManagedProcessGroupId(); + final String migratedCountProcessorId = getProcessorId(connectorId, managedGroupId, "StatefulCountProcessor"); + assertMigratedProcessorState(connectorId, migratedCountProcessorId); + + getClientUtil().startConnector(connectorId); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + assertEquals(Files.readString(SAMPLE_ASSET_FILE.toPath()).trim(), Files.readString(outputFile.toPath()).trim()); + } + + protected SourceFixture createSourceFixture(final String name, final FlowRegistryClientEntity registryClient, final boolean includeAsset, final File outputFile, + final boolean versionControlled) throws Exception { + final ProcessGroupEntity processGroup = getClientUtil().createProcessGroup(name, "root"); + + final ParameterContextEntity assetContext = getClientUtil().createParameterContext(name + "-asset", Map.of()); + if (includeAsset) { + final AssetEntity assetEntity = getNifiClient().getParamContextClient().createAsset(assetContext.getId(), SAMPLE_ASSET_FILE.getName(), SAMPLE_ASSET_FILE); + final ParameterContextUpdateRequestEntity requestEntity = getClientUtil().updateParameterAssetReferences( + assetContext, Map.of(ASSET_PARAMETER_NAME, List.of(assetEntity.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(assetContext.getId(), requestEntity.getRequest().getRequestId()); + } + + final ParameterContextEntity sourceContext = getClientUtil().createParameterContext( + name + "-source", + Map.of(SOURCE_TOPIC_PARAMETER_NAME, "orders"), + List.of(assetContext.getId()), + null); + getClientUtil().setParameterContext(processGroup.getId(), sourceContext); + + final ProcessorEntity statefulCount = getClientUtil().createProcessor("StatefulCountProcessor", processGroup.getId()); + final ProcessorEntity assetReader = getClientUtil().updateProcessorProperties(getClientUtil().createProcessor("AssetReadingProcessor", processGroup.getId()), Map.of( + "Source File", includeAsset ? "#{" + ASSET_PARAMETER_NAME + "}" : SAMPLE_ASSET_FILE.getAbsolutePath(), + "Output File", outputFile.getAbsolutePath())); + getClientUtil().setAutoTerminatedRelationships(assetReader, Set.of("success", "failure")); + + final ConnectionEntity connection = getClientUtil().createConnection(statefulCount, assetReader, "success"); + + if (versionControlled) { + assertNotNull(getClientUtil().startVersionControl(processGroup, registryClient, TEST_BUCKET, name)); + } + + return new SourceFixture(processGroup, connection); + } + + protected String getProcessorId(final String connectorId, final String groupId, final String processorTypeSuffix) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity flowEntity = getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + for (final ProcessorEntity processor : flowEntity.getProcessGroupFlow().getFlow().getProcessors()) { + if (processor.getComponent().getType().endsWith(processorTypeSuffix)) { + return processor.getId(); + } + } + + throw new IllegalStateException("Could not find processor ending with type " + processorTypeSuffix); + } + + protected boolean isSourceListed(final VersionedFlowMigrationSourcesEntity sourcesEntity, final String processGroupId) { + if (sourcesEntity.getMigrationSources() == null) { + return false; + } + + for (final VersionedFlowMigrationSourceDTO migrationSource : sourcesEntity.getMigrationSources()) { + if (processGroupId.equals(migrationSource.getProcessGroupId())) { + return true; + } + } + + return false; + } + + protected void assertMigratedProcessorState(final String connectorId, final String processorId) throws Exception { + if (!getNiFiInstance().isClustered()) { + final ComponentStateEntity stateEntity = getNifiClient().getConnectorClient().getProcessorState(connectorId, processorId); + assertStatePresent(stateEntity); + return; + } + + for (int nodeIndex = 1; nodeIndex <= getNumberOfNodes(); nodeIndex++) { + switchClientToNode(nodeIndex); + final ComponentStateEntity stateEntity = getNifiClient().getConnectorClient(DO_NOT_REPLICATE).getProcessorState(connectorId, processorId); + assertStatePresent(stateEntity); + assertNotNull(stateEntity.getComponentState().getClusterState()); + assertFalse(stateEntity.getComponentState().getClusterState().getState().isEmpty()); + } + } + + private void assertStatePresent(final ComponentStateEntity stateEntity) { + assertNotNull(stateEntity.getComponentState()); + assertNotNull(stateEntity.getComponentState().getLocalState()); + assertFalse(stateEntity.getComponentState().getLocalState().getState().isEmpty()); + } + + protected void prepareSourceForMigration(final SourceFixture sourceFixture, final File outputFile) throws Exception { + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + getClientUtil().assertFlowUpToDate(sourceFixture.processGroup().getId()); + } + + protected void assertConnectorFresh(final String connectorId) throws NiFiClientException, IOException { + final ConnectorClient connectorClient = getNifiClient().getConnectorClient(); + final ConnectorEntity connectorEntity = connectorClient.getConnector(connectorId); + + final String managedGroupId = connectorEntity.getComponent().getManagedProcessGroupId(); + final ProcessGroupFlowEntity flowEntity = connectorClient.getFlow(connectorId, managedGroupId); + assertTrue(flowEntity.getProcessGroupFlow().getFlow().getProcessors() == null + || flowEntity.getProcessGroupFlow().getFlow().getProcessors().isEmpty()); + assertTrue(flowEntity.getProcessGroupFlow().getFlow().getConnections() == null + || flowEntity.getProcessGroupFlow().getFlow().getConnections().isEmpty()); + + final AssetsEntity assetsEntity = connectorClient.getAssets(connectorId); + assertTrue(assetsEntity.getAssets() == null || assetsEntity.getAssets().isEmpty()); + } + + /** + * Asserts that the source process group has been left in the same state it was in before a failed + * migration attempt. The source must keep its original name, its original version-control state + * (registry coordinates and UP_TO_DATE state), its originally-applied parameter context, and must + * not have been disabled by the migration framework. + */ + protected void assertSourceUntouched(final SourceFixture sourceFixture, final String expectedName) throws NiFiClientException, IOException { + final ProcessGroupEntity processGroupEntity = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + assertEquals(expectedName, processGroupEntity.getComponent().getName()); + + // Version control coordinates and state must be unchanged. + assertNotNull(processGroupEntity.getComponent().getVersionControlInformation(), + "Source process group must remain under version control after a failed migration attempt"); + getClientUtil().assertFlowUpToDate(sourceFixture.processGroup().getId()); + + // Parameter context binding must be unchanged from the original setup. + final ProcessGroupEntity originalProcessGroupEntity = sourceFixture.processGroup(); + if (originalProcessGroupEntity.getComponent().getParameterContext() != null) { + assertNotNull(processGroupEntity.getComponent().getParameterContext(), + "Source process group must retain its parameter context after a failed migration attempt"); + assertEquals(originalProcessGroupEntity.getComponent().getParameterContext().getId(), + processGroupEntity.getComponent().getParameterContext().getId()); + } + + // Processors and controller services must NOT have been disabled by the migration framework. + // prepareSourceForMigration leaves processors STOPPED (started, output produced, then stopped), + // and any controller services in their original ENABLED state. A failed migration must leave + // those scheduled/enabled states unchanged. + final ProcessGroupFlowEntity flowEntity = + getNifiClient().getFlowClient().getProcessGroup(sourceFixture.processGroup().getId()); + for (final ProcessorEntity sourceProcessor : flowEntity.getProcessGroupFlow().getFlow().getProcessors()) { + final String state = sourceProcessor.getComponent().getState(); + assertNotEquals("DISABLED", state, + "Source processor " + sourceProcessor.getComponent().getName() + " must not be DISABLED after a failed migration attempt"); + assertEquals("STOPPED", state, + "Source processor " + sourceProcessor.getComponent().getName() + " must remain STOPPED after a failed migration attempt"); + } + final ControllerServicesEntity controllerServicesEntity = + getNifiClient().getFlowClient().getControllerServices(sourceFixture.processGroup().getId()); + if (controllerServicesEntity.getControllerServices() != null) { + for (final ControllerServiceEntity sourceService : controllerServicesEntity.getControllerServices()) { + final String state = sourceService.getComponent().getState(); + assertNotEquals("DISABLED", state, + "Source controller service " + sourceService.getComponent().getName() + " must not be DISABLED after a failed migration attempt"); + } + } + } + + protected record SourceFixture(ProcessGroupEntity processGroup, ConnectionEntity connection) { + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java new file mode 100644 index 000000000000..37d9500e1b9f --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorVersionedFlowMigrationUploadedPayloadIT.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConnectorVersionedFlowMigrationUploadedPayloadIT extends ConnectorVersionedFlowMigrationLocalIT { + @Test + public void testMigrateConnectorFromUploadedPayload() throws Exception { + final File outputFile = new File("target/migration/uploaded-output.txt"); + final File exportFile = new File("target/migration/uploaded-source.json"); + outputFile.delete(); + exportFile.delete(); + + final SourceFixture sourceFixture = createSourceFixture("UploadedMigrationSource", registerClient(), false, outputFile, false); + getClientUtil().startProcessGroupComponents(sourceFixture.processGroup().getId()); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + getClientUtil().stopProcessGroupComponents(sourceFixture.processGroup().getId()); + getClientUtil().emptyQueue(sourceFixture.connection().getId()); + + getNifiClient().getProcessGroupClient().exportProcessGroup(sourceFixture.processGroup().getId(), true, true, exportFile); + final ProcessGroupEntity sourceProcessGroup = getNifiClient().getProcessGroupClient().getProcessGroup(sourceFixture.processGroup().getId()); + getNifiClient().getProcessGroupClient().deleteProcessGroup(sourceProcessGroup); + + outputFile.delete(); + + final ConnectorEntity connector = getClientUtil().createConnector("MigrationTargetConnector"); + final String connectorId = connector.getId(); + final MigrationPayloadEntity payloadEntity = getClientUtil().uploadMigrationPayload(connectorId, exportFile); + assertNotNull(payloadEntity.getPayload()); + + final MigrationRequestEntity requestEntity = getClientUtil().startMigrationFromPayload(connectorId, payloadEntity.getPayload().getPayloadId()); + getClientUtil().waitForMigrationSuccess(connectorId, requestEntity.getRequest().getRequestId()); + + getClientUtil().startConnector(connectorId); + waitFor(() -> outputFile.exists() && outputFile.length() > 0); + assertEquals(Files.readString(SAMPLE_ASSET_FILE.toPath()).trim(), Files.readString(outputFile.toPath()).trim()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java index 05f9ed173c3d..73f05e9b8e5a 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java @@ -24,9 +24,12 @@ import org.apache.nifi.web.api.entity.ConnectorEntity; import org.apache.nifi.web.api.entity.ConnectorPropertyAllowableValuesEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import java.io.File; import java.io.IOException; @@ -259,6 +262,16 @@ VerifyConnectorConfigStepRequestEntity getConfigStepVerificationRequest(String c VerifyConnectorConfigStepRequestEntity deleteConfigStepVerificationRequest(String connectorId, String configurationStepName, String requestId) throws NiFiClientException, IOException; + VersionedFlowMigrationSourcesEntity listMigrationSources(String connectorId) throws NiFiClientException, IOException; + + MigrationPayloadEntity uploadMigrationPayload(String connectorId, File file) throws NiFiClientException, IOException; + + MigrationRequestEntity startMigration(MigrationRequestEntity requestEntity) throws NiFiClientException, IOException; + + MigrationRequestEntity getMigrationStatus(String connectorId, String requestId) throws NiFiClientException, IOException; + + MigrationRequestEntity cancelMigration(String connectorId, String requestId) throws NiFiClientException, IOException; + /** * Applies an update to a connector. * diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java index 7b648e7183e6..1ff058ec5851 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java @@ -34,9 +34,12 @@ import org.apache.nifi.web.api.entity.ConnectorPropertyAllowableValuesEntity; import org.apache.nifi.web.api.entity.ConnectorRunStatusEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; +import org.apache.nifi.web.api.entity.MigrationPayloadEntity; +import org.apache.nifi.web.api.entity.MigrationRequestEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ProcessGroupStatusEntity; import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity; +import org.apache.nifi.web.api.entity.VersionedFlowMigrationSourcesEntity; import java.io.File; import java.io.FileInputStream; @@ -402,6 +405,78 @@ public VerifyConnectorConfigStepRequestEntity deleteConfigStepVerificationReques }); } + @Override + public VersionedFlowMigrationSourcesEntity listMigrationSources(final String connectorId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + + return executeAction("Error retrieving connector migration sources", () -> { + final WebTarget target = connectorTarget + .path("/migration-sources") + .resolveTemplate("id", connectorId); + return getRequestBuilder(target).get(VersionedFlowMigrationSourcesEntity.class); + }); + } + + @Override + public MigrationPayloadEntity uploadMigrationPayload(final String connectorId, final File file) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(file, "Migration payload file required"); + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath()); + } + + try (final InputStream payloadInputStream = new FileInputStream(file)) { + return executeAction("Error uploading connector migration payload", () -> { + final WebTarget target = connectorTarget + .path("/migration-payloads") + .resolveTemplate("id", connectorId); + return getRequestBuilder(target).post(Entity.entity(payloadInputStream, MediaType.APPLICATION_OCTET_STREAM_TYPE), MigrationPayloadEntity.class); + }); + } + } + + @Override + public MigrationRequestEntity startMigration(final MigrationRequestEntity requestEntity) throws NiFiClientException, IOException { + Objects.requireNonNull(requestEntity, "Migration request entity required"); + Objects.requireNonNull(requestEntity.getRequest(), "Migration request required"); + Objects.requireNonNull(requestEntity.getRequest().getConnectorId(), "Connector ID required"); + + return executeAction("Error creating connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests") + .resolveTemplate("id", requestEntity.getRequest().getConnectorId()); + return getRequestBuilder(target).post(Entity.entity(requestEntity, MediaType.APPLICATION_JSON_TYPE), MigrationRequestEntity.class); + }); + } + + @Override + public MigrationRequestEntity getMigrationStatus(final String connectorId, final String requestId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(requestId, "Migration request ID required"); + + return executeAction("Error retrieving connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + return getRequestBuilder(target).get(MigrationRequestEntity.class); + }); + } + + @Override + public MigrationRequestEntity cancelMigration(final String connectorId, final String requestId) throws NiFiClientException, IOException { + Objects.requireNonNull(connectorId, "Connector ID required"); + Objects.requireNonNull(requestId, "Migration request ID required"); + + return executeAction("Error deleting connector migration request", () -> { + final WebTarget target = connectorTarget + .path("/migration-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + return getRequestBuilder(target).delete(MigrationRequestEntity.class); + }); + } + @Override public ConnectorEntity applyUpdate(final ConnectorEntity connectorEntity) throws NiFiClientException, IOException { if (connectorEntity == null) { diff --git a/pom.xml b/pom.xml index af98cd0c526d..87eadc2567cc 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ v24.14.1 - 2.8.0 + 2.9.0-SNAPSHOT 2.3.0