From d874338cb0d11d76af08af89a0500ff42cfe711d Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 22 Apr 2026 16:40:26 -0400 Subject: [PATCH 1/9] NIFI-15880: Allow Connectors to enter a Troublehshooting mode where the flow becomes modifiable and user-managed (temporarily) rather than Connector managed. --- .cursor/rules/code-style.mdc | 76 ++ .../connectors/AllowableValuesConnector.java | 5 + .../connectors/CronScheduleConnector.java | 13 + .../nifi/mock/connectors/GenerateAndLog.java | 5 + .../connectors/MissingBundleConnector.java | 5 + .../nifi/connectors/kafkas3/KafkaToS3.java | 8 + .../ConnectorConfigurationProvider.java | 27 +- .../connector/ConnectorSyncDirective.java | 26 +- .../endpoints/ConnectorEndpointMerger.java | 3 + .../nifi/groups/StandardProcessGroup.java | 59 ++ .../mapping/VersionedComponentFlowMapper.java | 28 +- .../components/connector/ConnectorNode.java | 40 + .../connector/ConnectorRepository.java | 35 + .../components/connector/ConnectorState.java | 3 +- .../connector/ConnectorSyncResult.java | 20 +- ...ameworkConnectorInitializationContext.java | 17 + .../connector/FrameworkFlowContext.java | 24 + .../org/apache/nifi/groups/ProcessGroup.java | 21 + .../components/connector/GhostConnector.java | 8 + ...tandardConnectorInitializationContext.java | 19 +- .../connector/StandardConnectorNode.java | 267 +++++- .../StandardConnectorRepository.java | 110 ++- .../connector/StandardFlowContext.java | 26 +- .../nifi/controller/FlowController.java | 48 + .../VersionedDataflowMapper.java | 10 +- .../VersionedFlowSynchronizer.java | 9 +- .../connector/BlockingConnector.java | 5 + .../DynamicAllowableValuesConnector.java | 13 + .../connector/DynamicFlowConnector.java | 6 + .../connector/MissingBundleConnector.java | 5 + .../OnPropertyModifiedConnector.java | 12 + .../connector/ParameterConnector.java | 7 + .../connector/SleepingConnector.java | 5 + .../connector/TestStandardConnectorNode.java | 20 + .../TestStandardConnectorRepository.java | 270 +++++- .../nifi/controller/flow/NopConnector.java | 5 + .../VersionedFlowSynchronizerTest.java | 21 +- .../service/mock/MockProcessGroup.java | 10 + .../org/apache/nifi/nar/DummyConnector.java | 5 + .../authorization/AuthorizableLookup.java | 45 + .../StandardAuthorizableLookup.java | 29 +- .../apache/nifi/web/NiFiServiceFacade.java | 8 + .../nifi/web/StandardNiFiServiceFacade.java | 124 ++- .../nifi/web/api/ConnectorResource.java | 164 +++- .../nifi/web/controller/ControllerFacade.java | 17 +- .../org/apache/nifi/web/dao/ConnectorDAO.java | 8 + .../nifi/web/dao/ControllerServiceDAO.java | 11 + .../org/apache/nifi/web/dao/FunnelDAO.java | 11 + .../org/apache/nifi/web/dao/LabelDAO.java | 11 + .../java/org/apache/nifi/web/dao/PortDAO.java | 11 + .../org/apache/nifi/web/dao/ProcessorDAO.java | 11 + .../nifi/web/dao/RemoteProcessGroupDAO.java | 11 + .../nifi/web/dao/impl/AbstractPortDAO.java | 16 +- .../nifi/web/dao/impl/ComponentDAO.java | 48 +- .../web/dao/impl/StandardConnectionDAO.java | 42 +- .../web/dao/impl/StandardConnectorDAO.java | 29 + .../impl/StandardControllerServiceDAO.java | 39 +- .../nifi/web/dao/impl/StandardFunnelDAO.java | 53 +- .../web/dao/impl/StandardInputPortDAO.java | 57 +- .../nifi/web/dao/impl/StandardLabelDAO.java | 53 +- .../web/dao/impl/StandardOutputPortDAO.java | 51 +- .../web/dao/impl/StandardProcessorDAO.java | 37 +- .../impl/StandardRemoteProcessGroupDAO.java | 53 +- .../StandardAuthorizableLookupTest.java | 2 +- .../web/StandardNiFiServiceFacadeTest.java | 7 +- .../dao/impl/StandardConnectionDAOTest.java | 21 +- .../dao/impl/StandardProcessGroupDAOTest.java | 24 +- .../tests/system/AssetConnector.java | 6 + .../system/BundleResolutionConnector.java | 7 + .../tests/system/CalculateConnector.java | 7 + .../system/ComponentLifecycleConnector.java | 10 +- .../tests/system/DataQueuingConnector.java | 6 + .../system/GatedDataQueuingConnector.java | 15 + .../system/NestedProcessGroupConnector.java | 6 + .../connectors/tests/system/NopConnector.java | 6 + .../system/ParameterContextConnector.java | 15 + .../tests/system/SelectiveDropConnector.java | 6 + .../nifi/tests/system/NiFiClientUtil.java | 70 +- .../system/connectors/ConnectorCrudIT.java | 28 +- .../ConnectorTroubleshootingIT.java | 852 ++++++++++++++++++ .../nifi/toolkit/client/ConnectorClient.java | 22 + .../client/impl/JerseyConnectorClient.java | 58 ++ pom.xml | 2 +- 83 files changed, 3176 insertions(+), 229 deletions(-) create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java diff --git a/.cursor/rules/code-style.mdc b/.cursor/rules/code-style.mdc index f1d22d363918..7171af0e1392 100644 --- a/.cursor/rules/code-style.mdc +++ b/.cursor/rules/code-style.mdc @@ -80,3 +80,79 @@ final List result = myList.stream() object, hold the object in a final reference and mutate the object's state. The same applies to `final Object[] holder = new Object[1]` for capturing a single reference; use `AtomicReference` instead. +17. Always place a blank line after a closing brace (`}`) that ends a control-flow construct (such as an + `if`, `else`, `for`, `while`, `switch`, or `try` / `catch` / `finally` block) when the next line is a + new statement or another control-flow construct at the same indentation level. This also applies to + the line following the closing brace of a nested block within a method. The goal is to clearly + separate logical sections of code. Exceptions: + - No blank line is required immediately before a closing brace of the enclosing block. + - No blank line is required between `} else {`, `} else if (...) {`, `} catch (...) {`, or + `} finally {` on the same chain. + + Bad: + ```java + for (final Connector connector : connectors) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + return false; + ``` + + Good: + ```java + for (final Connector connector : connectors) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + + return false; + ``` +18. Rule #11 (prefer importing a class rather than using a fully qualified classname inline) is not + optional. Even when a class is referenced only a single time, add an `import` statement rather than + referring to it by fully qualified name inline. The only acceptable use of a fully qualified + classname is when there is an unavoidable naming conflict with another imported class in the same + file. Fully qualified references scattered throughout code make it much harder to read and maintain. +19. Never combine a negative condition such as `if (x != null)`, `if (!collection.isEmpty())`, or + `if (!flag)` with an `else` clause. Negated conditions are harder to read, and pairing them with an + `else` branch forces the reader to mentally invert the predicate for the "happy path". Instead, + rewrite the condition in the positive form (for example, swap the branches so the `if` tests the + positive condition), or use an early `return`/`continue`/`throw` to eliminate the `else` entirely. + + Bad: + ```java + if (persistedManagedGroup != null) { + restore(persistedManagedGroup); + } else { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + } + ``` + + Good (positive predicate): + ```java + if (persistedManagedGroup == null) { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + } else { + restore(persistedManagedGroup); + } + ``` + + Or, preferably, use an early action and drop the `else`: + ```java + if (persistedManagedGroup == null) { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + return; + } + + restore(persistedManagedGroup); + ``` diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java index a263482eda28..d8172b7a2bbe 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java index 1e257c245c6d..e8473e274171 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java @@ -64,6 +64,19 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Cron_Schedule_Connector.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow is the flow template with the currently configured CRON expression + // applied as the Trigger Schedule parameter value. + final VersionedExternalFlow flow = getInitialFlow(); + final String triggerSchedule = activeFlowContext.getConfigurationContext().getProperty(SCHEDULE_STEP_NAME, TRIGGER_SCHEDULE_PARAM).getValue(); + if (triggerSchedule != null) { + VersionedFlowUtils.setParameterValue(flow, TRIGGER_SCHEDULE_PARAM, triggerSchedule); + } + + return flow; + } + @Override public List getConfigurationSteps() { return List.of(SCHEDULE_STEP); diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java index 96b9fa63c38e..40a6a5faa690 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java @@ -35,6 +35,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java index d49674ffd038..1dad60bc72f0 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java @@ -88,6 +88,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java index bb964a8181ba..f3871fdea1fa 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java @@ -112,6 +112,14 @@ public VersionedExternalFlow getInitialFlow() { return KafkaToS3FlowBuilder.loadInitialFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow for this Connector is the flow that is produced by applying the + // current configuration to the Kafka-to-S3 flow template. When the user exits Troubleshooting mode, + // this flow is reinstated to discard any manual edits made to the managed Process Group. + return buildFlow(activeFlowContext.getConfigurationContext()); + } + @Override public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException { final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext()); diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java index a0d5314a2f96..324934845a1d 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java @@ -17,7 +17,7 @@ package org.apache.nifi.components.connector; -import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import java.io.InputStream; import java.util.Optional; @@ -101,6 +101,25 @@ public interface ConnectorConfigurationProvider { */ void verifyCreate(String connectorId); + /** + * Verifies that the provider allows the connector with the given identifier to transition into + * Troubleshooting mode. This is called before the framework transitions a Connector into + * Troubleshooting, giving the provider an opportunity to reject the operation (for example, + * because the provider is currently performing a concurrent operation on the connector or the + * external system does not permit troubleshooting at this time). + * + *

If the provider cannot support the operation, it should throw a runtime exception + * (for example, {@link IllegalStateException} or + * {@link ConnectorConfigurationProviderException}) describing why the transition is not + * allowed. The exception message will be surfaced to the caller.

+ * + *

The default implementation is a no-op, allowing the transition to proceed.

+ * + * @param connectorId the identifier of the connector that is being transitioned into Troubleshooting mode + */ + default void verifyEnterTroubleshooting(final String connectorId) { + } + /** * Determines how the connector repository should handle synchronization for the given * connector during flow inheritance (cluster join). The provider examines the external @@ -112,7 +131,7 @@ public interface ConnectorConfigurationProvider { *
    *
  • A {@link ConnectorWorkingConfiguration} with the provider's working config and name * (overriding the potentially stale values from the versioned flow)
    • - *
  • A {@link ScheduledState} override (correcting stale run intent from the versioned flow)
    • + *
  • A {@link VersionedConnectorState} override (correcting stale run intent from the versioned flow)
    • *
* *

This method combines the verify and load operations into a single call to avoid @@ -123,10 +142,10 @@ public interface ConnectorConfigurationProvider { * behavior for Apache NiFi when no provider is configured.

* * @param connectorId the identifier of the connector to check - * @param proposedScheduledState the ScheduledState from the versioned flow + * @param proposedScheduledState the scheduled state from the versioned flow * @return a directive indicating how to handle this connector during sync */ - default ConnectorSyncDirective getSyncDirective(final String connectorId, final ScheduledState proposedScheduledState) { + default ConnectorSyncDirective getSyncDirective(final String connectorId, final VersionedConnectorState proposedScheduledState) { return ConnectorSyncDirective.allow(); } diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java index 3ca70c6ad114..6515971fa222 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java @@ -17,10 +17,10 @@ package org.apache.nifi.components.connector; -import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; /** - * Directive returned by {@link ConnectorConfigurationProvider#getSyncDirective(String, ScheduledState)} + * Directive returned by {@link ConnectorConfigurationProvider#getSyncDirective(String, VersionedConnectorState)} * indicating how the connector repository should handle synchronization for a connector during * flow inheritance. */ @@ -32,7 +32,7 @@ public class ConnectorSyncDirective { public enum Action { /** * Proceed with synchronization. The directive may optionally include a - * {@link ScheduledState} override and/or a {@link ConnectorWorkingConfiguration} + * {@link VersionedConnectorState} override and/or a {@link ConnectorWorkingConfiguration} * containing the provider's working config and name. */ ALLOW, @@ -57,10 +57,10 @@ public enum Action { private static final ConnectorSyncDirective REMOVE_DIRECTIVE = new ConnectorSyncDirective(Action.REMOVE, null, null); private final Action action; - private final ScheduledState scheduledStateOverride; + private final VersionedConnectorState scheduledStateOverride; private final ConnectorWorkingConfiguration workingConfiguration; - private ConnectorSyncDirective(final Action action, final ScheduledState scheduledStateOverride, + private ConnectorSyncDirective(final Action action, final VersionedConnectorState scheduledStateOverride, final ConnectorWorkingConfiguration workingConfiguration) { this.action = action; this.scheduledStateOverride = scheduledStateOverride; @@ -69,7 +69,7 @@ private ConnectorSyncDirective(final Action action, final ScheduledState schedul /** * Returns an ALLOW directive with no overrides. The connector repository will use the - * versioned flow's name, working config, and ScheduledState as-is. This is the default + * versioned flow's name, working config, and scheduled state as-is. This is the default * behavior when no {@link ConnectorConfigurationProvider} is configured (Apache NiFi). */ public static ConnectorSyncDirective allow() { @@ -78,7 +78,7 @@ public static ConnectorSyncDirective allow() { /** * Returns an ALLOW directive with the provider's working configuration (name + working - * config steps) and no ScheduledState override. + * config steps) and no scheduled state override. * * @param workingConfiguration the provider's working configuration including name */ @@ -88,14 +88,14 @@ public static ConnectorSyncDirective allow(final ConnectorWorkingConfiguration w /** * Returns an ALLOW directive with the provider's working configuration and a - * ScheduledState override. The override replaces the versioned flow's ScheduledState, + * scheduled state override. The override replaces the versioned flow's scheduled state, * which may be stale due to in-flight DPS tasks. * * @param workingConfiguration the provider's working configuration including name - * @param scheduledStateOverride the ScheduledState to use instead of the versioned flow's value + * @param scheduledStateOverride the scheduled state to use instead of the versioned flow's value */ public static ConnectorSyncDirective allow(final ConnectorWorkingConfiguration workingConfiguration, - final ScheduledState scheduledStateOverride) { + final VersionedConnectorState scheduledStateOverride) { return new ConnectorSyncDirective(Action.ALLOW, scheduledStateOverride, workingConfiguration); } @@ -119,10 +119,10 @@ public Action getAction() { } /** - * Returns the ScheduledState override, or {@code null} if the versioned flow's - * ScheduledState should be used. + * Returns the scheduled state override, or {@code null} if the versioned flow's + * scheduled state should be used. */ - public ScheduledState getScheduledStateOverride() { + public VersionedConnectorState getScheduledStateOverride() { return scheduledStateOverride; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java index a2481698e15c..38bfef8ccc30 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java @@ -32,6 +32,7 @@ public class ConnectorEndpointMerger extends AbstractSingleEntityEndpoint getConnectorIdentifier() { return Optional.ofNullable(connectorId); } + @Override + public Optional findOwningConnector() { + ProcessGroup group = this; + while (group != null) { + final Optional owningConnectorId = group.getConnectorIdentifier(); + if (owningConnectorId.isPresent()) { + final ConnectorNode connectorNode = flowManager.getConnector(owningConnectorId.get()); + return Optional.ofNullable(connectorNode); + } + + group = group.getParent(); + } + + return Optional.empty(); + } + @Override public void setPosition(final Position position) { this.position.set(position); @@ -3946,6 +3963,48 @@ public void updateFlow(final VersionedExternalFlow proposedSnapshot, final Strin synchronizeFlow(proposedSnapshot, synchronizationOptions, flowMappingOptions); } + @Override + public void restoreFlowPreservingIdentifiers(final VersionedExternalFlow proposedSnapshot) { + // Use the Instance Identifier captured in the persisted flow as the runtime identifier for every component. This is + // required so that Connection identifiers (and therefore FlowFile queue identifiers) match what was in use before + // the flow was persisted. Without this, queued FlowFiles in the FlowFile Repository cannot be re-associated with + // their Connections upon restore. + final ComponentIdGenerator idGenerator = (proposedId, instanceId, destinationGroupId) -> instanceId; + final VersionedComponentStateLookup stateLookup = VersionedComponentStateLookup.IDENTITY_LOOKUP; + final ComponentScheduler componentScheduler = new DefaultComponentScheduler(controllerServiceProvider, stateLookup); + + final FlowSynchronizationOptions synchronizationOptions = new FlowSynchronizationOptions.Builder() + .componentIdGenerator(idGenerator) + .componentComparisonIdLookup(VersionedComponent::getInstanceIdentifier) + .componentScheduler(componentScheduler) + .ignoreLocalModifications(true) + .updateDescendantVersionedFlows(true) + .updateGroupSettings(true) + .updateRpgUrls(false) + .propertyDecryptor(encryptor::decrypt) + .build(); + + // Sensitive property values in the proposed snapshot were encrypted using the same PropertyEncryptor when the snapshot + // was persisted (for example, when a Connector-managed flow is persisted in Troubleshooting mode). The currently loaded + // flow therefore must also be mapped with an equivalent SensitiveValueEncryptor so the comparison between "current" and + // "proposed" sensitive values operates on matching ciphertext; otherwise every sensitive property appears to differ and + // the decrypted value written back to the live component is the encrypted payload rather than the plaintext (or parameter + // reference) that was originally captured. + final FlowMappingOptions flowMappingOptions = new FlowMappingOptions.Builder() + .mapSensitiveConfiguration(true) + .mapPropertyDescriptors(true) + .stateLookup(stateLookup) + .sensitiveValueEncryptor(encryptor::encrypt) + .componentIdLookup(ComponentIdLookup.VERSIONED_OR_GENERATE) + .mapInstanceIdentifiers(true) + .mapControllerServiceReferencesToVersionedId(true) + .mapFlowRegistryClientId(false) + .mapAssetReferences(false) + .build(); + + synchronizeFlow(proposedSnapshot, synchronizationOptions, flowMappingOptions); + } + private ProcessContext createProcessContext(final ProcessorNode processorNode) { final org.apache.nifi.processor.Processor processor = processorNode.getProcessor(); final Class componentClass = processor == null ? null : processor.getClass(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index 216e4fd76428..f017a87c33e2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -73,6 +73,7 @@ import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; @@ -1095,6 +1096,20 @@ private org.apache.nifi.flow.ScheduledState mapScheduledState(final ScheduledSta } public VersionedConnector mapConnector(final ConnectorNode connectorNode) { + return mapConnector(connectorNode, null); + } + + /** + * Map the given ConnectorNode to a VersionedConnector. When the Connector is in Troubleshooting state, the Connector's + * Managed Process Group is also serialized into the VersionedConnector so that any user modifications made while in + * Troubleshooting mode survive a restart. The provided ControllerServiceProvider is required for mapping the Managed + * Process Group; if {@code null}, the Managed Process Group is not serialized even if the Connector is in Troubleshooting. + * + * @param connectorNode the connector node to map + * @param controllerServiceProvider the controller service provider used when mapping the Managed Process Group; may be null + * @return the mapped VersionedConnector + */ + public VersionedConnector mapConnector(final ConnectorNode connectorNode, final ControllerServiceProvider controllerServiceProvider) { final VersionedConnector versionedConnector = new VersionedConnector(); versionedConnector.setInstanceIdentifier(connectorNode.getIdentifier()); versionedConnector.setName(connectorNode.getName()); @@ -1108,6 +1123,12 @@ public VersionedConnector mapConnector(final ConnectorNode connectorNode) { final List workingFlowConfiguration = createVersionedConfigurationSteps(connectorNode.getWorkingFlowContext()); versionedConnector.setWorkingFlowConfiguration(workingFlowConfiguration); + if (connectorNode.getCurrentState() == ConnectorState.TROUBLESHOOTING && controllerServiceProvider != null) { + final ProcessGroup managedGroup = connectorNode.getActiveFlowContext().getManagedProcessGroup(); + final VersionedProcessGroup versionedManagedGroup = mapNonVersionedProcessGroup(managedGroup, controllerServiceProvider); + versionedConnector.setManagedProcessGroup(versionedManagedGroup); + } + return versionedConnector; } @@ -1158,14 +1179,15 @@ private Map mapPropertyValues(final St return versionedProperties; } - private org.apache.nifi.flow.ScheduledState mapConnectorState(final ConnectorState connectorState) { + private VersionedConnectorState mapConnectorState(final ConnectorState connectorState) { if (connectorState == null) { return null; } return switch (connectorState) { - case RUNNING, STARTING -> org.apache.nifi.flow.ScheduledState.RUNNING; - default -> org.apache.nifi.flow.ScheduledState.ENABLED; + case RUNNING, STARTING -> VersionedConnectorState.RUNNING; + case TROUBLESHOOTING -> VersionedConnectorState.TROUBLESHOOTING; + default -> VersionedConnectorState.ENABLED; }; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java index 08819d38857d..1c3c78073b6c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java @@ -307,4 +307,44 @@ void inheritConfiguration(List activeFlowConfigurati * @return the list of available actions */ List getAvailableActions(); + + /** + * Verifies that the Connector is in a state that allows it to be transitioned into TROUBLESHOOTING. + * @throws IllegalStateException if the Connector cannot enter Troubleshooting mode + */ + void verifyCanEnterTroubleshooting(); + + /** + * Verifies that the Connector is in a state that allows it to be transitioned out of TROUBLESHOOTING. + * The Connector must be in TROUBLESHOOTING. Additionally, all components within the Connector's Managed Process Group + * must be in a stopped / disabled state. + * @throws IllegalStateException if the Connector cannot exit Troubleshooting mode + */ + void verifyCanEndTroubleshooting(); + + /** + * Transitions the Connector into TROUBLESHOOTING state. This method should only be invoked via the ConnectorRepository. + * @throws IllegalStateException if the Connector cannot enter Troubleshooting mode + */ + void enterTroubleshooting(); + + /** + * Restores the Connector's state to TROUBLESHOOTING without stopping any components within the Managed Process Group + * and without running the pre-conditions enforced by {@link #enterTroubleshooting()}. This is intended to be used + * only by the flow synchronization layer when restoring a Connector that was persisted while in Troubleshooting + * mode so that components inside the Managed Process Group retain their persisted runtime state (for example, + * processors that were running when NiFi shut down stay running after restart). + */ + void restoreTroubleshootingState(); + + /** + * Transitions the Connector out of TROUBLESHOOTING state. The Connector's Managed Process Group will be restored + * to the Connector's authoritative view of the flow as reported by {@link Connector#getActiveFlow}. This method should + * only be invoked via the ConnectorRepository. + * + * @throws IllegalStateException if the Connector cannot exit Troubleshooting mode + * @throws FlowUpdateException if unable to apply the authoritative flow (for example because data is queued in a + * Connection that would be removed by the restore) + */ + void endTroubleshooting() throws FlowUpdateException; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java index d8d06bc79efa..14d8744f9b14 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java @@ -18,6 +18,7 @@ package org.apache.nifi.components.connector; import org.apache.nifi.asset.Asset; +import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.secrets.SecretsManager; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.VersionedConfigurationStep; @@ -151,6 +152,40 @@ void inheritConfiguration(ConnectorNode connector, Listnot call {@code ProcessGroup#verifyCanUpdate} and is intended for use only when restoring a Connector + * that was persisted while in Troubleshooting mode. The persisted user modifications are re-applied on top of the + * Connector's current Managed Process Group contents. + * + * @param troubleshootingProcessGroup the VersionedProcessGroup representing the Managed Process Group contents + * persisted while the Connector was in Troubleshooting mode + */ + void restoreTroubleshootingFlow(VersionedProcessGroup troubleshootingProcessGroup); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java index 91a0a295aa4f..058835c1c921 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java @@ -19,6 +19,7 @@ import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.resource.ComponentAuthorizable; import org.apache.nifi.components.VersionedComponent; +import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.FlowFileActivity; @@ -143,6 +144,16 @@ public interface ProcessGroup extends ComponentAuthorizable, Positionable, Versi */ Optional getConnectorIdentifier(); + /** + * Returns the owning Connector for this Process Group, traversing the Process Group hierarchy until a Process Group + * is found that is associated with a Connector. If no Process Group in the hierarchy is associated with a Connector, + * an empty Optional is returned. This is useful for determining whether a component is managed by a Connector. + * + * @return an Optional containing the owning ConnectorNode, or empty if this Process Group and all of its ancestors are + * not managed by a Connector + */ + Optional findOwningConnector(); + /** * @return the user-set comments about this ProcessGroup, or * null if no comments have been set @@ -948,6 +959,16 @@ default CompletableFuture stopComponents() { */ void updateFlow(VersionedExternalFlow proposedSnapshot, String componentIdSeed, boolean verifyNotDirty, boolean updateSettings, boolean updateDescendantVersionedFlows); + /** + * Updates the Process Group to match the proposed flow, using the Instance Identifier of each component in the + * proposed flow as the runtime identifier for that component. This is intended for use when restoring a previously + * persisted flow where the original runtime identifiers must be preserved (for example, so that queued FlowFiles + * in the FlowFile Repository can be re-associated with their Connections after a restart). + * + * @param proposedSnapshot the proposed flow whose Instance Identifiers should be used as runtime identifiers + */ + void restoreFlowPreservingIdentifiers(VersionedExternalFlow proposedSnapshot); + /** * Updates the Process Group to match the proposed flow * diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java index 0508821e9d1f..50acc4f42594 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java @@ -23,6 +23,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; import java.util.List; import java.util.Map; @@ -66,6 +67,13 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + final VersionedExternalFlow emptyFlow = new VersionedExternalFlow(); + emptyFlow.setFlowContents(new VersionedProcessGroup()); + return emptyFlow; + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java index 002f2a8aa2f8..c8b4a267fbf6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java @@ -81,14 +81,27 @@ public AssetManager getAssetManager() { @Override public void updateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow, final BundleCompatibility bundleCompatability) throws FlowUpdateException { - if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) { - throw new IllegalArgumentException("FlowContext is not an instance provided by the framework"); - } + final FrameworkFlowContext frameworkFlowContext = requireFrameworkFlowContext(flowContext); resolveBundles(versionedExternalFlow.getFlowContents(), bundleCompatability); frameworkFlowContext.updateFlow(versionedExternalFlow, assetManager); } + @Override + public void verifyUpdateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow, final BundleCompatibility bundleCompatability) throws FlowUpdateException { + final FrameworkFlowContext frameworkFlowContext = requireFrameworkFlowContext(flowContext); + + resolveBundles(versionedExternalFlow.getFlowContents(), bundleCompatability); + frameworkFlowContext.verifyUpdateFlow(versionedExternalFlow); + } + + private FrameworkFlowContext requireFrameworkFlowContext(final FlowContext flowContext) { + if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) { + throw new IllegalArgumentException("FlowContext is not an instance provided by the framework"); + } + return frameworkFlowContext; + } + protected void resolveBundles(final VersionedProcessGroup group, final BundleCompatibility bundleCompatability) { if (bundleCompatability == BundleCompatibility.REQUIRE_EXACT_BUNDLE) { return; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index b9659ebb4860..9cdb87a59ade 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -35,6 +35,7 @@ import org.apache.nifi.components.validation.ValidationTrigger; import org.apache.nifi.connectable.FlowFileActivity; import org.apache.nifi.connectable.FlowFileTransferCounts; +import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.queue.DropFlowFileStatus; @@ -50,6 +51,7 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flow.VersionedProcessor; import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; @@ -64,6 +66,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -84,6 +87,9 @@ public class StandardConnectorNode implements ConnectorNode { private static final Logger logger = LoggerFactory.getLogger(StandardConnectorNode.class); + private static final Set STOPPED_STATES = + EnumSet.of(org.apache.nifi.controller.ScheduledState.STOPPED, org.apache.nifi.controller.ScheduledState.DISABLED); + private final String identifier; private final FlowManager flowManager; private final ExtensionManager extensionManager; @@ -142,11 +148,19 @@ public String getName() { @Override public void setName(final String name) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot rename " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode before renaming the Connector."); + } + this.name = name; } @Override public void transitionStateForUpdating() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot apply an update to " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode before applying updates."); + } + final ConnectorState initialState = getCurrentState(); if (initialState == ConnectorState.UPDATING || initialState == ConnectorState.PREPARING_FOR_UPDATE) { return; @@ -329,6 +343,11 @@ public void markInvalid(final String subject, final String explanation) { @Override public void setConfiguration(final String stepName, final StepConfiguration configuration) throws FlowUpdateException { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot modify configuration step " + stepName + " for " + this + + " while it is in Troubleshooting mode; exit Troubleshooting mode before modifying Connector configuration."); + } + setConfiguration(stepName, configuration, false); } @@ -444,6 +463,10 @@ private void start(final FlowEngine scheduler, final CompletableFuture sta @Override public Future stop(final FlowEngine scheduler) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot stop " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode to resume normal lifecycle control."); + } + logger.info("Stopping {}", this); final CompletableFuture stopCompleteFuture = new CompletableFuture<>(); @@ -641,6 +664,9 @@ public void verifyCanDelete() { } final ConnectorState currentState = getCurrentState(); + if (currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot delete " + this + " because it is in Troubleshooting mode; exit Troubleshooting before deleting."); + } if (currentState == ConnectorState.STOPPED || currentState == ConnectorState.UPDATE_FAILED || currentState == ConnectorState.UPDATED) { return; } @@ -650,12 +676,155 @@ public void verifyCanDelete() { @Override public void verifyCanStart() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot start " + this + " because it is in Troubleshooting mode."); + } final ValidationState state = performValidation(); if (state.getStatus() != ValidationStatus.VALID) { throw new IllegalStateException("Cannot start " + this + " because it is not valid: " + state.getValidationErrors()); } } + @Override + public void verifyCanEnterTroubleshooting() { + if (isExtensionMissing()) { + throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because it is a Ghost Connector (its underlying extension is missing)."); + } + + final ConnectorState currentState = getCurrentState(); + switch (currentState) { + case TROUBLESHOOTING -> throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because it is already in Troubleshooting mode."); + case STARTING, STOPPING, DRAINING, PURGING, PREPARING_FOR_UPDATE, UPDATING -> + throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because its state is currently " + + currentState + "; it must be in a stable state before entering Troubleshooting."); + default -> { + // STOPPED, RUNNING, UPDATED, UPDATE_FAILED are all acceptable to enter Troubleshooting from + } + } + } + + @Override + public void verifyCanEndTroubleshooting() { + if (getCurrentState() != ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because it is not currently in Troubleshooting mode; current state is " + getCurrentState()); + } + + // Verify that every component inside the managed flow is stopped/disabled BEFORE doing any other validation. + // The flow-update check below relies on components being stopped/disabled in order to produce meaningful results. Otherwise, + // the update check may succeed and then the running flow puts it into a bad state. + final ProcessGroup managedGroup = getActiveFlowContext().getManagedProcessGroup(); + verifyAllComponentsStoppedAndDisabled(managedGroup); + + // After confirming all components are stopped or disabled, check if the managed flow can be safely reverted. + // Connector's authoritative flow would succeed. This mirrors exactly what endTroubleshooting() will do so that + // any problem (e.g. a Connection whose contents cannot be preserved or a component that cannot be replaced) + // is reported synchronously at the REST verify-phase rather than surfacing halfway through the state change. + final VersionedExternalFlow authoritativeFlow = resolveAuthoritativeFlow(); + try { + initializationContext.verifyUpdateFlow(activeFlowContext, authoritativeFlow, BundleCompatibility.RESOLVE_BUNDLE); + } catch (final FlowUpdateException e) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + + " because the Managed Process Group cannot be reverted to the Connector's authoritative flow: " + e.getMessage(), e); + } + } + + private VersionedExternalFlow resolveAuthoritativeFlow() { + final VersionedExternalFlow authoritativeFlow; + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { + authoritativeFlow = getConnector().getActiveFlow(activeFlowContext); + } + + if (authoritativeFlow == null || authoritativeFlow.getFlowContents() == null) { + logger.warn("Connector {} returned a null authoritative flow from getActiveFlow; using an empty flow.", this); + final VersionedExternalFlow empty = new VersionedExternalFlow(); + empty.setFlowContents(new VersionedProcessGroup()); + return empty; + } + + return authoritativeFlow; + } + + private void verifyAllComponentsStoppedAndDisabled(final ProcessGroup group) { + for (final ProcessorNode processor : group.getProcessors()) { + if (!STOPPED_STATES.contains(processor.getScheduledState())) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Processor " + processor.getIdentifier() + + " is in state " + processor.getScheduledState() + "; it must be STOPPED or DISABLED."); + } + } + + for (final Port port : group.getInputPorts()) { + if (!STOPPED_STATES.contains(port.getScheduledState())) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Input Port " + port.getIdentifier() + + " is in state " + port.getScheduledState() + "; it must be STOPPED or DISABLED."); + } + } + + for (final Port port : group.getOutputPorts()) { + if (!STOPPED_STATES.contains(port.getScheduledState())) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Output Port " + port.getIdentifier() + + " is in state " + port.getScheduledState() + "; it must be STOPPED or DISABLED."); + } + } + + for (final RemoteProcessGroup remoteProcessGroup : group.getRemoteProcessGroups()) { + if (remoteProcessGroup.isTransmitting()) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Remote Process Group " + + remoteProcessGroup.getIdentifier() + " is transmitting; it must be stopped."); + } + } + + for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) { + if (serviceNode.isActive()) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Controller Service " + serviceNode.getIdentifier() + + " is not disabled; all Controller Services within the managed flow must be disabled."); + } + } + + for (final ProcessGroup childGroup : group.getProcessGroups()) { + verifyAllComponentsStoppedAndDisabled(childGroup); + } + } + + @Override + public void enterTroubleshooting() { + verifyCanEnterTroubleshooting(); + logger.info("Transitioning {} into TROUBLESHOOTING state", this); + + // Deliberately do NOT stop or otherwise mutate the managed flow here. The explicit contract of Troubleshooting + // mode (NIP-28) is that a user can "break glass" on a live, running Connector to inspect or stabilize a + // production issue without first having to shut the flow down. The components that were running before entering + // Troubleshooting remain running; the user may stop individual components as needed in order to edit them, and + // they must all be stopped/disabled before the Connector can leave Troubleshooting mode (enforced in + // verifyCanEndTroubleshooting). + stateTransition.setDesiredState(ConnectorState.TROUBLESHOOTING); + stateTransition.setCurrentState(ConnectorState.TROUBLESHOOTING); + } + + @Override + public void restoreTroubleshootingState() { + logger.info("Restoring {} to TROUBLESHOOTING state from persisted flow", this); + stateTransition.setDesiredState(ConnectorState.TROUBLESHOOTING); + stateTransition.setCurrentState(ConnectorState.TROUBLESHOOTING); + } + + @Override + public void endTroubleshooting() throws FlowUpdateException { + verifyCanEndTroubleshooting(); + logger.info("Exiting TROUBLESHOOTING state for {} by restoring Connector's authoritative flow", this); + + final VersionedExternalFlow flowToApply = resolveAuthoritativeFlow(); + + // Route the update through the ConnectorInitializationContext so that bundle coordinates referenced by the + // authoritative flow are resolved against the currently-available bundles. This mirrors how the initial flow + // is applied in initializeConnector and avoids failing validation when the Connector hard-codes a bundle + // version that differs from the currently-installed NAR (which is common in test Connectors). + initializationContext.updateFlow(activeFlowContext, flowToApply, BundleCompatibility.RESOLVE_BUNDLE); + + stateTransition.setDesiredState(ConnectorState.STOPPED); + stateTransition.setCurrentState(ConnectorState.STOPPED); + logger.info("Successfully exited TROUBLESHOOTING state for {}; Connector is now STOPPED", this); + } + @Override public Connector getConnector() { return connectorDetails.getConnector(); @@ -914,6 +1083,11 @@ public boolean isValidationPaused() { @Override public List verifyConfigurationStep(final String stepName, final StepConfiguration configurationOverrides) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot verify configuration step " + stepName + " for " + this + + " while it is in Troubleshooting mode; exit Troubleshooting mode before running configuration verification."); + } + logger.debug("Verifying configuration step {} for {}", stepName, this); final List results = new ArrayList<>(); try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { @@ -1268,6 +1442,11 @@ public FrameworkFlowContext getWorkingFlowContext() { @Override public void discardWorkingConfiguration() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot discard the working configuration for " + this + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before discarding configuration changes."); + } + recreateWorkingFlowContext(); logger.debug("Discarded working configuration for {}", this); } @@ -1278,16 +1457,19 @@ public List getAvailableActions() { final ConnectorState currentState = getCurrentState(); final boolean dataQueued = activeFlowContext.getManagedProcessGroup().isDataQueued(); final boolean stopped = isStopped(); + final boolean troubleshooting = currentState == ConnectorState.TROUBLESHOOTING; - actions.add(createStartAction(stopped)); + actions.add(createStartAction(stopped && !troubleshooting, troubleshooting)); actions.add(createStopAction(currentState)); - actions.add(createConfigureAction()); - actions.add(createDiscardWorkingConfigAction()); - actions.add(createPurgeFlowFilesAction(stopped, dataQueued)); - actions.add(createDrainFlowFilesAction(stopped, dataQueued)); + actions.add(createConfigureAction(troubleshooting)); + actions.add(createDiscardWorkingConfigAction(troubleshooting)); + actions.add(createPurgeFlowFilesAction(stopped && !troubleshooting, dataQueued)); + actions.add(createDrainFlowFilesAction(stopped && !troubleshooting, dataQueued)); actions.add(createCancelDrainFlowFilesAction(currentState == ConnectorState.DRAINING)); - actions.add(createApplyUpdatesAction(currentState)); - actions.add(createDeleteAction(stopped, dataQueued)); + actions.add(createApplyUpdatesAction(currentState, troubleshooting)); + actions.add(createDeleteAction(stopped && !troubleshooting, dataQueued)); + actions.add(createEnterTroubleshootingAction(currentState)); + actions.add(createEndTroubleshootingAction(troubleshooting)); return actions; } @@ -1304,11 +1486,14 @@ private boolean isStopped() { return false; } - private ConnectorAction createStartAction(final boolean stopped) { + private ConnectorAction createStartAction(final boolean stopped, final boolean troubleshooting) { final boolean allowed; final String reason; - if (!stopped) { + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (!stopped) { allowed = false; reason = "Connector is not stopped"; } else { @@ -1329,25 +1514,70 @@ private ConnectorAction createStartAction(final boolean stopped) { private ConnectorAction createStopAction(final ConnectorState currentState) { final boolean allowed; - if (currentState == ConnectorState.RUNNING || currentState == ConnectorState.STARTING) { + final String reason; + if (currentState == ConnectorState.TROUBLESHOOTING) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (currentState == ConnectorState.RUNNING || currentState == ConnectorState.STARTING) { allowed = true; + reason = null; } else if (currentState == ConnectorState.UPDATED || currentState == ConnectorState.UPDATE_FAILED) { allowed = hasActiveThread(activeFlowContext.getManagedProcessGroup()); + reason = allowed ? null : "Connector is not running"; } else { allowed = false; + reason = "Connector is not running"; } - final String reason = allowed ? null : "Connector is not running"; return new StandardConnectorAction("STOP", "Stop the connector", allowed, reason); } - private ConnectorAction createConfigureAction() { + private ConnectorAction createConfigureAction(final boolean troubleshooting) { + if (troubleshooting) { + return new StandardConnectorAction("CONFIGURE", "Configure the connector", false, "Connector is in Troubleshooting mode"); + } return new StandardConnectorAction("CONFIGURE", "Configure the connector", true, null); } - private ConnectorAction createDiscardWorkingConfigAction() { - final boolean allowed = hasWorkingConfigurationChanges(); - final String reason = allowed ? null : "No pending changes to discard"; + private ConnectorAction createEnterTroubleshootingAction(final ConnectorState currentState) { + if (isExtensionMissing()) { + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector's extension is missing"); + } + switch (currentState) { + case TROUBLESHOOTING: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector is already in Troubleshooting mode"); + case STARTING, STOPPING, DRAINING, PURGING, PREPARING_FOR_UPDATE, UPDATING: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector is currently transitioning; current state is " + currentState); + default: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", true, null); + } + } + + private ConnectorAction createEndTroubleshootingAction(final boolean troubleshooting) { + if (!troubleshooting) { + return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", false, + "Connector is not in Troubleshooting mode"); + } + return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", true, null); + } + + private ConnectorAction createDiscardWorkingConfigAction(final boolean troubleshooting) { + final boolean allowed; + final String reason; + + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (!hasWorkingConfigurationChanges()) { + allowed = false; + reason = "No pending changes to discard"; + } else { + allowed = true; + reason = null; + } return new StandardConnectorAction("DISCARD_WORKING_CONFIGURATION", "Discard any changes made to the working configuration", allowed, reason); } @@ -1403,11 +1633,14 @@ private ConnectorAction createCancelDrainFlowFilesAction(final boolean draining) "Connector is not currently draining FlowFiles"); } - private ConnectorAction createApplyUpdatesAction(final ConnectorState currentState) { + private ConnectorAction createApplyUpdatesAction(final ConnectorState currentState, final boolean troubleshooting) { final boolean allowed; final String reason; - if (currentState == ConnectorState.PREPARING_FOR_UPDATE || currentState == ConnectorState.UPDATING) { + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (currentState == ConnectorState.PREPARING_FOR_UPDATE || currentState == ConnectorState.UPDATING) { allowed = false; reason = "Connector is updating"; } else if (!hasWorkingConfigurationChanges()) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index bdb527fecc81..9337cf17783e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -26,10 +26,11 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.engine.FlowEngine; import org.apache.nifi.flow.Bundle; -import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; +import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.util.BundleUtils; @@ -53,6 +54,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; @@ -110,7 +113,7 @@ public void restoreConnector(final ConnectorNode connector) { @Override public ConnectorSyncResult syncConnector(final VersionedConnector versionedConnector) { final String connectorId = versionedConnector.getInstanceIdentifier(); - final ScheduledState proposedScheduledState = versionedConnector.getScheduledState(); + final VersionedConnectorState proposedScheduledState = versionedConnector.getScheduledState(); logger.debug("syncConnector called for connector [{}]", connectorId); // Consult the provider for external state checks and working config @@ -207,18 +210,31 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne final List effectiveActiveConfig = versionedConnector.getActiveFlowConfiguration(); - final ScheduledState effectiveScheduledState = (directive.getScheduledStateOverride() != null) + final VersionedConnectorState effectiveScheduledState = (directive.getScheduledStateOverride() != null) ? directive.getScheduledStateOverride() : proposedScheduledState; // Set name locally (no provider.save()) connector.setName(effectiveName); - // Compare config and sync if changed final boolean wasRunning = currentState == ConnectorState.RUNNING; final boolean configChanged = isNewConnector || isConfigurationUpdated(connector, effectiveActiveConfig, effectiveWorkingConfig); - - if (configChanged) { + final boolean restoringTroubleshooting = effectiveScheduledState == VersionedConnectorState.TROUBLESHOOTING; + + // Configuration must be inherited even when the effective state is TROUBLESHOOTING. The Connector's managed + // Parameter Context is intentionally not registered with the global ParameterContextManager (see + // StandardFlowManager#createConnector) and therefore is not persisted in flow.json. The framework can only + // re-populate that Parameter Context by letting the Connector's applyUpdate run with the persisted active + // configuration, which is exactly what inheritConfiguration does. + // + // For a TROUBLESHOOTING restoration, the Connector's authoritative active flow that inheritConfiguration + // produces is then immediately overlaid by restoreTroubleshootingFlow with the user's persisted Managed + // Process Group snapshot. The structural overlay preserves the in-memory Parameter Context binding, so the + // Connector-supplied parameter values remain available, while the user's flow modifications are the ones + // running. The transient Connector-supplied flow shape is invisible outside this synchronization cycle: flow + // synchronization completes before the FlowFile Repository attaches FlowFiles to queues, so no FlowFile + // movement can observe it. + if (configChanged || restoringTroubleshooting) { logger.info("{} configuration needs synchronization", connector); try { inheritConfiguration(connector, effectiveActiveConfig, effectiveWorkingConfig, versionedConnector.getBundle()); @@ -237,6 +253,34 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne logger.debug("{} configuration is up to date, no update necessary", connector); } + if (restoringTroubleshooting) { + final VersionedProcessGroup persistedManagedGroup = versionedConnector.getManagedProcessGroup(); + if (persistedManagedGroup == null) { + logger.warn("{} effective scheduled state is TROUBLESHOOTING but no Managed Process Group snapshot was persisted; leaving Managed Process Group unchanged", connector); + } else { + logger.info("{} was persisted in Troubleshooting mode; restoring Managed Process Group from persisted snapshot", connector); + try { + connector.getActiveFlowContext().restoreTroubleshootingFlow(persistedManagedGroup); + } catch (final Exception e) { + logger.error("{} failed to restore Managed Process Group from Troubleshooting snapshot", connector, e); + connector.markInvalid("Flow Synchronization Failure", + "Failed to restore Managed Process Group from Troubleshooting snapshot: " + e.getMessage()); + return ConnectorSyncResult.failed(connector); + } + } + + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + // Use restoreTroubleshootingState rather than enterTroubleshooting because the former does not stop + // running components within the Managed Process Group. Components inside the Managed Process Group that + // were persisted as RUNNING have just been restored to the RUNNING state by restoreTroubleshootingFlow above; + // calling enterTroubleshooting here would immediately stop them, violating the contract that persisted + // user modifications (including running processors) should survive a NiFi restart while in Troubleshooting. + connector.restoreTroubleshootingState(); + } + + return ConnectorSyncResult.syncedConfigUnchanged(connector, effectiveScheduledState); + } + return configChanged ? ConnectorSyncResult.synced(connector, effectiveScheduledState) : ConnectorSyncResult.syncedConfigUnchanged(connector, effectiveScheduledState); @@ -429,9 +473,9 @@ private void stopConnectorAndAwait(final ConnectorNode connector) { logger.info("Stopping connector [{}] (current state: {}) and awaiting completion", connectorId, currentState); try { final Future stopFuture = stopConnector(connector); - stopFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + stopFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] stopped successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to stop", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -446,9 +490,9 @@ private void purgeConnectorAndAwait(final ConnectorNode connector) { try { logger.debug("Purging FlowFiles for connector [{}] before removal", connectorId); final Future purgeFuture = connector.purgeFlowFiles("Flow Synchronization"); - purgeFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + purgeFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] purged successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to purge FlowFiles", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -521,6 +565,16 @@ private void restartConnector(final ConnectorNode connector, final CompletableFu public void applyUpdate(final ConnectorNode connector, final ConnectorUpdateContext context) throws FlowUpdateException { logger.debug("Applying update to {}", connector); + // Refuse the update before any provider interaction or asset sync. Once a Connector enters Troubleshooting the + // user owns the managed flow (NIP-28) and the Connector configuration is locked. Deferring this check until + // transitionStateForUpdating() lets a provider that returns shouldApplyUpdate()=false cause the framework to + // silently treat the request as successful while the Connector is still in Troubleshooting, and also lets the + // provider observe and act on an update request that should never reach it in this state. + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot apply an update to " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before applying updates."); + } + if (configurationProvider != null && !configurationProvider.shouldApplyUpdate(connector.getIdentifier())) { logger.info("ConnectorConfigurationProvider indicated framework should not apply update for {}; skipping framework update process", connector); return; @@ -668,6 +722,11 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f @Override public void updateConnector(final ConnectorNode connector, final String name) { + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot update the configuration of " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before modifying the Connector configuration."); + } + if (configurationProvider != null) { // Load the latest provider state so that other in-flight working changes are not overwritten by a rename. final Optional externalConfig = configurationProvider.load(connector.getIdentifier()); @@ -680,6 +739,11 @@ public void updateConnector(final ConnectorNode connector, final String name) { @Override public void configureConnector(final ConnectorNode connector, final String stepName, final StepConfiguration configuration) throws FlowUpdateException { + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot modify the configuration of " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before modifying the Connector configuration."); + } + if (configurationProvider != null) { final ConnectorWorkingConfiguration mergedConfiguration = buildMergedWorkingConfiguration(connector, stepName, configuration); configurationProvider.save(connector.getIdentifier(), mergedConfiguration); @@ -710,6 +774,11 @@ public void inheritConfiguration(final ConnectorNode connector, final List()); } @@ -85,8 +85,13 @@ public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final try { managedProcessGroup.verifyCanUpdate(versionedExternalFlow, true, false); } catch (final IllegalStateException e) { - throw new FlowUpdateException("Flow is not in a state that allows the requested updated", e); + throw new FlowUpdateException("Flow is not in a state that allows the requested update", e); } + } + + @Override + public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final AssetManager assetManager) throws FlowUpdateException { + verifyUpdateFlow(versionedExternalFlow); final ParameterContext managedGroupParameterContext = managedProcessGroup.getParameterContext(); updateParameterContextNames(versionedExternalFlow.getFlowContents(), managedGroupParameterContext.getName()); @@ -104,6 +109,23 @@ public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final parameterContext = parameterContextFacadeFactory.create(managedProcessGroup); } + @Override + public void restoreTroubleshootingFlow(final VersionedProcessGroup troubleshootingProcessGroup) { + final VersionedExternalFlow externalFlow = new VersionedExternalFlow(); + externalFlow.setFlowContents(troubleshootingProcessGroup); + externalFlow.setParameterContexts(Map.of()); + + final ParameterContext managedGroupParameterContext = managedProcessGroup.getParameterContext(); + if (managedGroupParameterContext != null) { + updateParameterContextNames(troubleshootingProcessGroup, managedGroupParameterContext.getName()); + } + + managedProcessGroup.restoreFlowPreservingIdentifiers(externalFlow); + + rootGroup = groupFacadeFactory.create(managedProcessGroup, connectorLog); + parameterContext = parameterContextFacadeFactory.create(managedProcessGroup); + } + private void updateParameterContextNames(final VersionedProcessGroup group, final String parameterContextName) { group.setParameterContextName(parameterContextName); if (group.getProcessGroups() != null) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 259aed264641..c84f5efc1bc5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -1066,6 +1066,54 @@ public Connection findConnectionIncludingConnectorManaged(final String connectio return null; } + /** + * Finds an input Port by ID, searching both the root process group hierarchy and all connector-managed process + * groups. Returns null when the Port cannot be located. + */ + public Port findInputPortIncludingConnectorManaged(final String portId) { + final Port port = flowManager.getRootGroup().findInputPort(portId); + if (port != null) { + return port; + } + + for (final ConnectorNode connector : connectorRepository.getConnectors()) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + final Port managedPort = flowContext.getManagedProcessGroup().findInputPort(portId); + if (managedPort != null) { + return managedPort; + } + } + + return null; + } + + /** + * Finds an output Port by ID, searching both the root process group hierarchy and all connector-managed process + * groups. Returns null when the Port cannot be located. + */ + public Port findOutputPortIncludingConnectorManaged(final String portId) { + final Port port = flowManager.getRootGroup().findOutputPort(portId); + if (port != null) { + return port; + } + + for (final ConnectorNode connector : connectorRepository.getConnectors()) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + final Port managedPort = flowContext.getManagedProcessGroup().findOutputPort(portId); + if (managedPort != null) { + return managedPort; + } + } + + return null; + } + /** * Finds a RemoteGroupPort by ID, searching both the root process group hierarchy * and all connector-managed process groups. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index 7e74a7f2389b..3b84ff727201 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.serialization; import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.FlowAnalysisRuleNode; @@ -30,6 +31,7 @@ import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; @@ -99,9 +101,11 @@ private List mapConnectors() { final List connectors = new ArrayList<>(); for (final ConnectorNode connectorNode : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { - final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode); - if (flowController.isStartAfterInitialization(connectorNode)) { - versionedConnector.setScheduledState(ScheduledState.RUNNING); + final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode, flowController.getControllerServiceProvider()); + if (connectorNode.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + versionedConnector.setScheduledState(VersionedConnectorState.TROUBLESHOOTING); + } else if (flowController.isStartAfterInitialization(connectorNode)) { + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); } connectors.add(versionedConnector); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java index ba409ee7d448..d7a250d80c70 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java @@ -1047,10 +1047,11 @@ private void inheritConnectors(final FlowController flowController, final Versio logger.info("Connector [{}] sync result: {}", versionedConnector.getInstanceIdentifier(), result); if (result.getEffectiveScheduledState() != null && result.getConnectorNode() != null) { - if (result.getEffectiveScheduledState() == ScheduledState.RUNNING) { - flowController.startConnector(result.getConnectorNode()); - } else if (result.getEffectiveScheduledState() == ScheduledState.ENABLED) { - connectorRepository.stopConnector(result.getConnectorNode()); + switch (result.getEffectiveScheduledState()) { + case RUNNING -> flowController.startConnector(result.getConnectorNode()); + case ENABLED -> connectorRepository.stopConnector(result.getConnectorNode()); + case TROUBLESHOOTING -> logger.debug("Connector [{}] is in TROUBLESHOOTING state; leaving connector lifecycle alone", result.getConnectorNode().getIdentifier()); + default -> { } } } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java index 75c35154d693..e59f15da0ce2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java @@ -49,6 +49,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java index 404a78b9727f..9ecdf41c6807 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java @@ -79,6 +79,19 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // Build the flow that reflects the currently configured File path. This mirrors the logic in + // applyUpdate so that exiting Troubleshooting mode restores a flow equivalent to what would be + // installed by re-applying the active configuration. + final VersionedExternalFlow externalFlow = VersionedFlowUtils.loadFlowFromResource("flows/choose-color.json"); + final VersionedProcessGroup rootGroup = externalFlow.getFlowContents(); + final VersionedProcessor processor = rootGroup.getProcessors().iterator().next(); + final String filePath = activeFlowContext.getConfigurationContext().getProperty(FILE_STEP, FILE_PATH).getValue(); + processor.setProperties(Map.of("File", filePath == null ? "" : filePath)); + return externalFlow; + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java index bf8148214393..a4e32ad93299 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java @@ -130,6 +130,12 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-duplicate-log-flow.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative flow is the dynamically built flow based on the active configuration. + return getFlow(activeFlowContext); + } + public boolean isInitialized() { return initialized; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java index c21b55dfbbaf..55377dd15380 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java @@ -96,6 +96,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java index bf5f242a939e..0199a5907d9f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java @@ -60,6 +60,18 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/on-property-modified-tracker.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative flow has the Configured Number parameter set to the currently configured value. + final VersionedExternalFlow versionedExternalFlow = getInitialFlow(); + final String number = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, NUMBER_VALUE).getValue(); + if (number != null) { + VersionedFlowUtils.setParameterValue(versionedExternalFlow, PARAMETER_NAME, number); + } + + return versionedExternalFlow; + } + @Override public List getConfigurationSteps() { return List.of(CONFIG_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java index bbc071aca0eb..777cbd9b1997 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java @@ -76,6 +76,13 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-and-log-with-parameter.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The flow structure does not change with configuration; only parameter values do, which are applied + // to the active parameter context in applyUpdate. The authoritative flow definition matches the initial flow. + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(TEXT_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java index 6818b5b059cc..ef4f7dc0d680 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java @@ -44,6 +44,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java index 00e136a66358..5e5d9eabd136 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java @@ -881,6 +881,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -947,6 +952,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1124,6 +1134,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1168,6 +1183,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index cba4b12f5741..9eb0969a2a7a 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -28,17 +28,19 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.flow.Bundle; -import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.util.MockComponentLog; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -67,6 +69,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; @@ -575,6 +578,84 @@ public void testVerifyCreateProviderRejectsThrows() { assertThrows(ConnectorConfigurationProviderException.class, () -> repository.verifyCreate("connector-1")); } + @Test + public void testVerifyEnterTroubleshootingDelegatesToConnectorAndProvider() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.verifyEnterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + verify(provider).verifyEnterTroubleshooting("connector-1"); + } + + @Test + public void testVerifyEnterTroubleshootingWithoutProvider() { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.verifyEnterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + } + + @Test + public void testVerifyEnterTroubleshootingProviderVetoThrows() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + doThrow(new IllegalStateException("External system does not permit troubleshooting at this time")) + .when(provider).verifyEnterTroubleshooting("connector-1"); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> repository.verifyEnterTroubleshooting(connector)); + assertEquals("External system does not permit troubleshooting at this time", thrown.getMessage()); + + verify(connector).verifyCanEnterTroubleshooting(); + } + + @Test + public void testVerifyEnterTroubleshootingConnectorVetoDoesNotCallProvider() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + doThrow(new IllegalStateException("Connector is currently UPDATING")) + .when(connector).verifyCanEnterTroubleshooting(); + repository.restoreConnector(connector); + + assertThrows(IllegalStateException.class, () -> repository.verifyEnterTroubleshooting(connector)); + verify(provider, never()).verifyEnterTroubleshooting(anyString()); + } + + @Test + public void testEnterTroubleshootingInvokesVerifyThenTransition() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.enterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + verify(provider).verifyEnterTroubleshooting("connector-1"); + verify(connector).enterTroubleshooting(); + } + @Test public void testVerifyCreateExistingConnectorDoesNotCallProvider() { final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); @@ -929,6 +1010,27 @@ public void testApplyUpdateFailureCallsAbortUpdateButNotMarkInvalid() throws Flo verify(connector, never()).markInvalid(anyString(), anyString()); } + @Test + public void testApplyUpdateInTroubleshootingThrowsBeforeProviderConsultation() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + when(connector.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + repository.restoreConnector(connector); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> repository.applyUpdate(connector, mock(ConnectorUpdateContext.class))); + assertTrue(thrown.getMessage().contains("Troubleshooting"), + "Exception message should reference Troubleshooting: " + thrown.getMessage()); + + // Provider must not be consulted; allowing shouldApplyUpdate to silently veto would let the framework return + // success while the Connector was still in Troubleshooting. + verify(provider, never()).shouldApplyUpdate(anyString()); + verify(connector, never()).transitionStateForUpdating(); + } + // --- syncConnector tests --- @Test @@ -939,7 +1041,7 @@ public void testSyncConnectorStoppedWithConfigChange() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -956,7 +1058,7 @@ public void testSyncConnectorStoppedNoConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -972,7 +1074,7 @@ public void testSyncConnectorRunningWithConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -989,7 +1091,7 @@ public void testSyncConnectorDrainingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.DRAINING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1006,7 +1108,7 @@ public void testSyncConnectorPreparingForUpdateIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.PREPARING_FOR_UPDATE); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1022,7 +1124,7 @@ public void testSyncConnectorUpdatingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1037,7 +1139,7 @@ public void testSyncConnectorUpdateFailedRecovery() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATE_FAILED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("recovery-value"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1054,7 +1156,7 @@ public void testSyncConnectorUpdatedState() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1072,7 +1174,7 @@ public void testSyncConnectorInheritConfigurationFailureWhenRunning() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1091,7 +1193,7 @@ public void testSyncConnectorInheritConfigurationFailureWhenStopped() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1110,7 +1212,7 @@ public void testSyncConnectorProviderRejectsSync() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1130,7 +1232,7 @@ public void testSyncConnectorProviderThrowsException() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1149,7 +1251,7 @@ public void testSyncConnectorStartingWaitsForRunning() throws Exception { .thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1164,7 +1266,7 @@ public void testSyncConnectorStartingTimesOut() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STARTING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1188,7 +1290,7 @@ public void testSyncConnectorProviderReturnsRemoveStopsAndRemoves() throws Excep when(connector.stop(any())).thenReturn(CompletableFuture.completedFuture(null)); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1209,7 +1311,7 @@ public void testSyncConnectorProviderReturnsRemoveStoppedConnector() throws Exce when(connector.getConnector()).thenReturn(mockExtension); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1224,7 +1326,7 @@ public void testSyncConnectorProviderReturnsRemoveForNonExistent() throws Except when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.remove()); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1242,7 +1344,7 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1255,19 +1357,139 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t public void testSyncConnectorProviderAllowWithScheduledStateOverride() throws Exception { final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); when(provider.getSyncDirective(eq("connector-1"), any())) - .thenReturn(ConnectorSyncDirective.allow(null, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncDirective.allow(null, VersionedConnectorState.ENABLED)); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + assertEquals(VersionedConnectorState.ENABLED, result.getEffectiveScheduledState()); + } + + @Test + public void testSyncConnectorTroubleshootingInheritsConfigurationThenOverlaysSnapshotThenEntersTroubleshooting() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + // The Connector's managed Parameter Context is not registered with the global ParameterContextManager and is + // therefore not persisted in flow.json. When restoring a Connector whose effective scheduled state is + // TROUBLESHOOTING, the framework must run inheritConfiguration so the Connector re-populates its in-memory + // Parameter Context from the persisted active configuration. The Connector-supplied flow shape that produces + // is then overlaid by restoreTroubleshootingFlow with the user's persisted snapshot before the FlowFile Repository + // attaches FlowFiles to queues, so the transient Connector flow shape is invisible to data movement. + final List activeConfig = List.of( + createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value")))); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, activeConfig); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + assertEquals(VersionedConnectorState.TROUBLESHOOTING, result.getEffectiveScheduledState()); + + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + final InOrder inOrder = inOrder(connector, activeFlowContext); + inOrder.verify(connector).inheritConfiguration(eq(activeConfig), eq(activeConfig), any()); + inOrder.verify(activeFlowContext).restoreTroubleshootingFlow(persistedManagedGroup); + inOrder.verify(connector).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingPreservesExistingTroubleshootingState() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + verify(connector).inheritConfiguration(eq(List.of()), eq(List.of()), any()); + verify(connector.getActiveFlowContext()).restoreTroubleshootingFlow(persistedManagedGroup); + // Connector is already in TROUBLESHOOTING; restoreTroubleshootingState should not be invoked again. + verify(connector, never()).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingWithoutSnapshotLeavesManagedGroupUnchanged() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + repository.restoreConnector(connector); + + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); - assertEquals(ScheduledState.ENABLED, result.getEffectiveScheduledState()); + verify(connector).inheritConfiguration(eq(List.of()), eq(List.of()), any()); + verify(connector.getActiveFlowContext(), never()).restoreTroubleshootingFlow(any()); + verify(connector).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingSnapshotRestoreFailureMarksInvalid() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + doThrow(new RuntimeException("Snapshot restore failure")).when(activeFlowContext).restoreTroubleshootingFlow(any()); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); + verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); + verify(connector, never()).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingInheritConfigurationFailureSkipsSnapshotRestore() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + doThrow(new FlowUpdateException("Inherit failure")) + .when(connector).inheritConfiguration(any(), any(), any()); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); + // inheritConfiguration ran first and failed; the managed flow snapshot must not be overlaid and the + // Connector must not transition into TROUBLESHOOTING. + verify(connector.getActiveFlowContext(), never()).restoreTroubleshootingFlow(any()); + verify(connector, never()).restoreTroubleshootingState(); } @Test @@ -1286,7 +1508,7 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1453,7 +1675,7 @@ private VersionedConnectorValueReference createStringLiteralRef(final String val return ref; } - private VersionedConnector createVersionedConnector(final String id, final String name, final ScheduledState scheduledState, + private VersionedConnector createVersionedConnector(final String id, final String name, final VersionedConnectorState scheduledState, final List activeConfig) { final VersionedConnector vc = new VersionedConnector(); vc.setInstanceIdentifier(id); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java index c2dc1ab49a89..c37168fb1387 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java @@ -56,6 +56,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { if (!initialized) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java index 2684627afc3b..91ea1a874713 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java @@ -34,6 +34,7 @@ import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flow.VersionedReportingTask; @@ -306,12 +307,12 @@ void testSyncInheritConnectorDelegatesSyncToConnectorRepository() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.synced(syncedNode, ScheduledState.RUNNING)); + .thenReturn(ConnectorSyncResult.synced(syncedNode, VersionedConnectorState.RUNNING)); setFlowController(connectorRepository); when(versionedDataflow.getConnectors()).thenReturn(List.of(versionedConnector)); @@ -334,12 +335,12 @@ void testSyncInheritConnectorNotStartedWhenEnabled() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); setFlowController(connectorRepository); when(versionedDataflow.getConnectors()).thenReturn(List.of(versionedConnector)); @@ -363,7 +364,7 @@ void testSyncOrphanConnectorIsRemoved() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -372,7 +373,7 @@ void testSyncOrphanConnectorIsRemoved() { final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(proposedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); @@ -402,11 +403,11 @@ void testSyncOrphanConnectorNotRemovedWhenInProposedFlow() { versionedConnector.setName("Test Connector"); versionedConnector.setType("org.apache.nifi.connectors.TestConnector"); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); @@ -432,7 +433,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -443,7 +444,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(proposedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java index 6beb29d409eb..fe160b17e2b8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java @@ -19,6 +19,7 @@ import org.apache.nifi.authorization.Resource; import org.apache.nifi.authorization.resource.Authorizable; +import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.FlowFileActivity; @@ -132,6 +133,11 @@ public Optional getConnectorIdentifier() { return Optional.empty(); } + @Override + public Optional findOwningConnector() { + return Optional.empty(); + } + @Override public void setPosition(final Position position) { @@ -723,6 +729,10 @@ public ComponentAdditions addVersionedComponents(VersionedComponentAdditions add public void updateFlow(VersionedExternalFlow proposedFlow, String componentIdSeed, boolean verifyNotDirty, boolean updateSettings, boolean updateDescendantVersionedFlows) { } + @Override + public void restoreFlowPreservingIdentifiers(final VersionedExternalFlow proposedFlow) { + } + @Override public void synchronizeFlow(final VersionedExternalFlow proposedSnapshot, final FlowSynchronizationOptions synchronizationOptions, final FlowMappingOptions flowMappingOptions) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java index 2a2b21cde1b3..a81ec9043bc5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java @@ -51,6 +51,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext flowContext) throws FlowUpdateException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java index c1aaf3ad944e..3d62975b0bf2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java @@ -61,6 +61,15 @@ public interface AuthorizableLookup { */ ComponentAuthorizable getProcessor(String id); + /** + * Get the authorizable Processor, optionally including Connector-managed ProcessGroups in the search. + * + * @param id processor id + * @param includeConnectorManaged whether to search Connector-managed ProcessGroups + * @return authorizable + */ + ComponentAuthorizable getProcessor(String id, boolean includeConnectorManaged); + /** * Get the authorizable for querying Provenance. * @@ -120,6 +129,15 @@ public interface AuthorizableLookup { */ Authorizable getInputPort(String id); + /** + * Get the authorizable InputPort, optionally including Connector-managed ProcessGroups in the search. + * + * @param id input port id + * @param includeConnectorManaged whether to search Connector-managed ProcessGroups + * @return authorizable + */ + Authorizable getInputPort(String id, boolean includeConnectorManaged); + /** * Get the authorizable OutputPort. * @@ -128,6 +146,15 @@ public interface AuthorizableLookup { */ Authorizable getOutputPort(String id); + /** + * Get the authorizable OutputPort, optionally including Connector-managed ProcessGroups in the search. + * + * @param id output port id + * @param includeConnectorManaged whether to search Connector-managed ProcessGroups + * @return authorizable + */ + Authorizable getOutputPort(String id, boolean includeConnectorManaged); + /** * Get the authorizable Connection. * @@ -177,6 +204,15 @@ public interface AuthorizableLookup { */ Authorizable getRemoteProcessGroup(String id); + /** + * Get the authorizable RemoteProcessGroup, optionally including Connector-managed ProcessGroups in the search. + * + * @param id remote process group id + * @param includeConnectorManaged whether to search Connector-managed ProcessGroups + * @return authorizable + */ + Authorizable getRemoteProcessGroup(String id, boolean includeConnectorManaged); + /** * Get the authorizable Label. * @@ -210,6 +246,15 @@ public interface AuthorizableLookup { */ ComponentAuthorizable getControllerService(String id); + /** + * Get the authorizable ControllerService, optionally including Connector-managed ProcessGroups in the search. + * + * @param id controller service id + * @param includeConnectorManaged whether to search Connector-managed ProcessGroups + * @return authorizable + */ + ComponentAuthorizable getControllerService(String id, boolean includeConnectorManaged); + /** * Get the authorizable referencing component. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java index be703d121f43..bf28e55356c7 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java @@ -269,7 +269,12 @@ public ComponentAuthorizable getConfigurableComponent(ConfigurableComponent conf @Override public ComponentAuthorizable getProcessor(final String id) { - final ProcessorNode processorNode = processorDAO.getProcessor(id); + return getProcessor(id, false); + } + + @Override + public ComponentAuthorizable getProcessor(final String id, final boolean includeConnectorManaged) { + final ProcessorNode processorNode = processorDAO.getProcessor(id, includeConnectorManaged); return new ProcessorComponentAuthorizable(processorNode, controllerFacade.getExtensionManager()); } @@ -334,11 +339,21 @@ public Authorizable getInputPort(final String id) { return inputPortDAO.getPort(id); } + @Override + public Authorizable getInputPort(final String id, final boolean includeConnectorManaged) { + return inputPortDAO.getPort(id, includeConnectorManaged); + } + @Override public Authorizable getOutputPort(final String id) { return outputPortDAO.getPort(id); } + @Override + public Authorizable getOutputPort(final String id, final boolean includeConnectorManaged) { + return outputPortDAO.getPort(id, includeConnectorManaged); + } + @Override public ParameterContext getParameterContext(final String id) { return parameterContextDAO.getParameterContext(id); @@ -376,6 +391,11 @@ public Authorizable getRemoteProcessGroup(final String id) { return remoteProcessGroupDAO.getRemoteProcessGroup(id); } + @Override + public Authorizable getRemoteProcessGroup(final String id, final boolean includeConnectorManaged) { + return remoteProcessGroupDAO.getRemoteProcessGroup(id, includeConnectorManaged); + } + @Override public Authorizable getLabel(final String id) { return labelDAO.getLabel(id); @@ -421,7 +441,12 @@ public Resource getResource() { @Override public ComponentAuthorizable getControllerService(final String id) { - final ControllerServiceNode controllerService = controllerServiceDAO.getControllerService(id); + return getControllerService(id, false); + } + + @Override + public ComponentAuthorizable getControllerService(final String id, final boolean includeConnectorManaged) { + final ControllerServiceNode controllerService = controllerServiceDAO.getControllerService(id, includeConnectorManaged); return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 11f33b91bd24..d0490a6e6c1d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -220,6 +220,14 @@ public interface NiFiServiceFacade { ConnectorEntity cancelConnectorDrain(Revision revision, String id); + void verifyEnterConnectorTroubleshooting(String id); + + ConnectorEntity enterConnectorTroubleshooting(Revision revision, String id); + + void verifyEndConnectorTroubleshooting(String id); + + ConnectorEntity endConnectorTroubleshooting(Revision revision, String id); + ConfigurationStepNamesEntity getConnectorConfigurationSteps(String id); ConfigurationStepEntity getConnectorConfigurationStep(String id, String configurationStepName); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c13966b16abb..55aa655139e8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -3674,7 +3674,35 @@ public ConnectorEntity getConnector(final String id, final boolean clusterNodeRe @Override public void verifyUpdateConnector(final ConnectorDTO connectorDTO) { - // No-op placeholder for future detailed verification + final ConnectorNode connector = connectorDAO.getConnector(connectorDTO.getId()); + final ConnectorState currentState = connector.getCurrentState(); + + if (connectorDTO.getName() != null && currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot update Connector " + connectorDTO.getId() + + " while it is in Troubleshooting mode; exit Troubleshooting mode before modifying the Connector configuration."); + } + + if (connectorDTO.getState() != null) { + final ScheduledState desiredState; + try { + desiredState = ScheduledState.valueOf(connectorDTO.getState()); + } catch (final IllegalArgumentException iae) { + throw new IllegalArgumentException("Invalid run status specified for Connector " + connectorDTO.getId() + ": " + connectorDTO.getState()); + } + + if (currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot transition Connector " + connectorDTO.getId() + " to " + desiredState + + " while it is in Troubleshooting mode; exit Troubleshooting mode to resume normal lifecycle control."); + } + + switch (desiredState) { + case RUNNING -> connector.verifyCanStart(); + case STOPPED -> { + // Stop is valid from any non-Troubleshooting state; no additional verification required. + } + default -> throw new IllegalArgumentException("Unsupported scheduled state for Connector: " + desiredState); + } + } } @Override @@ -3702,7 +3730,8 @@ public ConnectorEntity updateConnector(final Revision revision, final ConnectorD @Override public void verifyDeleteConnector(final String id) { - // For now, DAO will enforce state; expose hook for symmetry + final ConnectorNode connector = connectorDAO.getConnector(id); + connector.verifyCanDelete(); } @Override @@ -3814,6 +3843,84 @@ public ConnectorEntity cancelConnectorDrain(final Revision revision, final Strin return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); } + @Override + public void verifyEnterConnectorTroubleshooting(final String id) { + connectorDAO.verifyEnterTroubleshooting(id); + } + + @Override + public ConnectorEntity enterConnectorTroubleshooting(final Revision revision, final String id) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final RevisionClaim claim = new StandardRevisionClaim(revision); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.enterTroubleshooting(id); + controllerFacade.save(); + + final ConnectorNode node = connectorDAO.getConnector(id); + final ConnectorDTO dto = dtoFactory.createConnectorDto(node); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode node = connectorDAO.getConnector(snapshot.getComponent().getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(node); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(node)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(node); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + + @Override + public void verifyEndConnectorTroubleshooting(final String id) { + // Verify that all cluster nodes are connected before exiting Troubleshooting mode. Otherwise, we run the risk of weird state transitions while the flow + // is in the middle of updating. + final List unconnectedNodes = getUnconnectedNodes(); + if (!unconnectedNodes.isEmpty()) { + throw new IllegalStateException("Cannot exit Troubleshooting mode because the following cluster nodes are not CONNECTED: " + + unconnectedNodes + ". All nodes must be CONNECTED before this operation may proceed."); + } + + connectorDAO.verifyEndTroubleshooting(id); + } + + private List getUnconnectedNodes() { + if (clusterCoordinator == null) { + return List.of(); + } + + final Map> connectionStates = clusterCoordinator.getConnectionStates(); + final List unconnectedNodes = new ArrayList<>(); + for (final Map.Entry> entry : connectionStates.entrySet()) { + if (entry.getKey() != NodeConnectionState.CONNECTED) { + unconnectedNodes.addAll(entry.getValue()); + } + } + + return unconnectedNodes; + } + + @Override + public ConnectorEntity endConnectorTroubleshooting(final Revision revision, final String id) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final RevisionClaim claim = new StandardRevisionClaim(revision); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.endTroubleshooting(id); + controllerFacade.save(); + + final ConnectorNode node = connectorDAO.getConnector(id); + final ConnectorDTO dto = dtoFactory.createConnectorDto(node); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode node = connectorDAO.getConnector(snapshot.getComponent().getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(node); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(node)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(node); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + @Override public ConfigurationStepNamesEntity getConnectorConfigurationSteps(final String id) { final ConnectorNode node = connectorDAO.getConnector(id); @@ -3903,7 +4010,11 @@ public ProcessGroupFlowEntity getConnectorFlow(final String connectorId, final S if (targetProcessGroup == null) { throw new ResourceNotFoundException("Process Group with ID " + processGroupId + " was not found within Connector " + connectorId); } - return createProcessGroupFlowEntity(targetProcessGroup, uiOnly); + + // Bulletin authorization within the managed flow resolves source components through the live group hierarchy + // (via authorizeBulletin's ProcessGroup overload), so the standard supplier works regardless of the + // Connector's state without needing DAO-level connector-managed lookups. + return createProcessGroupFlowEntity(targetProcessGroup, uiOnly, this::getProcessGroupBulletins); } @Override @@ -5655,6 +5766,11 @@ public ProcessGroupFlowEntity getProcessGroupFlow(final String groupId, final bo } private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup processGroup, final boolean uiOnly) { + return createProcessGroupFlowEntity(processGroup, uiOnly, this::getProcessGroupBulletins); + } + + private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup processGroup, final boolean uiOnly, + final Function> bulletinSupplier) { // Get the Process Group Status but we only need a status depth of one because for any child process group, // we ignore the status of each individual components. I.e., if Process Group A has child Group B, and child Group B // has a Processor, we don't care about the individual stats of that Processor because the ProcessGroupFlowEntity @@ -5663,7 +5779,7 @@ private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup p final RevisionDTO revision = dtoFactory.createRevisionDTO(revisionManager.getRevision(processGroup.getIdentifier())); final PermissionsDTO permissions = dtoFactory.createPermissionsDto(processGroup); return entityFactory.createProcessGroupFlowEntity(dtoFactory.createProcessGroupFlowDto(processGroup, groupStatus, - revisionManager, this::getProcessGroupBulletins, uiOnly), revision, permissions); + revisionManager, bulletinSupplier, uiOnly), revision, permissions); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java index 087a45fdfcf4..1b3685a72f92 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java @@ -807,6 +807,154 @@ public Response cancelDrain( ); } + /** + * Transitions the specified Connector into Troubleshooting mode. + * + * @param id The id of the connector. + * @param requestConnectorEntity A connectorEntity containing the revision. + * @return A connectorEntity. + */ + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/troubleshooting") + @Operation( + summary = "Transitions a Connector into Troubleshooting mode", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = ConnectorEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Places the Connector into Troubleshooting mode so that its managed flow may be directly modified. Standard lifecycle operations are disabled while in Troubleshooting mode.", + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid} or /operation/connectors/{uuid}") + } + ) + public Response enterTroubleshooting( + @Parameter( + description = "The connector id.", + required = true + ) + @PathParam("id") final String id, + @Parameter( + description = "The connector entity with revision.", + required = true + ) final ConnectorEntity requestConnectorEntity) { + + if (requestConnectorEntity == null || requestConnectorEntity.getRevision() == null) { + throw new IllegalArgumentException("Connector entity with revision must be specified."); + } + + if (requestConnectorEntity.getId() != null && !id.equals(requestConnectorEntity.getId())) { + throw new IllegalArgumentException(String.format("The connector id (%s) in the request body does not equal the " + + "connector id of the requested resource (%s).", requestConnectorEntity.getId(), id)); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, requestConnectorEntity); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(requestConnectorEntity.isDisconnectedNodeAcknowledged()); + } + + final Revision requestRevision = getRevision(requestConnectorEntity, id); + return withWriteLock( + serviceFacade, + requestConnectorEntity, + requestRevision, + lookup -> { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Authorizable connector = lookup.getConnector(id); + OperationAuthorizable.authorizeOperation(connector, authorizer, user); + }, + () -> serviceFacade.verifyEnterConnectorTroubleshooting(id), + (revision, connectorEntity) -> { + final ConnectorEntity entity = serviceFacade.enterConnectorTroubleshooting(revision, id); + populateRemainingConnectorEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + + /** + * Transitions the specified Connector out of Troubleshooting mode. + * + * @param version The revision is used to verify the client is working with the latest version of the flow. + * @param clientId Optional client id. + * @param id The id of the connector. + * @return A connectorEntity. + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/troubleshooting") + @Operation( + summary = "Transitions a Connector out of Troubleshooting mode", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = ConnectorEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Ends Troubleshooting mode for the Connector, restoring the authoritative flow. All components in the managed flow must be stopped and disabled, " + + "and the managed flow must have no active tasks.", + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid} or /operation/connectors/{uuid}") + } + ) + public Response endTroubleshooting( + @Parameter( + description = "The revision is used to verify the client is working with the latest version of the flow." + ) + @QueryParam(VERSION) final LongParameter version, + @Parameter( + description = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response." + ) + @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) final ClientIdParameter clientId, + @Parameter( + description = "Acknowledges that this node is disconnected to allow for mutable requests to proceed." + ) + @QueryParam(DISCONNECTED_NODE_ACKNOWLEDGED) @DefaultValue("false") final Boolean disconnectedNodeAcknowledged, + @Parameter( + description = "The connector id.", + required = true + ) + @PathParam("id") final String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(disconnectedNodeAcknowledged); + } + + final ConnectorEntity requestConnectorEntity = new ConnectorEntity(); + requestConnectorEntity.setId(id); + + final Revision requestRevision = new Revision(version == null ? null : version.getLong(), clientId.getClientId(), id); + return withWriteLock( + serviceFacade, + requestConnectorEntity, + requestRevision, + lookup -> { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Authorizable connector = lookup.getConnector(id); + OperationAuthorizable.authorizeOperation(connector, authorizer, user); + }, + () -> serviceFacade.verifyEndConnectorTroubleshooting(id), + (revision, connectorEntity) -> { + final ConnectorEntity entity = serviceFacade.endConnectorTroubleshooting(revision, id); + populateRemainingConnectorEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + @POST @Consumes(MediaType.WILDCARD) @@ -1799,7 +1947,9 @@ public Response getFlow( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the flow for the specified process group within the connector's hierarchy + // Get the flow for the specified process group within the connector's hierarchy. The facade method is + // Connector-aware and skips the normal Troubleshooting access gate at the DAO locate call sites for bulletin + // authorization lookups within the managed flow. final ProcessGroupFlowEntity entity = serviceFacade.getConnectorFlow(connectorId, processGroupId, uiOnly); flowResource.populateRemainingFlowContent(entity.getProcessGroupFlow()); return generateOkResponse(entity).build(); @@ -1861,9 +2011,11 @@ public Response getControllerServicesFromConnectorProcessGroup( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the controller services for the specified process group within the connector's hierarchy - final Set controllerServices = serviceFacade.getConnectorControllerServices( - connectorId, processGroupId, includeAncestorGroups, includeDescendantGroups, includeReferences); + // Get the controller services for the specified process group within the connector's hierarchy. The facade + // method operates on the ControllerServiceNodes resolved directly from the managed Process Group tree, so no + // DAO locate calls are made against Connector-managed components here. + final Set controllerServices = + serviceFacade.getConnectorControllerServices(connectorId, processGroupId, includeAncestorGroups, includeDescendantGroups, includeReferences); controllerServiceResource.populateRemainingControllerServiceEntitiesContent(controllerServices); // create the response entity @@ -2005,7 +2157,9 @@ public Response getConnectorStatus( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the status for the connector's managed process group + // Get the status for the Connector's managed Process Group. The facade method builds the status DTO from the + // resolved ProcessGroup directly (no DAO locate calls for Connector-managed components), so the normal + // Troubleshooting access gate does not apply. final ProcessGroupStatusEntity entity = serviceFacade.getConnectorProcessGroupStatus(id, recursive); return generateOkResponse(entity).build(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 4dfbbe89e062..ecbc773616fd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -860,7 +860,14 @@ public ProcessorStatus getProcessorStatus(final String processorId) { */ public ConnectionStatus getConnectionStatus(final String connectionId) { final ProcessGroup root = getRootGroup(); - final Connection connection = root.findConnection(connectionId); + Connection connection = root.findConnection(connectionId); + + // If the Connection was not found by traversing the root hierarchy, fall back to a direct FlowManager lookup. This + // is necessary because Connections that live inside a Connector's Managed Process Group are not part of the main + // root Process Group's parent hierarchy, but they are still registered with the FlowManager. + if (connection == null) { + connection = flowController.getFlowManager().getConnection(connectionId); + } // ensure the connection was found if (connection == null) { @@ -920,10 +927,8 @@ public StatusAnalytics getConnectionStatusAnalytics(final String connectionId) { * @return the status for the specified input port */ public PortStatus getInputPortStatus(final String portId) { - final ProcessGroup root = getRootGroup(); - final Port port = root.findInputPort(portId); + final Port port = flowController.findInputPortIncludingConnectorManaged(portId); - // ensure the input port was found if (port == null) { throw new ResourceNotFoundException(String.format("Unable to locate input port with id '%s'.", portId)); } @@ -949,10 +954,8 @@ public PortStatus getInputPortStatus(final String portId) { * @return the status for the specified output port */ public PortStatus getOutputPortStatus(final String portId) { - final ProcessGroup root = getRootGroup(); - final Port port = root.findOutputPort(portId); + final Port port = flowController.findOutputPortIncludingConnectorManaged(portId); - // ensure the output port was found if (port == null) { throw new ResourceNotFoundException(String.format("Unable to locate output port with id '%s'.", portId)); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java index acf2e7f53900..e63218ae1411 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java @@ -59,6 +59,14 @@ public interface ConnectorDAO { void verifyCancelDrainFlowFile(String id); + void verifyEnterTroubleshooting(String id); + + void enterTroubleshooting(String id); + + void verifyEndTroubleshooting(String id); + + void endTroubleshooting(String id); + void verifyPurgeFlowFiles(String id); void purgeFlowFiles(String id, String requestor); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java index 6dc41df450fc..18a501245eca 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java @@ -61,6 +61,17 @@ public interface ControllerServiceDAO { */ ControllerServiceNode getControllerService(String controllerServiceId); + /** + * Gets the specified controller service, optionally including Connector-managed Process Groups in the search. When + * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components + * within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param controllerServiceId The controller service id + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The controller service + */ + ControllerServiceNode getControllerService(String controllerServiceId, boolean includeConnectorManaged); + /** * Gets all of the controller services for the group with the given ID or all * controller-level services if the group id is null diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java index 858da8df818f..68c4b3f1d5e5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java @@ -46,6 +46,17 @@ public interface FunnelDAO { */ Funnel getFunnel(String funnelId); + /** + * Gets the specified funnel, optionally including Connector-managed Process Groups in the search. When + * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within + * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param funnelId The funnel id + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The funnel + */ + Funnel getFunnel(String funnelId, boolean includeConnectorManaged); + /** * Gets all of the funnels in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java index 515b0d49a9ab..a17cf2d8c7c0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java @@ -46,6 +46,17 @@ public interface LabelDAO { */ Label getLabel(String labelId); + /** + * Gets the specified label, optionally including Connector-managed Process Groups in the search. When + * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within + * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param labelId The label id + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The label + */ + Label getLabel(String labelId, boolean includeConnectorManaged); + /** * Gets all of the labels in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java index ed607db68016..62db8323c551 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java @@ -46,6 +46,17 @@ public interface PortDAO { */ Port getPort(String portId); + /** + * Gets the specified port, optionally including Connector-managed Process Groups in the search. When + * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within + * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param portId The port id + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The port + */ + Port getPort(String portId, boolean includeConnectorManaged); + /** * Gets all of the ports in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java index 72f7caee872e..48be11e93a5f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java @@ -59,6 +59,17 @@ public interface ProcessorDAO { */ ProcessorNode getProcessor(String id); + /** + * Gets the Processor transfer object for the specified id, optionally including Connector-managed Process Groups + * in the search. When {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access + * to components within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param id Id of the processor to return + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The Processor + */ + ProcessorNode getProcessor(String id, boolean includeConnectorManaged); + /** * Gets all the Processor transfer objects for this controller. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java index 7446a34f349c..25007b748aee 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java @@ -52,6 +52,17 @@ public interface RemoteProcessGroupDAO { */ RemoteProcessGroup getRemoteProcessGroup(String remoteProcessGroupId); + /** + * Gets the specified remote process group, optionally including Connector-managed Process Groups in the search. + * When {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components + * within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. + * + * @param remoteProcessGroupId The remote process group id + * @param includeConnectorManaged whether to search Connector-managed Process Groups + * @return The remote process group + */ + RemoteProcessGroup getRemoteProcessGroup(String remoteProcessGroupId, boolean includeConnectorManaged); + /** * Gets all of the remote process groups. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java index a41888bd3b98..33cba1c41942 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java @@ -40,7 +40,21 @@ public abstract class AbstractPortDAO extends ComponentDAO implements PortDAO { protected FlowController flowController; - protected abstract Port locatePort(final String portId); + protected Port locatePort(final String portId) { + return locatePort(portId, false); + } + + protected abstract Port locatePort(final String portId, final boolean includeConnectorManaged); + + @Override + public Port getPort(final String portId) { + return locatePort(portId); + } + + @Override + public Port getPort(final String portId, final boolean includeConnectorManaged) { + return locatePort(portId, includeConnectorManaged); + } @Override public void verifyUpdate(PortDTO portDTO) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java index 26f0de302520..0fffe87a0c51 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java @@ -18,6 +18,8 @@ import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.controller.FlowController; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; @@ -25,6 +27,7 @@ import org.apache.nifi.web.api.dto.BundleDTO; import java.util.List; +import java.util.Optional; public abstract class ComponentDAO { @@ -83,17 +86,54 @@ protected ProcessGroup locateProcessGroup(final FlowController flowController, f return group; } - // Optionally search Connector-managed ProcessGroups - if (includeConnectorManaged) { - group = flowController.getFlowManager().getGroup(groupId); - if (group != null) { + // Search Connector-managed ProcessGroups. The unconditional search is important so that if a component exists + // in a Connector-managed flow but the Connector is not in Troubleshooting mode, we can produce a clear 409 + // Conflict response rather than a 404 Not Found. + group = flowController.getFlowManager().getGroup(groupId); + if (group != null) { + if (includeConnectorManaged) { return group; } + + verifyAccessibleForComponentOperation(group, groupId); + return group; } throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); } + /** + * Verifies that the component represented by the given {@link ProcessGroup} (or a component contained within it) is + * accessible for a direct user-facing operation such as GET/PUT/POST/DELETE of the component itself. Components that + * live within a Connector's managed Process Group hierarchy are only accessible when the owning Connector is in + * {@link ConnectorState#TROUBLESHOOTING} mode. If the owning Connector is not in Troubleshooting mode, an + * {@link IllegalStateException} is thrown which is translated by the REST layer into a 409 Conflict response. + * + *

Connector-aware REST endpoints that need to read components within a managed flow regardless of + * the Connector's state must obtain those components through the {@code includeConnectorManaged} overloads on the + * relevant DAO (and {@link org.apache.nifi.authorization.AuthorizableLookup}) so that this verification is skipped + * at the locate call site rather than being bypassed globally for the current thread. + * + * @param group the ProcessGroup that owns (or is) the component being accessed + * @param componentId the identifier of the component being accessed (used in the error message) + */ + protected void verifyAccessibleForComponentOperation(final ProcessGroup group, final String componentId) { + if (group == null) { + return; + } + + final Optional owningConnector = group.findOwningConnector(); + if (owningConnector.isEmpty()) { + return; + } + + final ConnectorNode connector = owningConnector.get(); + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Component [" + componentId + "] is managed by Connector " + connector.getName() + " (" + + connector.getIdentifier() + "); the Connector must be in Troubleshooting mode for this component to be accessible."); + } + } + protected void verifyCreate(final ExtensionManager extensionManager, final String type, final BundleDTO bundle) { final List bundles = extensionManager.getBundles(type); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java index 5b4ab5c14205..3b9f80c21f5b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java @@ -24,6 +24,7 @@ import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Connectable; @@ -83,7 +84,6 @@ private Connection locateConnection(final String connectionId) { } private Connection locateConnection(final String connectionId, final boolean includeConnectorManaged) { - // First, search the main flow hierarchy final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Connection connection = rootGroup.findConnection(connectionId); @@ -91,17 +91,20 @@ private Connection locateConnection(final String connectionId, final boolean inc return connection; } - // Optionally search Connector-managed ProcessGroups - if (includeConnectorManaged) { - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { - final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); - if (flowContext != null) { - final ProcessGroup managedGroup = flowContext.getManagedProcessGroup(); - connection = managedGroup.findConnection(connectionId); - if (connection != null) { - return connection; - } + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + final ProcessGroup managedGroup = flowContext.getManagedProcessGroup(); + connection = managedGroup.findConnection(connectionId); + if (connection != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(connection.getProcessGroup(), connectionId); } + + return connection; } } @@ -111,7 +114,22 @@ private Connection locateConnection(final String connectionId, final boolean inc @Override public boolean hasConnection(String id) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findConnection(id) != null; + if (rootGroup.findConnection(id) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findConnection(id) != null) { + return true; + } + } + + return false; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index 9a507e16e5ce..af3069bd5526 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -27,6 +27,7 @@ import org.apache.nifi.components.connector.ConnectorUpdateContext; import org.apache.nifi.components.connector.ConnectorValueReference; import org.apache.nifi.components.connector.ConnectorValueType; +import org.apache.nifi.components.connector.FlowUpdateException; import org.apache.nifi.components.connector.SecretReference; import org.apache.nifi.components.connector.StepConfiguration; import org.apache.nifi.components.connector.StringLiteralValue; @@ -157,6 +158,34 @@ public void verifyCancelDrainFlowFile(final String id) { connector.verifyCancelDrainFlowFiles(); } + @Override + public void verifyEnterTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + getConnectorRepository().verifyEnterTroubleshooting(connector); + } + + @Override + public void enterTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + getConnectorRepository().enterTroubleshooting(connector); + } + + @Override + public void verifyEndTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + connector.verifyCanEndTroubleshooting(); + } + + @Override + public void endTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + try { + getConnectorRepository().endTroubleshooting(connector); + } catch (final FlowUpdateException e) { + throw new IllegalStateException("Failed to exit troubleshooting mode for Connector " + id + ": " + e, e); + } + } + @Override public void verifyPurgeFlowFiles(final String id) { final ConnectorNode connector = requireConnector(id, ConnectorSyncMode.LOCAL_ONLY); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java index 6c13a29d0b3e..b1a996e7ebb1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java @@ -71,14 +71,19 @@ public class StandardControllerServiceDAO extends ComponentDAO implements Contro private FlowController flowController; private ControllerServiceNode locateControllerService(final String controllerServiceId) { - // get the controller service + return locateControllerService(controllerServiceId, false); + } + + private ControllerServiceNode locateControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { final ControllerServiceNode controllerService = serviceProvider.getControllerServiceNode(controllerServiceId); - // ensure the controller service exists if (controllerService == null) { throw new ResourceNotFoundException(String.format("Unable to locate controller service with id '%s'.", controllerServiceId)); } + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(controllerService.getProcessGroup(), controllerServiceId); + } return controllerService; } @@ -116,11 +121,7 @@ public ControllerServiceNode createControllerService(final ControllerServiceDTO if (groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS)) { group = flowManager.getRootGroup(); } else { - group = flowManager.getRootGroup().findProcessGroup(groupId); - } - - if (group == null) { - throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); + group = locateProcessGroup(flowController, groupId); } group.addControllerService(controllerService); @@ -137,6 +138,11 @@ public ControllerServiceNode getControllerService(final String controllerService return locateControllerService(controllerServiceId); } + @Override + public ControllerServiceNode getControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { + return locateControllerService(controllerServiceId, includeConnectorManaged); + } + @Override public boolean hasControllerService(final String controllerServiceId) { return serviceProvider.getControllerServiceNode(controllerServiceId) != null; @@ -148,20 +154,17 @@ public Set getControllerServices(final String groupId, fi if (groupId == null) { return flowManager.getRootControllerServices(); - } else { - final String searchId = groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS) ? flowManager.getRootGroupId() : groupId; - final ProcessGroup procGroup = flowManager.getRootGroup().findProcessGroup(searchId); - if (procGroup == null) { - throw new ResourceNotFoundException("Could not find Process Group with ID " + groupId); - } + } - final Set serviceNodes = procGroup.getControllerServices(includeAncestorGroups); - if (includeDescendantGroups) { - serviceNodes.addAll(procGroup.findAllControllerServices()); - } + final String searchId = groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS) ? flowManager.getRootGroupId() : groupId; + final ProcessGroup procGroup = locateProcessGroup(flowController, searchId); - return serviceNodes; + final Set serviceNodes = procGroup.getControllerServices(includeAncestorGroups); + if (includeDescendantGroups) { + serviceNodes.addAll(procGroup.findAllControllerServices()); } + + return serviceNodes; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java index 186aa09a97d6..223cf7f1f505 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java @@ -16,6 +16,9 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.FlowController; @@ -34,20 +37,53 @@ public class StandardFunnelDAO extends ComponentDAO implements FunnelDAO { private FlowController flowController; private Funnel locateFunnel(final String funnelId) { - final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final Funnel funnel = rootGroup.findFunnel(funnelId); + return locateFunnel(funnelId, false); + } - if (funnel == null) { - throw new ResourceNotFoundException(String.format("Unable to find funnel with id '%s'.", funnelId)); - } else { + private Funnel locateFunnel(final String funnelId, final boolean includeConnectorManaged) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + Funnel funnel = rootGroup.findFunnel(funnelId); + if (funnel != null) { return funnel; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + funnel = flowContext.getManagedProcessGroup().findFunnel(funnelId); + if (funnel != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(funnel.getProcessGroup(), funnelId); + } + return funnel; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find funnel with id '%s'.", funnelId)); } @Override public boolean hasFunnel(String funnelId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findFunnel(funnelId) != null; + if (rootGroup.findFunnel(funnelId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + + return false; } @Override @@ -76,6 +112,11 @@ public Funnel getFunnel(String funnelId) { return locateFunnel(funnelId); } + @Override + public Funnel getFunnel(final String funnelId, final boolean includeConnectorManaged) { + return locateFunnel(funnelId, includeConnectorManaged); + } + @Override public Set getFunnels(String groupId) { ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java index cc84aae59a78..452e3fb1b4b8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java @@ -16,6 +16,9 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.ScheduledState; @@ -32,25 +35,60 @@ public class StandardInputPortDAO extends AbstractPortDAO implements PortDAO { @Override - protected Port locatePort(final String portId) { + protected Port locatePort(final String portId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Port port = rootGroup.findInputPort(portId); - if (port == null) { port = rootGroup.findOutputPort(portId); } - - if (port == null) { - throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); - } else { + if (port != null) { return port; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + final ProcessGroup managed = flowContext.getManagedProcessGroup(); + port = managed.findInputPort(portId); + if (port == null) { + port = managed.findOutputPort(portId); + } + if (port != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(port.getProcessGroup(), portId); + } + return port; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); } @Override public boolean hasPort(String portId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findInputPort(portId) != null || rootGroup.findOutputPort(portId) != null; + if (rootGroup.findInputPort(portId) != null || rootGroup.findOutputPort(portId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null) { + final ProcessGroup managed = flowContext.getManagedProcessGroup(); + if (managed.findInputPort(portId) != null || managed.findOutputPort(portId) != null) { + return true; + } + } + } + + return false; } @Override @@ -94,11 +132,6 @@ public Port createPort(String groupId, PortDTO portDTO) { return port; } - @Override - public Port getPort(String portId) { - return locatePort(portId); - } - @Override public Set getPorts(String groupId) { ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java index f1d0f80bf018..58a65aa36c93 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java @@ -16,6 +16,9 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; import org.apache.nifi.controller.FlowController; @@ -37,20 +40,53 @@ public class StandardLabelDAO extends ComponentDAO implements LabelDAO { private FlowController flowController; private Label locateLabel(final String labelId) { - final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final Label label = rootGroup.findLabel(labelId); + return locateLabel(labelId, false); + } - if (label == null) { - throw new ResourceNotFoundException(String.format("Unable to find label with id '%s'.", labelId)); - } else { + private Label locateLabel(final String labelId, final boolean includeConnectorManaged) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + Label label = rootGroup.findLabel(labelId); + if (label != null) { return label; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + label = flowContext.getManagedProcessGroup().findLabel(labelId); + if (label != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(label.getProcessGroup(), labelId); + } + return label; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find label with id '%s'.", labelId)); } @Override public boolean hasLabel(String labelId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findLabel(labelId) != null; + if (rootGroup.findLabel(labelId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findLabel(labelId) != null) { + return true; + } + } + + return false; } @Override @@ -85,6 +121,11 @@ public Label getLabel(String labelId) { return locateLabel(labelId); } + @Override + public Label getLabel(final String labelId, final boolean includeConnectorManaged) { + return locateLabel(labelId, includeConnectorManaged); + } + @Override public Set

    + *
  1. Create a Connector and configure it with non-default property values so its active configuration and + * active managed flow both differ from the unconfigured defaults.
    2. + *
  2. Apply the update so the active flow is the Connector's authoritative non-default flow.
    3. + *
  3. Enter Troubleshooting and modify the managed flow by adding a processor that the Connector's + * authoritative flow does not contain.
    4. + *
  4. Restart NiFi while still in Troubleshooting.
    5. + *
  5. Exit Troubleshooting.
    6. + *
  6. Assert the active configuration still contains the configured (non-default) values, the active flow is + * the Connector's authoritative non-default flow rather than the user-modified Troubleshooting flow, and + * finally that running the Connector produces output at the configured non-default destinations.
    7. + *
+ */ + @Test + public void testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRestart() throws NiFiClientException, IOException, InterruptedException { + // Use a secret name unique to this test so the underlying SecretsManager cannot return a value cached for the + // generic name "secret" by another test that ran earlier in the same JVM. + final String secretName = "configurationRestoreSecret"; + final String sensitiveSecretValue = "configured-secret-value"; + final String assetFileContent = "Hello, World!"; + final File configuredSensitiveOutput = new File("target/configuration-restore-sensitive.txt"); + final File configuredAssetOutput = new File("target/configuration-restore-asset.txt"); + configuredSensitiveOutput.delete(); + configuredAssetOutput.delete(); + + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", secretName + "=" + sensitiveSecretValue)); + + final ConnectorEntity connector = getClientUtil().createConnector("ParameterContextConnector"); + final String connectorId = connector.getId(); + + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity assetEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetFile.getName(), assetFile); + final String uploadedAssetId = assetEntity.getAsset().getId(); + + final ConnectorValueReferenceDTO secretRef = getClientUtil().createSecretValueReference( + paramProvider.getId(), secretName, "PropertiesParameterProvider.Parameters." + secretName); + final ConnectorValueReferenceDTO assetRef = new ConnectorValueReferenceDTO(); + assetRef.setValueType("ASSET_REFERENCE"); + assetRef.setAssetReferences(List.of(new AssetReferenceDTO(uploadedAssetId))); + + // The output file paths differ from the property descriptors' default values, so the active configuration + // after restart can be checked against these specific paths to prove the configured values were preserved + // instead of being overwritten by defaults. + final Map propertyValues = new HashMap<>(); + propertyValues.put("Sensitive Value", secretRef); + propertyValues.put("Asset File", assetRef); + propertyValues.put("Sensitive Output File", createStringLiteralRef(configuredSensitiveOutput.getAbsolutePath())); + propertyValues.put("Asset Output File", createStringLiteralRef(configuredAssetOutput.getAbsolutePath())); + + getClientUtil().configureConnectorWithReferences(connectorId, "Parameter Context Configuration", propertyValues); + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + assertNotNull(findProcessorByName(connectorId, "UpdateContent"), + "Active flow should contain UpdateContent before Troubleshooting"); + assertNotNull(findProcessorByName(connectorId, "ReplaceWithFile"), + "Active flow should contain ReplaceWithFile before Troubleshooting"); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // The Sleep processor is created in STOPPED state and is left disconnected so endTroubleshooting can later + // succeed without first having to stop or empty any user-introduced components. + final String managedGroupId = getNifiClient().getConnectorClient().getConnector(connectorId).getComponent().getManagedProcessGroupId(); + final ProcessorEntity troubleshootingProcessor = getClientUtil().createProcessor("Sleep", managedGroupId); + final String troubleshootingProcessorId = troubleshootingProcessor.getId(); + assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor should be present in the managed flow after adding it in Troubleshooting"); + + getNiFiInstance().stop(); + getNiFiInstance().start(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor should survive restart while in Troubleshooting"); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + final ConnectorEntity afterExit = getNifiClient().getConnectorClient().getConnector(connectorId); + final ConnectorConfigurationDTO activeConfig = afterExit.getComponent().getActiveConfiguration(); + final Map activeProperties = activeConfig.getConfigurationStepConfigurations().getFirst() + .getPropertyGroupConfigurations().getFirst().getPropertyValues(); + assertEquals(configuredSensitiveOutput.getAbsolutePath(), activeProperties.get("Sensitive Output File").getValue(), + "Active configuration must retain the configured Sensitive Output File path"); + assertEquals(configuredAssetOutput.getAbsolutePath(), activeProperties.get("Asset Output File").getValue(), + "Active configuration must retain the configured Asset Output File path"); + assertEquals("ASSET_REFERENCE", activeProperties.get("Asset File").getValueType(), + "Active configuration must retain the Asset reference for Asset File"); + assertEquals(uploadedAssetId, activeProperties.get("Asset File").getAssetReferences().get(0).getId(), + "Active configuration must retain the uploaded Asset id for Asset File"); + assertEquals("SECRET_REFERENCE", activeProperties.get("Sensitive Value").getValueType(), + "Active configuration must retain the Secret reference for Sensitive Value"); + + assertFalse(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor must be removed once the authoritative flow is restored on Troubleshooting exit"); + assertNotNull(findProcessorByName(connectorId, "UpdateContent"), + "Restored authoritative flow should contain UpdateContent"); + assertNotNull(findProcessorByName(connectorId, "ReplaceWithFile"), + "Restored authoritative flow should contain ReplaceWithFile"); + assertNotNull(findProcessorByName(connectorId, "GenerateFlowFile"), + "Restored authoritative flow should contain GenerateFlowFile"); + + getClientUtil().startConnector(connectorId); + assertConnectorState(connectorId, ConnectorState.RUNNING); + + waitFor(() -> configuredSensitiveOutput.exists() && configuredAssetOutput.exists()); + assertEquals(sensitiveSecretValue, Files.readString(configuredSensitiveOutput.toPath()).trim(), + "Running Connector must write the configured sensitive value to the configured Sensitive Output File"); + assertEquals(assetFileContent, Files.readString(configuredAssetOutput.toPath()).trim(), + "Running Connector must write the configured asset content to the configured Asset Output File"); + } + + private boolean containsProcessorId(final String connectorId, final String processorId) throws NiFiClientException, IOException { + for (final ProcessorEntity entity : findAllProcessors(connectorId)) { + if (processorId.equals(entity.getId())) { + return true; + } + } + return false; + } + + private void runManagedFlowAndAssertParameterValues(final String connectorId, final File sensitiveOutputFile, final File assetOutputFile, + final String expectedSensitiveValue, final String expectedAssetValue, final String phase) + throws NiFiClientException, IOException, InterruptedException { + + // Starting individual components is permitted while in Troubleshooting. The managed Process Group is a + // standard (non-stateless) group so processors and ports can be scheduled individually. The flow built by + // ParameterContextConnector routes FlowFiles through child group Input Ports, so every Port inside the + // managed flow must also be started for the pipeline to actually pass FlowFiles. + final List inputPorts = findAllInputPorts(connectorId); + final List outputPorts = findAllOutputPorts(connectorId); + final List processors = findAllProcessors(connectorId); + assertFalse(processors.isEmpty(), "Managed flow should contain processors " + phase); + + for (final PortEntity port : inputPorts) { + getNifiClient().getInputPortClient().startInputPort(port); + } + for (final PortEntity port : outputPorts) { + getNifiClient().getOutputPortClient().startOutputPort(port); + } + for (final ProcessorEntity processor : processors) { + getClientUtil().waitForValidProcessor(processor.getId()); + getClientUtil().startProcessor(processor); + } + + waitFor(() -> sensitiveOutputFile.exists() && assetOutputFile.exists()); + + assertEquals(expectedSensitiveValue, Files.readString(sensitiveOutputFile.toPath()).trim(), + "Sensitive output file must contain the configured sensitive parameter value " + phase); + assertEquals(expectedAssetValue, Files.readString(assetOutputFile.toPath()).trim(), + "Asset output file must contain the asset contents referenced by the asset parameter " + phase); + } + + private void stopAllManagedComponents(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + for (final ProcessorEntity processor : findAllProcessors(connectorId)) { + getClientUtil().stopProcessor(processor); + } + for (final PortEntity port : findAllInputPorts(connectorId)) { + getNifiClient().getInputPortClient().stopInputPort(port); + } + for (final PortEntity port : findAllOutputPorts(connectorId)) { + getNifiClient().getOutputPortClient().stopOutputPort(port); + } + for (final ProcessorEntity processor : findAllProcessors(connectorId)) { + waitForProcessorState(processor.getId(), ScheduledState.STOPPED); + } + } + + private List findAllInputPorts(final String connectorId) throws NiFiClientException, IOException { + final List result = new ArrayList<>(); + collectPorts(connectorId, null, true, result); + return result; + } + + private List findAllOutputPorts(final String connectorId) throws NiFiClientException, IOException { + final List result = new ArrayList<>(); + collectPorts(connectorId, null, false, result); + return result; + } + + private void collectPorts(final String connectorId, final String groupId, final boolean input, final List collected) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity entity = (groupId == null) ? getNifiClient().getConnectorClient().getFlow(connectorId) : getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + final FlowDTO flow = entity.getProcessGroupFlow().getFlow(); + collected.addAll(input ? flow.getInputPorts() : flow.getOutputPorts()); + + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + collectPorts(connectorId, child.getId(), input, collected); + } + } + + private ConnectorValueReferenceDTO createStringLiteralRef(final String value) { + final ConnectorValueReferenceDTO ref = new ConnectorValueReferenceDTO(); + ref.setValueType("STRING_LITERAL"); + ref.setValue(value); + return ref; + } +} diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java index 05f9ed173c3d..14a909fc3bc2 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java @@ -177,6 +177,28 @@ public interface ConnectorClient { */ ConnectorEntity cancelDrain(ConnectorEntity connectorEntity) throws NiFiClientException, IOException; + /** + * Transitions a connector into Troubleshooting mode. + * + * @param connectorEntity the connector entity (must contain id and revision) + * @return the updated connector entity + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + ConnectorEntity enterTroubleshooting(ConnectorEntity connectorEntity) throws NiFiClientException, IOException; + + /** + * Transitions a connector out of Troubleshooting mode. + * + * @param connectorId the connector ID + * @param clientId the client ID + * @param version the revision version + * @return the updated connector entity + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + ConnectorEntity endTroubleshooting(String connectorId, String clientId, long version) throws NiFiClientException, IOException; + /** * Gets the configuration step names for a connector. * diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java index 7b648e7183e6..d972f4891591 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java @@ -235,6 +235,64 @@ private ConnectorEntity cancelDrain(final String connectorId, final String clien }); } + @Override + public ConnectorEntity enterTroubleshooting(final ConnectorEntity connectorEntity) throws NiFiClientException, IOException { + return enterTroubleshooting(connectorEntity.getId(), connectorEntity.getRevision().getClientId(), + connectorEntity.getRevision().getVersion(), connectorEntity.isDisconnectedNodeAcknowledged()); + } + + private ConnectorEntity enterTroubleshooting(final String connectorId, final String clientId, final long version, + final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector ID cannot be null or blank"); + } + + return executeAction("Error entering Troubleshooting mode", () -> { + final WebTarget target = connectorTarget + .path("/troubleshooting") + .resolveTemplate("id", connectorId); + + final ConnectorEntity requestEntity = new ConnectorEntity(); + requestEntity.setId(connectorId); + requestEntity.setDisconnectedNodeAcknowledged(disconnectedNodeAcknowledged); + + final RevisionDTO revisionDto = new RevisionDTO(); + revisionDto.setClientId(clientId); + revisionDto.setVersion(version); + requestEntity.setRevision(revisionDto); + + return getRequestBuilder(target).post( + Entity.entity(requestEntity, MediaType.APPLICATION_JSON_TYPE), + ConnectorEntity.class); + }); + } + + @Override + public ConnectorEntity endTroubleshooting(final String connectorId, final String clientId, final long version) throws NiFiClientException, IOException { + return endTroubleshooting(connectorId, clientId, version, false); + } + + private ConnectorEntity endTroubleshooting(final String connectorId, final String clientId, final long version, + final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector ID cannot be null or blank"); + } + + return executeAction("Error ending Troubleshooting mode", () -> { + WebTarget target = connectorTarget + .path("/troubleshooting") + .queryParam("version", version) + .queryParam("clientId", clientId) + .resolveTemplate("id", connectorId); + + if (disconnectedNodeAcknowledged == Boolean.TRUE) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + + return getRequestBuilder(target).delete(ConnectorEntity.class); + }); + } + private ConnectorEntity updateConnectorRunStatus(final String connectorId, final String desiredState, final String clientId, final long version, final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { if (StringUtils.isBlank(connectorId)) { diff --git a/pom.xml b/pom.xml index af98cd0c526d..87eadc2567cc 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ v24.14.1 - 2.8.0 + 2.9.0-SNAPSHOT 2.3.0 From 42fbdd994a069299478f105d4e5a9ba22d2cfc46 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 11:47:51 -0400 Subject: [PATCH 2/9] NIFI-15880: Surface why END_TROUBLESHOOTING is unavailable in the allowable action; fix findProcessGroup for connector child groups - Refactored verifyCanEndTroubleshooting to delegate to a new getReasonCannotEndTroubleshooting() method that returns Optional. The verify method throws if the Optional is present; createEndTroubleshootingAction uses the same method to populate the reason field on the allowable action so callers see why the action is unavailable without having to attempt it. - Replaced the throwing verifyAllComponentsStoppedAndDisabled helper with findReasonComponentsNotStopped, which returns the first reason found as an Optional. The expensive flow-revert preflight only runs after the cheap component-state scan passes. - Fixed StandardProcessGroup.findProcessGroup to use the unconditional flowManager.getGroup(id) instead of the connector-ID-filtered overload. Only the root managed process group carries a connectorId; child groups do not, so the filtered lookup always returned null for non-root groups, causing a 404 when navigating into a child group in Troubleshooting mode. The existing isOwner check already enforces hierarchy ownership. --- .../nifi/groups/StandardProcessGroup.java | 2 +- .../connector/StandardConnectorNode.java | 81 +++++++++++-------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java index bc97980603f2..fa94289e5271 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java @@ -2141,7 +2141,7 @@ public ProcessGroup findProcessGroup(final String id) { return this; } - final ProcessGroup group = flowManager.getGroup(id, getConnectorIdentifier().orElse(null)); + final ProcessGroup group = flowManager.getGroup(id); if (group == null) { return null; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index 9cdb87a59ade..45f1286e3c2b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -705,27 +705,38 @@ public void verifyCanEnterTroubleshooting() { @Override public void verifyCanEndTroubleshooting() { - if (getCurrentState() != ConnectorState.TROUBLESHOOTING) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because it is not currently in Troubleshooting mode; current state is " + getCurrentState()); + final Optional reason = getReasonCannotEndTroubleshooting(); + if (reason.isPresent()) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + ": " + reason.get()); + } + } + + private Optional getReasonCannotEndTroubleshooting() { + final ConnectorState currentState = getCurrentState(); + if (currentState != ConnectorState.TROUBLESHOOTING) { + return Optional.of("Connector is not in Troubleshooting mode; current state is " + currentState); } // Verify that every component inside the managed flow is stopped/disabled BEFORE doing any other validation. - // The flow-update check below relies on components being stopped/disabled in order to produce meaningful results. Otherwise, - // the update check may succeed and then the running flow puts it into a bad state. - final ProcessGroup managedGroup = getActiveFlowContext().getManagedProcessGroup(); - verifyAllComponentsStoppedAndDisabled(managedGroup); - - // After confirming all components are stopped or disabled, check if the managed flow can be safely reverted. - // Connector's authoritative flow would succeed. This mirrors exactly what endTroubleshooting() will do so that - // any problem (e.g. a Connection whose contents cannot be preserved or a component that cannot be replaced) - // is reported synchronously at the REST verify-phase rather than surfacing halfway through the state change. + // The flow-update check below relies on components being stopped/disabled in order to produce meaningful results. + // Returning early here avoids running the expensive flow-revert preflight against a live, running flow. + final Optional componentReason = findReasonComponentsNotStopped(getActiveFlowContext().getManagedProcessGroup()); + if (componentReason.isPresent()) { + return componentReason; + } + + // After confirming all components are stopped or disabled, check if the managed flow can be safely reverted to the + // Connector's authoritative flow. This mirrors exactly what endTroubleshooting() will do so that any problem + // (e.g. a Connection whose contents cannot be preserved or a component that cannot be replaced) is reported + // synchronously rather than surfacing halfway through the state change. final VersionedExternalFlow authoritativeFlow = resolveAuthoritativeFlow(); try { initializationContext.verifyUpdateFlow(activeFlowContext, authoritativeFlow, BundleCompatibility.RESOLVE_BUNDLE); } catch (final FlowUpdateException e) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this - + " because the Managed Process Group cannot be reverted to the Connector's authoritative flow: " + e.getMessage(), e); + return Optional.of("The Managed Process Group cannot be reverted to the Connector's authoritative flow: " + e.getMessage()); } + + return Optional.empty(); } private VersionedExternalFlow resolveAuthoritativeFlow() { @@ -744,45 +755,48 @@ private VersionedExternalFlow resolveAuthoritativeFlow() { return authoritativeFlow; } - private void verifyAllComponentsStoppedAndDisabled(final ProcessGroup group) { + private Optional findReasonComponentsNotStopped(final ProcessGroup group) { for (final ProcessorNode processor : group.getProcessors()) { - if (!STOPPED_STATES.contains(processor.getScheduledState())) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Processor " + processor.getIdentifier() - + " is in state " + processor.getScheduledState() + "; it must be STOPPED or DISABLED."); + final ScheduledState scheduledState = processor.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Processor " + processor.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } } for (final Port port : group.getInputPorts()) { - if (!STOPPED_STATES.contains(port.getScheduledState())) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Input Port " + port.getIdentifier() - + " is in state " + port.getScheduledState() + "; it must be STOPPED or DISABLED."); + final ScheduledState scheduledState = port.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Input Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } } for (final Port port : group.getOutputPorts()) { - if (!STOPPED_STATES.contains(port.getScheduledState())) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Output Port " + port.getIdentifier() - + " is in state " + port.getScheduledState() + "; it must be STOPPED or DISABLED."); + final ScheduledState scheduledState = port.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Output Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } } for (final RemoteProcessGroup remoteProcessGroup : group.getRemoteProcessGroups()) { if (remoteProcessGroup.isTransmitting()) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Remote Process Group " - + remoteProcessGroup.getIdentifier() + " is transmitting; it must be stopped."); + return Optional.of("Remote Process Group " + remoteProcessGroup.getIdentifier() + " is transmitting; it must be stopped"); } } for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) { if (serviceNode.isActive()) { - throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + " because Controller Service " + serviceNode.getIdentifier() - + " is not disabled; all Controller Services within the managed flow must be disabled."); + return Optional.of("Controller Service " + serviceNode.getIdentifier() + " is active; all Controller Services within the managed flow must be disabled"); } } for (final ProcessGroup childGroup : group.getProcessGroups()) { - verifyAllComponentsStoppedAndDisabled(childGroup); + final Optional childReason = findReasonComponentsNotStopped(childGroup); + if (childReason.isPresent()) { + return childReason; + } } + + return Optional.empty(); } @Override @@ -1469,7 +1483,7 @@ public List getAvailableActions() { actions.add(createApplyUpdatesAction(currentState, troubleshooting)); actions.add(createDeleteAction(stopped && !troubleshooting, dataQueued)); actions.add(createEnterTroubleshootingAction(currentState)); - actions.add(createEndTroubleshootingAction(troubleshooting)); + actions.add(createEndTroubleshootingAction()); return actions; } @@ -1556,12 +1570,9 @@ private ConnectorAction createEnterTroubleshootingAction(final ConnectorState cu } } - private ConnectorAction createEndTroubleshootingAction(final boolean troubleshooting) { - if (!troubleshooting) { - return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", false, - "Connector is not in Troubleshooting mode"); - } - return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", true, null); + private ConnectorAction createEndTroubleshootingAction() { + final Optional reason = getReasonCannotEndTroubleshooting(); + return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", reason.isEmpty(), reason.orElse(null)); } private ConnectorAction createDiscardWorkingConfigAction(final boolean troubleshooting) { From c78637880eef75fca502b3849ad418aad0305de1 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 11:51:42 -0400 Subject: [PATCH 3/9] NIFI-15880: Add system test for fetching a non-root child process group via the Connector flow endpoint Adds testGetFlowForChildProcessGroupInTroubleshooting to ConnectorTroubleshootingIT to verify that the Connector's /flow/process-groups/{groupId} endpoint returns the correct flow for a child process group within the managed flow while in Troubleshooting mode. --- .../ConnectorTroubleshootingIT.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java index 0001a6ef6a82..98468c203b01 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java @@ -377,6 +377,36 @@ public void testNonProcessorComponentsInManagedFlow() throws NiFiClientException assertConflictExpected("GET process group", () -> getNifiClient().getProcessGroupClient().getProcessGroup(childGroupId)); } + /** + * Verify that the Connector's flow endpoint correctly returns the flow for a non-root child process + * group within the managed flow while in Troubleshooting mode. The fix that enables this is in + * StandardProcessGroup.findProcessGroup: child groups within a connector's managed hierarchy do not + * carry a connectorId field on their own ProcessGroup object, so the old connector-ID-filtered + * getGroup(id, connectorId) lookup always returned null for them, causing a 404. The unconditional + * getGroup(id) lookup combined with the existing isOwner hierarchy check is the correct approach. + */ + @Test + public void testGetFlowForChildProcessGroupInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + final ProcessGroupFlowEntity rootFlow = getNifiClient().getConnectorClient().getFlow(connectorId); + final List childGroups = new ArrayList<>(rootFlow.getProcessGroupFlow().getFlow().getProcessGroups()); + assertFalse(childGroups.isEmpty(), "ComponentLifecycleConnector managed flow must contain at least one child process group"); + + final String childGroupId = childGroups.get(0).getId(); + + final ProcessGroupFlowEntity childFlow = getNifiClient().getConnectorClient().getFlow(connectorId, childGroupId); + assertNotNull(childFlow); + assertEquals(childGroupId, childFlow.getProcessGroupFlow().getId()); + } + private void assertConnectorState(final String connectorId, final ConnectorState expected) throws NiFiClientException, IOException { final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); assertEquals(expected.name(), entity.getComponent().getState()); From 8842121314c8b70d7d97ed9f05c309b52c8dfc58 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 12:04:12 -0400 Subject: [PATCH 4/9] NIFI-15880: Fix compile errors in nifi-framework-core after nifi-api rebuild - Use fully-qualified org.apache.nifi.controller.ScheduledState for local variables in findReasonComponentsNotStopped to resolve the naming conflict with the imported org.apache.nifi.flow.ScheduledState. - Pass ConnectorSyncMode.LOCAL_ONLY to ConnectorRepository.getConnectors() in FlowController.findInputPortIncludingConnectorManaged and findOutputPortIncludingConnectorManaged; the no-arg overload was removed from the interface and these internal lookups do not need a provider sync. --- .../nifi/components/connector/StandardConnectorNode.java | 6 +++--- .../java/org/apache/nifi/controller/FlowController.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index 45f1286e3c2b..456c3e03a2a1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -757,21 +757,21 @@ private VersionedExternalFlow resolveAuthoritativeFlow() { private Optional findReasonComponentsNotStopped(final ProcessGroup group) { for (final ProcessorNode processor : group.getProcessors()) { - final ScheduledState scheduledState = processor.getScheduledState(); + final org.apache.nifi.controller.ScheduledState scheduledState = processor.getScheduledState(); if (!STOPPED_STATES.contains(scheduledState)) { return Optional.of("Processor " + processor.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } } for (final Port port : group.getInputPorts()) { - final ScheduledState scheduledState = port.getScheduledState(); + final org.apache.nifi.controller.ScheduledState scheduledState = port.getScheduledState(); if (!STOPPED_STATES.contains(scheduledState)) { return Optional.of("Input Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } } for (final Port port : group.getOutputPorts()) { - final ScheduledState scheduledState = port.getScheduledState(); + final org.apache.nifi.controller.ScheduledState scheduledState = port.getScheduledState(); if (!STOPPED_STATES.contains(scheduledState)) { return Optional.of("Output Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index c84f5efc1bc5..0804b1d56626 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -1076,7 +1076,7 @@ public Port findInputPortIncludingConnectorManaged(final String portId) { return port; } - for (final ConnectorNode connector : connectorRepository.getConnectors()) { + for (final ConnectorNode connector : connectorRepository.getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -1100,7 +1100,7 @@ public Port findOutputPortIncludingConnectorManaged(final String portId) { return port; } - for (final ConnectorNode connector : connectorRepository.getConnectors()) { + for (final ConnectorNode connector : connectorRepository.getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; From 1134ddf2fdcdf7f630a3b4c425685e674c4416c6 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 12:30:49 -0400 Subject: [PATCH 5/9] NIFI-15880: Add getActiveFlow override to test connector stubs DependentSecretVerifyConnector, RequiredSecretConnector, and TrackingConnector did not implement the new abstract Connector.getActiveFlow(FlowContext) method added to nifi-api. Add a stub returning null to each, since none of the tests exercised that method. --- .../connector/TestStandardConnectorNode.java | 10 ++++++++++ .../connector/TestStandardConnectorRepository.java | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java index 5e5d9eabd136..95afabd0c534 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java @@ -1007,6 +1007,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1078,6 +1083,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index 9eb0969a2a7a..4396b8a1ec9c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -1790,6 +1790,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } From b6b518a2b605c6373422b6aba5a786861eee08e8 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 14:17:26 -0400 Subject: [PATCH 6/9] NIFI-15880: Pass ConnectorSyncMode to ConnectorRepository.getConnectors in component DAOs After rebasing onto main (NIFI-15919), ConnectorRepository.getConnector and ConnectorRepository.getConnectors now require a ConnectorSyncMode. Update the connector-managed component lookups added to StandardFunnelDAO, StandardInputPortDAO, StandardLabelDAO, StandardOutputPortDAO, and StandardRemoteProcessGroupDAO to pass ConnectorSyncMode.LOCAL_ONLY. These are internal walks across in-memory connector state to locate a component by id and have no need to consult the configuration provider. --- .../java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java | 5 +++-- .../org/apache/nifi/web/dao/impl/StandardInputPortDAO.java | 5 +++-- .../java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java | 5 +++-- .../org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java | 5 +++-- .../nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java index 223cf7f1f505..dffe25495321 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Position; @@ -47,7 +48,7 @@ private Funnel locateFunnel(final String funnelId, final boolean includeConnecto return funnel; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -72,7 +73,7 @@ public boolean hasFunnel(String funnelId) { return true; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { continue; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java index 452e3fb1b4b8..3929807b87ba 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; @@ -45,7 +46,7 @@ protected Port locatePort(final String portId, final boolean includeConnectorMan return port; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -74,7 +75,7 @@ public boolean hasPort(String portId) { return true; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { continue; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java index 58a65aa36c93..c879144b6a73 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; @@ -50,7 +51,7 @@ private Label locateLabel(final String labelId, final boolean includeConnectorMa return label; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -75,7 +76,7 @@ public boolean hasLabel(String labelId) { return true; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { continue; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java index 5f58fc49aadd..dff622bd76cd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; @@ -42,7 +43,7 @@ protected Port locatePort(final String portId, final boolean includeConnectorMan return port; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -67,7 +68,7 @@ public boolean hasPort(String portId) { return true; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { continue; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java index a145d5caf70a..074dfe6971e3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; @@ -65,7 +66,7 @@ private RemoteProcessGroup locateRemoteProcessGroup(final String remoteProcessGr return remoteProcessGroup; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); if (flowContext == null) { continue; @@ -90,7 +91,7 @@ public boolean hasRemoteProcessGroup(String remoteProcessGroupId) { return true; } - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors()) { + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { continue; } From 332ec7b9f6ed5bcfbef15357f23f9a3cca08e64e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 27 May 2026 14:56:58 -0400 Subject: [PATCH 7/9] NIFI-15880: Fix broken assertion in testEndTroubleshootingBlockedByQueuedData assertEquals(-1, getQueuedCount(...)) compared an int against an OptionalInt, so the comparison was never meaningful. Replace it with assertFalse(getQueuedCount(...).isPresent(), ...) which expresses the intended check: after ending Troubleshooting the connection added during Troubleshooting must no longer exist in the managed flow. --- .../tests/system/connectors/ConnectorTroubleshootingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java index 98468c203b01..23f26ce184d7 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java @@ -158,7 +158,7 @@ public void testEndTroubleshootingBlockedByQueuedData() throws NiFiClientExcepti assertFalse(processorIdsAfterRestore.contains(generate.getId()), "GenerateFlowFile processor added in Troubleshooting should be removed once authoritative flow is restored"); assertFalse(processorIdsAfterRestore.contains(terminate.getId()), "TerminateFlowFile processor added in Troubleshooting should be removed once authoritative flow is restored"); - assertEquals(-1, getQueuedCount(connectorId, connectionId), "Connection added in Troubleshooting should be removed once authoritative flow is restored"); + assertFalse(getQueuedCount(connectorId, connectionId).isPresent(), "Connection added in Troubleshooting should be removed once authoritative flow is restored"); } /** From 6d0e6a10db6c90935a9db7a3750b0175302632c3 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 29 May 2026 11:08:07 -0400 Subject: [PATCH 8/9] NIFI-15880: Address Connector troubleshooting review feedback - Replace boolean includeConnectorManaged overloads on DAOs and AuthorizableLookup with new ConnectorManagedComponentLookup and ConnectorManagedAuthorizableLookup facades to keep connector-managed access off the primary lookup APIs. - Add ProcessGroup#findOwningConnector default implementation. - Deprecate one-arg VersionedComponentFlowMapper#mapConnector for removal to flag the latent footgun for callers. - Add ConnectorRepository#verifyEndTroubleshooting and a matching ConnectorConfigurationProvider hook so the repository runs verification symmetrically with enterTroubleshooting and notifies third-party providers of the transition. - Fix Javadoc typo and ScheduledState reference on ConnectorRepository. - Add ClusteredConnectorTroubleshootingIT to run the troubleshooting suite against a two-node cluster, and add restart helper that waits for all nodes to reconnect to avoid races against post-restart cluster rejoin. --- .../ConnectorConfigurationProvider.java | 19 +++ .../mapping/VersionedComponentFlowMapper.java | 8 + .../connector/ConnectorRepository.java | 18 +- .../org/apache/nifi/groups/ProcessGroup.java | 9 +- .../StandardConnectorRepository.java | 15 ++ .../service/mock/MockProcessGroup.java | 6 - .../authorization/AuthorizableLookup.java | 66 +------- .../ConnectorManagedAuthorizableLookup.java | 68 ++++++++ .../StandardAuthorizableLookup.java | 100 +++++++---- .../nifi/web/StandardNiFiServiceFacade.java | 15 +- .../nifi/web/api/FlowFileQueueResource.java | 12 +- .../nifi/web/api/ProcessGroupResource.java | 2 +- .../apache/nifi/web/dao/ConnectionDAO.java | 9 - .../dao/ConnectorManagedComponentLookup.java | 93 ++++++++++ .../nifi/web/dao/ControllerServiceDAO.java | 11 -- .../org/apache/nifi/web/dao/FunnelDAO.java | 11 -- .../org/apache/nifi/web/dao/LabelDAO.java | 11 -- .../java/org/apache/nifi/web/dao/PortDAO.java | 11 -- .../apache/nifi/web/dao/ProcessGroupDAO.java | 9 - .../org/apache/nifi/web/dao/ProcessorDAO.java | 11 -- .../nifi/web/dao/RemoteProcessGroupDAO.java | 11 -- .../nifi/web/dao/impl/AbstractPortDAO.java | 5 - .../nifi/web/dao/impl/ComponentDAO.java | 9 +- .../web/dao/impl/StandardConnectionDAO.java | 7 +- .../web/dao/impl/StandardConnectorDAO.java | 2 +- ...andardConnectorManagedComponentLookup.java | 140 +++++++++++++++ .../impl/StandardControllerServiceDAO.java | 7 +- .../nifi/web/dao/impl/StandardFunnelDAO.java | 7 +- .../nifi/web/dao/impl/StandardLabelDAO.java | 7 +- .../web/dao/impl/StandardProcessGroupDAO.java | 5 - .../web/dao/impl/StandardProcessorDAO.java | 7 +- .../impl/StandardRemoteProcessGroupDAO.java | 11 +- .../StandardAuthorizableLookupTest.java | 73 +++----- .../web/StandardNiFiServiceFacadeTest.java | 9 +- .../dao/impl/StandardConnectionDAOTest.java | 34 ++-- .../dao/impl/StandardProcessGroupDAOTest.java | 36 ++-- .../ClusteredConnectorTroubleshootingIT.java | 39 +++++ .../ConnectorTroubleshootingIT.java | 159 ++++++++++++++++-- 38 files changed, 706 insertions(+), 366 deletions(-) create mode 100644 nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorTroubleshootingIT.java diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java index 324934845a1d..a6327b2a8baf 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java @@ -120,6 +120,25 @@ public interface ConnectorConfigurationProvider { default void verifyEnterTroubleshooting(final String connectorId) { } + /** + * Verifies that the provider allows the connector with the given identifier to transition out of + * Troubleshooting mode. This is called before the framework restores the Connector's authoritative + * flow and transitions it back to its normal lifecycle, giving the provider an opportunity to + * reject the operation (for example, because the external system has placed the connector in a + * state that requires user attention before normal operation can resume). + * + *

If the provider cannot support the operation, it should throw a runtime exception + * (for example, {@link IllegalStateException} or + * {@link ConnectorConfigurationProviderException}) describing why the transition is not + * allowed. The exception message will be surfaced to the caller.

+ * + *

The default implementation is a no-op, allowing the transition to proceed.

+ * + * @param connectorId the identifier of the connector that is being transitioned out of Troubleshooting mode + */ + default void verifyEndTroubleshooting(final String connectorId) { + } + /** * Determines how the connector repository should handle synchronization for the given * connector during flow inheritance (cluster join). The provider examines the external diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index f017a87c33e2..2d12f29fa988 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -1095,6 +1095,14 @@ private org.apache.nifi.flow.ScheduledState mapScheduledState(final ScheduledSta : org.apache.nifi.flow.ScheduledState.ENABLED; } + /** + * @deprecated Use {@link #mapConnector(ConnectorNode, ControllerServiceProvider)} instead. This overload silently + * skips serializing the Connector's Managed Process Group when the Connector is in + * {@link org.apache.nifi.components.connector.ConnectorState#TROUBLESHOOTING}, because the Managed + * Process Group cannot be mapped without a ControllerServiceProvider. Callers that need a complete + * VersionedConnector (including troubleshooting state) must supply a ControllerServiceProvider. + */ + @Deprecated(forRemoval = true) public VersionedConnector mapConnector(final ConnectorNode connectorNode) { return mapConnector(connectorNode, null); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java index 14d8744f9b14..d81dbb0c2b68 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java @@ -74,9 +74,9 @@ public interface ConnectorRepository { * should be removed or rejected).

* *

When a {@link ConnectorConfigurationProvider} is configured, the provider's - * {@link ConnectorConfigurationProvider#getSyncDirective(String, org.apache.nifi.flow.ScheduledState)} + * {@link ConnectorConfigurationProvider#getSyncDirective(String, org.apache.nifi.flow.VersionedConnectorState)} * method is called to obtain a {@link ConnectorSyncDirective} that may override the - * working configuration, name, or ScheduledState from the versioned flow.

+ * working configuration, name, or scheduled state from the versioned flow.

* * @param versionedConnector the proposed connector from the versioned flow * @return a {@link ConnectorSyncResult} indicating the outcome @@ -174,13 +174,25 @@ void inheritConfiguration(ConnectorNode connector, ListThe default implementation returns {@link Optional#empty()}. Implementations that can resolve a + * {@link ConnectorNode} from a connector identifier (typically via a FlowManager) should override this method + * and walk the parent chain using {@link #getConnectorIdentifier()} and {@link #getParent()} to locate the + * owning Connector.

+ * * @return an Optional containing the owning ConnectorNode, or empty if this Process Group and all of its ancestors are * not managed by a Connector */ - Optional findOwningConnector(); + default Optional findOwningConnector() { + return Optional.empty(); + } /** * @return the user-set comments about this ProcessGroup, or diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index 9337cf17783e..1385dcb918cf 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -801,8 +801,23 @@ public void enterTroubleshooting(final ConnectorNode connector) { connector.enterTroubleshooting(); } + @Override + public void verifyEndTroubleshooting(final ConnectorNode connector) { + connector.verifyCanEndTroubleshooting(); + if (configurationProvider != null) { + configurationProvider.verifyEndTroubleshooting(connector.getIdentifier()); + } + } + @Override public void endTroubleshooting(final ConnectorNode connector) throws FlowUpdateException { + // Consult the optional provider hook for end-of-troubleshooting symmetry with enterTroubleshooting. + // The Connector's own pre-conditions (verifyCanEndTroubleshooting) are evaluated inside + // connector.endTroubleshooting(), so we do not call them a second time here in order to avoid + // double-evaluating the (relatively expensive) authoritative-flow preflight against the live managed group. + if (configurationProvider != null) { + configurationProvider.verifyEndTroubleshooting(connector.getIdentifier()); + } logger.info("Exiting Troubleshooting mode on Connector [{}]", connector.getIdentifier()); connector.endTroubleshooting(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java index fe160b17e2b8..f221ba28101f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java @@ -19,7 +19,6 @@ import org.apache.nifi.authorization.Resource; import org.apache.nifi.authorization.resource.Authorizable; -import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.FlowFileActivity; @@ -133,11 +132,6 @@ public Optional getConnectorIdentifier() { return Optional.empty(); } - @Override - public Optional findOwningConnector() { - return Optional.empty(); - } - @Override public void setPosition(final Position position) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java index 3d62975b0bf2..dc41df885604 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java @@ -61,15 +61,6 @@ public interface AuthorizableLookup { */ ComponentAuthorizable getProcessor(String id); - /** - * Get the authorizable Processor, optionally including Connector-managed ProcessGroups in the search. - * - * @param id processor id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - ComponentAuthorizable getProcessor(String id, boolean includeConnectorManaged); - /** * Get the authorizable for querying Provenance. * @@ -129,15 +120,6 @@ public interface AuthorizableLookup { */ Authorizable getInputPort(String id); - /** - * Get the authorizable InputPort, optionally including Connector-managed ProcessGroups in the search. - * - * @param id input port id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - Authorizable getInputPort(String id, boolean includeConnectorManaged); - /** * Get the authorizable OutputPort. * @@ -146,15 +128,6 @@ public interface AuthorizableLookup { */ Authorizable getOutputPort(String id); - /** - * Get the authorizable OutputPort, optionally including Connector-managed ProcessGroups in the search. - * - * @param id output port id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - Authorizable getOutputPort(String id, boolean includeConnectorManaged); - /** * Get the authorizable Connection. * @@ -163,15 +136,6 @@ public interface AuthorizableLookup { */ ConnectionAuthorizable getConnection(String id); - /** - * Get the authorizable Connection, optionally including Connector-managed ProcessGroups in the search. - * - * @param id connection id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - ConnectionAuthorizable getConnection(String id, boolean includeConnectorManaged); - /** * Get the authorizable root ProcessGroup. * @@ -187,15 +151,6 @@ public interface AuthorizableLookup { */ ProcessGroupAuthorizable getProcessGroup(String id); - /** - * Get the authorizable ProcessGroup, optionally including Connector-managed ProcessGroups in the search. - * - * @param id process group id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - ProcessGroupAuthorizable getProcessGroup(String id, boolean includeConnectorManaged); - /** * Get the authorizable RemoteProcessGroup. * @@ -204,15 +159,6 @@ public interface AuthorizableLookup { */ Authorizable getRemoteProcessGroup(String id); - /** - * Get the authorizable RemoteProcessGroup, optionally including Connector-managed ProcessGroups in the search. - * - * @param id remote process group id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - Authorizable getRemoteProcessGroup(String id, boolean includeConnectorManaged); - /** * Get the authorizable Label. * @@ -247,13 +193,15 @@ public interface AuthorizableLookup { ComponentAuthorizable getControllerService(String id); /** - * Get the authorizable ControllerService, optionally including Connector-managed ProcessGroups in the search. + * Returns a {@link ConnectorManagedAuthorizableLookup} view of this lookup that resolves components within + * Connector-managed Process Groups regardless of the owning Connector's state. Only Connector-scoped REST + * endpoints should use this view; standard endpoints continue to use the methods directly on this interface, + * which intentionally hide components inside Connector-managed flows unless the owning Connector is in + * Troubleshooting mode. * - * @param id controller service id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable + * @return the Connector-managed authorizable lookup facade */ - ComponentAuthorizable getControllerService(String id, boolean includeConnectorManaged); + ConnectorManagedAuthorizableLookup forConnectorManagedFlow(); /** * Get the authorizable referencing component. diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java new file mode 100644 index 000000000000..0add406c7db3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.authorization; + +import org.apache.nifi.authorization.resource.Authorizable; + +/** + * Facade exposing authorizable lookups that include components within Connector-managed Process Groups. + * + *

The standard {@link AuthorizableLookup} intentionally hides components inside Connector-managed Process Group + * hierarchies unless the owning Connector is in Troubleshooting mode. The few Connector-scoped REST endpoints that + * must authorize access to components inside a managed flow regardless of the Connector's state obtain those + * authorizables through this facade, so the standard {@link AuthorizableLookup} surface stays free of the + * "include connector managed" flag.

+ * + *

Obtain an instance via {@link AuthorizableLookup#forConnectorManagedFlow()}.

+ */ +public interface ConnectorManagedAuthorizableLookup { + + /** + * Get the authorizable Processor located within a Connector-managed Process Group. + */ + ComponentAuthorizable getProcessor(String id); + + /** + * Get the authorizable input Port located within a Connector-managed Process Group. + */ + Authorizable getInputPort(String id); + + /** + * Get the authorizable output Port located within a Connector-managed Process Group. + */ + Authorizable getOutputPort(String id); + + /** + * Get the authorizable Connection located within a Connector-managed Process Group. + */ + ConnectionAuthorizable getConnection(String id); + + /** + * Get the authorizable Process Group, including Connector-managed Process Groups. + */ + ProcessGroupAuthorizable getProcessGroup(String id); + + /** + * Get the authorizable Remote Process Group located within a Connector-managed Process Group. + */ + Authorizable getRemoteProcessGroup(String id); + + /** + * Get the authorizable Controller Service located within a Connector-managed Process Group. + */ + ComponentAuthorizable getControllerService(String id); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java index bf28e55356c7..a6becc883b33 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java @@ -57,6 +57,7 @@ import org.apache.nifi.web.dao.AccessPolicyDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ConnectorDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.ControllerServiceDAO; import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; @@ -247,6 +248,9 @@ public Resource getResource() { private AccessPolicyDAO accessPolicyDAO; private ParameterContextDAO parameterContextDAO; + private ConnectorManagedComponentLookup connectorManagedComponentLookup; + private final ConnectorManagedAuthorizableLookup connectorManagedAuthorizableLookup = new ConnectorManagedAuthorizableLookupImpl(); + @Override public Authorizable getController() { return controllerFacade; @@ -269,12 +273,7 @@ public ComponentAuthorizable getConfigurableComponent(ConfigurableComponent conf @Override public ComponentAuthorizable getProcessor(final String id) { - return getProcessor(id, false); - } - - @Override - public ComponentAuthorizable getProcessor(final String id, final boolean includeConnectorManaged) { - final ProcessorNode processorNode = processorDAO.getProcessor(id, includeConnectorManaged); + final ProcessorNode processorNode = processorDAO.getProcessor(id); return new ProcessorComponentAuthorizable(processorNode, controllerFacade.getExtensionManager()); } @@ -339,21 +338,11 @@ public Authorizable getInputPort(final String id) { return inputPortDAO.getPort(id); } - @Override - public Authorizable getInputPort(final String id, final boolean includeConnectorManaged) { - return inputPortDAO.getPort(id, includeConnectorManaged); - } - @Override public Authorizable getOutputPort(final String id) { return outputPortDAO.getPort(id); } - @Override - public Authorizable getOutputPort(final String id, final boolean includeConnectorManaged) { - return outputPortDAO.getPort(id, includeConnectorManaged); - } - @Override public ParameterContext getParameterContext(final String id) { return parameterContextDAO.getParameterContext(id); @@ -361,12 +350,7 @@ public ParameterContext getParameterContext(final String id) { @Override public ConnectionAuthorizable getConnection(final String id) { - return getConnection(id, false); - } - - @Override - public ConnectionAuthorizable getConnection(final String id, final boolean includeConnectorManaged) { - final Connection connection = connectionDAO.getConnection(id, includeConnectorManaged); + final Connection connection = connectionDAO.getConnection(id); return new StandardConnectionAuthorizable(connection); } @@ -377,12 +361,7 @@ public ProcessGroupAuthorizable getRootProcessGroup() { @Override public ProcessGroupAuthorizable getProcessGroup(final String id) { - return getProcessGroup(id, false); - } - - @Override - public ProcessGroupAuthorizable getProcessGroup(final String id, final boolean includeConnectorManaged) { - final ProcessGroup processGroup = processGroupDAO.getProcessGroup(id, includeConnectorManaged); + final ProcessGroup processGroup = processGroupDAO.getProcessGroup(id); return new StandardProcessGroupAuthorizable(processGroup, controllerFacade.getExtensionManager()); } @@ -391,11 +370,6 @@ public Authorizable getRemoteProcessGroup(final String id) { return remoteProcessGroupDAO.getRemoteProcessGroup(id); } - @Override - public Authorizable getRemoteProcessGroup(final String id, final boolean includeConnectorManaged) { - return remoteProcessGroupDAO.getRemoteProcessGroup(id, includeConnectorManaged); - } - @Override public Authorizable getLabel(final String id) { return labelDAO.getLabel(id); @@ -441,13 +415,60 @@ public Resource getResource() { @Override public ComponentAuthorizable getControllerService(final String id) { - return getControllerService(id, false); + final ControllerServiceNode controllerService = controllerServiceDAO.getControllerService(id); + return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); } @Override - public ComponentAuthorizable getControllerService(final String id, final boolean includeConnectorManaged) { - final ControllerServiceNode controllerService = controllerServiceDAO.getControllerService(id, includeConnectorManaged); - return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); + public ConnectorManagedAuthorizableLookup forConnectorManagedFlow() { + return connectorManagedAuthorizableLookup; + } + + /** + * Inner implementation of {@link ConnectorManagedAuthorizableLookup} that resolves components through the + * {@link ConnectorManagedComponentLookup} facade and wraps them in the same authorizable types as the surrounding + * {@link StandardAuthorizableLookup}. + */ + private final class ConnectorManagedAuthorizableLookupImpl implements ConnectorManagedAuthorizableLookup { + + @Override + public ComponentAuthorizable getProcessor(final String id) { + final ProcessorNode processorNode = connectorManagedComponentLookup.getProcessor(id); + return new ProcessorComponentAuthorizable(processorNode, controllerFacade.getExtensionManager()); + } + + @Override + public Authorizable getInputPort(final String id) { + return connectorManagedComponentLookup.getInputPort(id); + } + + @Override + public Authorizable getOutputPort(final String id) { + return connectorManagedComponentLookup.getOutputPort(id); + } + + @Override + public ConnectionAuthorizable getConnection(final String id) { + final Connection connection = connectorManagedComponentLookup.getConnection(id); + return new StandardConnectionAuthorizable(connection); + } + + @Override + public ProcessGroupAuthorizable getProcessGroup(final String id) { + final ProcessGroup processGroup = connectorManagedComponentLookup.getProcessGroup(id); + return new StandardProcessGroupAuthorizable(processGroup, controllerFacade.getExtensionManager()); + } + + @Override + public Authorizable getRemoteProcessGroup(final String id) { + return connectorManagedComponentLookup.getRemoteProcessGroup(id); + } + + @Override + public ComponentAuthorizable getControllerService(final String id) { + final ControllerServiceNode controllerService = connectorManagedComponentLookup.getControllerService(id); + return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); + } } @Override @@ -1388,6 +1409,11 @@ public void setProcessorDAO(ProcessorDAO processorDAO) { this.processorDAO = processorDAO; } + @Autowired + public void setConnectorManagedComponentLookup(final ConnectorManagedComponentLookup connectorManagedComponentLookup) { + this.connectorManagedComponentLookup = connectorManagedComponentLookup; + } + @Autowired public void setProcessGroupDAO(ProcessGroupDAO processGroupDAO) { this.processGroupDAO = processGroupDAO; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index 55aa655139e8..efde7d303049 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -420,6 +420,7 @@ import org.apache.nifi.web.dao.ComponentStateDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ConnectorDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.ControllerServiceDAO; import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; @@ -510,6 +511,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private PortDAO inputPortDAO; private PortDAO outputPortDAO; private ConnectionDAO connectionDAO; + private ConnectorManagedComponentLookup connectorManagedComponentLookup; private ControllerServiceDAO controllerServiceDAO; private ReportingTaskDAO reportingTaskDAO; private ConnectorDAO connectorDAO; @@ -2235,7 +2237,7 @@ public DropRequestDTO deleteFlowFileDropRequest(final String connectionId, final @Override public ListingRequestDTO deleteFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.deleteFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -2646,7 +2648,7 @@ public DropRequestDTO createFlowFileDropRequest(final String connectionId, final @Override public ListingRequestDTO createFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.createFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -4562,7 +4564,7 @@ public DropRequestDTO getFlowFileDropRequest(final String connectionId, final St @Override public ListingRequestDTO getFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.getFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -4864,7 +4866,7 @@ private ProcessGroup resolveOwningProcessGroupForBulletin(final Bulletin bulleti } try { - return processGroupDAO.getProcessGroup(groupId, true); + return connectorManagedComponentLookup.getProcessGroup(groupId); } catch (final ResourceNotFoundException e) { // Owning group was removed; fall back to global authorizable lookup. return null; @@ -8143,6 +8145,11 @@ public void setConnectionDAO(final ConnectionDAO connectionDAO) { this.connectionDAO = connectionDAO; } + @Autowired + public void setConnectorManagedComponentLookup(final ConnectorManagedComponentLookup connectorManagedComponentLookup) { + this.connectorManagedComponentLookup = connectorManagedComponentLookup; + } + @Autowired public void setAuditService(final AuditService auditService) { this.auditService = auditService; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java index 3175a306a2fe..d3d108e1d982 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java @@ -307,7 +307,7 @@ public Response createFlowFileListing( serviceFacade, requestConnectionEntity, lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(id, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(id); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }, @@ -374,7 +374,7 @@ public Response getListingRequest( // authorize access serviceFacade.authorizeAccess(lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); @@ -435,7 +435,7 @@ public Response deleteListingRequest( serviceFacade, new ListingEntity(connectionId, listingRequestId), lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }, @@ -522,7 +522,7 @@ public Response createDropRequest( serviceFacade, requestConnectionEntity, lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(id, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(id); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }, @@ -589,7 +589,7 @@ public Response getDropRequest( // authorize access serviceFacade.authorizeAccess(lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }); @@ -650,7 +650,7 @@ public Response removeDropRequest( serviceFacade, new DropEntity(connectionId, dropRequestId), lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }, diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java index c42ff3273a01..60033bc8116e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java @@ -885,7 +885,7 @@ public Response removeDropRequest( } private void authorizeHandleDropAllFlowFilesRequest(String processGroupId, AuthorizableLookup lookup) { - final ProcessGroupAuthorizable processGroup = lookup.getProcessGroup(processGroupId, true); + final ProcessGroupAuthorizable processGroup = lookup.forConnectorManagedFlow().getProcessGroup(processGroupId); authorizeProcessGroup(processGroup, authorizer, lookup, RequestAction.READ, false, false, false, false, false); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java index 8785d670a6c8..98caa34453ff 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java @@ -43,15 +43,6 @@ public interface ConnectionDAO { */ Connection getConnection(String id); - /** - * Gets the specified Connection, optionally including Connector-managed ProcessGroups in the search. - * - * @param id The connection id - * @param includeConnectorManaged Whether to search Connector-managed ProcessGroups - * @return The connection - */ - Connection getConnection(String id, boolean includeConnectorManaged); - /** * Gets the specified flow file drop request. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java new file mode 100644 index 000000000000..a016529eed33 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao; + +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.connectable.Funnel; +import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.label.Label; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; + +/** + * Facade exposing DAO-level component lookups that include Process Groups managed by a Connector. + * + *

The standard {@link ProcessorDAO}, {@link ConnectionDAO}, {@link ProcessGroupDAO}, etc. intentionally hide + * components that live inside a Connector's managed Process Group hierarchy unless the owning Connector is in + * Troubleshooting mode. The handful of REST endpoints that legitimately need to inspect or operate on components + * within a Connector-managed flow (the FlowFile queue endpoints under {@code /connectors/{id}/...} and the + * Connector-aware Process Group flow endpoint, for example) obtain those components through this facade so the + * standard DAO surface remains free of the "include connector managed" flag.

+ */ +public interface ConnectorManagedComponentLookup { + + /** + * Locate a Processor by identifier, including processors within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + ProcessorNode getProcessor(String id); + + /** + * Locate an input Port by identifier, including ports within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Port getInputPort(String id); + + /** + * Locate an output Port by identifier, including ports within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Port getOutputPort(String id); + + /** + * Locate a Connection by identifier, including connections within Connector-managed Process Groups regardless of + * the owning Connector's state. + */ + Connection getConnection(String id); + + /** + * Locate a Process Group by identifier, including Connector-managed Process Groups regardless of the owning + * Connector's state. + */ + ProcessGroup getProcessGroup(String id); + + /** + * Locate a Remote Process Group by identifier, including remote process groups within Connector-managed Process + * Groups regardless of the owning Connector's state. + */ + RemoteProcessGroup getRemoteProcessGroup(String id); + + /** + * Locate a Controller Service by identifier, including services within Connector-managed Process Groups regardless + * of the owning Connector's state. + */ + ControllerServiceNode getControllerService(String id); + + /** + * Locate a Label by identifier, including labels within Connector-managed Process Groups regardless of the owning + * Connector's state. + */ + Label getLabel(String id); + + /** + * Locate a Funnel by identifier, including funnels within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Funnel getFunnel(String id); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java index 18a501245eca..6dc41df450fc 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ControllerServiceDAO.java @@ -61,17 +61,6 @@ public interface ControllerServiceDAO { */ ControllerServiceNode getControllerService(String controllerServiceId); - /** - * Gets the specified controller service, optionally including Connector-managed Process Groups in the search. When - * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components - * within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param controllerServiceId The controller service id - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The controller service - */ - ControllerServiceNode getControllerService(String controllerServiceId, boolean includeConnectorManaged); - /** * Gets all of the controller services for the group with the given ID or all * controller-level services if the group id is null diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java index 68c4b3f1d5e5..858da8df818f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FunnelDAO.java @@ -46,17 +46,6 @@ public interface FunnelDAO { */ Funnel getFunnel(String funnelId); - /** - * Gets the specified funnel, optionally including Connector-managed Process Groups in the search. When - * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within - * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param funnelId The funnel id - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The funnel - */ - Funnel getFunnel(String funnelId, boolean includeConnectorManaged); - /** * Gets all of the funnels in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java index a17cf2d8c7c0..515b0d49a9ab 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/LabelDAO.java @@ -46,17 +46,6 @@ public interface LabelDAO { */ Label getLabel(String labelId); - /** - * Gets the specified label, optionally including Connector-managed Process Groups in the search. When - * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within - * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param labelId The label id - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The label - */ - Label getLabel(String labelId, boolean includeConnectorManaged); - /** * Gets all of the labels in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java index 62db8323c551..ed607db68016 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/PortDAO.java @@ -46,17 +46,6 @@ public interface PortDAO { */ Port getPort(String portId); - /** - * Gets the specified port, optionally including Connector-managed Process Groups in the search. When - * {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components within - * a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param portId The port id - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The port - */ - Port getPort(String portId, boolean includeConnectorManaged); - /** * Gets all of the ports in the specified group. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java index 1dd36a46b88b..df546810a6c9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java @@ -58,15 +58,6 @@ public interface ProcessGroupDAO { */ ProcessGroup getProcessGroup(String groupId); - /** - * Gets the specified process group, optionally including Connector-managed ProcessGroups in the search. - * - * @param groupId The process group id - * @param includeConnectorManaged Whether to search Connector-managed ProcessGroups - * @return The process group - */ - ProcessGroup getProcessGroup(String groupId, boolean includeConnectorManaged); - /** * Gets all of the process groups. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java index 48be11e93a5f..72f7caee872e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java @@ -59,17 +59,6 @@ public interface ProcessorDAO { */ ProcessorNode getProcessor(String id); - /** - * Gets the Processor transfer object for the specified id, optionally including Connector-managed Process Groups - * in the search. When {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access - * to components within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param id Id of the processor to return - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The Processor - */ - ProcessorNode getProcessor(String id, boolean includeConnectorManaged); - /** * Gets all the Processor transfer objects for this controller. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java index 25007b748aee..7446a34f349c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/RemoteProcessGroupDAO.java @@ -52,17 +52,6 @@ public interface RemoteProcessGroupDAO { */ RemoteProcessGroup getRemoteProcessGroup(String remoteProcessGroupId); - /** - * Gets the specified remote process group, optionally including Connector-managed Process Groups in the search. - * When {@code includeConnectorManaged} is {@code true}, the normal restriction that prevents access to components - * within a Connector-managed flow unless the owning Connector is in Troubleshooting mode is skipped. - * - * @param remoteProcessGroupId The remote process group id - * @param includeConnectorManaged whether to search Connector-managed Process Groups - * @return The remote process group - */ - RemoteProcessGroup getRemoteProcessGroup(String remoteProcessGroupId, boolean includeConnectorManaged); - /** * Gets all of the remote process groups. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java index 33cba1c41942..2be03fe0f9d6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java @@ -51,11 +51,6 @@ public Port getPort(final String portId) { return locatePort(portId); } - @Override - public Port getPort(final String portId, final boolean includeConnectorManaged) { - return locatePort(portId, includeConnectorManaged); - } - @Override public void verifyUpdate(PortDTO portDTO) { final Port port = locatePort(portDTO.getId()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java index 0fffe87a0c51..5d1e66dd7984 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java @@ -109,10 +109,11 @@ protected ProcessGroup locateProcessGroup(final FlowController flowController, f * {@link ConnectorState#TROUBLESHOOTING} mode. If the owning Connector is not in Troubleshooting mode, an * {@link IllegalStateException} is thrown which is translated by the REST layer into a 409 Conflict response. * - *

Connector-aware REST endpoints that need to read components within a managed flow regardless of - * the Connector's state must obtain those components through the {@code includeConnectorManaged} overloads on the - * relevant DAO (and {@link org.apache.nifi.authorization.AuthorizableLookup}) so that this verification is skipped - * at the locate call site rather than being bypassed globally for the current thread. + *

Connector-aware REST endpoints that need to read components within a managed flow regardless of the + * Connector's state must obtain those components through the + * {@link org.apache.nifi.web.dao.ConnectorManagedComponentLookup} facade (or, for authorization, through + * {@link org.apache.nifi.authorization.AuthorizableLookup#forConnectorManagedFlow()}) so that this verification is + * skipped at the locate call site rather than being bypassed globally for the current thread. * * @param group the ProcessGroup that owns (or is) the component being accessed * @param componentId the identifier of the component being accessed (used in the error message) diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java index 3b9f80c21f5b..61681598e7bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java @@ -83,7 +83,7 @@ private Connection locateConnection(final String connectionId) { return locateConnection(connectionId, false); } - private Connection locateConnection(final String connectionId, final boolean includeConnectorManaged) { + Connection locateConnection(final String connectionId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Connection connection = rootGroup.findConnection(connectionId); @@ -137,11 +137,6 @@ public Connection getConnection(final String id) { return locateConnection(id); } - @Override - public Connection getConnection(final String id, final boolean includeConnectorManaged) { - return locateConnection(id, includeConnectorManaged); - } - @Override public Set getConnections(final String groupId) { final ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index af3069bd5526..176cee91441f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -173,7 +173,7 @@ public void enterTroubleshooting(final String id) { @Override public void verifyEndTroubleshooting(final String id) { final ConnectorNode connector = getConnector(id); - connector.verifyCanEndTroubleshooting(); + getConnectorRepository().verifyEndTroubleshooting(connector); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java new file mode 100644 index 000000000000..ef271a8f4b3a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao.impl; + +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.connectable.Funnel; +import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.label.Label; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Default {@link ConnectorManagedComponentLookup} implementation. Delegates each lookup to the package-private + * {@code locate*(id, true)} helper on the corresponding {@code Standard*DAO}, which performs the same search the + * standard DAO surface performs but skips the verification that would otherwise reject access to a component inside a + * Connector-managed Process Group whose owning Connector is not in Troubleshooting mode. + */ +@Component +public class StandardConnectorManagedComponentLookup extends ComponentDAO implements ConnectorManagedComponentLookup { + + private FlowController flowController; + private StandardProcessorDAO processorDAO; + private StandardConnectionDAO connectionDAO; + private StandardInputPortDAO inputPortDAO; + private StandardOutputPortDAO outputPortDAO; + private StandardLabelDAO labelDAO; + private StandardFunnelDAO funnelDAO; + private StandardRemoteProcessGroupDAO remoteProcessGroupDAO; + private StandardControllerServiceDAO controllerServiceDAO; + + @Override + public ProcessorNode getProcessor(final String id) { + return processorDAO.locateProcessor(id, true); + } + + @Override + public Port getInputPort(final String id) { + return inputPortDAO.locatePort(id, true); + } + + @Override + public Port getOutputPort(final String id) { + return outputPortDAO.locatePort(id, true); + } + + @Override + public Connection getConnection(final String id) { + return connectionDAO.locateConnection(id, true); + } + + @Override + public ProcessGroup getProcessGroup(final String id) { + return locateProcessGroup(flowController, id, true); + } + + @Override + public RemoteProcessGroup getRemoteProcessGroup(final String id) { + return remoteProcessGroupDAO.locateRemoteProcessGroup(id, true); + } + + @Override + public ControllerServiceNode getControllerService(final String id) { + return controllerServiceDAO.locateControllerService(id, true); + } + + @Override + public Label getLabel(final String id) { + return labelDAO.locateLabel(id, true); + } + + @Override + public Funnel getFunnel(final String id) { + return funnelDAO.locateFunnel(id, true); + } + + @Autowired + public void setFlowController(final FlowController flowController) { + this.flowController = flowController; + } + + @Autowired + public void setProcessorDAO(final StandardProcessorDAO processorDAO) { + this.processorDAO = processorDAO; + } + + @Autowired + public void setConnectionDAO(final StandardConnectionDAO connectionDAO) { + this.connectionDAO = connectionDAO; + } + + @Autowired + public void setInputPortDAO(final StandardInputPortDAO inputPortDAO) { + this.inputPortDAO = inputPortDAO; + } + + @Autowired + public void setOutputPortDAO(final StandardOutputPortDAO outputPortDAO) { + this.outputPortDAO = outputPortDAO; + } + + @Autowired + public void setLabelDAO(final StandardLabelDAO labelDAO) { + this.labelDAO = labelDAO; + } + + @Autowired + public void setFunnelDAO(final StandardFunnelDAO funnelDAO) { + this.funnelDAO = funnelDAO; + } + + @Autowired + public void setRemoteProcessGroupDAO(final StandardRemoteProcessGroupDAO remoteProcessGroupDAO) { + this.remoteProcessGroupDAO = remoteProcessGroupDAO; + } + + @Autowired + public void setControllerServiceDAO(final StandardControllerServiceDAO controllerServiceDAO) { + this.controllerServiceDAO = controllerServiceDAO; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java index b1a996e7ebb1..7875221cec87 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java @@ -74,7 +74,7 @@ private ControllerServiceNode locateControllerService(final String controllerSer return locateControllerService(controllerServiceId, false); } - private ControllerServiceNode locateControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { + ControllerServiceNode locateControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { final ControllerServiceNode controllerService = serviceProvider.getControllerServiceNode(controllerServiceId); if (controllerService == null) { @@ -138,11 +138,6 @@ public ControllerServiceNode getControllerService(final String controllerService return locateControllerService(controllerServiceId); } - @Override - public ControllerServiceNode getControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { - return locateControllerService(controllerServiceId, includeConnectorManaged); - } - @Override public boolean hasControllerService(final String controllerServiceId) { return serviceProvider.getControllerServiceNode(controllerServiceId) != null; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java index dffe25495321..99ea168bf6a6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java @@ -41,7 +41,7 @@ private Funnel locateFunnel(final String funnelId) { return locateFunnel(funnelId, false); } - private Funnel locateFunnel(final String funnelId, final boolean includeConnectorManaged) { + Funnel locateFunnel(final String funnelId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Funnel funnel = rootGroup.findFunnel(funnelId); if (funnel != null) { @@ -113,11 +113,6 @@ public Funnel getFunnel(String funnelId) { return locateFunnel(funnelId); } - @Override - public Funnel getFunnel(final String funnelId, final boolean includeConnectorManaged) { - return locateFunnel(funnelId, includeConnectorManaged); - } - @Override public Set getFunnels(String groupId) { ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java index c879144b6a73..2aa7822033bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java @@ -44,7 +44,7 @@ private Label locateLabel(final String labelId) { return locateLabel(labelId, false); } - private Label locateLabel(final String labelId, final boolean includeConnectorManaged) { + Label locateLabel(final String labelId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Label label = rootGroup.findLabel(labelId); if (label != null) { @@ -122,11 +122,6 @@ public Label getLabel(String labelId) { return locateLabel(labelId); } - @Override - public Label getLabel(final String labelId, final boolean includeConnectorManaged) { - return locateLabel(labelId, includeConnectorManaged); - } - @Override public Set

This class deliberately does not redefine any of the parent's test methods. The verification of restart-while-in- + * Troubleshooting and authoritative-flow restoration that already exists in + * {@link ConnectorTroubleshootingIT#testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRestart()} (and + * the related restart tests) is the clustered coverage the user requested. Adding the override of + * {@link #getInstanceFactory()} below is sufficient to run all of those scenarios on a two-node cluster.

+ */ +public class ClusteredConnectorTroubleshootingIT extends ConnectorTroubleshootingIT { + + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java index 23f26ce184d7..e6cd77dab2da 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java @@ -30,6 +30,7 @@ import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; import org.apache.nifi.web.api.entity.ConnectionEntity; import org.apache.nifi.web.api.entity.ConnectorEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; @@ -138,8 +139,7 @@ public void testEndTroubleshootingBlockedByQueuedData() throws NiFiClientExcepti assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); - getNiFiInstance().stop(); - getNiFiInstance().start(); + restartNiFi(); assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); final OptionalInt queued = getQueuedCount(connectorId, connectionId); @@ -226,8 +226,7 @@ public void testProcessorsInTroubleshootingStillRunningAfterRestart() throws NiF waitForProcessorState(id, ScheduledState.RUNNING); } - getNiFiInstance().stop(); - getNiFiInstance().start(); + restartNiFi(); assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); for (final String id : startedProcessorIds) { @@ -263,8 +262,7 @@ public void testEnterTroubleshootingFromRunningKeepsProcessorsRunning() throws N "Processor " + refreshed.getComponent().getName() + " must remain RUNNING after entering Troubleshooting"); } - getNiFiInstance().stop(); - getNiFiInstance().start(); + restartNiFi(); assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); for (final ProcessorEntity processor : runningBeforeTroubleshooting) { @@ -412,6 +410,20 @@ private void assertConnectorState(final String connectorId, final ConnectorState assertEquals(expected.name(), entity.getComponent().getState()); } + /** + * Stop and restart the NiFi instance, then wait for all nodes to reconnect when running in a clustered + * environment. Subsequent flow-modifying requests (such as {@code endTroubleshooting}) would otherwise be rejected + * with HTTP 409 Conflict while one of the nodes is still in the CONNECTING state after the restart. + */ + private void restartNiFi() { + getNiFiInstance().stop(); + getNiFiInstance().start(); + + if (getNiFiInstance().isClustered()) { + waitForAllNodesConnected(); + } + } + private void assertConflict(final NiFiClientException e) { final Throwable cause = e.getCause(); if (cause instanceof WebApplicationException wae) { @@ -667,8 +679,7 @@ public void testParameterValuesResolvedBeforeAndAfterRestartInTroubleshooting() assertTrue(sensitiveOutputFile.delete() || !sensitiveOutputFile.exists(), "Failed to delete sensitive output file between runs"); assertTrue(assetOutputFile.delete() || !assetOutputFile.exists(), "Failed to delete asset output file between runs"); - getNiFiInstance().stop(); - getNiFiInstance().start(); + restartNiFi(); assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); @@ -751,8 +762,7 @@ public void testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRes assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), "User-added Sleep processor should be present in the managed flow after adding it in Troubleshooting"); - getNiFiInstance().stop(); - getNiFiInstance().start(); + restartNiFi(); assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), @@ -795,6 +805,135 @@ public void testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRes "Running Connector must write the configured asset content to the configured Asset Output File"); } + /** + * Validates that Connector Assets are retained across a NiFi restart while the Connector is in Troubleshooting mode + * and that, on exit, the Processor properties inside the managed flow are restored to reference the assets that the + * Connector's authoritative flow expects. + * + *

The scenario:

+ *
    + *
  1. Create a {@code ParameterContextConnector}, upload Asset A, and configure the Connector to use Asset A.
    2. + *
  2. Apply the update so the active flow's {@code ReplaceWithFile} processor's {@code Filename} property is + * {@code #{asset_param}}, where {@code asset_param} resolves to Asset A's file path.
    3. + *
  3. Enter Troubleshooting.
    4. + *
  4. Upload a second Asset B, and override the managed flow's {@code ReplaceWithFile.Filename} property to a + * literal that does not reference Asset A. This represents a user editing the Processor's asset-bearing + * property while the Connector is open for direct edits.
    5. + *
  5. Restart NiFi.
    6. + *
  6. After restart, assert that the Connector is still in Troubleshooting, both Assets A and B are still listed, + * and the user's override of {@code Filename} survived the restart.
    7. + *
  7. End Troubleshooting.
    8. + *
  8. Assert that the Connector's authoritative configuration is restored: {@code Asset File} still references + * Asset A, and {@code ReplaceWithFile.Filename} is back to {@code #{asset_param}}.
    9. + *
+ */ + @Test + public void testAssetsRetainedAcrossRestartInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final String secretName = "assetRetentionSecret"; + final String sensitiveSecretValue = "asset-retention-secret-value"; + final File sensitiveOutputFile = new File("target/asset-retention-sensitive.txt"); + final File assetOutputFile = new File("target/asset-retention-asset.txt"); + sensitiveOutputFile.delete(); + assetOutputFile.delete(); + + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", secretName + "=" + sensitiveSecretValue)); + + final ConnectorEntity connector = getClientUtil().createConnector("ParameterContextConnector"); + final String connectorId = connector.getId(); + + final File assetA = new File("src/test/resources/sample-assets/helloworld.txt"); + final File assetB = new File("src/test/resources/sample-assets/helloworld2.txt"); + final AssetEntity assetAEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetA.getName(), assetA); + final String assetAId = assetAEntity.getAsset().getId(); + + final ConnectorValueReferenceDTO secretRef = getClientUtil().createSecretValueReference( + paramProvider.getId(), secretName, "PropertiesParameterProvider.Parameters." + secretName); + final ConnectorValueReferenceDTO assetRef = new ConnectorValueReferenceDTO(); + assetRef.setValueType("ASSET_REFERENCE"); + assetRef.setAssetReferences(List.of(new AssetReferenceDTO(assetAId))); + + final Map propertyValues = new HashMap<>(); + propertyValues.put("Sensitive Value", secretRef); + propertyValues.put("Asset File", assetRef); + propertyValues.put("Sensitive Output File", createStringLiteralRef(sensitiveOutputFile.getAbsolutePath())); + propertyValues.put("Asset Output File", createStringLiteralRef(assetOutputFile.getAbsolutePath())); + + getClientUtil().configureConnectorWithReferences(connectorId, "Parameter Context Configuration", propertyValues); + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + final ProcessorEntity replaceWithFileBeforeTroubleshooting = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileBeforeTroubleshooting, "Active flow should contain ReplaceWithFile before Troubleshooting"); + final String authoritativeFilenameValue = replaceWithFileBeforeTroubleshooting.getComponent().getConfig().getProperties().get("Filename"); + assertEquals("#{asset_param}", authoritativeFilenameValue, + "Authoritative ReplaceWithFile.Filename should be parameterized as #{asset_param} before Troubleshooting"); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // Upload a second Asset while in Troubleshooting and confirm it joins the existing Asset. + final AssetEntity assetBEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetB.getName(), assetB); + final String assetBId = assetBEntity.getAsset().getId(); + assertAssetIds(connectorId, assetAId, assetBId); + + // Override the Processor's asset-bearing property to a literal that does not reference the Connector's Asset A. + final ProcessorEntity replaceWithFileInTroubleshooting = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileInTroubleshooting, "ReplaceWithFile should remain accessible while in Troubleshooting"); + final String overriddenFilenameValue = "target/asset-retention-override.txt"; + getClientUtil().updateProcessorProperties(replaceWithFileInTroubleshooting, Map.of("Filename", overriddenFilenameValue)); + + final ProcessorEntity afterPropertyOverride = findProcessorByName(connectorId, "ReplaceWithFile"); + assertEquals(overriddenFilenameValue, afterPropertyOverride.getComponent().getConfig().getProperties().get("Filename"), + "ReplaceWithFile.Filename should reflect the user override applied while in Troubleshooting"); + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + assertAssetIds(connectorId, assetAId, assetBId); + + final ProcessorEntity afterRestart = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(afterRestart, "ReplaceWithFile should be present after restart while in Troubleshooting"); + assertEquals(overriddenFilenameValue, afterRestart.getComponent().getConfig().getProperties().get("Filename"), + "User override of ReplaceWithFile.Filename must survive a restart while in Troubleshooting"); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + // After exit, the Connector's authoritative flow is restored. The Processor's Filename must reference the + // authoritative value (#{asset_param}), not the user override. + final ProcessorEntity replaceWithFileAfterExit = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileAfterExit, "Restored authoritative flow should contain ReplaceWithFile"); + assertEquals(authoritativeFilenameValue, replaceWithFileAfterExit.getComponent().getConfig().getProperties().get("Filename"), + "Authoritative ReplaceWithFile.Filename must be restored on Troubleshooting exit"); + + // Verify the active configuration still references Asset A (the Connector's expected asset). + final ConnectorEntity afterExit = getNifiClient().getConnectorClient().getConnector(connectorId); + final ConnectorConfigurationDTO activeConfig = afterExit.getComponent().getActiveConfiguration(); + final Map activeProperties = activeConfig.getConfigurationStepConfigurations().getFirst() + .getPropertyGroupConfigurations().getFirst().getPropertyValues(); + assertEquals("ASSET_REFERENCE", activeProperties.get("Asset File").getValueType(), + "Active configuration must retain the Asset reference for Asset File"); + assertEquals(assetAId, activeProperties.get("Asset File").getAssetReferences().get(0).getId(), + "Active configuration must continue to reference Asset A (the Connector's expected asset) after Troubleshooting exit"); + } + + private void assertAssetIds(final String connectorId, final String... expectedAssetIds) throws NiFiClientException, IOException { + final List actualAssetIds = new ArrayList<>(); + final AssetsEntity assetsEntity = getNifiClient().getConnectorClient().getAssets(connectorId); + if (assetsEntity != null && assetsEntity.getAssets() != null) { + for (final AssetEntity asset : assetsEntity.getAssets()) { + if (asset.getAsset() != null) { + actualAssetIds.add(asset.getAsset().getId()); + } + } + } + for (final String expected : expectedAssetIds) { + assertTrue(actualAssetIds.contains(expected), + "Expected Connector " + connectorId + " to contain Asset " + expected + " but found: " + actualAssetIds); + } + } + private boolean containsProcessorId(final String connectorId, final String processorId) throws NiFiClientException, IOException { for (final ProcessorEntity entity : findAllProcessors(connectorId)) { if (processorId.equals(entity.getId())) { From 6eb19430361cfd7f3518d6f61142c907789e7dbd Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 29 May 2026 11:30:38 -0400 Subject: [PATCH 9/9] NIFI-15880: Remove unused one-arg VersionedComponentFlowMapper#mapConnector The single-arg overload had no remaining callers after the two-arg form replaced it; it is framework-internal and not part of any public API, so delete it rather than carry it as a deprecated method. --- .../flow/mapping/VersionedComponentFlowMapper.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index 2d12f29fa988..ab1875e292d6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -1095,18 +1095,6 @@ private org.apache.nifi.flow.ScheduledState mapScheduledState(final ScheduledSta : org.apache.nifi.flow.ScheduledState.ENABLED; } - /** - * @deprecated Use {@link #mapConnector(ConnectorNode, ControllerServiceProvider)} instead. This overload silently - * skips serializing the Connector's Managed Process Group when the Connector is in - * {@link org.apache.nifi.components.connector.ConnectorState#TROUBLESHOOTING}, because the Managed - * Process Group cannot be mapped without a ControllerServiceProvider. Callers that need a complete - * VersionedConnector (including troubleshooting state) must supply a ControllerServiceProvider. - */ - @Deprecated(forRemoval = true) - public VersionedConnector mapConnector(final ConnectorNode connectorNode) { - return mapConnector(connectorNode, null); - } - /** * Map the given ConnectorNode to a VersionedConnector. When the Connector is in Troubleshooting state, the Connector's * Managed Process Group is also serialized into the VersionedConnector so that any user modifications made while in