diff --git a/.cursor/rules/code-style.mdc b/.cursor/rules/code-style.mdc index f1d22d363918..7171af0e1392 100644 --- a/.cursor/rules/code-style.mdc +++ b/.cursor/rules/code-style.mdc @@ -80,3 +80,79 @@ final List result = myList.stream() object, hold the object in a final reference and mutate the object's state. The same applies to `final Object[] holder = new Object[1]` for capturing a single reference; use `AtomicReference` instead. +17. Always place a blank line after a closing brace (`}`) that ends a control-flow construct (such as an + `if`, `else`, `for`, `while`, `switch`, or `try` / `catch` / `finally` block) when the next line is a + new statement or another control-flow construct at the same indentation level. This also applies to + the line following the closing brace of a nested block within a method. The goal is to clearly + separate logical sections of code. Exceptions: + - No blank line is required immediately before a closing brace of the enclosing block. + - No blank line is required between `} else {`, `} else if (...) {`, `} catch (...) {`, or + `} finally {` on the same chain. + + Bad: + ```java + for (final Connector connector : connectors) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + return false; + ``` + + Good: + ```java + for (final Connector connector : connectors) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + + return false; + ``` +18. Rule #11 (prefer importing a class rather than using a fully qualified classname inline) is not + optional. Even when a class is referenced only a single time, add an `import` statement rather than + referring to it by fully qualified name inline. The only acceptable use of a fully qualified + classname is when there is an unavoidable naming conflict with another imported class in the same + file. Fully qualified references scattered throughout code make it much harder to read and maintain. +19. Never combine a negative condition such as `if (x != null)`, `if (!collection.isEmpty())`, or + `if (!flag)` with an `else` clause. Negated conditions are harder to read, and pairing them with an + `else` branch forces the reader to mentally invert the predicate for the "happy path". Instead, + rewrite the condition in the positive form (for example, swap the branches so the `if` tests the + positive condition), or use an early `return`/`continue`/`throw` to eliminate the `else` entirely. + + Bad: + ```java + if (persistedManagedGroup != null) { + restore(persistedManagedGroup); + } else { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + } + ``` + + Good (positive predicate): + ```java + if (persistedManagedGroup == null) { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + } else { + restore(persistedManagedGroup); + } + ``` + + Or, preferably, use an early action and drop the `else`: + ```java + if (persistedManagedGroup == null) { + logger.warn("No snapshot was persisted; leaving Managed Process Group unchanged"); + return; + } + + restore(persistedManagedGroup); + ``` diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java index a263482eda28..d8172b7a2bbe 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/AllowableValuesConnector.java @@ -66,6 +66,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java index 1e257c245c6d..e8473e274171 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/CronScheduleConnector.java @@ -64,6 +64,19 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Cron_Schedule_Connector.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow is the flow template with the currently configured CRON expression + // applied as the Trigger Schedule parameter value. + final VersionedExternalFlow flow = getInitialFlow(); + final String triggerSchedule = activeFlowContext.getConfigurationContext().getProperty(SCHEDULE_STEP_NAME, TRIGGER_SCHEDULE_PARAM).getValue(); + if (triggerSchedule != null) { + VersionedFlowUtils.setParameterValue(flow, TRIGGER_SCHEDULE_PARAM, triggerSchedule); + } + + return flow; + } + @Override public List getConfigurationSteps() { return List.of(SCHEDULE_STEP); diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java index 96b9fa63c38e..40a6a5faa690 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java @@ -35,6 +35,11 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java index d49674ffd038..1dad60bc72f0 100644 --- a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java +++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/MissingBundleConnector.java @@ -88,6 +88,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override protected void onStepConfigured(final String stepName, final FlowContext flowContext) { } diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java index bb964a8181ba..f3871fdea1fa 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java @@ -112,6 +112,14 @@ public VersionedExternalFlow getInitialFlow() { return KafkaToS3FlowBuilder.loadInitialFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow for this Connector is the flow that is produced by applying the + // current configuration to the Kafka-to-S3 flow template. When the user exits Troubleshooting mode, + // this flow is reinstated to discard any manual edits made to the managed Process Group. + return buildFlow(activeFlowContext.getConfigurationContext()); + } + @Override public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException { final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext()); diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java index a0d5314a2f96..a6327b2a8baf 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorConfigurationProvider.java @@ -17,7 +17,7 @@ package org.apache.nifi.components.connector; -import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; import java.io.InputStream; import java.util.Optional; @@ -101,6 +101,44 @@ public interface ConnectorConfigurationProvider { */ void verifyCreate(String connectorId); + /** + * Verifies that the provider allows the connector with the given identifier to transition into + * Troubleshooting mode. This is called before the framework transitions a Connector into + * Troubleshooting, giving the provider an opportunity to reject the operation (for example, + * because the provider is currently performing a concurrent operation on the connector or the + * external system does not permit troubleshooting at this time). + * + *

If the provider cannot support the operation, it should throw a runtime exception + * (for example, {@link IllegalStateException} or + * {@link ConnectorConfigurationProviderException}) describing why the transition is not + * allowed. The exception message will be surfaced to the caller.

+ * + *

The default implementation is a no-op, allowing the transition to proceed.

+ * + * @param connectorId the identifier of the connector that is being transitioned into Troubleshooting mode + */ + default void verifyEnterTroubleshooting(final String connectorId) { + } + + /** + * Verifies that the provider allows the connector with the given identifier to transition out of + * Troubleshooting mode. This is called before the framework restores the Connector's authoritative + * flow and transitions it back to its normal lifecycle, giving the provider an opportunity to + * reject the operation (for example, because the external system has placed the connector in a + * state that requires user attention before normal operation can resume). + * + *

If the provider cannot support the operation, it should throw a runtime exception + * (for example, {@link IllegalStateException} or + * {@link ConnectorConfigurationProviderException}) describing why the transition is not + * allowed. The exception message will be surfaced to the caller.

+ * + *

The default implementation is a no-op, allowing the transition to proceed.

+ * + * @param connectorId the identifier of the connector that is being transitioned out of Troubleshooting mode + */ + default void verifyEndTroubleshooting(final String connectorId) { + } + /** * Determines how the connector repository should handle synchronization for the given * connector during flow inheritance (cluster join). The provider examines the external @@ -112,7 +150,7 @@ public interface ConnectorConfigurationProvider { *
    *
  • A {@link ConnectorWorkingConfiguration} with the provider's working config and name * (overriding the potentially stale values from the versioned flow)
    • - *
  • A {@link ScheduledState} override (correcting stale run intent from the versioned flow)
    • + *
  • A {@link VersionedConnectorState} override (correcting stale run intent from the versioned flow)
    • *
* *

This method combines the verify and load operations into a single call to avoid @@ -123,10 +161,10 @@ public interface ConnectorConfigurationProvider { * behavior for Apache NiFi when no provider is configured.

* * @param connectorId the identifier of the connector to check - * @param proposedScheduledState the ScheduledState from the versioned flow + * @param proposedScheduledState the scheduled state from the versioned flow * @return a directive indicating how to handle this connector during sync */ - default ConnectorSyncDirective getSyncDirective(final String connectorId, final ScheduledState proposedScheduledState) { + default ConnectorSyncDirective getSyncDirective(final String connectorId, final VersionedConnectorState proposedScheduledState) { return ConnectorSyncDirective.allow(); } diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java index 3ca70c6ad114..6515971fa222 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/connector/ConnectorSyncDirective.java @@ -17,10 +17,10 @@ package org.apache.nifi.components.connector; -import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnectorState; /** - * Directive returned by {@link ConnectorConfigurationProvider#getSyncDirective(String, ScheduledState)} + * Directive returned by {@link ConnectorConfigurationProvider#getSyncDirective(String, VersionedConnectorState)} * indicating how the connector repository should handle synchronization for a connector during * flow inheritance. */ @@ -32,7 +32,7 @@ public class ConnectorSyncDirective { public enum Action { /** * Proceed with synchronization. The directive may optionally include a - * {@link ScheduledState} override and/or a {@link ConnectorWorkingConfiguration} + * {@link VersionedConnectorState} override and/or a {@link ConnectorWorkingConfiguration} * containing the provider's working config and name. */ ALLOW, @@ -57,10 +57,10 @@ public enum Action { private static final ConnectorSyncDirective REMOVE_DIRECTIVE = new ConnectorSyncDirective(Action.REMOVE, null, null); private final Action action; - private final ScheduledState scheduledStateOverride; + private final VersionedConnectorState scheduledStateOverride; private final ConnectorWorkingConfiguration workingConfiguration; - private ConnectorSyncDirective(final Action action, final ScheduledState scheduledStateOverride, + private ConnectorSyncDirective(final Action action, final VersionedConnectorState scheduledStateOverride, final ConnectorWorkingConfiguration workingConfiguration) { this.action = action; this.scheduledStateOverride = scheduledStateOverride; @@ -69,7 +69,7 @@ private ConnectorSyncDirective(final Action action, final ScheduledState schedul /** * Returns an ALLOW directive with no overrides. The connector repository will use the - * versioned flow's name, working config, and ScheduledState as-is. This is the default + * versioned flow's name, working config, and scheduled state as-is. This is the default * behavior when no {@link ConnectorConfigurationProvider} is configured (Apache NiFi). */ public static ConnectorSyncDirective allow() { @@ -78,7 +78,7 @@ public static ConnectorSyncDirective allow() { /** * Returns an ALLOW directive with the provider's working configuration (name + working - * config steps) and no ScheduledState override. + * config steps) and no scheduled state override. * * @param workingConfiguration the provider's working configuration including name */ @@ -88,14 +88,14 @@ public static ConnectorSyncDirective allow(final ConnectorWorkingConfiguration w /** * Returns an ALLOW directive with the provider's working configuration and a - * ScheduledState override. The override replaces the versioned flow's ScheduledState, + * scheduled state override. The override replaces the versioned flow's scheduled state, * which may be stale due to in-flight DPS tasks. * * @param workingConfiguration the provider's working configuration including name - * @param scheduledStateOverride the ScheduledState to use instead of the versioned flow's value + * @param scheduledStateOverride the scheduled state to use instead of the versioned flow's value */ public static ConnectorSyncDirective allow(final ConnectorWorkingConfiguration workingConfiguration, - final ScheduledState scheduledStateOverride) { + final VersionedConnectorState scheduledStateOverride) { return new ConnectorSyncDirective(Action.ALLOW, scheduledStateOverride, workingConfiguration); } @@ -119,10 +119,10 @@ public Action getAction() { } /** - * Returns the ScheduledState override, or {@code null} if the versioned flow's - * ScheduledState should be used. + * Returns the scheduled state override, or {@code null} if the versioned flow's + * scheduled state should be used. */ - public ScheduledState getScheduledStateOverride() { + public VersionedConnectorState getScheduledStateOverride() { return scheduledStateOverride; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java index a2481698e15c..38bfef8ccc30 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ConnectorEndpointMerger.java @@ -32,6 +32,7 @@ public class ConnectorEndpointMerger extends AbstractSingleEntityEndpoint getConnectorIdentifier() { return Optional.ofNullable(connectorId); } + @Override + public Optional findOwningConnector() { + ProcessGroup group = this; + while (group != null) { + final Optional owningConnectorId = group.getConnectorIdentifier(); + if (owningConnectorId.isPresent()) { + final ConnectorNode connectorNode = flowManager.getConnector(owningConnectorId.get()); + return Optional.ofNullable(connectorNode); + } + + group = group.getParent(); + } + + return Optional.empty(); + } + @Override public void setPosition(final Position position) { this.position.set(position); @@ -2124,7 +2141,7 @@ public ProcessGroup findProcessGroup(final String id) { return this; } - final ProcessGroup group = flowManager.getGroup(id, getConnectorIdentifier().orElse(null)); + final ProcessGroup group = flowManager.getGroup(id); if (group == null) { return null; } @@ -3946,6 +3963,48 @@ public void updateFlow(final VersionedExternalFlow proposedSnapshot, final Strin synchronizeFlow(proposedSnapshot, synchronizationOptions, flowMappingOptions); } + @Override + public void restoreFlowPreservingIdentifiers(final VersionedExternalFlow proposedSnapshot) { + // Use the Instance Identifier captured in the persisted flow as the runtime identifier for every component. This is + // required so that Connection identifiers (and therefore FlowFile queue identifiers) match what was in use before + // the flow was persisted. Without this, queued FlowFiles in the FlowFile Repository cannot be re-associated with + // their Connections upon restore. + final ComponentIdGenerator idGenerator = (proposedId, instanceId, destinationGroupId) -> instanceId; + final VersionedComponentStateLookup stateLookup = VersionedComponentStateLookup.IDENTITY_LOOKUP; + final ComponentScheduler componentScheduler = new DefaultComponentScheduler(controllerServiceProvider, stateLookup); + + final FlowSynchronizationOptions synchronizationOptions = new FlowSynchronizationOptions.Builder() + .componentIdGenerator(idGenerator) + .componentComparisonIdLookup(VersionedComponent::getInstanceIdentifier) + .componentScheduler(componentScheduler) + .ignoreLocalModifications(true) + .updateDescendantVersionedFlows(true) + .updateGroupSettings(true) + .updateRpgUrls(false) + .propertyDecryptor(encryptor::decrypt) + .build(); + + // Sensitive property values in the proposed snapshot were encrypted using the same PropertyEncryptor when the snapshot + // was persisted (for example, when a Connector-managed flow is persisted in Troubleshooting mode). The currently loaded + // flow therefore must also be mapped with an equivalent SensitiveValueEncryptor so the comparison between "current" and + // "proposed" sensitive values operates on matching ciphertext; otherwise every sensitive property appears to differ and + // the decrypted value written back to the live component is the encrypted payload rather than the plaintext (or parameter + // reference) that was originally captured. + final FlowMappingOptions flowMappingOptions = new FlowMappingOptions.Builder() + .mapSensitiveConfiguration(true) + .mapPropertyDescriptors(true) + .stateLookup(stateLookup) + .sensitiveValueEncryptor(encryptor::encrypt) + .componentIdLookup(ComponentIdLookup.VERSIONED_OR_GENERATE) + .mapInstanceIdentifiers(true) + .mapControllerServiceReferencesToVersionedId(true) + .mapFlowRegistryClientId(false) + .mapAssetReferences(false) + .build(); + + synchronizeFlow(proposedSnapshot, synchronizationOptions, flowMappingOptions); + } + private ProcessContext createProcessContext(final ProcessorNode processorNode) { final org.apache.nifi.processor.Processor processor = processorNode.getProcessor(); final Class componentClass = processor == null ? null : processor.getClass(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java index 216e4fd76428..ab1875e292d6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java @@ -73,6 +73,7 @@ import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; @@ -1094,7 +1095,17 @@ private org.apache.nifi.flow.ScheduledState mapScheduledState(final ScheduledSta : org.apache.nifi.flow.ScheduledState.ENABLED; } - public VersionedConnector mapConnector(final ConnectorNode connectorNode) { + /** + * Map the given ConnectorNode to a VersionedConnector. When the Connector is in Troubleshooting state, the Connector's + * Managed Process Group is also serialized into the VersionedConnector so that any user modifications made while in + * Troubleshooting mode survive a restart. The provided ControllerServiceProvider is required for mapping the Managed + * Process Group; if {@code null}, the Managed Process Group is not serialized even if the Connector is in Troubleshooting. + * + * @param connectorNode the connector node to map + * @param controllerServiceProvider the controller service provider used when mapping the Managed Process Group; may be null + * @return the mapped VersionedConnector + */ + public VersionedConnector mapConnector(final ConnectorNode connectorNode, final ControllerServiceProvider controllerServiceProvider) { final VersionedConnector versionedConnector = new VersionedConnector(); versionedConnector.setInstanceIdentifier(connectorNode.getIdentifier()); versionedConnector.setName(connectorNode.getName()); @@ -1108,6 +1119,12 @@ public VersionedConnector mapConnector(final ConnectorNode connectorNode) { final List workingFlowConfiguration = createVersionedConfigurationSteps(connectorNode.getWorkingFlowContext()); versionedConnector.setWorkingFlowConfiguration(workingFlowConfiguration); + if (connectorNode.getCurrentState() == ConnectorState.TROUBLESHOOTING && controllerServiceProvider != null) { + final ProcessGroup managedGroup = connectorNode.getActiveFlowContext().getManagedProcessGroup(); + final VersionedProcessGroup versionedManagedGroup = mapNonVersionedProcessGroup(managedGroup, controllerServiceProvider); + versionedConnector.setManagedProcessGroup(versionedManagedGroup); + } + return versionedConnector; } @@ -1158,14 +1175,15 @@ private Map mapPropertyValues(final St return versionedProperties; } - private org.apache.nifi.flow.ScheduledState mapConnectorState(final ConnectorState connectorState) { + private VersionedConnectorState mapConnectorState(final ConnectorState connectorState) { if (connectorState == null) { return null; } return switch (connectorState) { - case RUNNING, STARTING -> org.apache.nifi.flow.ScheduledState.RUNNING; - default -> org.apache.nifi.flow.ScheduledState.ENABLED; + case RUNNING, STARTING -> VersionedConnectorState.RUNNING; + case TROUBLESHOOTING -> VersionedConnectorState.TROUBLESHOOTING; + default -> VersionedConnectorState.ENABLED; }; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java index 08819d38857d..1c3c78073b6c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorNode.java @@ -307,4 +307,44 @@ void inheritConfiguration(List activeFlowConfigurati * @return the list of available actions */ List getAvailableActions(); + + /** + * Verifies that the Connector is in a state that allows it to be transitioned into TROUBLESHOOTING. + * @throws IllegalStateException if the Connector cannot enter Troubleshooting mode + */ + void verifyCanEnterTroubleshooting(); + + /** + * Verifies that the Connector is in a state that allows it to be transitioned out of TROUBLESHOOTING. + * The Connector must be in TROUBLESHOOTING. Additionally, all components within the Connector's Managed Process Group + * must be in a stopped / disabled state. + * @throws IllegalStateException if the Connector cannot exit Troubleshooting mode + */ + void verifyCanEndTroubleshooting(); + + /** + * Transitions the Connector into TROUBLESHOOTING state. This method should only be invoked via the ConnectorRepository. + * @throws IllegalStateException if the Connector cannot enter Troubleshooting mode + */ + void enterTroubleshooting(); + + /** + * Restores the Connector's state to TROUBLESHOOTING without stopping any components within the Managed Process Group + * and without running the pre-conditions enforced by {@link #enterTroubleshooting()}. This is intended to be used + * only by the flow synchronization layer when restoring a Connector that was persisted while in Troubleshooting + * mode so that components inside the Managed Process Group retain their persisted runtime state (for example, + * processors that were running when NiFi shut down stay running after restart). + */ + void restoreTroubleshootingState(); + + /** + * Transitions the Connector out of TROUBLESHOOTING state. The Connector's Managed Process Group will be restored + * to the Connector's authoritative view of the flow as reported by {@link Connector#getActiveFlow}. This method should + * only be invoked via the ConnectorRepository. + * + * @throws IllegalStateException if the Connector cannot exit Troubleshooting mode + * @throws FlowUpdateException if unable to apply the authoritative flow (for example because data is queued in a + * Connection that would be removed by the restore) + */ + void endTroubleshooting() throws FlowUpdateException; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java index d8d06bc79efa..d81dbb0c2b68 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/ConnectorRepository.java @@ -18,6 +18,7 @@ package org.apache.nifi.components.connector; import org.apache.nifi.asset.Asset; +import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.components.connector.secrets.SecretsManager; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.VersionedConfigurationStep; @@ -73,9 +74,9 @@ public interface ConnectorRepository { * should be removed or rejected).

* *

When a {@link ConnectorConfigurationProvider} is configured, the provider's - * {@link ConnectorConfigurationProvider#getSyncDirective(String, org.apache.nifi.flow.ScheduledState)} + * {@link ConnectorConfigurationProvider#getSyncDirective(String, org.apache.nifi.flow.VersionedConnectorState)} * method is called to obtain a {@link ConnectorSyncDirective} that may override the - * working configuration, name, or ScheduledState from the versioned flow.

+ * working configuration, name, or scheduled state from the versioned flow.

* * @param versionedConnector the proposed connector from the versioned flow * @return a {@link ConnectorSyncResult} indicating the outcome @@ -151,6 +152,52 @@ void inheritConfiguration(ConnectorNode connector, Listnot call {@code ProcessGroup#verifyCanUpdate} and is intended for use only when restoring a Connector + * that was persisted while in Troubleshooting mode. The persisted user modifications are re-applied on top of the + * Connector's current Managed Process Group contents. + * + * @param troubleshootingProcessGroup the VersionedProcessGroup representing the Managed Process Group contents + * persisted while the Connector was in Troubleshooting mode + */ + void restoreTroubleshootingFlow(VersionedProcessGroup troubleshootingProcessGroup); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java index 91a0a295aa4f..02bd828e8f35 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java @@ -19,6 +19,7 @@ import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.resource.ComponentAuthorizable; import org.apache.nifi.components.VersionedComponent; +import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.FlowFileActivity; @@ -143,6 +144,23 @@ public interface ProcessGroup extends ComponentAuthorizable, Positionable, Versi */ Optional getConnectorIdentifier(); + /** + * Returns the owning Connector for this Process Group, traversing the Process Group hierarchy until a Process Group + * is found that is associated with a Connector. If no Process Group in the hierarchy is associated with a Connector, + * an empty Optional is returned. This is useful for determining whether a component is managed by a Connector. + * + *

The default implementation returns {@link Optional#empty()}. Implementations that can resolve a + * {@link ConnectorNode} from a connector identifier (typically via a FlowManager) should override this method + * and walk the parent chain using {@link #getConnectorIdentifier()} and {@link #getParent()} to locate the + * owning Connector.

+ * + * @return an Optional containing the owning ConnectorNode, or empty if this Process Group and all of its ancestors are + * not managed by a Connector + */ + default Optional findOwningConnector() { + return Optional.empty(); + } + /** * @return the user-set comments about this ProcessGroup, or * null if no comments have been set @@ -948,6 +966,16 @@ default CompletableFuture stopComponents() { */ void updateFlow(VersionedExternalFlow proposedSnapshot, String componentIdSeed, boolean verifyNotDirty, boolean updateSettings, boolean updateDescendantVersionedFlows); + /** + * Updates the Process Group to match the proposed flow, using the Instance Identifier of each component in the + * proposed flow as the runtime identifier for that component. This is intended for use when restoring a previously + * persisted flow where the original runtime identifiers must be preserved (for example, so that queued FlowFiles + * in the FlowFile Repository can be re-associated with their Connections after a restart). + * + * @param proposedSnapshot the proposed flow whose Instance Identifiers should be used as runtime identifiers + */ + void restoreFlowPreservingIdentifiers(VersionedExternalFlow proposedSnapshot); + /** * Updates the Process Group to match the proposed flow * diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java index 0508821e9d1f..50acc4f42594 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/GhostConnector.java @@ -23,6 +23,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; import java.util.List; import java.util.Map; @@ -66,6 +67,13 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + final VersionedExternalFlow emptyFlow = new VersionedExternalFlow(); + emptyFlow.setFlowContents(new VersionedProcessGroup()); + return emptyFlow; + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java index 002f2a8aa2f8..c8b4a267fbf6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorInitializationContext.java @@ -81,14 +81,27 @@ public AssetManager getAssetManager() { @Override public void updateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow, final BundleCompatibility bundleCompatability) throws FlowUpdateException { - if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) { - throw new IllegalArgumentException("FlowContext is not an instance provided by the framework"); - } + final FrameworkFlowContext frameworkFlowContext = requireFrameworkFlowContext(flowContext); resolveBundles(versionedExternalFlow.getFlowContents(), bundleCompatability); frameworkFlowContext.updateFlow(versionedExternalFlow, assetManager); } + @Override + public void verifyUpdateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow, final BundleCompatibility bundleCompatability) throws FlowUpdateException { + final FrameworkFlowContext frameworkFlowContext = requireFrameworkFlowContext(flowContext); + + resolveBundles(versionedExternalFlow.getFlowContents(), bundleCompatability); + frameworkFlowContext.verifyUpdateFlow(versionedExternalFlow); + } + + private FrameworkFlowContext requireFrameworkFlowContext(final FlowContext flowContext) { + if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) { + throw new IllegalArgumentException("FlowContext is not an instance provided by the framework"); + } + return frameworkFlowContext; + } + protected void resolveBundles(final VersionedProcessGroup group, final BundleCompatibility bundleCompatability) { if (bundleCompatability == BundleCompatibility.REQUIRE_EXACT_BUNDLE) { return; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index b9659ebb4860..456c3e03a2a1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -35,6 +35,7 @@ import org.apache.nifi.components.validation.ValidationTrigger; import org.apache.nifi.connectable.FlowFileActivity; import org.apache.nifi.connectable.FlowFileTransferCounts; +import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.queue.DropFlowFileStatus; @@ -50,6 +51,7 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flow.VersionedProcessor; import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; @@ -64,6 +66,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -84,6 +87,9 @@ public class StandardConnectorNode implements ConnectorNode { private static final Logger logger = LoggerFactory.getLogger(StandardConnectorNode.class); + private static final Set STOPPED_STATES = + EnumSet.of(org.apache.nifi.controller.ScheduledState.STOPPED, org.apache.nifi.controller.ScheduledState.DISABLED); + private final String identifier; private final FlowManager flowManager; private final ExtensionManager extensionManager; @@ -142,11 +148,19 @@ public String getName() { @Override public void setName(final String name) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot rename " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode before renaming the Connector."); + } + this.name = name; } @Override public void transitionStateForUpdating() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot apply an update to " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode before applying updates."); + } + final ConnectorState initialState = getCurrentState(); if (initialState == ConnectorState.UPDATING || initialState == ConnectorState.PREPARING_FOR_UPDATE) { return; @@ -329,6 +343,11 @@ public void markInvalid(final String subject, final String explanation) { @Override public void setConfiguration(final String stepName, final StepConfiguration configuration) throws FlowUpdateException { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot modify configuration step " + stepName + " for " + this + + " while it is in Troubleshooting mode; exit Troubleshooting mode before modifying Connector configuration."); + } + setConfiguration(stepName, configuration, false); } @@ -444,6 +463,10 @@ private void start(final FlowEngine scheduler, final CompletableFuture sta @Override public Future stop(final FlowEngine scheduler) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot stop " + this + " while it is in Troubleshooting mode; exit Troubleshooting mode to resume normal lifecycle control."); + } + logger.info("Stopping {}", this); final CompletableFuture stopCompleteFuture = new CompletableFuture<>(); @@ -641,6 +664,9 @@ public void verifyCanDelete() { } final ConnectorState currentState = getCurrentState(); + if (currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot delete " + this + " because it is in Troubleshooting mode; exit Troubleshooting before deleting."); + } if (currentState == ConnectorState.STOPPED || currentState == ConnectorState.UPDATE_FAILED || currentState == ConnectorState.UPDATED) { return; } @@ -650,12 +676,169 @@ public void verifyCanDelete() { @Override public void verifyCanStart() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot start " + this + " because it is in Troubleshooting mode."); + } final ValidationState state = performValidation(); if (state.getStatus() != ValidationStatus.VALID) { throw new IllegalStateException("Cannot start " + this + " because it is not valid: " + state.getValidationErrors()); } } + @Override + public void verifyCanEnterTroubleshooting() { + if (isExtensionMissing()) { + throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because it is a Ghost Connector (its underlying extension is missing)."); + } + + final ConnectorState currentState = getCurrentState(); + switch (currentState) { + case TROUBLESHOOTING -> throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because it is already in Troubleshooting mode."); + case STARTING, STOPPING, DRAINING, PURGING, PREPARING_FOR_UPDATE, UPDATING -> + throw new IllegalStateException("Cannot enter Troubleshooting mode for " + this + " because its state is currently " + + currentState + "; it must be in a stable state before entering Troubleshooting."); + default -> { + // STOPPED, RUNNING, UPDATED, UPDATE_FAILED are all acceptable to enter Troubleshooting from + } + } + } + + @Override + public void verifyCanEndTroubleshooting() { + final Optional reason = getReasonCannotEndTroubleshooting(); + if (reason.isPresent()) { + throw new IllegalStateException("Cannot end Troubleshooting mode for " + this + ": " + reason.get()); + } + } + + private Optional getReasonCannotEndTroubleshooting() { + final ConnectorState currentState = getCurrentState(); + if (currentState != ConnectorState.TROUBLESHOOTING) { + return Optional.of("Connector is not in Troubleshooting mode; current state is " + currentState); + } + + // Verify that every component inside the managed flow is stopped/disabled BEFORE doing any other validation. + // The flow-update check below relies on components being stopped/disabled in order to produce meaningful results. + // Returning early here avoids running the expensive flow-revert preflight against a live, running flow. + final Optional componentReason = findReasonComponentsNotStopped(getActiveFlowContext().getManagedProcessGroup()); + if (componentReason.isPresent()) { + return componentReason; + } + + // After confirming all components are stopped or disabled, check if the managed flow can be safely reverted to the + // Connector's authoritative flow. This mirrors exactly what endTroubleshooting() will do so that any problem + // (e.g. a Connection whose contents cannot be preserved or a component that cannot be replaced) is reported + // synchronously rather than surfacing halfway through the state change. + final VersionedExternalFlow authoritativeFlow = resolveAuthoritativeFlow(); + try { + initializationContext.verifyUpdateFlow(activeFlowContext, authoritativeFlow, BundleCompatibility.RESOLVE_BUNDLE); + } catch (final FlowUpdateException e) { + return Optional.of("The Managed Process Group cannot be reverted to the Connector's authoritative flow: " + e.getMessage()); + } + + return Optional.empty(); + } + + private VersionedExternalFlow resolveAuthoritativeFlow() { + final VersionedExternalFlow authoritativeFlow; + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { + authoritativeFlow = getConnector().getActiveFlow(activeFlowContext); + } + + if (authoritativeFlow == null || authoritativeFlow.getFlowContents() == null) { + logger.warn("Connector {} returned a null authoritative flow from getActiveFlow; using an empty flow.", this); + final VersionedExternalFlow empty = new VersionedExternalFlow(); + empty.setFlowContents(new VersionedProcessGroup()); + return empty; + } + + return authoritativeFlow; + } + + private Optional findReasonComponentsNotStopped(final ProcessGroup group) { + for (final ProcessorNode processor : group.getProcessors()) { + final org.apache.nifi.controller.ScheduledState scheduledState = processor.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Processor " + processor.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); + } + } + + for (final Port port : group.getInputPorts()) { + final org.apache.nifi.controller.ScheduledState scheduledState = port.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Input Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); + } + } + + for (final Port port : group.getOutputPorts()) { + final org.apache.nifi.controller.ScheduledState scheduledState = port.getScheduledState(); + if (!STOPPED_STATES.contains(scheduledState)) { + return Optional.of("Output Port " + port.getIdentifier() + " is in state " + scheduledState + "; it must be STOPPED or DISABLED"); + } + } + + for (final RemoteProcessGroup remoteProcessGroup : group.getRemoteProcessGroups()) { + if (remoteProcessGroup.isTransmitting()) { + return Optional.of("Remote Process Group " + remoteProcessGroup.getIdentifier() + " is transmitting; it must be stopped"); + } + } + + for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) { + if (serviceNode.isActive()) { + return Optional.of("Controller Service " + serviceNode.getIdentifier() + " is active; all Controller Services within the managed flow must be disabled"); + } + } + + for (final ProcessGroup childGroup : group.getProcessGroups()) { + final Optional childReason = findReasonComponentsNotStopped(childGroup); + if (childReason.isPresent()) { + return childReason; + } + } + + return Optional.empty(); + } + + @Override + public void enterTroubleshooting() { + verifyCanEnterTroubleshooting(); + logger.info("Transitioning {} into TROUBLESHOOTING state", this); + + // Deliberately do NOT stop or otherwise mutate the managed flow here. The explicit contract of Troubleshooting + // mode (NIP-28) is that a user can "break glass" on a live, running Connector to inspect or stabilize a + // production issue without first having to shut the flow down. The components that were running before entering + // Troubleshooting remain running; the user may stop individual components as needed in order to edit them, and + // they must all be stopped/disabled before the Connector can leave Troubleshooting mode (enforced in + // verifyCanEndTroubleshooting). + stateTransition.setDesiredState(ConnectorState.TROUBLESHOOTING); + stateTransition.setCurrentState(ConnectorState.TROUBLESHOOTING); + } + + @Override + public void restoreTroubleshootingState() { + logger.info("Restoring {} to TROUBLESHOOTING state from persisted flow", this); + stateTransition.setDesiredState(ConnectorState.TROUBLESHOOTING); + stateTransition.setCurrentState(ConnectorState.TROUBLESHOOTING); + } + + @Override + public void endTroubleshooting() throws FlowUpdateException { + verifyCanEndTroubleshooting(); + logger.info("Exiting TROUBLESHOOTING state for {} by restoring Connector's authoritative flow", this); + + final VersionedExternalFlow flowToApply = resolveAuthoritativeFlow(); + + // Route the update through the ConnectorInitializationContext so that bundle coordinates referenced by the + // authoritative flow are resolved against the currently-available bundles. This mirrors how the initial flow + // is applied in initializeConnector and avoids failing validation when the Connector hard-codes a bundle + // version that differs from the currently-installed NAR (which is common in test Connectors). + initializationContext.updateFlow(activeFlowContext, flowToApply, BundleCompatibility.RESOLVE_BUNDLE); + + stateTransition.setDesiredState(ConnectorState.STOPPED); + stateTransition.setCurrentState(ConnectorState.STOPPED); + logger.info("Successfully exited TROUBLESHOOTING state for {}; Connector is now STOPPED", this); + } + @Override public Connector getConnector() { return connectorDetails.getConnector(); @@ -914,6 +1097,11 @@ public boolean isValidationPaused() { @Override public List verifyConfigurationStep(final String stepName, final StepConfiguration configurationOverrides) { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot verify configuration step " + stepName + " for " + this + + " while it is in Troubleshooting mode; exit Troubleshooting mode before running configuration verification."); + } + logger.debug("Verifying configuration step {} for {}", stepName, this); final List results = new ArrayList<>(); try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { @@ -1268,6 +1456,11 @@ public FrameworkFlowContext getWorkingFlowContext() { @Override public void discardWorkingConfiguration() { + if (getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot discard the working configuration for " + this + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before discarding configuration changes."); + } + recreateWorkingFlowContext(); logger.debug("Discarded working configuration for {}", this); } @@ -1278,16 +1471,19 @@ public List getAvailableActions() { final ConnectorState currentState = getCurrentState(); final boolean dataQueued = activeFlowContext.getManagedProcessGroup().isDataQueued(); final boolean stopped = isStopped(); + final boolean troubleshooting = currentState == ConnectorState.TROUBLESHOOTING; - actions.add(createStartAction(stopped)); + actions.add(createStartAction(stopped && !troubleshooting, troubleshooting)); actions.add(createStopAction(currentState)); - actions.add(createConfigureAction()); - actions.add(createDiscardWorkingConfigAction()); - actions.add(createPurgeFlowFilesAction(stopped, dataQueued)); - actions.add(createDrainFlowFilesAction(stopped, dataQueued)); + actions.add(createConfigureAction(troubleshooting)); + actions.add(createDiscardWorkingConfigAction(troubleshooting)); + actions.add(createPurgeFlowFilesAction(stopped && !troubleshooting, dataQueued)); + actions.add(createDrainFlowFilesAction(stopped && !troubleshooting, dataQueued)); actions.add(createCancelDrainFlowFilesAction(currentState == ConnectorState.DRAINING)); - actions.add(createApplyUpdatesAction(currentState)); - actions.add(createDeleteAction(stopped, dataQueued)); + actions.add(createApplyUpdatesAction(currentState, troubleshooting)); + actions.add(createDeleteAction(stopped && !troubleshooting, dataQueued)); + actions.add(createEnterTroubleshootingAction(currentState)); + actions.add(createEndTroubleshootingAction()); return actions; } @@ -1304,11 +1500,14 @@ private boolean isStopped() { return false; } - private ConnectorAction createStartAction(final boolean stopped) { + private ConnectorAction createStartAction(final boolean stopped, final boolean troubleshooting) { final boolean allowed; final String reason; - if (!stopped) { + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (!stopped) { allowed = false; reason = "Connector is not stopped"; } else { @@ -1329,25 +1528,67 @@ private ConnectorAction createStartAction(final boolean stopped) { private ConnectorAction createStopAction(final ConnectorState currentState) { final boolean allowed; - if (currentState == ConnectorState.RUNNING || currentState == ConnectorState.STARTING) { + final String reason; + if (currentState == ConnectorState.TROUBLESHOOTING) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (currentState == ConnectorState.RUNNING || currentState == ConnectorState.STARTING) { allowed = true; + reason = null; } else if (currentState == ConnectorState.UPDATED || currentState == ConnectorState.UPDATE_FAILED) { allowed = hasActiveThread(activeFlowContext.getManagedProcessGroup()); + reason = allowed ? null : "Connector is not running"; } else { allowed = false; + reason = "Connector is not running"; } - final String reason = allowed ? null : "Connector is not running"; return new StandardConnectorAction("STOP", "Stop the connector", allowed, reason); } - private ConnectorAction createConfigureAction() { + private ConnectorAction createConfigureAction(final boolean troubleshooting) { + if (troubleshooting) { + return new StandardConnectorAction("CONFIGURE", "Configure the connector", false, "Connector is in Troubleshooting mode"); + } return new StandardConnectorAction("CONFIGURE", "Configure the connector", true, null); } - private ConnectorAction createDiscardWorkingConfigAction() { - final boolean allowed = hasWorkingConfigurationChanges(); - final String reason = allowed ? null : "No pending changes to discard"; + private ConnectorAction createEnterTroubleshootingAction(final ConnectorState currentState) { + if (isExtensionMissing()) { + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector's extension is missing"); + } + switch (currentState) { + case TROUBLESHOOTING: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector is already in Troubleshooting mode"); + case STARTING, STOPPING, DRAINING, PURGING, PREPARING_FOR_UPDATE, UPDATING: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", false, + "Connector is currently transitioning; current state is " + currentState); + default: + return new StandardConnectorAction("ENTER_TROUBLESHOOTING", "Enter Troubleshooting mode for the connector", true, null); + } + } + + private ConnectorAction createEndTroubleshootingAction() { + final Optional reason = getReasonCannotEndTroubleshooting(); + return new StandardConnectorAction("END_TROUBLESHOOTING", "Exit Troubleshooting mode for the connector", reason.isEmpty(), reason.orElse(null)); + } + + private ConnectorAction createDiscardWorkingConfigAction(final boolean troubleshooting) { + final boolean allowed; + final String reason; + + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (!hasWorkingConfigurationChanges()) { + allowed = false; + reason = "No pending changes to discard"; + } else { + allowed = true; + reason = null; + } return new StandardConnectorAction("DISCARD_WORKING_CONFIGURATION", "Discard any changes made to the working configuration", allowed, reason); } @@ -1403,11 +1644,14 @@ private ConnectorAction createCancelDrainFlowFilesAction(final boolean draining) "Connector is not currently draining FlowFiles"); } - private ConnectorAction createApplyUpdatesAction(final ConnectorState currentState) { + private ConnectorAction createApplyUpdatesAction(final ConnectorState currentState, final boolean troubleshooting) { final boolean allowed; final String reason; - if (currentState == ConnectorState.PREPARING_FOR_UPDATE || currentState == ConnectorState.UPDATING) { + if (troubleshooting) { + allowed = false; + reason = "Connector is in Troubleshooting mode"; + } else if (currentState == ConnectorState.PREPARING_FOR_UPDATE || currentState == ConnectorState.UPDATING) { allowed = false; reason = "Connector is updating"; } else if (!hasWorkingConfigurationChanges()) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index bdb527fecc81..1385dcb918cf 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -26,10 +26,11 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.engine.FlowEngine; import org.apache.nifi.flow.Bundle; -import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; +import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.util.BundleUtils; @@ -53,6 +54,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; @@ -110,7 +113,7 @@ public void restoreConnector(final ConnectorNode connector) { @Override public ConnectorSyncResult syncConnector(final VersionedConnector versionedConnector) { final String connectorId = versionedConnector.getInstanceIdentifier(); - final ScheduledState proposedScheduledState = versionedConnector.getScheduledState(); + final VersionedConnectorState proposedScheduledState = versionedConnector.getScheduledState(); logger.debug("syncConnector called for connector [{}]", connectorId); // Consult the provider for external state checks and working config @@ -207,18 +210,31 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne final List effectiveActiveConfig = versionedConnector.getActiveFlowConfiguration(); - final ScheduledState effectiveScheduledState = (directive.getScheduledStateOverride() != null) + final VersionedConnectorState effectiveScheduledState = (directive.getScheduledStateOverride() != null) ? directive.getScheduledStateOverride() : proposedScheduledState; // Set name locally (no provider.save()) connector.setName(effectiveName); - // Compare config and sync if changed final boolean wasRunning = currentState == ConnectorState.RUNNING; final boolean configChanged = isNewConnector || isConfigurationUpdated(connector, effectiveActiveConfig, effectiveWorkingConfig); - - if (configChanged) { + final boolean restoringTroubleshooting = effectiveScheduledState == VersionedConnectorState.TROUBLESHOOTING; + + // Configuration must be inherited even when the effective state is TROUBLESHOOTING. The Connector's managed + // Parameter Context is intentionally not registered with the global ParameterContextManager (see + // StandardFlowManager#createConnector) and therefore is not persisted in flow.json. The framework can only + // re-populate that Parameter Context by letting the Connector's applyUpdate run with the persisted active + // configuration, which is exactly what inheritConfiguration does. + // + // For a TROUBLESHOOTING restoration, the Connector's authoritative active flow that inheritConfiguration + // produces is then immediately overlaid by restoreTroubleshootingFlow with the user's persisted Managed + // Process Group snapshot. The structural overlay preserves the in-memory Parameter Context binding, so the + // Connector-supplied parameter values remain available, while the user's flow modifications are the ones + // running. The transient Connector-supplied flow shape is invisible outside this synchronization cycle: flow + // synchronization completes before the FlowFile Repository attaches FlowFiles to queues, so no FlowFile + // movement can observe it. + if (configChanged || restoringTroubleshooting) { logger.info("{} configuration needs synchronization", connector); try { inheritConfiguration(connector, effectiveActiveConfig, effectiveWorkingConfig, versionedConnector.getBundle()); @@ -237,6 +253,34 @@ public ConnectorSyncResult syncConnector(final VersionedConnector versionedConne logger.debug("{} configuration is up to date, no update necessary", connector); } + if (restoringTroubleshooting) { + final VersionedProcessGroup persistedManagedGroup = versionedConnector.getManagedProcessGroup(); + if (persistedManagedGroup == null) { + logger.warn("{} effective scheduled state is TROUBLESHOOTING but no Managed Process Group snapshot was persisted; leaving Managed Process Group unchanged", connector); + } else { + logger.info("{} was persisted in Troubleshooting mode; restoring Managed Process Group from persisted snapshot", connector); + try { + connector.getActiveFlowContext().restoreTroubleshootingFlow(persistedManagedGroup); + } catch (final Exception e) { + logger.error("{} failed to restore Managed Process Group from Troubleshooting snapshot", connector, e); + connector.markInvalid("Flow Synchronization Failure", + "Failed to restore Managed Process Group from Troubleshooting snapshot: " + e.getMessage()); + return ConnectorSyncResult.failed(connector); + } + } + + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + // Use restoreTroubleshootingState rather than enterTroubleshooting because the former does not stop + // running components within the Managed Process Group. Components inside the Managed Process Group that + // were persisted as RUNNING have just been restored to the RUNNING state by restoreTroubleshootingFlow above; + // calling enterTroubleshooting here would immediately stop them, violating the contract that persisted + // user modifications (including running processors) should survive a NiFi restart while in Troubleshooting. + connector.restoreTroubleshootingState(); + } + + return ConnectorSyncResult.syncedConfigUnchanged(connector, effectiveScheduledState); + } + return configChanged ? ConnectorSyncResult.synced(connector, effectiveScheduledState) : ConnectorSyncResult.syncedConfigUnchanged(connector, effectiveScheduledState); @@ -429,9 +473,9 @@ private void stopConnectorAndAwait(final ConnectorNode connector) { logger.info("Stopping connector [{}] (current state: {}) and awaiting completion", connectorId, currentState); try { final Future stopFuture = stopConnector(connector); - stopFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + stopFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] stopped successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to stop", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -446,9 +490,9 @@ private void purgeConnectorAndAwait(final ConnectorNode connector) { try { logger.debug("Purging FlowFiles for connector [{}] before removal", connectorId); final Future purgeFuture = connector.purgeFlowFiles("Flow Synchronization"); - purgeFuture.get(syncTimeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS); + purgeFuture.get(syncTimeout.toMillis(), TimeUnit.MILLISECONDS); logger.debug("Connector [{}] purged successfully", connectorId); - } catch (final java.util.concurrent.TimeoutException e) { + } catch (final TimeoutException e) { logger.warn("Timed out waiting for connector [{}] to purge FlowFiles", connectorId); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -521,6 +565,16 @@ private void restartConnector(final ConnectorNode connector, final CompletableFu public void applyUpdate(final ConnectorNode connector, final ConnectorUpdateContext context) throws FlowUpdateException { logger.debug("Applying update to {}", connector); + // Refuse the update before any provider interaction or asset sync. Once a Connector enters Troubleshooting the + // user owns the managed flow (NIP-28) and the Connector configuration is locked. Deferring this check until + // transitionStateForUpdating() lets a provider that returns shouldApplyUpdate()=false cause the framework to + // silently treat the request as successful while the Connector is still in Troubleshooting, and also lets the + // provider observe and act on an update request that should never reach it in this state. + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot apply an update to " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before applying updates."); + } + if (configurationProvider != null && !configurationProvider.shouldApplyUpdate(connector.getIdentifier())) { logger.info("ConnectorConfigurationProvider indicated framework should not apply update for {}; skipping framework update process", connector); return; @@ -668,6 +722,11 @@ private void collectReferencedAssetIds(final FrameworkFlowContext flowContext, f @Override public void updateConnector(final ConnectorNode connector, final String name) { + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot update the configuration of " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before modifying the Connector configuration."); + } + if (configurationProvider != null) { // Load the latest provider state so that other in-flight working changes are not overwritten by a rename. final Optional externalConfig = configurationProvider.load(connector.getIdentifier()); @@ -680,6 +739,11 @@ public void updateConnector(final ConnectorNode connector, final String name) { @Override public void configureConnector(final ConnectorNode connector, final String stepName, final StepConfiguration configuration) throws FlowUpdateException { + if (connector.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot modify the configuration of " + connector + " while it is in Troubleshooting mode; " + + "exit Troubleshooting mode before modifying the Connector configuration."); + } + if (configurationProvider != null) { final ConnectorWorkingConfiguration mergedConfiguration = buildMergedWorkingConfiguration(connector, stepName, configuration); configurationProvider.save(connector.getIdentifier(), mergedConfiguration); @@ -710,6 +774,11 @@ public void inheritConfiguration(final ConnectorNode connector, final List()); } @@ -85,8 +85,13 @@ public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final try { managedProcessGroup.verifyCanUpdate(versionedExternalFlow, true, false); } catch (final IllegalStateException e) { - throw new FlowUpdateException("Flow is not in a state that allows the requested updated", e); + throw new FlowUpdateException("Flow is not in a state that allows the requested update", e); } + } + + @Override + public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final AssetManager assetManager) throws FlowUpdateException { + verifyUpdateFlow(versionedExternalFlow); final ParameterContext managedGroupParameterContext = managedProcessGroup.getParameterContext(); updateParameterContextNames(versionedExternalFlow.getFlowContents(), managedGroupParameterContext.getName()); @@ -104,6 +109,23 @@ public void updateFlow(final VersionedExternalFlow versionedExternalFlow, final parameterContext = parameterContextFacadeFactory.create(managedProcessGroup); } + @Override + public void restoreTroubleshootingFlow(final VersionedProcessGroup troubleshootingProcessGroup) { + final VersionedExternalFlow externalFlow = new VersionedExternalFlow(); + externalFlow.setFlowContents(troubleshootingProcessGroup); + externalFlow.setParameterContexts(Map.of()); + + final ParameterContext managedGroupParameterContext = managedProcessGroup.getParameterContext(); + if (managedGroupParameterContext != null) { + updateParameterContextNames(troubleshootingProcessGroup, managedGroupParameterContext.getName()); + } + + managedProcessGroup.restoreFlowPreservingIdentifiers(externalFlow); + + rootGroup = groupFacadeFactory.create(managedProcessGroup, connectorLog); + parameterContext = parameterContextFacadeFactory.create(managedProcessGroup); + } + private void updateParameterContextNames(final VersionedProcessGroup group, final String parameterContextName) { group.setParameterContextName(parameterContextName); if (group.getProcessGroups() != null) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 259aed264641..0804b1d56626 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -1066,6 +1066,54 @@ public Connection findConnectionIncludingConnectorManaged(final String connectio return null; } + /** + * Finds an input Port by ID, searching both the root process group hierarchy and all connector-managed process + * groups. Returns null when the Port cannot be located. + */ + public Port findInputPortIncludingConnectorManaged(final String portId) { + final Port port = flowManager.getRootGroup().findInputPort(portId); + if (port != null) { + return port; + } + + for (final ConnectorNode connector : connectorRepository.getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + final Port managedPort = flowContext.getManagedProcessGroup().findInputPort(portId); + if (managedPort != null) { + return managedPort; + } + } + + return null; + } + + /** + * Finds an output Port by ID, searching both the root process group hierarchy and all connector-managed process + * groups. Returns null when the Port cannot be located. + */ + public Port findOutputPortIncludingConnectorManaged(final String portId) { + final Port port = flowManager.getRootGroup().findOutputPort(portId); + if (port != null) { + return port; + } + + for (final ConnectorNode connector : connectorRepository.getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + final Port managedPort = flowContext.getManagedProcessGroup().findOutputPort(portId); + if (managedPort != null) { + return managedPort; + } + } + + return null; + } + /** * Finds a RemoteGroupPort by ID, searching both the root process group hierarchy * and all connector-managed process groups. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index 7e74a7f2389b..3b84ff727201 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.serialization; import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.FlowAnalysisRuleNode; @@ -30,6 +31,7 @@ import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; @@ -99,9 +101,11 @@ private List mapConnectors() { final List connectors = new ArrayList<>(); for (final ConnectorNode connectorNode : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { - final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode); - if (flowController.isStartAfterInitialization(connectorNode)) { - versionedConnector.setScheduledState(ScheduledState.RUNNING); + final VersionedConnector versionedConnector = flowMapper.mapConnector(connectorNode, flowController.getControllerServiceProvider()); + if (connectorNode.getCurrentState() == ConnectorState.TROUBLESHOOTING) { + versionedConnector.setScheduledState(VersionedConnectorState.TROUBLESHOOTING); + } else if (flowController.isStartAfterInitialization(connectorNode)) { + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); } connectors.add(versionedConnector); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java index ba409ee7d448..d7a250d80c70 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java @@ -1047,10 +1047,11 @@ private void inheritConnectors(final FlowController flowController, final Versio logger.info("Connector [{}] sync result: {}", versionedConnector.getInstanceIdentifier(), result); if (result.getEffectiveScheduledState() != null && result.getConnectorNode() != null) { - if (result.getEffectiveScheduledState() == ScheduledState.RUNNING) { - flowController.startConnector(result.getConnectorNode()); - } else if (result.getEffectiveScheduledState() == ScheduledState.ENABLED) { - connectorRepository.stopConnector(result.getConnectorNode()); + switch (result.getEffectiveScheduledState()) { + case RUNNING -> flowController.startConnector(result.getConnectorNode()); + case ENABLED -> connectorRepository.stopConnector(result.getConnectorNode()); + case TROUBLESHOOTING -> logger.debug("Connector [{}] is in TROUBLESHOOTING state; leaving connector lifecycle alone", result.getConnectorNode().getIdentifier()); + default -> { } } } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java index 75c35154d693..e59f15da0ce2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/BlockingConnector.java @@ -49,6 +49,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java index 404a78b9727f..9ecdf41c6807 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicAllowableValuesConnector.java @@ -79,6 +79,19 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // Build the flow that reflects the currently configured File path. This mirrors the logic in + // applyUpdate so that exiting Troubleshooting mode restores a flow equivalent to what would be + // installed by re-applying the active configuration. + final VersionedExternalFlow externalFlow = VersionedFlowUtils.loadFlowFromResource("flows/choose-color.json"); + final VersionedProcessGroup rootGroup = externalFlow.getFlowContents(); + final VersionedProcessor processor = rootGroup.getProcessors().iterator().next(); + final String filePath = activeFlowContext.getConfigurationContext().getProperty(FILE_STEP, FILE_PATH).getValue(); + processor.setProperties(Map.of("File", filePath == null ? "" : filePath)); + return externalFlow; + } + @Override public List getConfigurationSteps() { return CONFIGURATION_STEPS; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java index bf8148214393..a4e32ad93299 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/DynamicFlowConnector.java @@ -130,6 +130,12 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-duplicate-log-flow.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative flow is the dynamically built flow based on the active configuration. + return getFlow(activeFlowContext); + } + public boolean isInitialized() { return initialized; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java index c21b55dfbbaf..55377dd15380 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/MissingBundleConnector.java @@ -96,6 +96,11 @@ public VersionedExternalFlow getInitialFlow() { return externalFlow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java index bf5f242a939e..0199a5907d9f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/OnPropertyModifiedConnector.java @@ -60,6 +60,18 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/on-property-modified-tracker.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative flow has the Configured Number parameter set to the currently configured value. + final VersionedExternalFlow versionedExternalFlow = getInitialFlow(); + final String number = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, NUMBER_VALUE).getValue(); + if (number != null) { + VersionedFlowUtils.setParameterValue(versionedExternalFlow, PARAMETER_NAME, number); + } + + return versionedExternalFlow; + } + @Override public List getConfigurationSteps() { return List.of(CONFIG_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java index bbc071aca0eb..777cbd9b1997 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/ParameterConnector.java @@ -76,6 +76,13 @@ public VersionedExternalFlow getInitialFlow() { return VersionedFlowUtils.loadFlowFromResource("flows/generate-and-log-with-parameter.json"); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The flow structure does not change with configuration; only parameter values do, which are applied + // to the active parameter context in applyUpdate. The authoritative flow definition matches the initial flow. + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(TEXT_STEP); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java index 6818b5b059cc..ef4f7dc0d680 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/SleepingConnector.java @@ -44,6 +44,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { try { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java index 00e136a66358..95afabd0c534 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java @@ -881,6 +881,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -947,6 +952,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -997,6 +1007,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1068,6 +1083,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1124,6 +1144,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } @@ -1168,6 +1193,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index cba4b12f5741..4396b8a1ec9c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -28,17 +28,19 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.flow.Bundle; -import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConfigurationStep; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedConnectorValueReference; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.util.MockComponentLog; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -67,6 +69,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; @@ -575,6 +578,84 @@ public void testVerifyCreateProviderRejectsThrows() { assertThrows(ConnectorConfigurationProviderException.class, () -> repository.verifyCreate("connector-1")); } + @Test + public void testVerifyEnterTroubleshootingDelegatesToConnectorAndProvider() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.verifyEnterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + verify(provider).verifyEnterTroubleshooting("connector-1"); + } + + @Test + public void testVerifyEnterTroubleshootingWithoutProvider() { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.verifyEnterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + } + + @Test + public void testVerifyEnterTroubleshootingProviderVetoThrows() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + doThrow(new IllegalStateException("External system does not permit troubleshooting at this time")) + .when(provider).verifyEnterTroubleshooting("connector-1"); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> repository.verifyEnterTroubleshooting(connector)); + assertEquals("External system does not permit troubleshooting at this time", thrown.getMessage()); + + verify(connector).verifyCanEnterTroubleshooting(); + } + + @Test + public void testVerifyEnterTroubleshootingConnectorVetoDoesNotCallProvider() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + doThrow(new IllegalStateException("Connector is currently UPDATING")) + .when(connector).verifyCanEnterTroubleshooting(); + repository.restoreConnector(connector); + + assertThrows(IllegalStateException.class, () -> repository.verifyEnterTroubleshooting(connector)); + verify(provider, never()).verifyEnterTroubleshooting(anyString()); + } + + @Test + public void testEnterTroubleshootingInvokesVerifyThenTransition() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + repository.restoreConnector(connector); + + repository.enterTroubleshooting(connector); + + verify(connector).verifyCanEnterTroubleshooting(); + verify(provider).verifyEnterTroubleshooting("connector-1"); + verify(connector).enterTroubleshooting(); + } + @Test public void testVerifyCreateExistingConnectorDoesNotCallProvider() { final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); @@ -929,6 +1010,27 @@ public void testApplyUpdateFailureCallsAbortUpdateButNotMarkInvalid() throws Flo verify(connector, never()).markInvalid(anyString(), anyString()); } + @Test + public void testApplyUpdateInTroubleshootingThrowsBeforeProviderConsultation() { + final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); + final StandardConnectorRepository repository = createRepositoryWithProvider(provider); + + final ConnectorNode connector = mock(ConnectorNode.class); + when(connector.getIdentifier()).thenReturn("connector-1"); + when(connector.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + repository.restoreConnector(connector); + + final IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> repository.applyUpdate(connector, mock(ConnectorUpdateContext.class))); + assertTrue(thrown.getMessage().contains("Troubleshooting"), + "Exception message should reference Troubleshooting: " + thrown.getMessage()); + + // Provider must not be consulted; allowing shouldApplyUpdate to silently veto would let the framework return + // success while the Connector was still in Troubleshooting. + verify(provider, never()).shouldApplyUpdate(anyString()); + verify(connector, never()).transitionStateForUpdating(); + } + // --- syncConnector tests --- @Test @@ -939,7 +1041,7 @@ public void testSyncConnectorStoppedWithConfigChange() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -956,7 +1058,7 @@ public void testSyncConnectorStoppedNoConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -972,7 +1074,7 @@ public void testSyncConnectorRunningWithConfigChange() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -989,7 +1091,7 @@ public void testSyncConnectorDrainingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.DRAINING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1006,7 +1108,7 @@ public void testSyncConnectorPreparingForUpdateIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.PREPARING_FOR_UPDATE); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1022,7 +1124,7 @@ public void testSyncConnectorUpdatingIsRejected() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1037,7 +1139,7 @@ public void testSyncConnectorUpdateFailedRecovery() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.UPDATE_FAILED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("recovery-value"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1054,7 +1156,7 @@ public void testSyncConnectorUpdatedState() throws Exception { final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1072,7 +1174,7 @@ public void testSyncConnectorInheritConfigurationFailureWhenRunning() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1091,7 +1193,7 @@ public void testSyncConnectorInheritConfigurationFailureWhenStopped() throws Exc .when(connector).inheritConfiguration(any(), any(), any()); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of(createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("value1"))))); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1110,7 +1212,7 @@ public void testSyncConnectorProviderRejectsSync() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1130,7 +1232,7 @@ public void testSyncConnectorProviderThrowsException() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1149,7 +1251,7 @@ public void testSyncConnectorStartingWaitsForRunning() throws Exception { .thenReturn(ConnectorState.RUNNING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1164,7 +1266,7 @@ public void testSyncConnectorStartingTimesOut() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STARTING); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1188,7 +1290,7 @@ public void testSyncConnectorProviderReturnsRemoveStopsAndRemoves() throws Excep when(connector.stop(any())).thenReturn(CompletableFuture.completedFuture(null)); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1209,7 +1311,7 @@ public void testSyncConnectorProviderReturnsRemoveStoppedConnector() throws Exce when(connector.getConnector()).thenReturn(mockExtension); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1224,7 +1326,7 @@ public void testSyncConnectorProviderReturnsRemoveForNonExistent() throws Except when(provider.getSyncDirective(eq("connector-1"), any())).thenReturn(ConnectorSyncDirective.remove()); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1242,7 +1344,7 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(flowManager.createConnector(anyString(), eq("connector-1"), any(), anyBoolean(), anyBoolean())).thenReturn(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1255,19 +1357,139 @@ public void testSyncConnectorProviderReturnsRejectCreatesNodeForNewConnector() t public void testSyncConnectorProviderAllowWithScheduledStateOverride() throws Exception { final ConnectorConfigurationProvider provider = mock(ConnectorConfigurationProvider.class); when(provider.getSyncDirective(eq("connector-1"), any())) - .thenReturn(ConnectorSyncDirective.allow(null, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncDirective.allow(null, VersionedConnectorState.ENABLED)); final StandardConnectorRepository repository = createRepositoryWithProvider(provider); final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.RUNNING, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.RUNNING, List.of()); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + assertEquals(VersionedConnectorState.ENABLED, result.getEffectiveScheduledState()); + } + + @Test + public void testSyncConnectorTroubleshootingInheritsConfigurationThenOverlaysSnapshotThenEntersTroubleshooting() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + // The Connector's managed Parameter Context is not registered with the global ParameterContextManager and is + // therefore not persisted in flow.json. When restoring a Connector whose effective scheduled state is + // TROUBLESHOOTING, the framework must run inheritConfiguration so the Connector re-populates its in-memory + // Parameter Context from the persisted active configuration. The Connector-supplied flow shape that produces + // is then overlaid by restoreTroubleshootingFlow with the user's persisted snapshot before the FlowFile Repository + // attaches FlowFiles to queues, so the transient Connector flow shape is invisible to data movement. + final List activeConfig = List.of( + createVersionedStep("step1", Map.of("prop1", createStringLiteralRef("new-value")))); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, activeConfig); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + assertEquals(VersionedConnectorState.TROUBLESHOOTING, result.getEffectiveScheduledState()); + + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + final InOrder inOrder = inOrder(connector, activeFlowContext); + inOrder.verify(connector).inheritConfiguration(eq(activeConfig), eq(activeConfig), any()); + inOrder.verify(activeFlowContext).restoreTroubleshootingFlow(persistedManagedGroup); + inOrder.verify(connector).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingPreservesExistingTroubleshootingState() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); + verify(connector).inheritConfiguration(eq(List.of()), eq(List.of()), any()); + verify(connector.getActiveFlowContext()).restoreTroubleshootingFlow(persistedManagedGroup); + // Connector is already in TROUBLESHOOTING; restoreTroubleshootingState should not be invoked again. + verify(connector, never()).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingWithoutSnapshotLeavesManagedGroupUnchanged() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + repository.restoreConnector(connector); + + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); assertEquals(ConnectorSyncResult.Outcome.SYNCED_CONFIG_UNCHANGED, result.getOutcome()); - assertEquals(ScheduledState.ENABLED, result.getEffectiveScheduledState()); + verify(connector).inheritConfiguration(eq(List.of()), eq(List.of()), any()); + verify(connector.getActiveFlowContext(), never()).restoreTroubleshootingFlow(any()); + verify(connector).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingSnapshotRestoreFailureMarksInvalid() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + final FrameworkFlowContext activeFlowContext = connector.getActiveFlowContext(); + doThrow(new RuntimeException("Snapshot restore failure")).when(activeFlowContext).restoreTroubleshootingFlow(any()); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); + verify(connector).markInvalid(eq("Flow Synchronization Failure"), anyString()); + verify(connector, never()).restoreTroubleshootingState(); + } + + @Test + public void testSyncConnectorTroubleshootingInheritConfigurationFailureSkipsSnapshotRestore() throws Exception { + final StandardConnectorRepository repository = createRepositoryWithProvider(null); + + final ConnectorNode connector = createConnectorNodeWithEmptyWorkingConfig("connector-1", "Test Connector"); + when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); + doThrow(new FlowUpdateException("Inherit failure")) + .when(connector).inheritConfiguration(any(), any(), any()); + repository.restoreConnector(connector); + + final VersionedProcessGroup persistedManagedGroup = new VersionedProcessGroup(); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", + VersionedConnectorState.TROUBLESHOOTING, List.of()); + versioned.setManagedProcessGroup(persistedManagedGroup); + + final ConnectorSyncResult result = repository.syncConnector(versioned); + + assertEquals(ConnectorSyncResult.Outcome.FAILED, result.getOutcome()); + // inheritConfiguration ran first and failed; the managed flow snapshot must not be overlaid and the + // Connector must not transition into TROUBLESHOOTING. + verify(connector.getActiveFlowContext(), never()).restoreTroubleshootingFlow(any()); + verify(connector, never()).restoreTroubleshootingState(); } @Test @@ -1286,7 +1508,7 @@ public void testSyncConnectorProviderAllowWithWorkingConfig() throws Exception { when(connector.getCurrentState()).thenReturn(ConnectorState.STOPPED); repository.restoreConnector(connector); - final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", ScheduledState.ENABLED, List.of()); + final VersionedConnector versioned = createVersionedConnector("connector-1", "Test Connector", VersionedConnectorState.ENABLED, List.of()); final ConnectorSyncResult result = repository.syncConnector(versioned); @@ -1453,7 +1675,7 @@ private VersionedConnectorValueReference createStringLiteralRef(final String val return ref; } - private VersionedConnector createVersionedConnector(final String id, final String name, final ScheduledState scheduledState, + private VersionedConnector createVersionedConnector(final String id, final String name, final VersionedConnectorState scheduledState, final List activeConfig) { final VersionedConnector vc = new VersionedConnector(); vc.setInstanceIdentifier(id); @@ -1568,6 +1790,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java index c2dc1ab49a89..c37168fb1387 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/NopConnector.java @@ -56,6 +56,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext activeContext) throws FlowUpdateException { if (!initialized) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java index 2684627afc3b..91ea1a874713 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizerTest.java @@ -34,6 +34,7 @@ import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorState; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flow.VersionedReportingTask; @@ -306,12 +307,12 @@ void testSyncInheritConnectorDelegatesSyncToConnectorRepository() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.RUNNING); + versionedConnector.setScheduledState(VersionedConnectorState.RUNNING); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.synced(syncedNode, ScheduledState.RUNNING)); + .thenReturn(ConnectorSyncResult.synced(syncedNode, VersionedConnectorState.RUNNING)); setFlowController(connectorRepository); when(versionedDataflow.getConnectors()).thenReturn(List.of(versionedConnector)); @@ -334,12 +335,12 @@ void testSyncInheritConnectorNotStartedWhenEnabled() { versionedConnector.setName("Test Connector"); versionedConnector.setType(connectorType); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorRepository connectorRepository = mock(ConnectorRepository.class); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); setFlowController(connectorRepository); when(versionedDataflow.getConnectors()).thenReturn(List.of(versionedConnector)); @@ -363,7 +364,7 @@ void testSyncOrphanConnectorIsRemoved() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -372,7 +373,7 @@ void testSyncOrphanConnectorIsRemoved() { final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(proposedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); @@ -402,11 +403,11 @@ void testSyncOrphanConnectorNotRemovedWhenInProposedFlow() { versionedConnector.setName("Test Connector"); versionedConnector.setType("org.apache.nifi.connectors.TestConnector"); versionedConnector.setBundle(CORE_BUNDLE); - versionedConnector.setScheduledState(ScheduledState.ENABLED); + versionedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(versionedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); @@ -432,7 +433,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { proposedConnector.setName("Proposed Connector"); proposedConnector.setType("org.apache.nifi.connectors.TestConnector"); proposedConnector.setBundle(CORE_BUNDLE); - proposedConnector.setScheduledState(ScheduledState.ENABLED); + proposedConnector.setScheduledState(VersionedConnectorState.ENABLED); final ConnectorNode orphanConnector = mock(ConnectorNode.class); org.mockito.Mockito.lenient().when(orphanConnector.getIdentifier()).thenReturn("orphan-connector-id"); @@ -443,7 +444,7 @@ void testSyncOrphanRemovalFailureMarksInvalid() { final ConnectorNode syncedNode = mock(ConnectorNode.class); when(connectorRepository.syncConnector(proposedConnector)) - .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, ScheduledState.ENABLED)); + .thenReturn(ConnectorSyncResult.syncedConfigUnchanged(syncedNode, VersionedConnectorState.ENABLED)); when(connectorRepository.stopConnector(syncedNode)) .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java index 6beb29d409eb..f221ba28101f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java @@ -723,6 +723,10 @@ public ComponentAdditions addVersionedComponents(VersionedComponentAdditions add public void updateFlow(VersionedExternalFlow proposedFlow, String componentIdSeed, boolean verifyNotDirty, boolean updateSettings, boolean updateDescendantVersionedFlows) { } + @Override + public void restoreFlowPreservingIdentifiers(final VersionedExternalFlow proposedFlow) { + } + @Override public void synchronizeFlow(final VersionedExternalFlow proposedSnapshot, final FlowSynchronizationOptions synchronizationOptions, final FlowMappingOptions flowMappingOptions) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java index 2a2b21cde1b3..a81ec9043bc5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/DummyConnector.java @@ -51,6 +51,11 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + @Override public void start(final FlowContext flowContext) throws FlowUpdateException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java index c1aaf3ad944e..dc41df885604 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java @@ -136,15 +136,6 @@ public interface AuthorizableLookup { */ ConnectionAuthorizable getConnection(String id); - /** - * Get the authorizable Connection, optionally including Connector-managed ProcessGroups in the search. - * - * @param id connection id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - ConnectionAuthorizable getConnection(String id, boolean includeConnectorManaged); - /** * Get the authorizable root ProcessGroup. * @@ -160,15 +151,6 @@ public interface AuthorizableLookup { */ ProcessGroupAuthorizable getProcessGroup(String id); - /** - * Get the authorizable ProcessGroup, optionally including Connector-managed ProcessGroups in the search. - * - * @param id process group id - * @param includeConnectorManaged whether to search Connector-managed ProcessGroups - * @return authorizable - */ - ProcessGroupAuthorizable getProcessGroup(String id, boolean includeConnectorManaged); - /** * Get the authorizable RemoteProcessGroup. * @@ -210,6 +192,17 @@ public interface AuthorizableLookup { */ ComponentAuthorizable getControllerService(String id); + /** + * Returns a {@link ConnectorManagedAuthorizableLookup} view of this lookup that resolves components within + * Connector-managed Process Groups regardless of the owning Connector's state. Only Connector-scoped REST + * endpoints should use this view; standard endpoints continue to use the methods directly on this interface, + * which intentionally hide components inside Connector-managed flows unless the owning Connector is in + * Troubleshooting mode. + * + * @return the Connector-managed authorizable lookup facade + */ + ConnectorManagedAuthorizableLookup forConnectorManagedFlow(); + /** * Get the authorizable referencing component. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java new file mode 100644 index 000000000000..0add406c7db3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectorManagedAuthorizableLookup.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.authorization; + +import org.apache.nifi.authorization.resource.Authorizable; + +/** + * Facade exposing authorizable lookups that include components within Connector-managed Process Groups. + * + *

The standard {@link AuthorizableLookup} intentionally hides components inside Connector-managed Process Group + * hierarchies unless the owning Connector is in Troubleshooting mode. The few Connector-scoped REST endpoints that + * must authorize access to components inside a managed flow regardless of the Connector's state obtain those + * authorizables through this facade, so the standard {@link AuthorizableLookup} surface stays free of the + * "include connector managed" flag.

+ * + *

Obtain an instance via {@link AuthorizableLookup#forConnectorManagedFlow()}.

+ */ +public interface ConnectorManagedAuthorizableLookup { + + /** + * Get the authorizable Processor located within a Connector-managed Process Group. + */ + ComponentAuthorizable getProcessor(String id); + + /** + * Get the authorizable input Port located within a Connector-managed Process Group. + */ + Authorizable getInputPort(String id); + + /** + * Get the authorizable output Port located within a Connector-managed Process Group. + */ + Authorizable getOutputPort(String id); + + /** + * Get the authorizable Connection located within a Connector-managed Process Group. + */ + ConnectionAuthorizable getConnection(String id); + + /** + * Get the authorizable Process Group, including Connector-managed Process Groups. + */ + ProcessGroupAuthorizable getProcessGroup(String id); + + /** + * Get the authorizable Remote Process Group located within a Connector-managed Process Group. + */ + Authorizable getRemoteProcessGroup(String id); + + /** + * Get the authorizable Controller Service located within a Connector-managed Process Group. + */ + ComponentAuthorizable getControllerService(String id); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java index be703d121f43..a6becc883b33 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java @@ -57,6 +57,7 @@ import org.apache.nifi.web.dao.AccessPolicyDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ConnectorDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.ControllerServiceDAO; import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; @@ -247,6 +248,9 @@ public Resource getResource() { private AccessPolicyDAO accessPolicyDAO; private ParameterContextDAO parameterContextDAO; + private ConnectorManagedComponentLookup connectorManagedComponentLookup; + private final ConnectorManagedAuthorizableLookup connectorManagedAuthorizableLookup = new ConnectorManagedAuthorizableLookupImpl(); + @Override public Authorizable getController() { return controllerFacade; @@ -346,12 +350,7 @@ public ParameterContext getParameterContext(final String id) { @Override public ConnectionAuthorizable getConnection(final String id) { - return getConnection(id, false); - } - - @Override - public ConnectionAuthorizable getConnection(final String id, final boolean includeConnectorManaged) { - final Connection connection = connectionDAO.getConnection(id, includeConnectorManaged); + final Connection connection = connectionDAO.getConnection(id); return new StandardConnectionAuthorizable(connection); } @@ -362,12 +361,7 @@ public ProcessGroupAuthorizable getRootProcessGroup() { @Override public ProcessGroupAuthorizable getProcessGroup(final String id) { - return getProcessGroup(id, false); - } - - @Override - public ProcessGroupAuthorizable getProcessGroup(final String id, final boolean includeConnectorManaged) { - final ProcessGroup processGroup = processGroupDAO.getProcessGroup(id, includeConnectorManaged); + final ProcessGroup processGroup = processGroupDAO.getProcessGroup(id); return new StandardProcessGroupAuthorizable(processGroup, controllerFacade.getExtensionManager()); } @@ -425,6 +419,58 @@ public ComponentAuthorizable getControllerService(final String id) { return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); } + @Override + public ConnectorManagedAuthorizableLookup forConnectorManagedFlow() { + return connectorManagedAuthorizableLookup; + } + + /** + * Inner implementation of {@link ConnectorManagedAuthorizableLookup} that resolves components through the + * {@link ConnectorManagedComponentLookup} facade and wraps them in the same authorizable types as the surrounding + * {@link StandardAuthorizableLookup}. + */ + private final class ConnectorManagedAuthorizableLookupImpl implements ConnectorManagedAuthorizableLookup { + + @Override + public ComponentAuthorizable getProcessor(final String id) { + final ProcessorNode processorNode = connectorManagedComponentLookup.getProcessor(id); + return new ProcessorComponentAuthorizable(processorNode, controllerFacade.getExtensionManager()); + } + + @Override + public Authorizable getInputPort(final String id) { + return connectorManagedComponentLookup.getInputPort(id); + } + + @Override + public Authorizable getOutputPort(final String id) { + return connectorManagedComponentLookup.getOutputPort(id); + } + + @Override + public ConnectionAuthorizable getConnection(final String id) { + final Connection connection = connectorManagedComponentLookup.getConnection(id); + return new StandardConnectionAuthorizable(connection); + } + + @Override + public ProcessGroupAuthorizable getProcessGroup(final String id) { + final ProcessGroup processGroup = connectorManagedComponentLookup.getProcessGroup(id); + return new StandardProcessGroupAuthorizable(processGroup, controllerFacade.getExtensionManager()); + } + + @Override + public Authorizable getRemoteProcessGroup(final String id) { + return connectorManagedComponentLookup.getRemoteProcessGroup(id); + } + + @Override + public ComponentAuthorizable getControllerService(final String id) { + final ControllerServiceNode controllerService = connectorManagedComponentLookup.getControllerService(id); + return new ControllerServiceComponentAuthorizable(controllerService, controllerFacade.getExtensionManager()); + } + } + @Override public Authorizable getProvenance() { return PROVENANCE_AUTHORIZABLE; @@ -1363,6 +1409,11 @@ public void setProcessorDAO(ProcessorDAO processorDAO) { this.processorDAO = processorDAO; } + @Autowired + public void setConnectorManagedComponentLookup(final ConnectorManagedComponentLookup connectorManagedComponentLookup) { + this.connectorManagedComponentLookup = connectorManagedComponentLookup; + } + @Autowired public void setProcessGroupDAO(ProcessGroupDAO processGroupDAO) { this.processGroupDAO = processGroupDAO; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 11f33b91bd24..d0490a6e6c1d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -220,6 +220,14 @@ public interface NiFiServiceFacade { ConnectorEntity cancelConnectorDrain(Revision revision, String id); + void verifyEnterConnectorTroubleshooting(String id); + + ConnectorEntity enterConnectorTroubleshooting(Revision revision, String id); + + void verifyEndConnectorTroubleshooting(String id); + + ConnectorEntity endConnectorTroubleshooting(Revision revision, String id); + ConfigurationStepNamesEntity getConnectorConfigurationSteps(String id); ConfigurationStepEntity getConnectorConfigurationStep(String id, String configurationStepName); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c13966b16abb..efde7d303049 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -420,6 +420,7 @@ import org.apache.nifi.web.dao.ComponentStateDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ConnectorDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.ControllerServiceDAO; import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; @@ -510,6 +511,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private PortDAO inputPortDAO; private PortDAO outputPortDAO; private ConnectionDAO connectionDAO; + private ConnectorManagedComponentLookup connectorManagedComponentLookup; private ControllerServiceDAO controllerServiceDAO; private ReportingTaskDAO reportingTaskDAO; private ConnectorDAO connectorDAO; @@ -2235,7 +2237,7 @@ public DropRequestDTO deleteFlowFileDropRequest(final String connectionId, final @Override public ListingRequestDTO deleteFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.deleteFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -2646,7 +2648,7 @@ public DropRequestDTO createFlowFileDropRequest(final String connectionId, final @Override public ListingRequestDTO createFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.createFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -3674,7 +3676,35 @@ public ConnectorEntity getConnector(final String id, final boolean clusterNodeRe @Override public void verifyUpdateConnector(final ConnectorDTO connectorDTO) { - // No-op placeholder for future detailed verification + final ConnectorNode connector = connectorDAO.getConnector(connectorDTO.getId()); + final ConnectorState currentState = connector.getCurrentState(); + + if (connectorDTO.getName() != null && currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot update Connector " + connectorDTO.getId() + + " while it is in Troubleshooting mode; exit Troubleshooting mode before modifying the Connector configuration."); + } + + if (connectorDTO.getState() != null) { + final ScheduledState desiredState; + try { + desiredState = ScheduledState.valueOf(connectorDTO.getState()); + } catch (final IllegalArgumentException iae) { + throw new IllegalArgumentException("Invalid run status specified for Connector " + connectorDTO.getId() + ": " + connectorDTO.getState()); + } + + if (currentState == ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Cannot transition Connector " + connectorDTO.getId() + " to " + desiredState + + " while it is in Troubleshooting mode; exit Troubleshooting mode to resume normal lifecycle control."); + } + + switch (desiredState) { + case RUNNING -> connector.verifyCanStart(); + case STOPPED -> { + // Stop is valid from any non-Troubleshooting state; no additional verification required. + } + default -> throw new IllegalArgumentException("Unsupported scheduled state for Connector: " + desiredState); + } + } } @Override @@ -3702,7 +3732,8 @@ public ConnectorEntity updateConnector(final Revision revision, final ConnectorD @Override public void verifyDeleteConnector(final String id) { - // For now, DAO will enforce state; expose hook for symmetry + final ConnectorNode connector = connectorDAO.getConnector(id); + connector.verifyCanDelete(); } @Override @@ -3814,6 +3845,84 @@ public ConnectorEntity cancelConnectorDrain(final Revision revision, final Strin return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); } + @Override + public void verifyEnterConnectorTroubleshooting(final String id) { + connectorDAO.verifyEnterTroubleshooting(id); + } + + @Override + public ConnectorEntity enterConnectorTroubleshooting(final Revision revision, final String id) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final RevisionClaim claim = new StandardRevisionClaim(revision); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.enterTroubleshooting(id); + controllerFacade.save(); + + final ConnectorNode node = connectorDAO.getConnector(id); + final ConnectorDTO dto = dtoFactory.createConnectorDto(node); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode node = connectorDAO.getConnector(snapshot.getComponent().getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(node); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(node)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(node); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + + @Override + public void verifyEndConnectorTroubleshooting(final String id) { + // Verify that all cluster nodes are connected before exiting Troubleshooting mode. Otherwise, we run the risk of weird state transitions while the flow + // is in the middle of updating. + final List unconnectedNodes = getUnconnectedNodes(); + if (!unconnectedNodes.isEmpty()) { + throw new IllegalStateException("Cannot exit Troubleshooting mode because the following cluster nodes are not CONNECTED: " + + unconnectedNodes + ". All nodes must be CONNECTED before this operation may proceed."); + } + + connectorDAO.verifyEndTroubleshooting(id); + } + + private List getUnconnectedNodes() { + if (clusterCoordinator == null) { + return List.of(); + } + + final Map> connectionStates = clusterCoordinator.getConnectionStates(); + final List unconnectedNodes = new ArrayList<>(); + for (final Map.Entry> entry : connectionStates.entrySet()) { + if (entry.getKey() != NodeConnectionState.CONNECTED) { + unconnectedNodes.addAll(entry.getValue()); + } + } + + return unconnectedNodes; + } + + @Override + public ConnectorEntity endConnectorTroubleshooting(final Revision revision, final String id) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final RevisionClaim claim = new StandardRevisionClaim(revision); + + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + connectorDAO.endTroubleshooting(id); + controllerFacade.save(); + + final ConnectorNode node = connectorDAO.getConnector(id); + final ConnectorDTO dto = dtoFactory.createConnectorDto(node); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final ConnectorNode node = connectorDAO.getConnector(snapshot.getComponent().getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(node); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(node)); + final ConnectorStatusDTO statusDto = createConnectorStatusDto(node); + return entityFactory.createConnectorEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, statusDto); + } + @Override public ConfigurationStepNamesEntity getConnectorConfigurationSteps(final String id) { final ConnectorNode node = connectorDAO.getConnector(id); @@ -3903,7 +4012,11 @@ public ProcessGroupFlowEntity getConnectorFlow(final String connectorId, final S if (targetProcessGroup == null) { throw new ResourceNotFoundException("Process Group with ID " + processGroupId + " was not found within Connector " + connectorId); } - return createProcessGroupFlowEntity(targetProcessGroup, uiOnly); + + // Bulletin authorization within the managed flow resolves source components through the live group hierarchy + // (via authorizeBulletin's ProcessGroup overload), so the standard supplier works regardless of the + // Connector's state without needing DAO-level connector-managed lookups. + return createProcessGroupFlowEntity(targetProcessGroup, uiOnly, this::getProcessGroupBulletins); } @Override @@ -4451,7 +4564,7 @@ public DropRequestDTO getFlowFileDropRequest(final String connectionId, final St @Override public ListingRequestDTO getFlowFileListingRequest(final String connectionId, final String listingRequestId) { - final Connection connection = connectionDAO.getConnection(connectionId, true); + final Connection connection = connectorManagedComponentLookup.getConnection(connectionId); final ListingRequestDTO listRequest = dtoFactory.createListingRequestDTO(connectionDAO.getFlowFileListingRequest(connectionId, listingRequestId)); // include whether the source and destination are running @@ -4753,7 +4866,7 @@ private ProcessGroup resolveOwningProcessGroupForBulletin(final Bulletin bulleti } try { - return processGroupDAO.getProcessGroup(groupId, true); + return connectorManagedComponentLookup.getProcessGroup(groupId); } catch (final ResourceNotFoundException e) { // Owning group was removed; fall back to global authorizable lookup. return null; @@ -5655,6 +5768,11 @@ public ProcessGroupFlowEntity getProcessGroupFlow(final String groupId, final bo } private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup processGroup, final boolean uiOnly) { + return createProcessGroupFlowEntity(processGroup, uiOnly, this::getProcessGroupBulletins); + } + + private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup processGroup, final boolean uiOnly, + final Function> bulletinSupplier) { // Get the Process Group Status but we only need a status depth of one because for any child process group, // we ignore the status of each individual components. I.e., if Process Group A has child Group B, and child Group B // has a Processor, we don't care about the individual stats of that Processor because the ProcessGroupFlowEntity @@ -5663,7 +5781,7 @@ private ProcessGroupFlowEntity createProcessGroupFlowEntity(final ProcessGroup p final RevisionDTO revision = dtoFactory.createRevisionDTO(revisionManager.getRevision(processGroup.getIdentifier())); final PermissionsDTO permissions = dtoFactory.createPermissionsDto(processGroup); return entityFactory.createProcessGroupFlowEntity(dtoFactory.createProcessGroupFlowDto(processGroup, groupStatus, - revisionManager, this::getProcessGroupBulletins, uiOnly), revision, permissions); + revisionManager, bulletinSupplier, uiOnly), revision, permissions); } @Override @@ -8027,6 +8145,11 @@ public void setConnectionDAO(final ConnectionDAO connectionDAO) { this.connectionDAO = connectionDAO; } + @Autowired + public void setConnectorManagedComponentLookup(final ConnectorManagedComponentLookup connectorManagedComponentLookup) { + this.connectorManagedComponentLookup = connectorManagedComponentLookup; + } + @Autowired public void setAuditService(final AuditService auditService) { this.auditService = auditService; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java index 087a45fdfcf4..1b3685a72f92 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java @@ -807,6 +807,154 @@ public Response cancelDrain( ); } + /** + * Transitions the specified Connector into Troubleshooting mode. + * + * @param id The id of the connector. + * @param requestConnectorEntity A connectorEntity containing the revision. + * @return A connectorEntity. + */ + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/troubleshooting") + @Operation( + summary = "Transitions a Connector into Troubleshooting mode", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = ConnectorEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Places the Connector into Troubleshooting mode so that its managed flow may be directly modified. Standard lifecycle operations are disabled while in Troubleshooting mode.", + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid} or /operation/connectors/{uuid}") + } + ) + public Response enterTroubleshooting( + @Parameter( + description = "The connector id.", + required = true + ) + @PathParam("id") final String id, + @Parameter( + description = "The connector entity with revision.", + required = true + ) final ConnectorEntity requestConnectorEntity) { + + if (requestConnectorEntity == null || requestConnectorEntity.getRevision() == null) { + throw new IllegalArgumentException("Connector entity with revision must be specified."); + } + + if (requestConnectorEntity.getId() != null && !id.equals(requestConnectorEntity.getId())) { + throw new IllegalArgumentException(String.format("The connector id (%s) in the request body does not equal the " + + "connector id of the requested resource (%s).", requestConnectorEntity.getId(), id)); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, requestConnectorEntity); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(requestConnectorEntity.isDisconnectedNodeAcknowledged()); + } + + final Revision requestRevision = getRevision(requestConnectorEntity, id); + return withWriteLock( + serviceFacade, + requestConnectorEntity, + requestRevision, + lookup -> { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Authorizable connector = lookup.getConnector(id); + OperationAuthorizable.authorizeOperation(connector, authorizer, user); + }, + () -> serviceFacade.verifyEnterConnectorTroubleshooting(id), + (revision, connectorEntity) -> { + final ConnectorEntity entity = serviceFacade.enterConnectorTroubleshooting(revision, id); + populateRemainingConnectorEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + + /** + * Transitions the specified Connector out of Troubleshooting mode. + * + * @param version The revision is used to verify the client is working with the latest version of the flow. + * @param clientId Optional client id. + * @param id The id of the connector. + * @return A connectorEntity. + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/troubleshooting") + @Operation( + summary = "Transitions a Connector out of Troubleshooting mode", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = ConnectorEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Ends Troubleshooting mode for the Connector, restoring the authoritative flow. All components in the managed flow must be stopped and disabled, " + + "and the managed flow must have no active tasks.", + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid} or /operation/connectors/{uuid}") + } + ) + public Response endTroubleshooting( + @Parameter( + description = "The revision is used to verify the client is working with the latest version of the flow." + ) + @QueryParam(VERSION) final LongParameter version, + @Parameter( + description = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response." + ) + @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) final ClientIdParameter clientId, + @Parameter( + description = "Acknowledges that this node is disconnected to allow for mutable requests to proceed." + ) + @QueryParam(DISCONNECTED_NODE_ACKNOWLEDGED) @DefaultValue("false") final Boolean disconnectedNodeAcknowledged, + @Parameter( + description = "The connector id.", + required = true + ) + @PathParam("id") final String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(disconnectedNodeAcknowledged); + } + + final ConnectorEntity requestConnectorEntity = new ConnectorEntity(); + requestConnectorEntity.setId(id); + + final Revision requestRevision = new Revision(version == null ? null : version.getLong(), clientId.getClientId(), id); + return withWriteLock( + serviceFacade, + requestConnectorEntity, + requestRevision, + lookup -> { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final Authorizable connector = lookup.getConnector(id); + OperationAuthorizable.authorizeOperation(connector, authorizer, user); + }, + () -> serviceFacade.verifyEndConnectorTroubleshooting(id), + (revision, connectorEntity) -> { + final ConnectorEntity entity = serviceFacade.endConnectorTroubleshooting(revision, id); + populateRemainingConnectorEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + @POST @Consumes(MediaType.WILDCARD) @@ -1799,7 +1947,9 @@ public Response getFlow( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the flow for the specified process group within the connector's hierarchy + // Get the flow for the specified process group within the connector's hierarchy. The facade method is + // Connector-aware and skips the normal Troubleshooting access gate at the DAO locate call sites for bulletin + // authorization lookups within the managed flow. final ProcessGroupFlowEntity entity = serviceFacade.getConnectorFlow(connectorId, processGroupId, uiOnly); flowResource.populateRemainingFlowContent(entity.getProcessGroupFlow()); return generateOkResponse(entity).build(); @@ -1861,9 +2011,11 @@ public Response getControllerServicesFromConnectorProcessGroup( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the controller services for the specified process group within the connector's hierarchy - final Set controllerServices = serviceFacade.getConnectorControllerServices( - connectorId, processGroupId, includeAncestorGroups, includeDescendantGroups, includeReferences); + // Get the controller services for the specified process group within the connector's hierarchy. The facade + // method operates on the ControllerServiceNodes resolved directly from the managed Process Group tree, so no + // DAO locate calls are made against Connector-managed components here. + final Set controllerServices = + serviceFacade.getConnectorControllerServices(connectorId, processGroupId, includeAncestorGroups, includeDescendantGroups, includeReferences); controllerServiceResource.populateRemainingControllerServiceEntitiesContent(controllerServices); // create the response entity @@ -2005,7 +2157,9 @@ public Response getConnectorStatus( connector.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); - // get the status for the connector's managed process group + // Get the status for the Connector's managed Process Group. The facade method builds the status DTO from the + // resolved ProcessGroup directly (no DAO locate calls for Connector-managed components), so the normal + // Troubleshooting access gate does not apply. final ProcessGroupStatusEntity entity = serviceFacade.getConnectorProcessGroupStatus(id, recursive); return generateOkResponse(entity).build(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java index 3175a306a2fe..d3d108e1d982 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowFileQueueResource.java @@ -307,7 +307,7 @@ public Response createFlowFileListing( serviceFacade, requestConnectionEntity, lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(id, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(id); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }, @@ -374,7 +374,7 @@ public Response getListingRequest( // authorize access serviceFacade.authorizeAccess(lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }); @@ -435,7 +435,7 @@ public Response deleteListingRequest( serviceFacade, new ListingEntity(connectionId, listingRequestId), lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); }, @@ -522,7 +522,7 @@ public Response createDropRequest( serviceFacade, requestConnectionEntity, lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(id, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(id); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }, @@ -589,7 +589,7 @@ public Response getDropRequest( // authorize access serviceFacade.authorizeAccess(lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }); @@ -650,7 +650,7 @@ public Response removeDropRequest( serviceFacade, new DropEntity(connectionId, dropRequestId), lookup -> { - final ConnectionAuthorizable connAuth = lookup.getConnection(connectionId, true); + final ConnectionAuthorizable connAuth = lookup.forConnectorManagedFlow().getConnection(connectionId); final Authorizable dataAuthorizable = connAuth.getSourceData(); dataAuthorizable.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); }, diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java index c42ff3273a01..60033bc8116e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java @@ -885,7 +885,7 @@ public Response removeDropRequest( } private void authorizeHandleDropAllFlowFilesRequest(String processGroupId, AuthorizableLookup lookup) { - final ProcessGroupAuthorizable processGroup = lookup.getProcessGroup(processGroupId, true); + final ProcessGroupAuthorizable processGroup = lookup.forConnectorManagedFlow().getProcessGroup(processGroupId); authorizeProcessGroup(processGroup, authorizer, lookup, RequestAction.READ, false, false, false, false, false); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 4dfbbe89e062..ecbc773616fd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -860,7 +860,14 @@ public ProcessorStatus getProcessorStatus(final String processorId) { */ public ConnectionStatus getConnectionStatus(final String connectionId) { final ProcessGroup root = getRootGroup(); - final Connection connection = root.findConnection(connectionId); + Connection connection = root.findConnection(connectionId); + + // If the Connection was not found by traversing the root hierarchy, fall back to a direct FlowManager lookup. This + // is necessary because Connections that live inside a Connector's Managed Process Group are not part of the main + // root Process Group's parent hierarchy, but they are still registered with the FlowManager. + if (connection == null) { + connection = flowController.getFlowManager().getConnection(connectionId); + } // ensure the connection was found if (connection == null) { @@ -920,10 +927,8 @@ public StatusAnalytics getConnectionStatusAnalytics(final String connectionId) { * @return the status for the specified input port */ public PortStatus getInputPortStatus(final String portId) { - final ProcessGroup root = getRootGroup(); - final Port port = root.findInputPort(portId); + final Port port = flowController.findInputPortIncludingConnectorManaged(portId); - // ensure the input port was found if (port == null) { throw new ResourceNotFoundException(String.format("Unable to locate input port with id '%s'.", portId)); } @@ -949,10 +954,8 @@ public PortStatus getInputPortStatus(final String portId) { * @return the status for the specified output port */ public PortStatus getOutputPortStatus(final String portId) { - final ProcessGroup root = getRootGroup(); - final Port port = root.findOutputPort(portId); + final Port port = flowController.findOutputPortIncludingConnectorManaged(portId); - // ensure the output port was found if (port == null) { throw new ResourceNotFoundException(String.format("Unable to locate output port with id '%s'.", portId)); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java index 8785d670a6c8..98caa34453ff 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java @@ -43,15 +43,6 @@ public interface ConnectionDAO { */ Connection getConnection(String id); - /** - * Gets the specified Connection, optionally including Connector-managed ProcessGroups in the search. - * - * @param id The connection id - * @param includeConnectorManaged Whether to search Connector-managed ProcessGroups - * @return The connection - */ - Connection getConnection(String id, boolean includeConnectorManaged); - /** * Gets the specified flow file drop request. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java index acf2e7f53900..e63218ae1411 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorDAO.java @@ -59,6 +59,14 @@ public interface ConnectorDAO { void verifyCancelDrainFlowFile(String id); + void verifyEnterTroubleshooting(String id); + + void enterTroubleshooting(String id); + + void verifyEndTroubleshooting(String id); + + void endTroubleshooting(String id); + void verifyPurgeFlowFiles(String id); void purgeFlowFiles(String id, String requestor); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java new file mode 100644 index 000000000000..a016529eed33 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectorManagedComponentLookup.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao; + +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.connectable.Funnel; +import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.label.Label; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; + +/** + * Facade exposing DAO-level component lookups that include Process Groups managed by a Connector. + * + *

The standard {@link ProcessorDAO}, {@link ConnectionDAO}, {@link ProcessGroupDAO}, etc. intentionally hide + * components that live inside a Connector's managed Process Group hierarchy unless the owning Connector is in + * Troubleshooting mode. The handful of REST endpoints that legitimately need to inspect or operate on components + * within a Connector-managed flow (the FlowFile queue endpoints under {@code /connectors/{id}/...} and the + * Connector-aware Process Group flow endpoint, for example) obtain those components through this facade so the + * standard DAO surface remains free of the "include connector managed" flag.

+ */ +public interface ConnectorManagedComponentLookup { + + /** + * Locate a Processor by identifier, including processors within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + ProcessorNode getProcessor(String id); + + /** + * Locate an input Port by identifier, including ports within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Port getInputPort(String id); + + /** + * Locate an output Port by identifier, including ports within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Port getOutputPort(String id); + + /** + * Locate a Connection by identifier, including connections within Connector-managed Process Groups regardless of + * the owning Connector's state. + */ + Connection getConnection(String id); + + /** + * Locate a Process Group by identifier, including Connector-managed Process Groups regardless of the owning + * Connector's state. + */ + ProcessGroup getProcessGroup(String id); + + /** + * Locate a Remote Process Group by identifier, including remote process groups within Connector-managed Process + * Groups regardless of the owning Connector's state. + */ + RemoteProcessGroup getRemoteProcessGroup(String id); + + /** + * Locate a Controller Service by identifier, including services within Connector-managed Process Groups regardless + * of the owning Connector's state. + */ + ControllerServiceNode getControllerService(String id); + + /** + * Locate a Label by identifier, including labels within Connector-managed Process Groups regardless of the owning + * Connector's state. + */ + Label getLabel(String id); + + /** + * Locate a Funnel by identifier, including funnels within Connector-managed Process Groups regardless of the + * owning Connector's state. + */ + Funnel getFunnel(String id); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java index 1dd36a46b88b..df546810a6c9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessGroupDAO.java @@ -58,15 +58,6 @@ public interface ProcessGroupDAO { */ ProcessGroup getProcessGroup(String groupId); - /** - * Gets the specified process group, optionally including Connector-managed ProcessGroups in the search. - * - * @param groupId The process group id - * @param includeConnectorManaged Whether to search Connector-managed ProcessGroups - * @return The process group - */ - ProcessGroup getProcessGroup(String groupId, boolean includeConnectorManaged); - /** * Gets all of the process groups. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java index a41888bd3b98..2be03fe0f9d6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java @@ -40,7 +40,16 @@ public abstract class AbstractPortDAO extends ComponentDAO implements PortDAO { protected FlowController flowController; - protected abstract Port locatePort(final String portId); + protected Port locatePort(final String portId) { + return locatePort(portId, false); + } + + protected abstract Port locatePort(final String portId, final boolean includeConnectorManaged); + + @Override + public Port getPort(final String portId) { + return locatePort(portId); + } @Override public void verifyUpdate(PortDTO portDTO) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java index 26f0de302520..5d1e66dd7984 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/ComponentDAO.java @@ -18,6 +18,8 @@ import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.controller.FlowController; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; @@ -25,6 +27,7 @@ import org.apache.nifi.web.api.dto.BundleDTO; import java.util.List; +import java.util.Optional; public abstract class ComponentDAO { @@ -83,17 +86,55 @@ protected ProcessGroup locateProcessGroup(final FlowController flowController, f return group; } - // Optionally search Connector-managed ProcessGroups - if (includeConnectorManaged) { - group = flowController.getFlowManager().getGroup(groupId); - if (group != null) { + // Search Connector-managed ProcessGroups. The unconditional search is important so that if a component exists + // in a Connector-managed flow but the Connector is not in Troubleshooting mode, we can produce a clear 409 + // Conflict response rather than a 404 Not Found. + group = flowController.getFlowManager().getGroup(groupId); + if (group != null) { + if (includeConnectorManaged) { return group; } + + verifyAccessibleForComponentOperation(group, groupId); + return group; } throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); } + /** + * Verifies that the component represented by the given {@link ProcessGroup} (or a component contained within it) is + * accessible for a direct user-facing operation such as GET/PUT/POST/DELETE of the component itself. Components that + * live within a Connector's managed Process Group hierarchy are only accessible when the owning Connector is in + * {@link ConnectorState#TROUBLESHOOTING} mode. If the owning Connector is not in Troubleshooting mode, an + * {@link IllegalStateException} is thrown which is translated by the REST layer into a 409 Conflict response. + * + *

Connector-aware REST endpoints that need to read components within a managed flow regardless of the + * Connector's state must obtain those components through the + * {@link org.apache.nifi.web.dao.ConnectorManagedComponentLookup} facade (or, for authorization, through + * {@link org.apache.nifi.authorization.AuthorizableLookup#forConnectorManagedFlow()}) so that this verification is + * skipped at the locate call site rather than being bypassed globally for the current thread. + * + * @param group the ProcessGroup that owns (or is) the component being accessed + * @param componentId the identifier of the component being accessed (used in the error message) + */ + protected void verifyAccessibleForComponentOperation(final ProcessGroup group, final String componentId) { + if (group == null) { + return; + } + + final Optional owningConnector = group.findOwningConnector(); + if (owningConnector.isEmpty()) { + return; + } + + final ConnectorNode connector = owningConnector.get(); + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + throw new IllegalStateException("Component [" + componentId + "] is managed by Connector " + connector.getName() + " (" + + connector.getIdentifier() + "); the Connector must be in Troubleshooting mode for this component to be accessible."); + } + } + protected void verifyCreate(final ExtensionManager extensionManager, final String type, final BundleDTO bundle) { final List bundles = extensionManager.getBundles(type); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java index 5b4ab5c14205..61681598e7bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java @@ -24,6 +24,7 @@ import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Connectable; @@ -82,8 +83,7 @@ private Connection locateConnection(final String connectionId) { return locateConnection(connectionId, false); } - private Connection locateConnection(final String connectionId, final boolean includeConnectorManaged) { - // First, search the main flow hierarchy + Connection locateConnection(final String connectionId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Connection connection = rootGroup.findConnection(connectionId); @@ -91,17 +91,20 @@ private Connection locateConnection(final String connectionId, final boolean inc return connection; } - // Optionally search Connector-managed ProcessGroups - if (includeConnectorManaged) { - for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { - final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); - if (flowContext != null) { - final ProcessGroup managedGroup = flowContext.getManagedProcessGroup(); - connection = managedGroup.findConnection(connectionId); - if (connection != null) { - return connection; - } + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + final ProcessGroup managedGroup = flowContext.getManagedProcessGroup(); + connection = managedGroup.findConnection(connectionId); + if (connection != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(connection.getProcessGroup(), connectionId); } + + return connection; } } @@ -111,7 +114,22 @@ private Connection locateConnection(final String connectionId, final boolean inc @Override public boolean hasConnection(String id) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findConnection(id) != null; + if (rootGroup.findConnection(id) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findConnection(id) != null) { + return true; + } + } + + return false; } @Override @@ -119,11 +137,6 @@ public Connection getConnection(final String id) { return locateConnection(id); } - @Override - public Connection getConnection(final String id, final boolean includeConnectorManaged) { - return locateConnection(id, includeConnectorManaged); - } - @Override public Set getConnections(final String groupId) { final ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java index 9a507e16e5ce..176cee91441f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorDAO.java @@ -27,6 +27,7 @@ import org.apache.nifi.components.connector.ConnectorUpdateContext; import org.apache.nifi.components.connector.ConnectorValueReference; import org.apache.nifi.components.connector.ConnectorValueType; +import org.apache.nifi.components.connector.FlowUpdateException; import org.apache.nifi.components.connector.SecretReference; import org.apache.nifi.components.connector.StepConfiguration; import org.apache.nifi.components.connector.StringLiteralValue; @@ -157,6 +158,34 @@ public void verifyCancelDrainFlowFile(final String id) { connector.verifyCancelDrainFlowFiles(); } + @Override + public void verifyEnterTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + getConnectorRepository().verifyEnterTroubleshooting(connector); + } + + @Override + public void enterTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + getConnectorRepository().enterTroubleshooting(connector); + } + + @Override + public void verifyEndTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + getConnectorRepository().verifyEndTroubleshooting(connector); + } + + @Override + public void endTroubleshooting(final String id) { + final ConnectorNode connector = getConnector(id); + try { + getConnectorRepository().endTroubleshooting(connector); + } catch (final FlowUpdateException e) { + throw new IllegalStateException("Failed to exit troubleshooting mode for Connector " + id + ": " + e, e); + } + } + @Override public void verifyPurgeFlowFiles(final String id) { final ConnectorNode connector = requireConnector(id, ConnectorSyncMode.LOCAL_ONLY); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java new file mode 100644 index 000000000000..ef271a8f4b3a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectorManagedComponentLookup.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao.impl; + +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.connectable.Funnel; +import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.label.Label; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Default {@link ConnectorManagedComponentLookup} implementation. Delegates each lookup to the package-private + * {@code locate*(id, true)} helper on the corresponding {@code Standard*DAO}, which performs the same search the + * standard DAO surface performs but skips the verification that would otherwise reject access to a component inside a + * Connector-managed Process Group whose owning Connector is not in Troubleshooting mode. + */ +@Component +public class StandardConnectorManagedComponentLookup extends ComponentDAO implements ConnectorManagedComponentLookup { + + private FlowController flowController; + private StandardProcessorDAO processorDAO; + private StandardConnectionDAO connectionDAO; + private StandardInputPortDAO inputPortDAO; + private StandardOutputPortDAO outputPortDAO; + private StandardLabelDAO labelDAO; + private StandardFunnelDAO funnelDAO; + private StandardRemoteProcessGroupDAO remoteProcessGroupDAO; + private StandardControllerServiceDAO controllerServiceDAO; + + @Override + public ProcessorNode getProcessor(final String id) { + return processorDAO.locateProcessor(id, true); + } + + @Override + public Port getInputPort(final String id) { + return inputPortDAO.locatePort(id, true); + } + + @Override + public Port getOutputPort(final String id) { + return outputPortDAO.locatePort(id, true); + } + + @Override + public Connection getConnection(final String id) { + return connectionDAO.locateConnection(id, true); + } + + @Override + public ProcessGroup getProcessGroup(final String id) { + return locateProcessGroup(flowController, id, true); + } + + @Override + public RemoteProcessGroup getRemoteProcessGroup(final String id) { + return remoteProcessGroupDAO.locateRemoteProcessGroup(id, true); + } + + @Override + public ControllerServiceNode getControllerService(final String id) { + return controllerServiceDAO.locateControllerService(id, true); + } + + @Override + public Label getLabel(final String id) { + return labelDAO.locateLabel(id, true); + } + + @Override + public Funnel getFunnel(final String id) { + return funnelDAO.locateFunnel(id, true); + } + + @Autowired + public void setFlowController(final FlowController flowController) { + this.flowController = flowController; + } + + @Autowired + public void setProcessorDAO(final StandardProcessorDAO processorDAO) { + this.processorDAO = processorDAO; + } + + @Autowired + public void setConnectionDAO(final StandardConnectionDAO connectionDAO) { + this.connectionDAO = connectionDAO; + } + + @Autowired + public void setInputPortDAO(final StandardInputPortDAO inputPortDAO) { + this.inputPortDAO = inputPortDAO; + } + + @Autowired + public void setOutputPortDAO(final StandardOutputPortDAO outputPortDAO) { + this.outputPortDAO = outputPortDAO; + } + + @Autowired + public void setLabelDAO(final StandardLabelDAO labelDAO) { + this.labelDAO = labelDAO; + } + + @Autowired + public void setFunnelDAO(final StandardFunnelDAO funnelDAO) { + this.funnelDAO = funnelDAO; + } + + @Autowired + public void setRemoteProcessGroupDAO(final StandardRemoteProcessGroupDAO remoteProcessGroupDAO) { + this.remoteProcessGroupDAO = remoteProcessGroupDAO; + } + + @Autowired + public void setControllerServiceDAO(final StandardControllerServiceDAO controllerServiceDAO) { + this.controllerServiceDAO = controllerServiceDAO; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java index 6c13a29d0b3e..7875221cec87 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardControllerServiceDAO.java @@ -71,14 +71,19 @@ public class StandardControllerServiceDAO extends ComponentDAO implements Contro private FlowController flowController; private ControllerServiceNode locateControllerService(final String controllerServiceId) { - // get the controller service + return locateControllerService(controllerServiceId, false); + } + + ControllerServiceNode locateControllerService(final String controllerServiceId, final boolean includeConnectorManaged) { final ControllerServiceNode controllerService = serviceProvider.getControllerServiceNode(controllerServiceId); - // ensure the controller service exists if (controllerService == null) { throw new ResourceNotFoundException(String.format("Unable to locate controller service with id '%s'.", controllerServiceId)); } + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(controllerService.getProcessGroup(), controllerServiceId); + } return controllerService; } @@ -116,11 +121,7 @@ public ControllerServiceNode createControllerService(final ControllerServiceDTO if (groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS)) { group = flowManager.getRootGroup(); } else { - group = flowManager.getRootGroup().findProcessGroup(groupId); - } - - if (group == null) { - throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); + group = locateProcessGroup(flowController, groupId); } group.addControllerService(controllerService); @@ -148,20 +149,17 @@ public Set getControllerServices(final String groupId, fi if (groupId == null) { return flowManager.getRootControllerServices(); - } else { - final String searchId = groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS) ? flowManager.getRootGroupId() : groupId; - final ProcessGroup procGroup = flowManager.getRootGroup().findProcessGroup(searchId); - if (procGroup == null) { - throw new ResourceNotFoundException("Could not find Process Group with ID " + groupId); - } + } - final Set serviceNodes = procGroup.getControllerServices(includeAncestorGroups); - if (includeDescendantGroups) { - serviceNodes.addAll(procGroup.findAllControllerServices()); - } + final String searchId = groupId.equals(FlowManager.ROOT_GROUP_ID_ALIAS) ? flowManager.getRootGroupId() : groupId; + final ProcessGroup procGroup = locateProcessGroup(flowController, searchId); - return serviceNodes; + final Set serviceNodes = procGroup.getControllerServices(includeAncestorGroups); + if (includeDescendantGroups) { + serviceNodes.addAll(procGroup.findAllControllerServices()); } + + return serviceNodes; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java index 186aa09a97d6..99ea168bf6a6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFunnelDAO.java @@ -16,6 +16,10 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.FlowController; @@ -34,20 +38,53 @@ public class StandardFunnelDAO extends ComponentDAO implements FunnelDAO { private FlowController flowController; private Funnel locateFunnel(final String funnelId) { - final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final Funnel funnel = rootGroup.findFunnel(funnelId); + return locateFunnel(funnelId, false); + } - if (funnel == null) { - throw new ResourceNotFoundException(String.format("Unable to find funnel with id '%s'.", funnelId)); - } else { + Funnel locateFunnel(final String funnelId, final boolean includeConnectorManaged) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + Funnel funnel = rootGroup.findFunnel(funnelId); + if (funnel != null) { return funnel; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + funnel = flowContext.getManagedProcessGroup().findFunnel(funnelId); + if (funnel != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(funnel.getProcessGroup(), funnelId); + } + return funnel; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find funnel with id '%s'.", funnelId)); } @Override public boolean hasFunnel(String funnelId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findFunnel(funnelId) != null; + if (rootGroup.findFunnel(funnelId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findFunnel(funnelId) != null) { + return true; + } + } + + return false; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java index cc84aae59a78..3929807b87ba 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardInputPortDAO.java @@ -16,6 +16,10 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.ScheduledState; @@ -32,25 +36,60 @@ public class StandardInputPortDAO extends AbstractPortDAO implements PortDAO { @Override - protected Port locatePort(final String portId) { + protected Port locatePort(final String portId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); Port port = rootGroup.findInputPort(portId); - if (port == null) { port = rootGroup.findOutputPort(portId); } - - if (port == null) { - throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); - } else { + if (port != null) { return port; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + final ProcessGroup managed = flowContext.getManagedProcessGroup(); + port = managed.findInputPort(portId); + if (port == null) { + port = managed.findOutputPort(portId); + } + if (port != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(port.getProcessGroup(), portId); + } + return port; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); } @Override public boolean hasPort(String portId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findInputPort(portId) != null || rootGroup.findOutputPort(portId) != null; + if (rootGroup.findInputPort(portId) != null || rootGroup.findOutputPort(portId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null) { + final ProcessGroup managed = flowContext.getManagedProcessGroup(); + if (managed.findInputPort(portId) != null || managed.findOutputPort(portId) != null) { + return true; + } + } + } + + return false; } @Override @@ -94,11 +133,6 @@ public Port createPort(String groupId, PortDTO portDTO) { return port; } - @Override - public Port getPort(String portId) { - return locatePort(portId); - } - @Override public Set getPorts(String groupId) { ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java index f1d0f80bf018..2aa7822033bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardLabelDAO.java @@ -16,6 +16,10 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; import org.apache.nifi.controller.FlowController; @@ -37,20 +41,53 @@ public class StandardLabelDAO extends ComponentDAO implements LabelDAO { private FlowController flowController; private Label locateLabel(final String labelId) { - final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final Label label = rootGroup.findLabel(labelId); + return locateLabel(labelId, false); + } - if (label == null) { - throw new ResourceNotFoundException(String.format("Unable to find label with id '%s'.", labelId)); - } else { + Label locateLabel(final String labelId, final boolean includeConnectorManaged) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + Label label = rootGroup.findLabel(labelId); + if (label != null) { return label; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + label = flowContext.getManagedProcessGroup().findLabel(labelId); + if (label != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(label.getProcessGroup(), labelId); + } + return label; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find label with id '%s'.", labelId)); } @Override public boolean hasLabel(String labelId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findLabel(labelId) != null; + if (rootGroup.findLabel(labelId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findLabel(labelId) != null) { + return true; + } + } + + return false; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java index 67f50489a54c..dff622bd76cd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardOutputPortDAO.java @@ -16,6 +16,10 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.ScheduledState; @@ -32,21 +36,50 @@ public class StandardOutputPortDAO extends AbstractPortDAO implements PortDAO { @Override - protected Port locatePort(final String portId) { + protected Port locatePort(final String portId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final Port port = rootGroup.findOutputPort(portId); - - if (port == null) { - throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); - } else { + Port port = rootGroup.findOutputPort(portId); + if (port != null) { return port; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + port = flowContext.getManagedProcessGroup().findOutputPort(portId); + if (port != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(port.getProcessGroup(), portId); + } + return port; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find port with id '%s'.", portId)); } @Override public boolean hasPort(String portId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findOutputPort(portId) != null; + if (rootGroup.findOutputPort(portId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findOutputPort(portId) != null) { + return true; + } + } + + return false; } @Override @@ -90,11 +123,6 @@ public Port createPort(String groupId, PortDTO portDTO) { return port; } - @Override - public Port getPort(String portId) { - return locatePort(portId); - } - @Override public Set getPorts(String groupId) { ProcessGroup group = locateProcessGroup(flowController, groupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java index b6612c3126d4..4bc2eb28e855 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAO.java @@ -187,11 +187,6 @@ public ProcessGroup getProcessGroup(String groupId) { return locateProcessGroup(flowController, groupId); } - @Override - public ProcessGroup getProcessGroup(String groupId, boolean includeConnectorManaged) { - return locateProcessGroup(flowController, groupId, includeConnectorManaged); - } - @Override public Set getProcessGroups(final String parentGroupId, final ProcessGroupRecursivity processGroupRecursivity) { ProcessGroup group = locateProcessGroup(flowController, parentGroupId); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java index ebdccd1c15c6..3b386f244630 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java @@ -20,6 +20,7 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigurableComponent; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; import org.apache.nifi.connectable.Connection; @@ -79,20 +80,41 @@ public class StandardProcessorDAO extends ComponentDAO implements ProcessorDAO { private ComponentStateDAO componentStateDAO; private ProcessorNode locateProcessor(final String processorId) { + return locateProcessor(processorId, false); + } + + ProcessorNode locateProcessor(final String processorId, final boolean includeConnectorManaged) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); final ProcessorNode processor = rootGroup.findProcessor(processorId); - - if (processor == null) { - throw new ResourceNotFoundException(String.format("Unable to find processor with id '%s'.", processorId)); - } else { + if (processor != null) { return processor; } + + final ProcessorNode globalProcessor = flowController.getFlowManager().getProcessorNode(processorId); + if (globalProcessor != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(globalProcessor.getProcessGroup(), processorId); + } + return globalProcessor; + } + + throw new ResourceNotFoundException(String.format("Unable to find processor with id '%s'.", processorId)); } @Override public boolean hasProcessor(String id) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findProcessor(id) != null; + if (rootGroup.findProcessor(id) != null) { + return true; + } + + final ProcessorNode globalProcessor = flowController.getFlowManager().getProcessorNode(id); + if (globalProcessor == null) { + return false; + } + return globalProcessor.getProcessGroup().findOwningConnector() + .map(owningConnector -> owningConnector.getCurrentState() == ConnectorState.TROUBLESHOOTING) + .orElse(false); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java index f8888154defa..4c8bfae96014 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardRemoteProcessGroupDAO.java @@ -17,6 +17,10 @@ package org.apache.nifi.web.dao.impl; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.components.connector.ConnectorSyncMode; +import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; import org.apache.nifi.connectable.Position; @@ -52,20 +56,53 @@ public class StandardRemoteProcessGroupDAO extends ComponentDAO implements Remot private ComponentStateDAO componentStateDAO; private RemoteProcessGroup locateRemoteProcessGroup(final String remoteProcessGroupId) { - final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - final RemoteProcessGroup remoteProcessGroup = rootGroup.findRemoteProcessGroup(remoteProcessGroupId); + return locateRemoteProcessGroup(remoteProcessGroupId, false); + } - if (remoteProcessGroup == null) { - throw new ResourceNotFoundException(String.format("Unable to find remote process group with id '%s'.", remoteProcessGroupId)); - } else { + RemoteProcessGroup locateRemoteProcessGroup(final String remoteProcessGroupId, final boolean includeConnectorManaged) { + final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); + RemoteProcessGroup remoteProcessGroup = rootGroup.findRemoteProcessGroup(remoteProcessGroupId); + if (remoteProcessGroup != null) { return remoteProcessGroup; } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext == null) { + continue; + } + + remoteProcessGroup = flowContext.getManagedProcessGroup().findRemoteProcessGroup(remoteProcessGroupId); + if (remoteProcessGroup != null) { + if (!includeConnectorManaged) { + verifyAccessibleForComponentOperation(remoteProcessGroup.getProcessGroup(), remoteProcessGroupId); + } + return remoteProcessGroup; + } + } + + throw new ResourceNotFoundException(String.format("Unable to find remote process group with id '%s'.", remoteProcessGroupId)); } @Override public boolean hasRemoteProcessGroup(String remoteProcessGroupId) { final ProcessGroup rootGroup = flowController.getFlowManager().getRootGroup(); - return rootGroup.findRemoteProcessGroup(remoteProcessGroupId) != null; + if (rootGroup.findRemoteProcessGroup(remoteProcessGroupId) != null) { + return true; + } + + for (final ConnectorNode connector : flowController.getConnectorRepository().getConnectors(ConnectorSyncMode.LOCAL_ONLY)) { + if (connector.getCurrentState() != ConnectorState.TROUBLESHOOTING) { + continue; + } + + final FrameworkFlowContext flowContext = connector.getActiveFlowContext(); + if (flowContext != null && flowContext.getManagedProcessGroup().findRemoteProcessGroup(remoteProcessGroupId) != null) { + return true; + } + } + + return false; } /** @@ -108,9 +145,7 @@ public RemoteProcessGroup createRemoteProcessGroup(String groupId, RemoteProcess */ @Override public RemoteProcessGroup getRemoteProcessGroup(String remoteProcessGroupId) { - final RemoteProcessGroup remoteProcessGroup = locateRemoteProcessGroup(remoteProcessGroupId); - - return remoteProcessGroup; + return locateRemoteProcessGroup(remoteProcessGroupId); } /** diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/authorization/StandardAuthorizableLookupTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/authorization/StandardAuthorizableLookupTest.java index 9a7bfc6cdced..b44bc1c13a7d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/authorization/StandardAuthorizableLookupTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/authorization/StandardAuthorizableLookupTest.java @@ -32,6 +32,7 @@ import org.apache.nifi.registry.flow.FlowRegistryClientNode; import org.apache.nifi.web.controller.ControllerFacade; import org.apache.nifi.web.dao.ConnectionDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; import org.apache.nifi.web.dao.ProcessGroupDAO; @@ -111,13 +112,13 @@ void testGetAuthorizableFromResourceFlowAnalysisRule() { } @Test - void testGetConnectionWithoutIncludeConnectorManaged() { + void testGetConnectionResolvesThroughConnectionDAO() { final StandardAuthorizableLookup lookup = getLookup(); final ConnectionDAO connectionDAO = mock(ConnectionDAO.class); final Connection connection = mock(Connection.class); final Connectable sourceConnectable = mock(Connectable.class); - when(connectionDAO.getConnection(eq(COMPONENT_ID), eq(false))).thenReturn(connection); + when(connectionDAO.getConnection(eq(COMPONENT_ID))).thenReturn(connection); when(connection.getSource()).thenReturn(sourceConnectable); when(connection.getDestination()).thenReturn(sourceConnectable); when(connection.getSourceAuthorizable()).thenReturn(sourceConnectable); @@ -126,90 +127,56 @@ void testGetConnectionWithoutIncludeConnectorManaged() { final ConnectionAuthorizable result = lookup.getConnection(COMPONENT_ID); assertNotNull(result); - verify(connectionDAO).getConnection(eq(COMPONENT_ID), eq(false)); + verify(connectionDAO).getConnection(eq(COMPONENT_ID)); } @Test - void testGetConnectionWithIncludeConnectorManagedTrue() { + void testGetConnectionThroughConnectorManagedFlowFacade() { final StandardAuthorizableLookup lookup = getLookup(); - final ConnectionDAO connectionDAO = mock(ConnectionDAO.class); - final Connection connection = mock(Connection.class); - final Connectable sourceConnectable = mock(Connectable.class); - - when(connectionDAO.getConnection(eq(COMPONENT_ID), eq(true))).thenReturn(connection); - when(connection.getSource()).thenReturn(sourceConnectable); - when(connection.getDestination()).thenReturn(sourceConnectable); - when(connection.getSourceAuthorizable()).thenReturn(sourceConnectable); - lookup.setConnectionDAO(connectionDAO); - - final ConnectionAuthorizable result = lookup.getConnection(COMPONENT_ID, true); - - assertNotNull(result); - verify(connectionDAO).getConnection(eq(COMPONENT_ID), eq(true)); - } - - @Test - void testGetConnectionWithIncludeConnectorManagedFalse() { - final StandardAuthorizableLookup lookup = getLookup(); - final ConnectionDAO connectionDAO = mock(ConnectionDAO.class); + final ConnectorManagedComponentLookup connectorManagedComponentLookup = mock(ConnectorManagedComponentLookup.class); final Connection connection = mock(Connection.class); final Connectable sourceConnectable = mock(Connectable.class); - when(connectionDAO.getConnection(eq(COMPONENT_ID), eq(false))).thenReturn(connection); + when(connectorManagedComponentLookup.getConnection(eq(COMPONENT_ID))).thenReturn(connection); when(connection.getSource()).thenReturn(sourceConnectable); when(connection.getDestination()).thenReturn(sourceConnectable); when(connection.getSourceAuthorizable()).thenReturn(sourceConnectable); - lookup.setConnectionDAO(connectionDAO); + lookup.setConnectorManagedComponentLookup(connectorManagedComponentLookup); - final ConnectionAuthorizable result = lookup.getConnection(COMPONENT_ID, false); + final ConnectionAuthorizable result = lookup.forConnectorManagedFlow().getConnection(COMPONENT_ID); assertNotNull(result); - verify(connectionDAO).getConnection(eq(COMPONENT_ID), eq(false)); + verify(connectorManagedComponentLookup).getConnection(eq(COMPONENT_ID)); } @Test - void testGetProcessGroupWithoutIncludeConnectorManaged() { + void testGetProcessGroupResolvesThroughProcessGroupDAO() { final StandardAuthorizableLookup lookup = getLookup(); final ProcessGroupDAO processGroupDAO = mock(ProcessGroupDAO.class); final ProcessGroup processGroup = mock(ProcessGroup.class); - when(processGroupDAO.getProcessGroup(eq(COMPONENT_ID), eq(false))).thenReturn(processGroup); + when(processGroupDAO.getProcessGroup(eq(COMPONENT_ID))).thenReturn(processGroup); lookup.setProcessGroupDAO(processGroupDAO); final ProcessGroupAuthorizable result = lookup.getProcessGroup(COMPONENT_ID); assertNotNull(result); - verify(processGroupDAO).getProcessGroup(eq(COMPONENT_ID), eq(false)); + verify(processGroupDAO).getProcessGroup(eq(COMPONENT_ID)); } @Test - void testGetProcessGroupWithIncludeConnectorManagedTrue() { + void testGetProcessGroupThroughConnectorManagedFlowFacade() { final StandardAuthorizableLookup lookup = getLookup(); - final ProcessGroupDAO processGroupDAO = mock(ProcessGroupDAO.class); + final ConnectorManagedComponentLookup connectorManagedComponentLookup = mock(ConnectorManagedComponentLookup.class); final ProcessGroup processGroup = mock(ProcessGroup.class); - when(processGroupDAO.getProcessGroup(eq(COMPONENT_ID), eq(true))).thenReturn(processGroup); - lookup.setProcessGroupDAO(processGroupDAO); - - final ProcessGroupAuthorizable result = lookup.getProcessGroup(COMPONENT_ID, true); - - assertNotNull(result); - verify(processGroupDAO).getProcessGroup(eq(COMPONENT_ID), eq(true)); - } - - @Test - void testGetProcessGroupWithIncludeConnectorManagedFalse() { - final StandardAuthorizableLookup lookup = getLookup(); - final ProcessGroupDAO processGroupDAO = mock(ProcessGroupDAO.class); - final ProcessGroup processGroup = mock(ProcessGroup.class); - - when(processGroupDAO.getProcessGroup(eq(COMPONENT_ID), eq(false))).thenReturn(processGroup); - lookup.setProcessGroupDAO(processGroupDAO); + when(connectorManagedComponentLookup.getProcessGroup(eq(COMPONENT_ID))).thenReturn(processGroup); + lookup.setConnectorManagedComponentLookup(connectorManagedComponentLookup); - final ProcessGroupAuthorizable result = lookup.getProcessGroup(COMPONENT_ID, false); + final ProcessGroupAuthorizable result = lookup.forConnectorManagedFlow().getProcessGroup(COMPONENT_ID); assertNotNull(result); - verify(processGroupDAO).getProcessGroup(eq(COMPONENT_ID), eq(false)); + verify(connectorManagedComponentLookup).getProcessGroup(eq(COMPONENT_ID)); } private StandardAuthorizableLookup getLookup() { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index c1cba635aecf..0a8076239901 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -130,6 +130,7 @@ import org.apache.nifi.web.controller.ControllerFacade; import org.apache.nifi.web.dao.ComponentStateDAO; import org.apache.nifi.web.dao.ConnectorDAO; +import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.ProcessGroupDAO; import org.apache.nifi.web.dao.RemoteProcessGroupDAO; import org.apache.nifi.web.dao.UserDAO; @@ -144,6 +145,7 @@ import org.mockito.Answers; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.mockito.stubbing.Answer; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -224,6 +226,7 @@ public class StandardNiFiServiceFacadeTest { private Authorizer authorizer; private FlowController flowController; private ProcessGroupDAO processGroupDAO; + private ConnectorManagedComponentLookup connectorManagedComponentLookup; private RuleViolationsManager ruleViolationsManager; @BeforeEach @@ -250,7 +253,7 @@ public void setUp() throws Exception { // authorizable lookup final AuthorizableLookup authorizableLookup = mock(AuthorizableLookup.class); - when(authorizableLookup.getProcessor(Mockito.anyString())).then(getProcessorInvocation -> { + final Answer processorLookupAnswer = getProcessorInvocation -> { final String processorId = getProcessorInvocation.getArgument(0); // processor-2 is no longer part of the flow @@ -277,7 +280,8 @@ public Resource getResource() { }); return componentAuthorizable; - }); + }; + when(authorizableLookup.getProcessor(Mockito.anyString())).then(processorLookupAnswer); // authorizer authorizer = mock(Authorizer.class); @@ -323,6 +327,7 @@ public Resource getResource() { processGroupDAO = mock(ProcessGroupDAO.class, Answers.RETURNS_DEEP_STUBS); ruleViolationsManager = mock(RuleViolationsManager.class); + connectorManagedComponentLookup = mock(ConnectorManagedComponentLookup.class); serviceFacade = new StandardNiFiServiceFacade(); serviceFacade.setAuditService(auditService); @@ -333,6 +338,7 @@ public Resource getResource() { serviceFacade.setControllerFacade(controllerFacade); serviceFacade.setProcessGroupDAO(processGroupDAO); serviceFacade.setRuleViolationsManager(ruleViolationsManager); + serviceFacade.setConnectorManagedComponentLookup(connectorManagedComponentLookup); } @@ -1114,7 +1120,7 @@ public void testGetBulletinBoard_BulletinAuthorizedViaOwningProcessGroupWhenAuth final ProcessorNode approvingProcessor = approvingProcessorNode(); when(processGroup.findProcessor(PROCESSOR_ID_2)).thenReturn(approvingProcessor); - when(processGroupDAO.getProcessGroup(groupId, true)).thenReturn(processGroup); + when(connectorManagedComponentLookup.getProcessGroup(groupId)).thenReturn(processGroup); final StandardNiFiServiceFacade serviceFacadeSpy = spy(serviceFacade); final MockTestBulletinRepository bulletinRepository = new MockTestBulletinRepository(); @@ -1131,7 +1137,7 @@ public void testGetBulletinBoard_BulletinAuthorizedViaOwningProcessGroupWhenAuth assertEquals(1, board.getBulletins().size()); assertTrue(board.getBulletins().get(0).getCanRead(), "Bulletin board canRead should be true when the source is resolved via the owning ProcessGroup"); - verify(processGroupDAO).getProcessGroup(groupId, true); + verify(connectorManagedComponentLookup).getProcessGroup(groupId); } private ProcessGroupEntity invokeUpdateProcessGroupWithBulletin(final String groupId, final ProcessGroup processGroup, diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectionDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectionDAOTest.java index db8a22f34e9c..20df04a91601 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectionDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectionDAOTest.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorRepository; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.components.connector.ConnectorSyncMode; import org.apache.nifi.components.connector.FrameworkFlowContext; import org.apache.nifi.connectable.Connection; @@ -34,6 +35,7 @@ import org.mockito.quality.Strictness; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -46,6 +48,7 @@ class StandardConnectionDAOTest { private StandardConnectionDAO connectionDAO; + private StandardConnectorManagedComponentLookup connectorManagedComponentLookup; @Mock private FlowController flowController; @@ -83,6 +86,9 @@ void setUp() { connectionDAO = new StandardConnectionDAO(); connectionDAO.setFlowController(flowController); + connectorManagedComponentLookup = new StandardConnectorManagedComponentLookup(); + connectorManagedComponentLookup.setConnectionDAO(connectionDAO); + when(flowController.getFlowManager()).thenReturn(flowManager); when(flowManager.getRootGroup()).thenReturn(rootGroup); when(flowController.getConnectorRepository()).thenReturn(connectorRepository); @@ -92,13 +98,18 @@ void setUp() { when(rootGroup.findConnection(CONNECTOR_CONNECTION_ID)).thenReturn(null); when(rootGroup.findConnection(NON_EXISTENT_ID)).thenReturn(null); - // Setup connector managed group when(connectorRepository.getConnectors(ConnectorSyncMode.LOCAL_ONLY)).thenReturn(List.of(connectorNode)); when(connectorNode.getActiveFlowContext()).thenReturn(frameworkFlowContext); when(frameworkFlowContext.getManagedProcessGroup()).thenReturn(connectorManagedGroup); when(connectorManagedGroup.findConnection(CONNECTOR_CONNECTION_ID)).thenReturn(connectorConnection); when(connectorManagedGroup.findConnection(ROOT_CONNECTION_ID)).thenReturn(null); when(connectorManagedGroup.findConnection(NON_EXISTENT_ID)).thenReturn(null); + + // Attach the connector-managed Connection to a ProcessGroup owned by a Connector that is not in + // Troubleshooting mode so that access checks behave as they would for a real connector-managed component. + when(connectorConnection.getProcessGroup()).thenReturn(connectorManagedGroup); + when(connectorManagedGroup.findOwningConnector()).thenReturn(Optional.of(connectorNode)); + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.STOPPED); } @Test @@ -109,44 +120,38 @@ void testGetConnectionFromRootGroup() { } @Test - void testGetConnectionFromRootGroupWithIncludeConnectorManagedFalse() { - final Connection result = connectionDAO.getConnection(ROOT_CONNECTION_ID, false); - - assertEquals(rootConnection, result); - } - - @Test - void testGetConnectionFromRootGroupWithIncludeConnectorManagedTrue() { - final Connection result = connectionDAO.getConnection(ROOT_CONNECTION_ID, true); + void testConnectorManagedLookupReturnsRootConnection() { + final Connection result = connectorManagedComponentLookup.getConnection(ROOT_CONNECTION_ID); assertEquals(rootConnection, result); } @Test - void testGetConnectionFromConnectorManagedGroupWithIncludeConnectorManagedTrue() { - final Connection result = connectionDAO.getConnection(CONNECTOR_CONNECTION_ID, true); + void testConnectorManagedLookupReturnsConnectorConnectionRegardlessOfConnectorState() { + final Connection result = connectorManagedComponentLookup.getConnection(CONNECTOR_CONNECTION_ID); assertEquals(connectorConnection, result); } @Test - void testGetConnectionFromConnectorManagedGroupWithIncludeConnectorManagedFalseThrows() { - assertThrows(ResourceNotFoundException.class, () -> - connectionDAO.getConnection(CONNECTOR_CONNECTION_ID, false) + void testGetConnectionOnConnectorManagedConnectionThrowsWhenConnectorNotTroubleshooting() { + assertThrows(IllegalStateException.class, () -> + connectionDAO.getConnection(CONNECTOR_CONNECTION_ID) ); } @Test - void testGetConnectionWithDefaultDoesNotFindConnectorManagedConnection() { - assertThrows(ResourceNotFoundException.class, () -> - connectionDAO.getConnection(CONNECTOR_CONNECTION_ID) - ); + void testGetConnectionFromConnectorManagedConnectionInTroubleshootingReturnsConnection() { + when(connectorNode.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + + final Connection result = connectionDAO.getConnection(CONNECTOR_CONNECTION_ID); + assertEquals(connectorConnection, result); } @Test void testGetConnectionWithNonExistentIdThrows() { assertThrows(ResourceNotFoundException.class, () -> - connectionDAO.getConnection(NON_EXISTENT_ID, true) + connectorManagedComponentLookup.getConnection(NON_EXISTENT_ID) ); } @@ -172,7 +177,7 @@ void testGetConnectionFromConnectorWithNullActiveFlowContext() { when(connectorNode.getActiveFlowContext()).thenReturn(null); assertThrows(ResourceNotFoundException.class, () -> - connectionDAO.getConnection(CONNECTOR_CONNECTION_ID, true) + connectorManagedComponentLookup.getConnection(CONNECTOR_CONNECTION_ID) ); } @@ -190,7 +195,7 @@ void testGetConnectionWithMultipleConnectors() { when(flowContext2.getManagedProcessGroup()).thenReturn(managedGroup2); when(managedGroup2.findConnection(secondConnectorConnectionId)).thenReturn(connectionInSecondConnector); - final Connection result = connectionDAO.getConnection(secondConnectorConnectionId, true); + final Connection result = connectorManagedComponentLookup.getConnection(secondConnectorConnectionId); assertEquals(connectionInSecondConnector, result); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAOTest.java index cf5441f199ab..9e6767f3239d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessGroupDAOTest.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.components.connector.ConnectorState; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.groups.ProcessGroup; @@ -28,6 +30,8 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import java.util.Optional; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; @@ -37,6 +41,7 @@ class StandardProcessGroupDAOTest { private StandardProcessGroupDAO processGroupDAO; + private StandardConnectorManagedComponentLookup connectorManagedComponentLookup; @Mock private FlowController flowController; @@ -50,6 +55,9 @@ class StandardProcessGroupDAOTest { @Mock private ProcessGroup connectorManagedGroup; + @Mock + private ConnectorNode owningConnector; + private static final String ROOT_GROUP_ID = "root-group-id"; private static final String CONNECTOR_GROUP_ID = "connector-group-id"; private static final String NON_EXISTENT_ID = "non-existent-id"; @@ -59,17 +67,21 @@ void setUp() { processGroupDAO = new StandardProcessGroupDAO(); processGroupDAO.setFlowController(flowController); + connectorManagedComponentLookup = new StandardConnectorManagedComponentLookup(); + connectorManagedComponentLookup.setFlowController(flowController); + when(flowController.getFlowManager()).thenReturn(flowManager); - // Setup root group lookup (non-connector managed) when(flowManager.getGroup(ROOT_GROUP_ID, null)).thenReturn(rootGroup); when(flowManager.getGroup(CONNECTOR_GROUP_ID, null)).thenReturn(null); when(flowManager.getGroup(NON_EXISTENT_ID, null)).thenReturn(null); - // Setup connector-managed group lookup (includes all groups) when(flowManager.getGroup(ROOT_GROUP_ID)).thenReturn(rootGroup); when(flowManager.getGroup(CONNECTOR_GROUP_ID)).thenReturn(connectorManagedGroup); when(flowManager.getGroup(NON_EXISTENT_ID)).thenReturn(null); + + when(connectorManagedGroup.findOwningConnector()).thenReturn(Optional.of(owningConnector)); + when(owningConnector.getCurrentState()).thenReturn(ConnectorState.STOPPED); } @Test @@ -80,51 +92,45 @@ void testGetProcessGroupFromRootHierarchy() { } @Test - void testGetProcessGroupFromRootHierarchyWithIncludeConnectorManagedFalse() { - final ProcessGroup result = processGroupDAO.getProcessGroup(ROOT_GROUP_ID, false); - - assertEquals(rootGroup, result); - } - - @Test - void testGetProcessGroupFromRootHierarchyWithIncludeConnectorManagedTrue() { - final ProcessGroup result = processGroupDAO.getProcessGroup(ROOT_GROUP_ID, true); + void testConnectorManagedLookupReturnsRootGroup() { + final ProcessGroup result = connectorManagedComponentLookup.getProcessGroup(ROOT_GROUP_ID); assertEquals(rootGroup, result); } @Test - void testGetProcessGroupFromConnectorManagedWithIncludeConnectorManagedTrue() { - final ProcessGroup result = processGroupDAO.getProcessGroup(CONNECTOR_GROUP_ID, true); + void testConnectorManagedLookupReturnsConnectorManagedGroupRegardlessOfConnectorState() { + final ProcessGroup result = connectorManagedComponentLookup.getProcessGroup(CONNECTOR_GROUP_ID); assertEquals(connectorManagedGroup, result); } @Test - void testGetProcessGroupFromConnectorManagedWithIncludeConnectorManagedFalseThrows() { - assertThrows(ResourceNotFoundException.class, () -> - processGroupDAO.getProcessGroup(CONNECTOR_GROUP_ID, false) + void testGetProcessGroupOnConnectorManagedGroupThrowsWhenConnectorNotTroubleshooting() { + assertThrows(IllegalStateException.class, () -> + processGroupDAO.getProcessGroup(CONNECTOR_GROUP_ID) ); } @Test - void testGetProcessGroupWithDefaultDoesNotFindConnectorManagedGroup() { - assertThrows(ResourceNotFoundException.class, () -> - processGroupDAO.getProcessGroup(CONNECTOR_GROUP_ID) - ); + void testGetProcessGroupFromConnectorManagedWhenInTroubleshootingReturnsGroup() { + when(owningConnector.getCurrentState()).thenReturn(ConnectorState.TROUBLESHOOTING); + + final ProcessGroup result = processGroupDAO.getProcessGroup(CONNECTOR_GROUP_ID); + assertEquals(connectorManagedGroup, result); } @Test - void testGetProcessGroupWithNonExistentIdThrows() { + void testGetProcessGroupWithNonExistentIdThrowsThroughConnectorManagedLookup() { assertThrows(ResourceNotFoundException.class, () -> - processGroupDAO.getProcessGroup(NON_EXISTENT_ID, true) + connectorManagedComponentLookup.getProcessGroup(NON_EXISTENT_ID) ); } @Test - void testGetProcessGroupWithNonExistentIdAndIncludeConnectorManagedFalseThrows() { + void testGetProcessGroupWithNonExistentIdThrows() { assertThrows(ResourceNotFoundException.class, () -> - processGroupDAO.getProcessGroup(NON_EXISTENT_ID, false) + processGroupDAO.getProcessGroup(NON_EXISTENT_ID) ); } } diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java index 2e927bb038a7..4b710ae53bc5 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/AssetConnector.java @@ -63,6 +63,12 @@ public VersionedExternalFlow getInitialFlow() { return null; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // This Connector does not manage a flow; there is no authoritative Active flow. + return null; + } + @Override public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException { // No-op: this connector does not manipulate the flow. diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java index f15d668ef51f..9106e9eeb5fd 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BundleResolutionConnector.java @@ -110,6 +110,13 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // After configuration, the authoritative flow is the one produced by createFlowWithBundleScenarios, + // which is what applyUpdate installs into the managed Process Group. + return createFlowWithBundleScenarios(); + } + private VersionedExternalFlow createFlowWithBundleScenarios() { final VersionedProcessGroup group = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Bundle Resolution Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java index bb767219154d..c5ac49bb9afc 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/CalculateConnector.java @@ -172,6 +172,13 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The flow structure does not depend on the Connector's configuration; applyUpdate only invokes a + // processor method and writes output to a file, it does not modify the flow itself. + return getInitialFlow(); + } + @Override public List getConfigurationSteps() { return List.of(CALCULATION_STEP); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java index 81c6af171b13..9eddbf597e02 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ComponentLifecycleConnector.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; /** * Test Connector designed to verify the complete component lifecycle. @@ -54,6 +53,7 @@ public class ComponentLifecycleConnector extends AbstractConnector { private static final Bundle SYSTEM_TEST_EXTENSIONS_BUNDLE = new Bundle("org.apache.nifi", "nifi-system-test-extensions-nar", "2.8.0-SNAPSHOT"); + private static final String ROOT_GROUP_ID = "component-lifecycle-root-group"; @Override protected void onStepConfigured(final String stepName, final FlowContext workingContext) { @@ -67,8 +67,14 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // This Connector's flow is fully determined statically and does not depend on runtime configuration. + return getInitialFlow(); + } + private VersionedProcessGroup createRootGroup() { - final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(UUID.randomUUID().toString(), "Component Lifecycle Root"); + final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(ROOT_GROUP_ID, "Component Lifecycle Root"); rootGroup.setPosition(new Position(0, 0)); rootGroup.setRemoteProcessGroups(new HashSet<>()); rootGroup.setScheduledState(ScheduledState.ENABLED); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java index 1c50941c8371..8d331bab4306 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/DataQueuingConnector.java @@ -63,6 +63,12 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // This Connector's flow is static and does not depend on runtime configuration. + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java index 8c7f680d86ba..9abcbf87c59a 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/GatedDataQueuingConnector.java @@ -94,6 +94,21 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative flow is the initial flow with the currently configured Gate File Path applied to the + // TerminateFlowFile processor. This mirrors the behavior of applyUpdate so that exiting Troubleshooting + // restores a flow equivalent to re-applying the active configuration. + final VersionedExternalFlow flow = getInitialFlow(); + final String gateFilePath = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, GATE_FILE_PATH).getValue(); + if (gateFilePath != null) { + VersionedFlowUtils.findProcessor(flow.getFlowContents(), processor -> processor.getType().endsWith("TerminateFlowFile")) + .ifPresent(processor -> processor.getProperties().put("Gate File", gateFilePath)); + } + + return flow; + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java index e3903a38f1de..1070e28dc8c3 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NestedProcessGroupConnector.java @@ -58,6 +58,12 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // This Connector's flow structure is static and does not depend on runtime configuration. + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java index 3d28240ca256..2409c3200465 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java @@ -100,6 +100,12 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // No-op Connector: the authoritative flow is always the initial flow and does not depend on configuration. + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(new ConfigVerificationResult.Builder() diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java index 9d160a11f1c2..fc2047c444dc 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/ParameterContextConnector.java @@ -134,6 +134,21 @@ public VersionedExternalFlow getInitialFlow() { return createEmptyFlow(); } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow is the fully-constructed flow produced by applying the current + // configuration values. Before configuration has been applied, fall back to the initial empty flow. + final String sensitiveValue = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, SENSITIVE_VALUE).getValue(); + final String assetFilePath = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, ASSET_FILE).getValue(); + if (sensitiveValue == null || assetFilePath == null) { + return createEmptyFlow(); + } + + final String sensitiveOutputFile = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, SENSITIVE_OUTPUT_FILE).getValue(); + final String assetOutputFile = activeFlowContext.getConfigurationContext().getProperty(CONFIG_STEP, ASSET_OUTPUT_FILE).getValue(); + return createFlow(sensitiveValue, assetFilePath, sensitiveOutputFile, assetOutputFile); + } + private VersionedExternalFlow createEmptyFlow() { final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup(ROOT_GROUP_ID, "Parameter Context Test Flow"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java index 441f4bc3d47f..61d9eabb8958 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/SelectiveDropConnector.java @@ -83,6 +83,12 @@ public VersionedExternalFlow getInitialFlow() { return flow; } + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // This Connector's flow is fully determined statically and does not depend on runtime configuration. + return getInitialFlow(); + } + @Override public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { return List.of(); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 8960e202ce3e..02c020df9a1e 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -426,8 +426,59 @@ public void stopConnectors() throws NiFiClientException, IOException, Interrupte final ConnectorsEntity connectorsEntity = nifiClient.getFlowClient().getConnectors(); for (final ConnectorEntity connector : connectorsEntity.getConnectors()) { connector.setDisconnectedNodeAcknowledged(true); - getConnectorClient().stopConnector(connector); - waitForConnectorStopped(connector.getId()); + final String state = connector.getComponent() == null ? null : connector.getComponent().getState(); + + if (ConnectorState.TROUBLESHOOTING.name().equals(state)) { + final String managedGroupId = connector.getComponent().getManagedProcessGroupId(); + if (managedGroupId != null) { + try { + stopProcessGroupComponents(managedGroupId); + disableControllerServices(managedGroupId, true); + emptyConnectorManagedQueues(connector.getId(), managedGroupId); + } catch (final Exception stopException) { + logger.warn("Failed to prepare managed Process Group [{}] for Connector [{}] during teardown", + managedGroupId, connector.getId(), stopException); + } + } + + try { + endTroubleshooting(connector.getId()); + } catch (final Exception endException) { + logger.warn("Failed to end Troubleshooting for Connector [{}] during teardown", connector.getId(), endException); + } + + continue; + } + + try { + getConnectorClient().stopConnector(connector); + waitForConnectorStopped(connector.getId()); + } catch (final Exception stopException) { + logger.warn("Failed to stop Connector [{}] during teardown", connector.getId(), stopException); + } + } + } + + private void emptyConnectorManagedQueues(final String connectorId, final String groupId) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity flowEntity = getConnectorClient().getFlow(connectorId, groupId); + if (flowEntity == null || flowEntity.getProcessGroupFlow() == null || flowEntity.getProcessGroupFlow().getFlow() == null) { + return; + } + + final FlowDTO flow = flowEntity.getProcessGroupFlow().getFlow(); + if (flow.getConnections() != null) { + for (final ConnectionEntity connection : flow.getConnections()) { + try { + emptyQueue(connection.getId()); + } catch (final Exception ignored) { + } + } + } + + if (flow.getProcessGroups() != null) { + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + emptyConnectorManagedQueues(connectorId, child.getId()); + } } } @@ -490,6 +541,21 @@ public void waitForConnectorDraining(final String connectorId) throws NiFiClient waitForConnectorState(connectorId, ConnectorState.DRAINING); } + public ConnectorEntity enterTroubleshooting(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity entity = getConnectorClient().getConnector(connectorId); + entity.setDisconnectedNodeAcknowledged(true); + final ConnectorEntity result = getConnectorClient().enterTroubleshooting(entity); + waitForConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + return result; + } + + public ConnectorEntity endTroubleshooting(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity entity = getConnectorClient().getConnector(connectorId); + final ConnectorEntity result = getConnectorClient().endTroubleshooting(connectorId, entity.getRevision().getClientId(), entity.getRevision().getVersion()); + waitForConnectorStopped(connectorId); + return result; + } + public ParameterProviderEntity createParameterProvider(final String simpleTypeName) throws NiFiClientException, IOException { return createParameterProvider(NiFiSystemIT.TEST_PARAM_PROVIDERS_PACKAGE + "." + simpleTypeName, NiFiSystemIT.NIFI_GROUP_ID, NiFiSystemIT.TEST_EXTENSIONS_ARTIFACT_ID, nifiVersion); } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorTroubleshootingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorTroubleshootingIT.java new file mode 100644 index 000000000000..7c73e0800ee8 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorTroubleshootingIT.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; + +/** + * Clustered variant of {@link ConnectorTroubleshootingIT}. All Troubleshooting lifecycle tests defined on the parent + * class are re-executed against a two-node NiFi cluster so that the cluster-replication and node-restart paths are + * exercised in addition to the standalone paths. + * + *

This class deliberately does not redefine any of the parent's test methods. The verification of restart-while-in- + * Troubleshooting and authoritative-flow restoration that already exists in + * {@link ConnectorTroubleshootingIT#testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRestart()} (and + * the related restart tests) is the clustered coverage the user requested. Adding the override of + * {@link #getInstanceFactory()} below is sufficient to run all of those scenarios on a two-node cluster.

+ */ +public class ClusteredConnectorTroubleshootingIT extends ConnectorTroubleshootingIT { + + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java index 3dec256e24f5..38a0d1c1fd7f 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java @@ -17,7 +17,8 @@ package org.apache.nifi.tests.system.connectors; -import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.components.connector.BundleCompatibility; import org.apache.nifi.components.connector.ConnectorState; @@ -194,12 +195,8 @@ public void testProcessGroupAccessibility() throws NiFiClientException, IOExcept assertNotNull(connectorFlow); assertEquals(managedProcessGroupId, connectorFlow.getId()); - try { - getNifiClient().getFlowClient().getProcessGroup(managedProcessGroupId); - fail("Was able to retrieve connector-managed process group via FlowClient"); - } catch (final NiFiClientException e) { - assertInstanceOf(NotFoundException.class, e.getCause()); - } + assertConnectorManagedAccessRejected(() -> getNifiClient().getFlowClient().getProcessGroup(managedProcessGroupId), + "Was able to retrieve connector-managed process group via FlowClient"); final Set childGroups = connectorFlow.getFlow().getProcessGroups(); assertEquals(1, childGroups.size(), "Expected exactly one child process group"); @@ -212,14 +209,25 @@ public void testProcessGroupAccessibility() throws NiFiClientException, IOExcept assertNotNull(childFlowEntity); assertEquals(childGroupId, childFlowEntity.getProcessGroupFlow().getId()); + assertConnectorManagedAccessRejected(() -> getNifiClient().getFlowClient().getProcessGroup(childGroupId), + "Was able to retrieve child process group of connector-managed flow via FlowClient"); + } + + private void assertConnectorManagedAccessRejected(final FlowClientCall call, final String failureMessage) throws IOException { try { - getNifiClient().getFlowClient().getProcessGroup(childGroupId); - fail("Was able to retrieve child process group of connector-managed flow via FlowClient"); + call.invoke(); + fail(failureMessage); } catch (final NiFiClientException e) { - assertInstanceOf(NotFoundException.class, e.getCause()); + final WebApplicationException cause = assertInstanceOf(WebApplicationException.class, e.getCause()); + assertEquals(Response.Status.CONFLICT.getStatusCode(), cause.getResponse().getStatus()); } } + @FunctionalInterface + private interface FlowClientCall { + void invoke() throws NiFiClientException, IOException; + } + @Test public void testBundleResolutionRequireExactBundle() throws NiFiClientException, IOException, InterruptedException { final ConnectorAndProcessor connectorAndProcessor = createBundleResolutionConnector(BundleCompatibility.REQUIRE_EXACT_BUNDLE); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java new file mode 100644 index 000000000000..e6cd77dab2da --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorTroubleshootingIT.java @@ -0,0 +1,1021 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.tests.system.connectors; + +import jakarta.ws.rs.WebApplicationException; +import org.apache.nifi.components.connector.ConnectorState; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; +import org.apache.nifi.web.api.dto.ConnectorConfigurationDTO; +import org.apache.nifi.web.api.dto.ConnectorValueReferenceDTO; +import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.web.api.dto.ProcessGroupDTO; +import org.apache.nifi.web.api.dto.ProcessorDTO; +import org.apache.nifi.web.api.dto.flow.FlowDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.ConnectionEntity; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.ControllerServiceEntity; +import org.apache.nifi.web.api.entity.ParameterProviderEntity; +import org.apache.nifi.web.api.entity.PortEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * System tests that validate the Troubleshooting lifecycle of Connectors. + */ +public class ConnectorTroubleshootingIT extends NiFiSystemIT { + + /** + * Transition a Connector into Troubleshooting, modify a processor inside the managed flow, then transition back + * out. The Connector's authoritative flow should be restored on exit and the Connector should start smoothly. + */ + @Test + public void testEnterAndExitTroubleshootingRestoresFlow() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + final List originalProcessors = findAllProcessors(connectorId); + assertFalse(originalProcessors.isEmpty(), "Managed flow should contain processors"); + final ProcessorEntity originalProcessor = originalProcessors.get(0); + final String originalSchedulingPeriod = originalProcessor.getComponent().getConfig().getSchedulingPeriod(); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + final ProcessorEntity fetchedProcessor = getNifiClient().getProcessorClient().getProcessor(originalProcessor.getId()); + assertNotNull(fetchedProcessor); + + fetchedProcessor.getComponent().getConfig().setSchedulingPeriod("42 sec"); + final ProcessorEntity updatedProcessor = getNifiClient().getProcessorClient().updateProcessor(fetchedProcessor); + assertEquals("42 sec", updatedProcessor.getComponent().getConfig().getSchedulingPeriod()); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + final ProcessorEntity restoredProcessor = findProcessorByName(connectorId, originalProcessor.getComponent().getName()); + assertNotNull(restoredProcessor, "Expected original processor to be restored by authoritative flow"); + assertEquals(originalSchedulingPeriod, restoredProcessor.getComponent().getConfig().getSchedulingPeriod(), + "Scheduling period should be restored to authoritative value"); + + getClientUtil().startConnector(connectorId); + assertConnectorState(connectorId, ConnectorState.RUNNING); + } + + /** + * Transition into Troubleshooting, add a new Connection, queue up data in that Connection, and verify that ending + * Troubleshooting fails with a 409. Restart NiFi and verify the data is still queued. Drop all FlowFiles, then end + * Troubleshooting successfully. + */ + @Test + public void testEndTroubleshootingBlockedByQueuedData() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + getClientUtil().enterTroubleshooting(connectorId); + + final String managedGroupId = getNifiClient().getConnectorClient().getConnector(connectorId).getComponent().getManagedProcessGroupId(); + assertNotNull(managedGroupId); + + final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile", managedGroupId); + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile", managedGroupId); + getClientUtil().updateProcessorProperties(generate, Map.of("File Size", "10 B")); + getClientUtil().updateProcessorSchedulingPeriod(generate, "10 ms"); + + final ConnectionEntity connection = getClientUtil().createConnection(generate, terminate, "success", managedGroupId); + final String connectionId = connection.getId(); + + getClientUtil().startProcessor(generate); + waitForQueuedFlowFiles(connectorId, connectionId, 1); + getClientUtil().stopProcessor(generate); + + try { + getClientUtil().endTroubleshooting(connectorId); + fail("Expected endTroubleshooting to fail with 409 while connection has queued data"); + } catch (final NiFiClientException e) { + assertConflict(e); + } + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + final OptionalInt queued = getQueuedCount(connectorId, connectionId); + assertTrue(queued.isPresent(), "Connection must still be present after restart"); + assertTrue(queued.getAsInt() > 0, "Queued data must survive restart"); + + getClientUtil().emptyQueue(connectionId); + waitForQueuedFlowFiles(connectorId, connectionId, 0); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + final List processorIdsAfterRestore = findAllProcessors(connectorId).stream() + .map(ProcessorEntity::getId) + .toList(); + + assertFalse(processorIdsAfterRestore.contains(generate.getId()), "GenerateFlowFile processor added in Troubleshooting should be removed once authoritative flow is restored"); + assertFalse(processorIdsAfterRestore.contains(terminate.getId()), "TerminateFlowFile processor added in Troubleshooting should be removed once authoritative flow is restored"); + assertFalse(getQueuedCount(connectorId, connectionId).isPresent(), "Connection added in Troubleshooting should be removed once authoritative flow is restored"); + } + + /** + * Verify that GET/PUT/POST/DELETE on a processor that lives within a Connector's managed flow is denied with a 409 + * when the Connector is not in Troubleshooting mode, and succeeds once it is. + */ + @Test + public void testComponentAccessBlockedWhenNotInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + final List processors = findAllProcessors(connectorId); + assertFalse(processors.isEmpty()); + final String processorId = processors.get(0).getId(); + + try { + getNifiClient().getProcessorClient().getProcessor(processorId); + fail("Expected 409 Conflict retrieving processor managed by Connector when not in Troubleshooting"); + } catch (final NiFiClientException e) { + assertConflict(e); + } + + getClientUtil().enterTroubleshooting(connectorId); + final ProcessorEntity processor = getNifiClient().getProcessorClient().getProcessor(processorId); + assertNotNull(processor); + assertEquals(processorId, processor.getId()); + } + + /** + * Stop a connector, enter Troubleshooting, start processors inside the managed flow, restart NiFi, and ensure the + * processors are still running and the Connector remains in Troubleshooting. + */ + @Test + public void testProcessorsInTroubleshootingStillRunningAfterRestart() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + getClientUtil().enterTroubleshooting(connectorId); + + final List processors = findAllProcessors(connectorId); + assertFalse(processors.isEmpty()); + + final List startedProcessorIds = new ArrayList<>(); + for (final ProcessorEntity processor : processors) { + if (ScheduledState.DISABLED.name().equals(processor.getComponent().getState())) { + continue; + } + + try { + getClientUtil().startProcessor(processor); + startedProcessorIds.add(processor.getId()); + } catch (final Exception ignored) { + } + } + + assertFalse(startedProcessorIds.isEmpty(), "Expected at least one processor to be started"); + + for (final String id : startedProcessorIds) { + waitForProcessorState(id, ScheduledState.RUNNING); + } + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + for (final String id : startedProcessorIds) { + waitForProcessorState(id, ScheduledState.RUNNING); + } + } + + /** + * A running Connector must be able to transition into + * Troubleshooting mode without the framework stopping any of the components inside the managed flow. Any component + * that was RUNNING prior to entering Troubleshooting must remain RUNNING after the transition, and must also + * survive a restart of NiFi while the Connector stays in Troubleshooting. + */ + @Test + public void testEnterTroubleshootingFromRunningKeepsProcessorsRunning() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + getClientUtil().startConnector(connectorId); + assertConnectorState(connectorId, ConnectorState.RUNNING); + + final List runningBeforeTroubleshooting = findProcessorsInState(connectorId, ScheduledState.RUNNING); + assertFalse(runningBeforeTroubleshooting.isEmpty(), "Expected at least one processor to be RUNNING before entering Troubleshooting"); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + for (final ProcessorEntity processor : runningBeforeTroubleshooting) { + final ProcessorEntity refreshed = getNifiClient().getProcessorClient().getProcessor(processor.getId()); + assertEquals(ScheduledState.RUNNING.name(), refreshed.getComponent().getState(), + "Processor " + refreshed.getComponent().getName() + " must remain RUNNING after entering Troubleshooting"); + } + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + for (final ProcessorEntity processor : runningBeforeTroubleshooting) { + waitForProcessorState(processor.getId(), ScheduledState.RUNNING); + } + } + + /** + * While a Connector is in Troubleshooting mode, lifecycle operations on the Connector itself (start, stop, + * applyUpdate, drain, purge, and delete) must be blocked with a 409 Conflict. Once Troubleshooting is exited those + * operations become available again. + */ + @Test + public void testConnectorLifecycleBlockedDuringTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + assertConflictExpected("startConnector", () -> { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + getNifiClient().getConnectorClient().startConnector(entity); + }); + + assertConflictExpected("stopConnector", () -> { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + getNifiClient().getConnectorClient().stopConnector(entity); + }); + + assertConflictExpected("applyUpdate", () -> { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + getNifiClient().getConnectorClient().applyUpdate(entity); + }); + + assertConflictExpected("drainConnector", () -> { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + getNifiClient().getConnectorClient().drainConnector(entity); + }); + + assertConflictExpected("createPurgeRequest", () -> getNifiClient().getConnectorClient().createPurgeRequest(connectorId)); + + assertConflictExpected("deleteConnector", () -> { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + getNifiClient().getConnectorClient().deleteConnector(entity); + }); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + getClientUtil().startConnector(connectorId); + assertConnectorState(connectorId, ConnectorState.RUNNING); + } + + /** + * Verify that non-Processor component types inside a Connector's managed flow are reachable through the standard + * component REST endpoints while the Connector is in Troubleshooting mode, and that access is blocked with a 409 + * Conflict when the Connector is not in Troubleshooting mode. Covers Connections, Ports (input and output), + * ControllerServices, and child ProcessGroups. + */ + @Test + public void testNonProcessorComponentsInManagedFlow() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + // Enter Troubleshooting briefly to discover the IDs of components inside the managed flow. This is the only + // straightforward way to obtain the ControllerService ID; the rest are discoverable via the Connector's flow + // endpoint, but using Troubleshooting here keeps the discovery uniform. + getClientUtil().enterTroubleshooting(connectorId); + final String connectionId = findFirstConnectionId(connectorId); + final String inputPortId = findFirstInputPortId(connectorId); + final String outputPortId = findFirstOutputPortId(connectorId); + final String controllerServiceId = findFirstControllerServiceId(connectorId); + final String childGroupId = findFirstChildProcessGroupId(connectorId); + + assertNotNull(connectionId, "Managed flow should contain at least one Connection"); + assertNotNull(inputPortId, "Managed flow should contain at least one InputPort"); + assertNotNull(outputPortId, "Managed flow should contain at least one OutputPort"); + assertNotNull(controllerServiceId, "Managed flow should contain at least one ControllerService"); + assertNotNull(childGroupId, "Managed flow should contain at least one child ProcessGroup"); + + final ConnectionEntity connection = getNifiClient().getConnectionClient().getConnection(connectionId); + assertEquals(connectionId, connection.getId()); + + final PortEntity inputPort = getNifiClient().getInputPortClient().getInputPort(inputPortId); + assertEquals(inputPortId, inputPort.getId()); + + final PortEntity outputPort = getNifiClient().getOutputPortClient().getOutputPort(outputPortId); + assertEquals(outputPortId, outputPort.getId()); + + final ControllerServiceEntity controllerService = getNifiClient().getControllerServicesClient().getControllerService(controllerServiceId); + assertEquals(controllerServiceId, controllerService.getId()); + + final ProcessGroupEntity childGroup = getNifiClient().getProcessGroupClient().getProcessGroup(childGroupId); + assertEquals(childGroupId, childGroup.getId()); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + assertConflictExpected("GET connection", () -> getNifiClient().getConnectionClient().getConnection(connectionId)); + assertConflictExpected("GET input port", () -> getNifiClient().getInputPortClient().getInputPort(inputPortId)); + assertConflictExpected("GET output port", () -> getNifiClient().getOutputPortClient().getOutputPort(outputPortId)); + assertConflictExpected("GET controller service", () -> getNifiClient().getControllerServicesClient().getControllerService(controllerServiceId)); + assertConflictExpected("GET process group", () -> getNifiClient().getProcessGroupClient().getProcessGroup(childGroupId)); + } + + /** + * Verify that the Connector's flow endpoint correctly returns the flow for a non-root child process + * group within the managed flow while in Troubleshooting mode. The fix that enables this is in + * StandardProcessGroup.findProcessGroup: child groups within a connector's managed hierarchy do not + * carry a connectorId field on their own ProcessGroup object, so the old connector-ID-filtered + * getGroup(id, connectorId) lookup always returned null for them, causing a 404. The unconditional + * getGroup(id) lookup combined with the existing isOwner hierarchy check is the correct approach. + */ + @Test + public void testGetFlowForChildProcessGroupInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = getClientUtil().createConnector("ComponentLifecycleConnector"); + final String connectorId = connector.getId(); + + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + final ProcessGroupFlowEntity rootFlow = getNifiClient().getConnectorClient().getFlow(connectorId); + final List childGroups = new ArrayList<>(rootFlow.getProcessGroupFlow().getFlow().getProcessGroups()); + assertFalse(childGroups.isEmpty(), "ComponentLifecycleConnector managed flow must contain at least one child process group"); + + final String childGroupId = childGroups.get(0).getId(); + + final ProcessGroupFlowEntity childFlow = getNifiClient().getConnectorClient().getFlow(connectorId, childGroupId); + assertNotNull(childFlow); + assertEquals(childGroupId, childFlow.getProcessGroupFlow().getId()); + } + + private void assertConnectorState(final String connectorId, final ConnectorState expected) throws NiFiClientException, IOException { + final ConnectorEntity entity = getNifiClient().getConnectorClient().getConnector(connectorId); + assertEquals(expected.name(), entity.getComponent().getState()); + } + + /** + * Stop and restart the NiFi instance, then wait for all nodes to reconnect when running in a clustered + * environment. Subsequent flow-modifying requests (such as {@code endTroubleshooting}) would otherwise be rejected + * with HTTP 409 Conflict while one of the nodes is still in the CONNECTING state after the restart. + */ + private void restartNiFi() { + getNiFiInstance().stop(); + getNiFiInstance().start(); + + if (getNiFiInstance().isClustered()) { + waitForAllNodesConnected(); + } + } + + private void assertConflict(final NiFiClientException e) { + final Throwable cause = e.getCause(); + if (cause instanceof WebApplicationException wae) { + assertEquals(409, wae.getResponse().getStatus(), "Expected 409 Conflict, got: " + wae.getResponse().getStatus()); + return; + } + + fail("Expected WebApplicationException 409, got: " + cause); + } + + private List findAllProcessors(final String connectorId) throws NiFiClientException, IOException { + final List result = new ArrayList<>(); + collectProcessors(connectorId, null, result); + return result; + } + + private void collectProcessors(final String connectorId, final String groupId, final List collected) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity entity = (groupId == null) ? getNifiClient().getConnectorClient().getFlow(connectorId) : getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + final FlowDTO flow = entity.getProcessGroupFlow().getFlow(); + collected.addAll(flow.getProcessors()); + + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + collectProcessors(connectorId, child.getId(), collected); + } + } + + private ProcessorEntity findProcessorByName(final String connectorId, final String name) throws NiFiClientException, IOException { + for (final ProcessorEntity entity : findAllProcessors(connectorId)) { + final ProcessorDTO dto = entity.getComponent(); + if (name.equals(dto.getName())) { + return entity; + } + } + + return null; + } + + private OptionalInt getQueuedCount(final String connectorId, final String connectionId) throws NiFiClientException, IOException { + return collectQueuedCount(connectorId, null, connectionId); + } + + private OptionalInt collectQueuedCount(final String connectorId, final String groupId, final String connectionId) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity entity = (groupId == null) ? getNifiClient().getConnectorClient().getFlow(connectorId) : getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + + final FlowDTO flow = entity.getProcessGroupFlow().getFlow(); + for (final ConnectionEntity connection : flow.getConnections()) { + if (connectionId.equals(connection.getId())) { + final String queued = connection.getStatus().getAggregateSnapshot().getQueued(); + final String count = queued.substring(0, queued.indexOf(' ')); + return OptionalInt.of(Integer.parseInt(count.replace(",", ""))); + } + } + + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + final OptionalInt childResult = collectQueuedCount(connectorId, child.getId(), connectionId); + if (childResult.isPresent()) { + return childResult; + } + } + + return OptionalInt.empty(); + } + + private void waitForQueuedFlowFiles(final String connectorId, final String connectionId, final int minCount) throws InterruptedException { + waitFor(() -> { + try { + final OptionalInt queuedCount = getQueuedCount(connectorId, connectionId); + if (queuedCount.isEmpty()) { + return false; + } + final int count = queuedCount.getAsInt(); + if (minCount == 0) { + return count == 0; + } + return count >= minCount; + } catch (final Exception e) { + return false; + } + }); + } + + private void waitForProcessorState(final String processorId, final ScheduledState desired) throws InterruptedException { + waitFor(() -> { + try { + final ProcessorEntity entity = getNifiClient().getProcessorClient().getProcessor(processorId); + return desired.name().equals(entity.getComponent().getState()); + } catch (final Exception e) { + return false; + } + }); + } + + private List findProcessorsInState(final String connectorId, final ScheduledState state) throws NiFiClientException, IOException { + final List matches = new ArrayList<>(); + for (final ProcessorEntity entity : findAllProcessors(connectorId)) { + if (state.name().equals(entity.getComponent().getState())) { + matches.add(entity); + } + } + return matches; + } + + private String findFirstConnectionId(final String connectorId) throws NiFiClientException, IOException { + final FlowDTO rootFlow = getNifiClient().getConnectorClient().getFlow(connectorId).getProcessGroupFlow().getFlow(); + if (!rootFlow.getConnections().isEmpty()) { + return rootFlow.getConnections().iterator().next().getId(); + } + for (final ProcessGroupEntity child : rootFlow.getProcessGroups()) { + final FlowDTO childFlow = getNifiClient().getConnectorClient().getFlow(connectorId, child.getId()).getProcessGroupFlow().getFlow(); + if (!childFlow.getConnections().isEmpty()) { + return childFlow.getConnections().iterator().next().getId(); + } + } + return null; + } + + private String findFirstInputPortId(final String connectorId) throws NiFiClientException, IOException { + return findFirstPortId(connectorId, true); + } + + private String findFirstOutputPortId(final String connectorId) throws NiFiClientException, IOException { + return findFirstPortId(connectorId, false); + } + + private String findFirstPortId(final String connectorId, final boolean input) throws NiFiClientException, IOException { + final FlowDTO rootFlow = getNifiClient().getConnectorClient().getFlow(connectorId).getProcessGroupFlow().getFlow(); + for (final PortEntity port : input ? rootFlow.getInputPorts() : rootFlow.getOutputPorts()) { + return port.getId(); + } + for (final ProcessGroupEntity child : rootFlow.getProcessGroups()) { + final FlowDTO childFlow = getNifiClient().getConnectorClient().getFlow(connectorId, child.getId()).getProcessGroupFlow().getFlow(); + for (final PortEntity port : input ? childFlow.getInputPorts() : childFlow.getOutputPorts()) { + return port.getId(); + } + } + return null; + } + + private String findFirstControllerServiceId(final String connectorId) throws NiFiClientException, IOException { + final String managedGroupId = getNifiClient().getConnectorClient().getConnector(connectorId).getComponent().getManagedProcessGroupId(); + return collectFirstControllerServiceId(managedGroupId); + } + + private String collectFirstControllerServiceId(final String groupId) throws NiFiClientException, IOException { + for (final ControllerServiceEntity entity : getNifiClient().getFlowClient().getControllerServices(groupId).getControllerServices()) { + final ControllerServiceDTO dto = entity.getComponent(); + if (dto != null) { + return entity.getId(); + } + } + + final ProcessGroupFlowEntity flow = getNifiClient().getFlowClient().getProcessGroup(groupId); + for (final ProcessGroupEntity child : flow.getProcessGroupFlow().getFlow().getProcessGroups()) { + final String childServiceId = collectFirstControllerServiceId(child.getId()); + if (childServiceId != null) { + return childServiceId; + } + } + + return null; + } + + private String findFirstChildProcessGroupId(final String connectorId) throws NiFiClientException, IOException { + final FlowDTO flow = getNifiClient().getConnectorClient().getFlow(connectorId).getProcessGroupFlow().getFlow(); + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + final ProcessGroupDTO dto = child.getComponent(); + if (dto != null) { + return child.getId(); + } + } + return null; + } + + private void assertConflictExpected(final String description, final ConflictingCall call) { + try { + call.run(); + fail("Expected 409 Conflict for " + description + " but request succeeded"); + } catch (final NiFiClientException e) { + assertConflict(e); + } catch (final IOException e) { + fail("Unexpected IOException while invoking " + description + ": " + e.getMessage()); + } + } + + @FunctionalInterface + private interface ConflictingCall { + void run() throws NiFiClientException, IOException; + } + + /** + * Validates that a Connector whose managed flow references parameters correctly resolves those parameter values + * both before and after a NiFi restart while in Troubleshooting mode. Parameter values live inside the managed + * Process Group's ParameterContext. The Connector's managed Parameter Context is intentionally not registered + * with the global ParameterContextManager and is therefore not persisted in flow.json, so on every restart the + * Connector's lifecycle must re-populate it. The restart path for a Connector whose effective state is + * TROUBLESHOOTING goes through {@code StandardConnectorRepository#syncConnector}, which calls + * {@code ConnectorNode#inheritConfiguration} so the Connector's applyUpdate re-populates the Parameter Context + * from the persisted active configuration, and then immediately overlays the persisted Troubleshooting snapshot + * via {@code StandardFlowContext#restoreTroubleshootingFlow}. The structural overlay preserves the in-memory Parameter + * Context binding, so the Connector-supplied parameter values remain available while the user's flow + * modifications are the ones that run. The transient Connector-supplied flow shape is invisible outside this + * synchronization cycle: flow synchronization completes before the FlowFile Repository attaches FlowFiles to + * queues, so no FlowFile movement can observe it. + */ + @Test + public void testParameterValuesResolvedBeforeAndAfterRestartInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + // Use a secret name unique to this test so the underlying SecretsManager cannot return a value cached for the + // generic name "secret" by another test that ran earlier in the same JVM. + final String secretName = "parameterResolutionSecret"; + final String sensitiveSecretValue = "my-super-secret-value"; + final String assetFileContent = "Hello, World!"; + final File sensitiveOutputFile = new File("target/troubleshooting-sensitive.txt"); + final File assetOutputFile = new File("target/troubleshooting-asset.txt"); + sensitiveOutputFile.delete(); + assetOutputFile.delete(); + + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", secretName + "=" + sensitiveSecretValue)); + + final ConnectorEntity connector = getClientUtil().createConnector("ParameterContextConnector"); + final String connectorId = connector.getId(); + + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity assetEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetFile.getName(), assetFile); + final String uploadedAssetId = assetEntity.getAsset().getId(); + + final ConnectorValueReferenceDTO secretRef = getClientUtil().createSecretValueReference( + paramProvider.getId(), secretName, "PropertiesParameterProvider.Parameters." + secretName); + final ConnectorValueReferenceDTO assetRef = new ConnectorValueReferenceDTO(); + assetRef.setValueType("ASSET_REFERENCE"); + assetRef.setAssetReferences(List.of(new AssetReferenceDTO(uploadedAssetId))); + + final Map propertyValues = new HashMap<>(); + propertyValues.put("Sensitive Value", secretRef); + propertyValues.put("Asset File", assetRef); + propertyValues.put("Sensitive Output File", createStringLiteralRef(sensitiveOutputFile.getAbsolutePath())); + propertyValues.put("Asset Output File", createStringLiteralRef(assetOutputFile.getAbsolutePath())); + + getClientUtil().configureConnectorWithReferences(connectorId, "Parameter Context Configuration", propertyValues); + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + // Transition into Troubleshooting from STOPPED; components inside the managed flow are not yet running. + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // First verification: parameter values resolve correctly on the initial flow, before any restart. + runManagedFlowAndAssertParameterValues(connectorId, sensitiveOutputFile, assetOutputFile, sensitiveSecretValue, assetFileContent, "before restart"); + + // Stop every component and clear the output files so that the post-restart run can prove the parameter + // values were re-populated correctly rather than simply finding the files produced by the pre-restart run. + stopAllManagedComponents(connectorId); + assertTrue(sensitiveOutputFile.delete() || !sensitiveOutputFile.exists(), "Failed to delete sensitive output file between runs"); + assertTrue(assetOutputFile.delete() || !assetOutputFile.exists(), "Failed to delete asset output file between runs"); + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // Second verification: after the restore path has been exercised, the Parameter Context must still produce + // the correct resolved values when the processors are started again. + runManagedFlowAndAssertParameterValues(connectorId, sensitiveOutputFile, assetOutputFile, sensitiveSecretValue, assetFileContent, "after restart"); + } + + /** + * Validates that a Connector's configured (non-default) Active configuration survives a NiFi restart while the + * Connector is in Troubleshooting mode, and that flow modifications made during Troubleshooting are discarded + * when Troubleshooting exits. The end-to-end scenario: + *
    + *
  1. Create a Connector and configure it with non-default property values so its active configuration and + * active managed flow both differ from the unconfigured defaults.
    2. + *
  2. Apply the update so the active flow is the Connector's authoritative non-default flow.
    3. + *
  3. Enter Troubleshooting and modify the managed flow by adding a processor that the Connector's + * authoritative flow does not contain.
    4. + *
  4. Restart NiFi while still in Troubleshooting.
    5. + *
  5. Exit Troubleshooting.
    6. + *
  6. Assert the active configuration still contains the configured (non-default) values, the active flow is + * the Connector's authoritative non-default flow rather than the user-modified Troubleshooting flow, and + * finally that running the Connector produces output at the configured non-default destinations.
    7. + *
+ */ + @Test + public void testConfigurationAndAuthoritativeFlowRestoredAfterTroubleshootingRestart() throws NiFiClientException, IOException, InterruptedException { + // Use a secret name unique to this test so the underlying SecretsManager cannot return a value cached for the + // generic name "secret" by another test that ran earlier in the same JVM. + final String secretName = "configurationRestoreSecret"; + final String sensitiveSecretValue = "configured-secret-value"; + final String assetFileContent = "Hello, World!"; + final File configuredSensitiveOutput = new File("target/configuration-restore-sensitive.txt"); + final File configuredAssetOutput = new File("target/configuration-restore-asset.txt"); + configuredSensitiveOutput.delete(); + configuredAssetOutput.delete(); + + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", secretName + "=" + sensitiveSecretValue)); + + final ConnectorEntity connector = getClientUtil().createConnector("ParameterContextConnector"); + final String connectorId = connector.getId(); + + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity assetEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetFile.getName(), assetFile); + final String uploadedAssetId = assetEntity.getAsset().getId(); + + final ConnectorValueReferenceDTO secretRef = getClientUtil().createSecretValueReference( + paramProvider.getId(), secretName, "PropertiesParameterProvider.Parameters." + secretName); + final ConnectorValueReferenceDTO assetRef = new ConnectorValueReferenceDTO(); + assetRef.setValueType("ASSET_REFERENCE"); + assetRef.setAssetReferences(List.of(new AssetReferenceDTO(uploadedAssetId))); + + // The output file paths differ from the property descriptors' default values, so the active configuration + // after restart can be checked against these specific paths to prove the configured values were preserved + // instead of being overwritten by defaults. + final Map propertyValues = new HashMap<>(); + propertyValues.put("Sensitive Value", secretRef); + propertyValues.put("Asset File", assetRef); + propertyValues.put("Sensitive Output File", createStringLiteralRef(configuredSensitiveOutput.getAbsolutePath())); + propertyValues.put("Asset Output File", createStringLiteralRef(configuredAssetOutput.getAbsolutePath())); + + getClientUtil().configureConnectorWithReferences(connectorId, "Parameter Context Configuration", propertyValues); + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + assertNotNull(findProcessorByName(connectorId, "UpdateContent"), + "Active flow should contain UpdateContent before Troubleshooting"); + assertNotNull(findProcessorByName(connectorId, "ReplaceWithFile"), + "Active flow should contain ReplaceWithFile before Troubleshooting"); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // The Sleep processor is created in STOPPED state and is left disconnected so endTroubleshooting can later + // succeed without first having to stop or empty any user-introduced components. + final String managedGroupId = getNifiClient().getConnectorClient().getConnector(connectorId).getComponent().getManagedProcessGroupId(); + final ProcessorEntity troubleshootingProcessor = getClientUtil().createProcessor("Sleep", managedGroupId); + final String troubleshootingProcessorId = troubleshootingProcessor.getId(); + assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor should be present in the managed flow after adding it in Troubleshooting"); + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + assertTrue(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor should survive restart while in Troubleshooting"); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + final ConnectorEntity afterExit = getNifiClient().getConnectorClient().getConnector(connectorId); + final ConnectorConfigurationDTO activeConfig = afterExit.getComponent().getActiveConfiguration(); + final Map activeProperties = activeConfig.getConfigurationStepConfigurations().getFirst() + .getPropertyGroupConfigurations().getFirst().getPropertyValues(); + assertEquals(configuredSensitiveOutput.getAbsolutePath(), activeProperties.get("Sensitive Output File").getValue(), + "Active configuration must retain the configured Sensitive Output File path"); + assertEquals(configuredAssetOutput.getAbsolutePath(), activeProperties.get("Asset Output File").getValue(), + "Active configuration must retain the configured Asset Output File path"); + assertEquals("ASSET_REFERENCE", activeProperties.get("Asset File").getValueType(), + "Active configuration must retain the Asset reference for Asset File"); + assertEquals(uploadedAssetId, activeProperties.get("Asset File").getAssetReferences().get(0).getId(), + "Active configuration must retain the uploaded Asset id for Asset File"); + assertEquals("SECRET_REFERENCE", activeProperties.get("Sensitive Value").getValueType(), + "Active configuration must retain the Secret reference for Sensitive Value"); + + assertFalse(containsProcessorId(connectorId, troubleshootingProcessorId), + "User-added Sleep processor must be removed once the authoritative flow is restored on Troubleshooting exit"); + assertNotNull(findProcessorByName(connectorId, "UpdateContent"), + "Restored authoritative flow should contain UpdateContent"); + assertNotNull(findProcessorByName(connectorId, "ReplaceWithFile"), + "Restored authoritative flow should contain ReplaceWithFile"); + assertNotNull(findProcessorByName(connectorId, "GenerateFlowFile"), + "Restored authoritative flow should contain GenerateFlowFile"); + + getClientUtil().startConnector(connectorId); + assertConnectorState(connectorId, ConnectorState.RUNNING); + + waitFor(() -> configuredSensitiveOutput.exists() && configuredAssetOutput.exists()); + assertEquals(sensitiveSecretValue, Files.readString(configuredSensitiveOutput.toPath()).trim(), + "Running Connector must write the configured sensitive value to the configured Sensitive Output File"); + assertEquals(assetFileContent, Files.readString(configuredAssetOutput.toPath()).trim(), + "Running Connector must write the configured asset content to the configured Asset Output File"); + } + + /** + * Validates that Connector Assets are retained across a NiFi restart while the Connector is in Troubleshooting mode + * and that, on exit, the Processor properties inside the managed flow are restored to reference the assets that the + * Connector's authoritative flow expects. + * + *

The scenario:

+ *
    + *
  1. Create a {@code ParameterContextConnector}, upload Asset A, and configure the Connector to use Asset A.
    2. + *
  2. Apply the update so the active flow's {@code ReplaceWithFile} processor's {@code Filename} property is + * {@code #{asset_param}}, where {@code asset_param} resolves to Asset A's file path.
    3. + *
  3. Enter Troubleshooting.
    4. + *
  4. Upload a second Asset B, and override the managed flow's {@code ReplaceWithFile.Filename} property to a + * literal that does not reference Asset A. This represents a user editing the Processor's asset-bearing + * property while the Connector is open for direct edits.
    5. + *
  5. Restart NiFi.
    6. + *
  6. After restart, assert that the Connector is still in Troubleshooting, both Assets A and B are still listed, + * and the user's override of {@code Filename} survived the restart.
    7. + *
  7. End Troubleshooting.
    8. + *
  8. Assert that the Connector's authoritative configuration is restored: {@code Asset File} still references + * Asset A, and {@code ReplaceWithFile.Filename} is back to {@code #{asset_param}}.
    9. + *
+ */ + @Test + public void testAssetsRetainedAcrossRestartInTroubleshooting() throws NiFiClientException, IOException, InterruptedException { + final String secretName = "assetRetentionSecret"; + final String sensitiveSecretValue = "asset-retention-secret-value"; + final File sensitiveOutputFile = new File("target/asset-retention-sensitive.txt"); + final File assetOutputFile = new File("target/asset-retention-asset.txt"); + sensitiveOutputFile.delete(); + assetOutputFile.delete(); + + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", secretName + "=" + sensitiveSecretValue)); + + final ConnectorEntity connector = getClientUtil().createConnector("ParameterContextConnector"); + final String connectorId = connector.getId(); + + final File assetA = new File("src/test/resources/sample-assets/helloworld.txt"); + final File assetB = new File("src/test/resources/sample-assets/helloworld2.txt"); + final AssetEntity assetAEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetA.getName(), assetA); + final String assetAId = assetAEntity.getAsset().getId(); + + final ConnectorValueReferenceDTO secretRef = getClientUtil().createSecretValueReference( + paramProvider.getId(), secretName, "PropertiesParameterProvider.Parameters." + secretName); + final ConnectorValueReferenceDTO assetRef = new ConnectorValueReferenceDTO(); + assetRef.setValueType("ASSET_REFERENCE"); + assetRef.setAssetReferences(List.of(new AssetReferenceDTO(assetAId))); + + final Map propertyValues = new HashMap<>(); + propertyValues.put("Sensitive Value", secretRef); + propertyValues.put("Asset File", assetRef); + propertyValues.put("Sensitive Output File", createStringLiteralRef(sensitiveOutputFile.getAbsolutePath())); + propertyValues.put("Asset Output File", createStringLiteralRef(assetOutputFile.getAbsolutePath())); + + getClientUtil().configureConnectorWithReferences(connectorId, "Parameter Context Configuration", propertyValues); + getClientUtil().applyConnectorUpdate(connector); + getClientUtil().waitForValidConnector(connectorId); + + final ProcessorEntity replaceWithFileBeforeTroubleshooting = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileBeforeTroubleshooting, "Active flow should contain ReplaceWithFile before Troubleshooting"); + final String authoritativeFilenameValue = replaceWithFileBeforeTroubleshooting.getComponent().getConfig().getProperties().get("Filename"); + assertEquals("#{asset_param}", authoritativeFilenameValue, + "Authoritative ReplaceWithFile.Filename should be parameterized as #{asset_param} before Troubleshooting"); + + getClientUtil().enterTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + + // Upload a second Asset while in Troubleshooting and confirm it joins the existing Asset. + final AssetEntity assetBEntity = getNifiClient().getConnectorClient().createAsset(connectorId, assetB.getName(), assetB); + final String assetBId = assetBEntity.getAsset().getId(); + assertAssetIds(connectorId, assetAId, assetBId); + + // Override the Processor's asset-bearing property to a literal that does not reference the Connector's Asset A. + final ProcessorEntity replaceWithFileInTroubleshooting = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileInTroubleshooting, "ReplaceWithFile should remain accessible while in Troubleshooting"); + final String overriddenFilenameValue = "target/asset-retention-override.txt"; + getClientUtil().updateProcessorProperties(replaceWithFileInTroubleshooting, Map.of("Filename", overriddenFilenameValue)); + + final ProcessorEntity afterPropertyOverride = findProcessorByName(connectorId, "ReplaceWithFile"); + assertEquals(overriddenFilenameValue, afterPropertyOverride.getComponent().getConfig().getProperties().get("Filename"), + "ReplaceWithFile.Filename should reflect the user override applied while in Troubleshooting"); + + restartNiFi(); + + assertConnectorState(connectorId, ConnectorState.TROUBLESHOOTING); + assertAssetIds(connectorId, assetAId, assetBId); + + final ProcessorEntity afterRestart = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(afterRestart, "ReplaceWithFile should be present after restart while in Troubleshooting"); + assertEquals(overriddenFilenameValue, afterRestart.getComponent().getConfig().getProperties().get("Filename"), + "User override of ReplaceWithFile.Filename must survive a restart while in Troubleshooting"); + + getClientUtil().endTroubleshooting(connectorId); + assertConnectorState(connectorId, ConnectorState.STOPPED); + + // After exit, the Connector's authoritative flow is restored. The Processor's Filename must reference the + // authoritative value (#{asset_param}), not the user override. + final ProcessorEntity replaceWithFileAfterExit = findProcessorByName(connectorId, "ReplaceWithFile"); + assertNotNull(replaceWithFileAfterExit, "Restored authoritative flow should contain ReplaceWithFile"); + assertEquals(authoritativeFilenameValue, replaceWithFileAfterExit.getComponent().getConfig().getProperties().get("Filename"), + "Authoritative ReplaceWithFile.Filename must be restored on Troubleshooting exit"); + + // Verify the active configuration still references Asset A (the Connector's expected asset). + final ConnectorEntity afterExit = getNifiClient().getConnectorClient().getConnector(connectorId); + final ConnectorConfigurationDTO activeConfig = afterExit.getComponent().getActiveConfiguration(); + final Map activeProperties = activeConfig.getConfigurationStepConfigurations().getFirst() + .getPropertyGroupConfigurations().getFirst().getPropertyValues(); + assertEquals("ASSET_REFERENCE", activeProperties.get("Asset File").getValueType(), + "Active configuration must retain the Asset reference for Asset File"); + assertEquals(assetAId, activeProperties.get("Asset File").getAssetReferences().get(0).getId(), + "Active configuration must continue to reference Asset A (the Connector's expected asset) after Troubleshooting exit"); + } + + private void assertAssetIds(final String connectorId, final String... expectedAssetIds) throws NiFiClientException, IOException { + final List actualAssetIds = new ArrayList<>(); + final AssetsEntity assetsEntity = getNifiClient().getConnectorClient().getAssets(connectorId); + if (assetsEntity != null && assetsEntity.getAssets() != null) { + for (final AssetEntity asset : assetsEntity.getAssets()) { + if (asset.getAsset() != null) { + actualAssetIds.add(asset.getAsset().getId()); + } + } + } + for (final String expected : expectedAssetIds) { + assertTrue(actualAssetIds.contains(expected), + "Expected Connector " + connectorId + " to contain Asset " + expected + " but found: " + actualAssetIds); + } + } + + private boolean containsProcessorId(final String connectorId, final String processorId) throws NiFiClientException, IOException { + for (final ProcessorEntity entity : findAllProcessors(connectorId)) { + if (processorId.equals(entity.getId())) { + return true; + } + } + return false; + } + + private void runManagedFlowAndAssertParameterValues(final String connectorId, final File sensitiveOutputFile, final File assetOutputFile, + final String expectedSensitiveValue, final String expectedAssetValue, final String phase) + throws NiFiClientException, IOException, InterruptedException { + + // Starting individual components is permitted while in Troubleshooting. The managed Process Group is a + // standard (non-stateless) group so processors and ports can be scheduled individually. The flow built by + // ParameterContextConnector routes FlowFiles through child group Input Ports, so every Port inside the + // managed flow must also be started for the pipeline to actually pass FlowFiles. + final List inputPorts = findAllInputPorts(connectorId); + final List outputPorts = findAllOutputPorts(connectorId); + final List processors = findAllProcessors(connectorId); + assertFalse(processors.isEmpty(), "Managed flow should contain processors " + phase); + + for (final PortEntity port : inputPorts) { + getNifiClient().getInputPortClient().startInputPort(port); + } + for (final PortEntity port : outputPorts) { + getNifiClient().getOutputPortClient().startOutputPort(port); + } + for (final ProcessorEntity processor : processors) { + getClientUtil().waitForValidProcessor(processor.getId()); + getClientUtil().startProcessor(processor); + } + + waitFor(() -> sensitiveOutputFile.exists() && assetOutputFile.exists()); + + assertEquals(expectedSensitiveValue, Files.readString(sensitiveOutputFile.toPath()).trim(), + "Sensitive output file must contain the configured sensitive parameter value " + phase); + assertEquals(expectedAssetValue, Files.readString(assetOutputFile.toPath()).trim(), + "Asset output file must contain the asset contents referenced by the asset parameter " + phase); + } + + private void stopAllManagedComponents(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + for (final ProcessorEntity processor : findAllProcessors(connectorId)) { + getClientUtil().stopProcessor(processor); + } + for (final PortEntity port : findAllInputPorts(connectorId)) { + getNifiClient().getInputPortClient().stopInputPort(port); + } + for (final PortEntity port : findAllOutputPorts(connectorId)) { + getNifiClient().getOutputPortClient().stopOutputPort(port); + } + for (final ProcessorEntity processor : findAllProcessors(connectorId)) { + waitForProcessorState(processor.getId(), ScheduledState.STOPPED); + } + } + + private List findAllInputPorts(final String connectorId) throws NiFiClientException, IOException { + final List result = new ArrayList<>(); + collectPorts(connectorId, null, true, result); + return result; + } + + private List findAllOutputPorts(final String connectorId) throws NiFiClientException, IOException { + final List result = new ArrayList<>(); + collectPorts(connectorId, null, false, result); + return result; + } + + private void collectPorts(final String connectorId, final String groupId, final boolean input, final List collected) throws NiFiClientException, IOException { + final ProcessGroupFlowEntity entity = (groupId == null) ? getNifiClient().getConnectorClient().getFlow(connectorId) : getNifiClient().getConnectorClient().getFlow(connectorId, groupId); + final FlowDTO flow = entity.getProcessGroupFlow().getFlow(); + collected.addAll(input ? flow.getInputPorts() : flow.getOutputPorts()); + + for (final ProcessGroupEntity child : flow.getProcessGroups()) { + collectPorts(connectorId, child.getId(), input, collected); + } + } + + private ConnectorValueReferenceDTO createStringLiteralRef(final String value) { + final ConnectorValueReferenceDTO ref = new ConnectorValueReferenceDTO(); + ref.setValueType("STRING_LITERAL"); + ref.setValue(value); + return ref; + } +} diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java index 05f9ed173c3d..14a909fc3bc2 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java @@ -177,6 +177,28 @@ public interface ConnectorClient { */ ConnectorEntity cancelDrain(ConnectorEntity connectorEntity) throws NiFiClientException, IOException; + /** + * Transitions a connector into Troubleshooting mode. + * + * @param connectorEntity the connector entity (must contain id and revision) + * @return the updated connector entity + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + ConnectorEntity enterTroubleshooting(ConnectorEntity connectorEntity) throws NiFiClientException, IOException; + + /** + * Transitions a connector out of Troubleshooting mode. + * + * @param connectorId the connector ID + * @param clientId the client ID + * @param version the revision version + * @return the updated connector entity + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + ConnectorEntity endTroubleshooting(String connectorId, String clientId, long version) throws NiFiClientException, IOException; + /** * Gets the configuration step names for a connector. * diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java index 7b648e7183e6..d972f4891591 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java @@ -235,6 +235,64 @@ private ConnectorEntity cancelDrain(final String connectorId, final String clien }); } + @Override + public ConnectorEntity enterTroubleshooting(final ConnectorEntity connectorEntity) throws NiFiClientException, IOException { + return enterTroubleshooting(connectorEntity.getId(), connectorEntity.getRevision().getClientId(), + connectorEntity.getRevision().getVersion(), connectorEntity.isDisconnectedNodeAcknowledged()); + } + + private ConnectorEntity enterTroubleshooting(final String connectorId, final String clientId, final long version, + final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector ID cannot be null or blank"); + } + + return executeAction("Error entering Troubleshooting mode", () -> { + final WebTarget target = connectorTarget + .path("/troubleshooting") + .resolveTemplate("id", connectorId); + + final ConnectorEntity requestEntity = new ConnectorEntity(); + requestEntity.setId(connectorId); + requestEntity.setDisconnectedNodeAcknowledged(disconnectedNodeAcknowledged); + + final RevisionDTO revisionDto = new RevisionDTO(); + revisionDto.setClientId(clientId); + revisionDto.setVersion(version); + requestEntity.setRevision(revisionDto); + + return getRequestBuilder(target).post( + Entity.entity(requestEntity, MediaType.APPLICATION_JSON_TYPE), + ConnectorEntity.class); + }); + } + + @Override + public ConnectorEntity endTroubleshooting(final String connectorId, final String clientId, final long version) throws NiFiClientException, IOException { + return endTroubleshooting(connectorId, clientId, version, false); + } + + private ConnectorEntity endTroubleshooting(final String connectorId, final String clientId, final long version, + final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector ID cannot be null or blank"); + } + + return executeAction("Error ending Troubleshooting mode", () -> { + WebTarget target = connectorTarget + .path("/troubleshooting") + .queryParam("version", version) + .queryParam("clientId", clientId) + .resolveTemplate("id", connectorId); + + if (disconnectedNodeAcknowledged == Boolean.TRUE) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + + return getRequestBuilder(target).delete(ConnectorEntity.class); + }); + } + private ConnectorEntity updateConnectorRunStatus(final String connectorId, final String desiredState, final String clientId, final long version, final Boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { if (StringUtils.isBlank(connectorId)) { diff --git a/pom.xml b/pom.xml index af98cd0c526d..87eadc2567cc 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ v24.14.1 - 2.8.0 + 2.9.0-SNAPSHOT 2.3.0