From 4597dc2220c0a2b666403d1d2660b2ba7f996f29 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:47:21 -0700 Subject: [PATCH 1/3] chore: remove aliases --- .github/workflows/aurora_performance.yml | 14 +- .github/workflows/dependabot_auto_merge.yml | 2 +- .github/workflows/integration_tests.yml | 149 +++++++++++++++--- .github/workflows/main.yml | 6 +- .github/workflows/release.yml | 6 +- .github/workflows/release_draft.yml | 8 +- .github/workflows/remove-old-artifacts.yml | 2 +- .github/workflows/run-autoscaling-tests.yml | 14 +- common/lib/host_info.ts | 39 +---- .../lib/host_list_provider/topology_utils.ts | 1 - common/lib/partial_plugin_service.ts | 52 ++---- common/lib/plugin_service.ts | 62 ++++---- ...rora_initial_connection_strategy_plugin.ts | 20 +-- .../routing/substitute_connect_routing.ts | 1 - .../aurora_connection_tracker_plugin.ts | 82 ++++++++-- .../opened_connection_tracker.ts | 141 +++++++++-------- common/lib/plugins/default_plugin.ts | 2 +- .../efm/host_monitoring_connection_plugin.ts | 27 ++-- .../lib/plugins/failover/failover_plugin.ts | 13 +- .../failover/reader_failover_handler.ts | 6 +- .../failover/writer_failover_handler.ts | 18 ++- .../lib/plugins/failover2/failover2_plugin.ts | 6 +- .../limitless/limitless_connection_context.ts | 11 ++ .../limitless/limitless_connection_plugin.ts | 3 + .../limitless/limitless_router_service.ts | 16 +- common/lib/utils/full_services_container.ts | 3 + common/lib/utils/host_id_cache_service.ts | 137 ++++++++++++++++ common/lib/utils/messages.ts | 1 + common/lib/utils/service_utils.ts | 3 + mysql/lib/client.ts | 5 +- pg/lib/client.ts | 5 +- 31 files changed, 563 insertions(+), 292 deletions(-) create mode 100644 common/lib/utils/host_id_cache_service.ts diff --git a/.github/workflows/aurora_performance.yml b/.github/workflows/aurora_performance.yml index 72ebb9317..5d0bd0825 100644 --- a/.github/workflows/aurora_performance.yml +++ b/.github/workflows/aurora_performance.yml @@ -18,25 +18,25 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_aurora_perf_tests @@ -59,7 +59,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -73,7 +73,7 @@ jobs: - name: Archive Performance results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.db }}-performance-results path: ./tests/integration/container/reports diff --git a/.github/workflows/dependabot_auto_merge.yml b/.github/workflows/dependabot_auto_merge.yml index c7574baba..536f793be 100644 --- a/.github/workflows/dependabot_auto_merge.yml +++ b/.github/workflows/dependabot_auto_merge.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1 + uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v2 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve PR diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index fc7390759..2218d9eac 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,50 +5,157 @@ on: push: branches: - main + - chore/remove-aliases paths-ignore: - - "**/*.md" - - "**/*.jpg" - - "**/README.txt" - - "**/LICENSE.txt" - - "docs/**" - - "ISSUE_TEMPLATE/**" - - "**/remove-old-artifacts.yml" + - '**/*.md' + - '**/*.jpg' + - '**/README.txt' + - '**/LICENSE.txt' + - 'docs/**' + - 'ISSUE_TEMPLATE/**' + - '**/remove-old-artifacts.yml' + pull_request_target: + branches: + - main + paths-ignore: + - '**/*.md' + - '**/*.jpg' + - '**/README.txt' + - '**/LICENSE.txt' + - 'docs/**' + - 'ISSUE_TEMPLATE/**' + - '**/remove-old-artifacts.yml' permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout jobs: - run-integration-tests: - name: Run Integration Tests + approve: + # Auto-approve for non-fork scenarios + if: > + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - run: echo "Approved — not a fork PR" + approve-fork: + # Require manual approval for fork PRs + if: > + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + environment: integration-tests + steps: + - run: echo "Fork PR approved by maintainer" + run-integration-tests-default: + name: Run Integration Tests (Default) + needs: [ approve, approve-fork ] + if: | + always() && + (needs.approve.result == 'success' || needs.approve-fork.result == 'success') && + !(needs.approve.result == 'failure' || needs.approve-fork.result == 'failure') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dbEngine: ["aurora-mysql", "aurora-postgres"] + + steps: + - name: 'Clone repository' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: "Set up JDK 8" + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: "corretto" + java-version: 8 + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "22.x" + - name: Install dependencies + run: npm install --no-save + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: nodejs_int_default_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + output-credentials: true + + - name: Run Integration Tests + run: | + ./gradlew --no-parallel --no-daemon test-${{ matrix.dbEngine }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + AURORA_MYSQL_DB_ENGINE_VERSION: default + AURORA_PG_DB_ENGINE_VERSION: default + + - name: "Get Github Action IP" + if: always() + id: ip + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 + + - name: "Remove Github Action IP" + if: always() + run: | + aws ec2 revoke-security-group-ingress \ + --group-name default \ + --protocol -1 \ + --port -1 \ + --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ + 2>&1 > /dev/null; + + - name: Archive results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-report-default-${{ matrix.dbEngine }} + path: ./tests/integration/container/reports + retention-days: 5 + + run-integration-tests-latest: + name: Run Integration Tests (Latest) runs-on: ubuntu-latest + needs: run-integration-tests-default + if: | + always() && + needs.run-integration-tests-default.result == 'success' strategy: fail-fast: false matrix: - versions: [ "default", "latest" ] dbEngine: ["aurora-mysql", "aurora-postgres" ] steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} - role-session-name: nodejs_int_default_tests + role-session-name: nodejs_int_latest_tests role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} output-credentials: true @@ -61,13 +168,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - AURORA_MYSQL_DB_ENGINE_VERSION: ${{ matrix.dbEngine }} - AURORA_PG_DB_ENGINE_VERSION: ${{ matrix.versions }} + AURORA_MYSQL_DB_ENGINE_VERSION: latest + AURORA_PG_DB_ENGINE_VERSION: latest - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -81,8 +188,8 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: integration-report-default-${{ matrix.dbEngine }}-${{ matrix.versions}} + name: integration-report-latest-${{ matrix.dbEngine }} path: ./tests/integration/container/reports retention-days: 5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed517c71a..7ba7b23ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,11 +16,11 @@ jobs: run-checks-and-unit-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Run eslint - linting diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aedac0853..dcd54a9a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,11 +11,11 @@ jobs: packages: write steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" registry-url: "https://registry.npmjs.org" - name: "Install dependencies" run: npm install --no-save diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml index fb2c2207a..85973b38a 100644 --- a/.github/workflows/release_draft.yml +++ b/.github/workflows/release_draft.yml @@ -19,11 +19,11 @@ jobs: runs-on: ubuntu-latest steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: "Install dependencies" run: npm install --no-save - name: "Run eslint - linting" @@ -46,7 +46,7 @@ jobs: touch RELEASE_DETAILS.md echo "$RELEASE_DETAILS" > RELEASE_DETAILS.md - name: "Upload to Draft Release" - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1 with: draft: true name: "AWS Advanced NodeJS Wrapper - v${{ env.RELEASE_VERSION }}" diff --git a/.github/workflows/remove-old-artifacts.yml b/.github/workflows/remove-old-artifacts.yml index 5899e8671..71d497c8d 100644 --- a/.github/workflows/remove-old-artifacts.yml +++ b/.github/workflows/remove-old-artifacts.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Remove Old Artifacts - uses: c-hive/gha-remove-artifacts@v1 + uses: c-hive/gha-remove-artifacts@44fc7acaf1b3d0987da0e8d4707a989d80e9554b # v1 with: age: "1 week" skip-tags: true diff --git a/.github/workflows/run-autoscaling-tests.yml b/.github/workflows/run-autoscaling-tests.yml index 32d243db2..0c65d3aec 100644 --- a/.github/workflows/run-autoscaling-tests.yml +++ b/.github/workflows/run-autoscaling-tests.yml @@ -21,21 +21,21 @@ jobs: dbEngine: [ "mysql", "postgres" ] steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Configure AWS credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_autoscaling_tests @@ -56,7 +56,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() run: | @@ -68,7 +68,7 @@ jobs: 2>&1 > /dev/null; - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: autoscaling-report path: ./tests/integration/container/reports diff --git a/common/lib/host_info.ts b/common/lib/host_info.ts index 3ac577518..4d332a384 100644 --- a/common/lib/host_info.ts +++ b/common/lib/host_info.ts @@ -25,15 +25,13 @@ export class HostInfo { public static readonly NO_PORT: number = -1; public static readonly DEFAULT_WEIGHT: number = 100; - readonly host: string; + readonly host: string; // full domain name readonly port: number; role: HostRole; readonly weight: number; // Greater or equal 0. Lesser the weight, the healthier host. readonly lastUpdateTime: number; availability: HostAvailability; - aliases: Set = new Set(); - allAliases: Set = new Set(); - hostId: string; + hostId: string; // id; could be a host name, host domain name, or a unique string hostAvailabilityStrategy: HostAvailabilityStrategy; constructor( @@ -57,7 +55,6 @@ export class HostInfo { this.weight = weight; this.lastUpdateTime = lastUpdateTime; this.hostAvailabilityStrategy = hostAvailabilityStrategy; - this.allAliases.add(this.asAlias).add(this.host); this.hostId = hostId; } @@ -65,38 +62,6 @@ export class HostInfo { return this.port != HostInfo.NO_PORT; } - addAlias(...alias: string[]) { - if (!alias || alias.length < 1) { - return; - } - - alias.forEach((x) => { - this.aliases.add(x); - this.allAliases.add(x); - }); - } - - removeAlias(aliases: string[]) { - if (!aliases || aliases.length < 1) { - return; - } - - aliases.forEach((x) => { - this.aliases.delete(x); - this.allAliases.delete(x); - }); - } - - resetAliases() { - this.aliases.clear(); - this.allAliases.clear(); - this.allAliases.add(this.asAlias); - } - - get asAlias() { - return this.isPortSpecified() ? `${this.host}:${this.port}` : this.host; - } - get url() { let url = this.hostAndPort; if (!url.endsWith("/")) { diff --git a/common/lib/host_list_provider/topology_utils.ts b/common/lib/host_list_provider/topology_utils.ts index 500f3046d..3cddf4860 100644 --- a/common/lib/host_list_provider/topology_utils.ts +++ b/common/lib/host_list_provider/topology_utils.ts @@ -130,7 +130,6 @@ export abstract class TopologyUtils { .withLastUpdateTime(lastUpdateTime) .withHostId(finalInstanceId) .build(); - host.addAlias(finalEndpoint); return host; } diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts index b3b8f7295..4203b2309 100644 --- a/common/lib/partial_plugin_service.ts +++ b/common/lib/partial_plugin_service.ts @@ -40,6 +40,7 @@ import { FullServicesContainer } from "./utils/full_services_container"; import { HostListProviderService } from "./host_list_provider_service"; import { StorageService } from "./utils/storage/storage_service"; import { CoreServicesContainer } from "./utils/core_services_container"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; /** * A PluginService containing some methods that are not intended to be called. This class is intended to be used @@ -62,6 +63,7 @@ export class PartialPluginService implements PluginService, HostListProviderServ protected readonly driverDialect: DriverDialect; protected allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; private _isPooledClient: boolean = false; + private _trackedConnectionHost: TrackedConnectionListHost | null = null; private connectionUrlParser: ConnectionUrlParser; constructor( @@ -221,15 +223,11 @@ export class PartialPluginService implements PluginService, HostListProviderServ return hosts; } - setAvailability(hostAliases: Set, availability: HostAvailability): void { - if (hostAliases.size === 0) { - return; - } - + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void { const hostsToChange = [ ...new Set( this.getAllHosts().filter( - (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) ) ) ]; @@ -430,36 +428,12 @@ export class PartialPluginService implements PluginService, HostListProviderServ return provider.identifyConnection(targetClient); } - async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise { - if (!hostInfo) { - return; - } - - if (hostInfo.aliases.size > 0) { - logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); - return; - } - - hostInfo.addAlias(hostInfo.asAlias); - - try { - const res = await this.dialect.getHostAliasAndParseResults(targetClient); - if (res) { - hostInfo.addAlias(res); - } - } catch (error) { - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + getRoutedHostInfo(): HostInfo | null { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getRoutedHostInfo")); + } - try { - const host = await this.identifyConnection(targetClient); - if (host && host.allAliases) { - hostInfo.addAlias(...host.allAliases); - } - } catch (error) { - // Ignore errors from identifyConnection - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setRoutedHostInfo")); } getHostInfoBuilder(): HostInfoBuilder { @@ -547,4 +521,12 @@ export class PartialPluginService implements PluginService, HostListProviderServ setIsPooledClient(isPooledClient: boolean): void { this._isPooledClient = isPooledClient; } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; + } } diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 8e8a5a1a8..1b80affbb 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -47,7 +47,7 @@ import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; import { ConnectionPlugin } from "./connection_plugin"; import { FullServicesContainer } from "./utils/full_services_container"; import { StorageService } from "./utils/storage/storage_service"; -import { CoreServicesContainer } from "./utils/core_services_container"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; export interface PluginService extends ErrorHandler { isInTransaction(): boolean; @@ -96,14 +96,20 @@ export interface PluginService extends ErrorHandler { getHosts(): HostInfo[]; - setAvailability(hostAliases: Set, availability: HostAvailability): void; + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void; updateConfigWithProperties(props: Map): void; - fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise; + getRoutedHostInfo(): HostInfo | null; + + setRoutedHostInfo(routedHostInfo: HostInfo | null); identifyConnection(targetClient: ClientWrapper): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo | null): Promise; + + identifyConnection(targetClient: ClientWrapper, connectionHostInfo?: HostInfo | null): Promise; + connect(hostInfo: HostInfo, props: Map): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; @@ -151,6 +157,10 @@ export interface PluginService extends ErrorHandler { isPooledClient(): boolean; setIsPooledClient(isPooledClient: boolean): void; + + getTrackedConnectionHost(): TrackedConnectionListHost | null; + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void; } export class PluginServiceImpl implements PluginService, HostListProviderService { @@ -162,6 +172,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private _isInTransaction: boolean = false; private servicesContainer: FullServicesContainer; protected hosts: HostInfo[] = []; + protected routedHostInfo: HostInfo | null; private dbDialectProvider: DatabaseDialectProvider; private readonly initialHost: string; private dialect: DatabaseDialect; @@ -173,6 +184,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; protected _isPooledClient: boolean = false; + protected _trackedConnectionHost: TrackedConnectionListHost | null = null; constructor( container: FullServicesContainer, @@ -459,15 +471,11 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return hosts; } - setAvailability(hostAliases: Set, availability: HostAvailability) { - if (hostAliases.size === 0) { - return; - } - + setAvailability(hostInfo: HostInfo, availability: HostAvailability) { const hostsToChange = [ ...new Set( this.getAllHosts().filter( - (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) ) ) ]; @@ -496,30 +504,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this._currentClient.config = Object.fromEntries(props.entries()); } - async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo) { - if (hostInfo == null) { - return; - } - - if (hostInfo.aliases.size > 0) { - logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); - return; - } - - hostInfo.addAlias(hostInfo.asAlias); - - // Add the host name and port, this host name is usually the internal IP address. - try { - const res: string = await this.dialect.getHostAliasAndParseResults(targetClient); - hostInfo.addAlias(res); - } catch (error) { - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + getRoutedHostInfo(): HostInfo | null { + return this.routedHostInfo; + } - const host: HostInfo | void | null = await this.identifyConnection(targetClient); - if (host) { - hostInfo.addAlias(...host.allAliases); - } + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + this.routedHostInfo = routedHostInfo; } identifyConnection(targetClient: ClientWrapper): Promise { @@ -806,4 +796,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService setIsPooledClient(isPooledClient: boolean): void { this._isPooledClient = isPooledClient; } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; + } } diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts index 7d7891a6a..cdabc7d1a 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts @@ -121,9 +121,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu continue; } - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); } return writerCandidateClient; } @@ -139,16 +137,14 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu } // Writer connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); return writerCandidateClient; } catch (error: any) { await this.pluginService.abortTargetClient(writerCandidateClient); if (this.pluginService.isLoginError(error) || !writerCandidate) { throw error; } else if (writerCandidate) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); } } } @@ -185,9 +181,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic // and return the current (writer) client. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); return readerCandidateClient; } await this.pluginService.abortTargetClient(readerCandidateClient); @@ -196,9 +190,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu } // Reader connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); } else { logger.debug("Reader candidate not found"); } @@ -233,7 +225,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.pluginService.isLoginError(error) || !readerCandidate) { throw error; } else if (readerCandidate) { - this.pluginService.setAvailability(readerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(readerCandidate, HostAvailability.NOT_AVAILABLE); } } } diff --git a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts index 94f356770..812873d7a 100644 --- a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts @@ -73,7 +73,6 @@ export class SubstituteConnectRouting extends BaseConnectRouting { .withHostId(iamHost.hostId) .withAvailability(HostAvailability.AVAILABLE) .build(); - reroutedHostInfo.addAlias(iamHost.host); const reroutedProperties: Map = new Map(properties); reroutedProperties.set(WrapperProperties.HOST.name, iamHost.host); diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts index 651bbea84..d256d8bce 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts @@ -28,7 +28,15 @@ import { HostRole } from "../../host_role"; import { OpenedConnectionTracker } from "./opened_connection_tracker"; export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - private static readonly subscribedMethods = new Set(["notifyHostListChanged", "connect", "query", "rollback"]); + private static readonly subscribedMethods = new Set([ + ...SubscribedMethodHelper.NETWORK_BOUND_METHODS, + "end", + "abort", + "notifyHostListChanged" + ]); + private static readonly CLOSING_METHODS = new Set(["end", "abort"]); + private static readonly TOPOLOGY_CHANGES_EXPECTED_TIME_NS = BigInt(3 * 60 * 1_000_000_000); + private static hostListRefreshEndTimeNs: bigint = 0n; private readonly pluginService: PluginService; private readonly rdsUtils: RdsUtils; @@ -56,46 +64,92 @@ export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin impl connectFunc: () => Promise ): Promise { const targetClient = await connectFunc(); - - if (targetClient) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); - if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + let connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; + + if (targetClient && !this.pluginService.isPooledClient()) { + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); + if (type.isRdsCluster || type === RdsUrlType.OTHER || type === RdsUrlType.IP_ADDRESS) { + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + connectionHostInfo = identifiedHostInfo; + await this.pluginService.setRoutedHostInfo(connectionHostInfo); + } } - await this.tracker.populateOpenedConnectionQueue(hostInfo, targetClient); + const host = this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); + this.pluginService.setTrackedConnectionHost(host); } return targetClient; } override async execute(methodName: string, methodFunc: () => Promise, methodArgs: any[]): Promise { + const currentHostInfo = this.pluginService.getCurrentHostInfo(); this.rememberWriter(); + const isClosing = AuroraConnectionTrackerPlugin.CLOSING_METHODS.has(methodName); + try { + if (!isClosing) { + let needRefreshHostList = false; + const localRefreshEndTimeNs = AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs; + if (localRefreshEndTimeNs > 0n) { + if (localRefreshEndTimeNs > process.hrtime.bigint()) { + needRefreshHostList = true; + } else { + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; + } + } + if (this.needUpdateCurrentWriter || needRefreshHostList) { + await this.checkWriterChanged(needRefreshHostList); + } + } + const result = await methodFunc(); - if (this.needUpdateCurrentWriter) { - await this.checkWriterChanged(); + + if (isClosing) { + const host = this.pluginService.getTrackedConnectionHost(); + if (host) { + this.tracker.removeConnectionTracking(host); + this.pluginService.setTrackedConnectionHost(null); + } else if (currentHostInfo) { + this.tracker.removeConnectionTrackingByHost(currentHostInfo, this.pluginService.getCurrentClient()?.targetClient); + } } return result; } catch (error) { if (error instanceof FailoverError) { - await this.checkWriterChanged(); + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = + process.hrtime.bigint() + AuroraConnectionTrackerPlugin.TOPOLOGY_CHANGES_EXPECTED_TIME_NS; + // This call may effectively close/abort the current connection. + await this.checkWriterChanged(true); } throw error; } } - private async checkWriterChanged(): Promise { + private async checkWriterChanged(needRefreshHostList: boolean): Promise { + if (needRefreshHostList) { + try { + await this.pluginService.refreshHostList(); + } catch (error) { + // Ignore: continue with whatever topology is currently available. + } + } + const hostInfoAfterFailover = this.getWriter(this.pluginService.getAllHosts()); + if (hostInfoAfterFailover === null) { + return; + } + if (this.currentWriter === null) { this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; - } else if (!this.currentWriter.equals(hostInfoAfterFailover!)) { - // writer changed + } else if (this.currentWriter.hostAndPort !== hostInfoAfterFailover.hostAndPort) { + // The writer changed. await this.tracker.invalidateAllConnections(this.currentWriter); this.tracker.logOpenedConnections(); this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; } } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 6c8b6a580..6efa08391 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -21,9 +21,10 @@ import { logger } from "../../../logutils"; import { MapUtils } from "../../utils/map_utils"; import { Messages } from "../../utils/messages"; import { PluginService } from "../../plugin_service"; +import { TrackedConnectionList, TrackedConnectionListHost } from "./tracked_connection_list"; export class OpenedConnectionTracker { - static readonly openedConnections: Map>> = new Map>>(); + static readonly openedConnections: Map = new Map(); readonly pluginService: PluginService; private static readonly rdsUtils = new RdsUtils(); @@ -31,101 +32,113 @@ export class OpenedConnectionTracker { this.pluginService = pluginService; } - async populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): Promise { - const aliases = hostInfo.aliases; + populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): TrackedConnectionListHost | null { + if (!hostInfo || !client) { + return null; + } - // Check if the connection was established using an instance endpoint + // Check if the connection was established using an instance endpoint. if (OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host)) { - this.trackConnection(hostInfo.hostAndPort, client); - return; + const host = this.trackConnection(hostInfo.hostAndPort, client); + this.logOpenedConnections(); + return host; } - const instanceEndpoint = [...aliases] - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .reduce((max, s) => (s > max ? s : max), ""); + // It might be a custom domain name. Let's track by hostId and custom domain name. + let lastHost: TrackedConnectionListHost | null = null; + if (hostInfo.hostId) { + lastHost = this.trackConnection(hostInfo.hostId, client); + } + if (hostInfo.hostAndPort) { + lastHost = this.trackConnection(hostInfo.hostAndPort, client); + } + this.logOpenedConnections(); + return lastHost; + } - if (!instanceEndpoint) { - logger.debug(Messages.get("OpenedConnectionTracker.unableToPopulateOpenedConnectionQueue", hostInfo.host)); + async invalidateAllConnections(hostInfo: HostInfo): Promise { + if (!hostInfo) { return; } + await this.invalidateAllConnectionsMultipleHosts(hostInfo.hostAndPort, hostInfo.host, hostInfo.hostId); + } - this.trackConnection(instanceEndpoint, client); + async invalidateAllConnectionsMultipleHosts(...keys: string[]): Promise { + for (const key of keys) { + if (!key) { + continue; + } + try { + const connectionList = OpenedConnectionTracker.openedConnections.get(key); + this.logConnectionList(key, connectionList); + await this.invalidateConnections(connectionList); + } catch (error) { + // Ignore and continue with the remaining keys. + } + } } - async invalidateAllConnections(hostInfo: HostInfo): Promise { - await this.invalidateAllConnectionsMultipleHosts(hostInfo.asAlias); - await this.invalidateAllConnectionsMultipleHosts(...Array.from(hostInfo.aliases)); + removeConnectionTracking(host: TrackedConnectionListHost | null): void { + host?.remove(); } - async invalidateAllConnectionsMultipleHosts(...hosts: string[]): Promise { - try { - const instanceEndpoint = hosts - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .at(0); - if (!instanceEndpoint) { - return; - } - const connectionQueue = OpenedConnectionTracker.openedConnections.get(instanceEndpoint); - this.logConnectionQueue(instanceEndpoint, connectionQueue!); - await this.invalidateConnections(connectionQueue!); - } catch (error) { - // ignore + removeConnectionTrackingByHost(hostInfo: HostInfo, client: ClientWrapper | undefined | null): void { + const hostAndPort = OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host) ? hostInfo.hostAndPort : null; + if (!hostAndPort) { + return; + } + + const connectionList = OpenedConnectionTracker.openedConnections.get(hostAndPort); + if (connectionList) { + connectionList.removeIf((ref) => { + const conn = ref.deref(); + return !conn || conn === client; + }); } } - private trackConnection(instanceEndpoint: string, client: ClientWrapper): void { - const connectionQueue = MapUtils.computeIfAbsent( - OpenedConnectionTracker.openedConnections, - instanceEndpoint, - (k) => new Array>() - ); - connectionQueue!.push(new WeakRef(client)); - this.logOpenedConnections(); + private trackConnection(instanceEndpoint: string, client: ClientWrapper): TrackedConnectionListHost { + const connectionList = MapUtils.computeIfAbsent(OpenedConnectionTracker.openedConnections, instanceEndpoint, (_) => new TrackedConnectionList()); + return connectionList!.add(client); } - private async invalidateConnections(connectionQueue: Array>): Promise { - let clientRef: WeakRef | undefined; - while ((clientRef = connectionQueue?.shift()) != null) { - const client = clientRef?.deref() ?? null; - if (!client) { - continue; - } + private async invalidateConnections(connectionList: TrackedConnectionList | undefined): Promise { + if (!connectionList || connectionList.isEmpty()) { + return; + } + + const connections = connectionList.drainAll(); + for (const client of connections) { await this.pluginService.abortTargetClient(client); } } logOpenedConnections(): void { - let str = ""; - const hostList = []; - - for (const queue of OpenedConnectionTracker.openedConnections.values()) { - if (queue.length !== 0) { - for (const connRef of queue) { - const conn = connRef?.deref() ?? null; - if (conn) { - hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); - } - } - str = hostList.join("\n\t"); + const hostList: string[] = []; + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + for (const conn of connectionList.getConnections()) { + hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); } } - logger.debug(`Opened Connections Tracked: \n\t${str}`); + logger.debug(`Opened Connections Tracked: \n\t${hostList.join("\n\t")}`); } - private logConnectionQueue(host: string, queue: Array>): void { - if (!queue || queue.length === 0) { + private logConnectionList(host: string, connectionList: TrackedConnectionList | undefined): void { + if (!connectionList || connectionList.isEmpty()) { return; } - logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${queue.map((x) => x.deref()!.hostInfo).join()}\n]`)); + const connections = connectionList.getConnections().map((conn) => conn.hostInfo); + logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${connections.join()}\n]`)); } pruneNullConnections(): void { - for (const [key, queue] of OpenedConnectionTracker.openedConnections) { - OpenedConnectionTracker.openedConnections.set( - key, - queue.filter((connWeakRef: WeakRef) => connWeakRef?.deref() ?? null) - ); + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + connectionList.removeIf((ref) => !ref.deref()); } } + + static clearCache(): void { + OpenedConnectionTracker.openedConnections.clear(); + } } diff --git a/common/lib/plugins/default_plugin.ts b/common/lib/plugins/default_plugin.ts index 2c3b45d8a..ecd5ae0a7 100644 --- a/common/lib/plugins/default_plugin.ts +++ b/common/lib/plugins/default_plugin.ts @@ -84,7 +84,7 @@ export class DefaultPlugin extends AbstractConnectionPlugin { ); const result: ConnectionInfo = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(hostInfo, HostAvailability.AVAILABLE); this.pluginService.setIsPooledClient(result.isPooled); await this.pluginService.updateDialect(result.client); return result.client; diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts index 753e2f695..59ad986fb 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts @@ -59,10 +59,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp ): Promise { const targetClient = await connectFunc(); if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); + const connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + this.pluginService.setRoutedHostInfo(identifiedHostInfo); + } } } return targetClient; @@ -110,10 +113,6 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (monitorContext.isHostUnhealthy) { const monitoringHostInfo = await this.getMonitoringHostInfo(); - if (monitoringHostInfo) { - this.pluginService.setAvailability(monitoringHostInfo.allAliases, HostAvailability.NOT_AVAILABLE); - } - const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; if (targetClient) { @@ -125,7 +124,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp await this.pluginService.abortTargetClient(targetClient); } // eslint-disable-next-line no-unsafe-finally - throw new UnavailableHostError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo.host)); + throw new UnavailableHostError( + Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo?.host ?? "Unknown host") + ); } } } @@ -149,7 +150,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (this.monitoringHostInfo) { return this.monitoringHostInfo; } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); + this.monitoringHostInfo = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getCurrentHostInfo(); if (this.monitoringHostInfo == null) { this.throwUnableToIdentifyConnection(null); } @@ -158,12 +159,16 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp try { if (rdsUrlType.isRdsCluster) { logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); + this.monitoringHostInfo = await this.pluginService.identifyConnection( + this.pluginService.getCurrentClient().targetClient!, + this.pluginService.getCurrentHostInfo() + ); if (this.monitoringHostInfo == null) { const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); this.throwUnableToIdentifyConnection(host); } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); + // Update identified HostInfo for the current connection + await this.pluginService.setCurrentClient(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); } } catch (error: any) { if (!(error instanceof AwsWrapperError)) { diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index ceefae36c..73014d2dc 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -184,12 +184,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { if (this.isHostStillValid(url, changes)) { return Promise.resolve(); } - - for (const alias of currentHost.allAliases) { - if (this.isHostStillValid(alias + "/", changes)) { - return Promise.resolve(); - } - } } logger.info(Messages.get("Failover.invalidHost", currentHost?.host ?? "empty host")); return Promise.resolve(); @@ -279,7 +273,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.invalidateCurrentClient(); const currentHostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } this._lastError = e; @@ -291,7 +285,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { } async failover(failedHost: HostInfo) { - this.pluginService.setAvailability(failedHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(failedHost, HostAvailability.NOT_AVAILABLE); if (this.failoverMode === FailoverMode.STRICT_WRITER) { await this.failoverWriter(); @@ -308,8 +302,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { const telemetryContext = telemetryFactory.openTelemetryContext(FailoverPlugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); - let failedHost = null; if (failedHostInfo && failedHostInfo.getRawAvailability() === HostAvailability.AVAILABLE) { failedHost = failedHostInfo; @@ -334,7 +326,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.updateTopology(true); this.failoverWriterSuccessCounter.inc(); } catch (error: any) { diff --git a/common/lib/plugins/failover/reader_failover_handler.ts b/common/lib/plugins/failover/reader_failover_handler.ts index 21c82b1e5..fddf12a60 100644 --- a/common/lib/plugins/failover/reader_failover_handler.ts +++ b/common/lib/plugins/failover/reader_failover_handler.ts @@ -141,7 +141,7 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler async failoverInternal(hosts: HostInfo[], currentHost: HostInfo | null): Promise { if (currentHost) { - this.pluginService.setAvailability(currentHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHost, HostAvailability.NOT_AVAILABLE); } const hostsByPriority = this.getHostsByPriority(hosts); return this.getConnectionFromHostGroup(hostsByPriority); @@ -336,7 +336,7 @@ class ConnectionAttemptTask { try { this.targetClient = await this.pluginService.forceConnect(this.newHost, copy); - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.AVAILABLE); logger.info(Messages.get("ClusterAwareReaderFailoverHandler.successfulReaderConnection", this.newHost.host)); if (this.taskHandler.getSelectedConnectionAttemptTask(this.failoverTaskId) === -1) { this.taskHandler.setSelectedConnectionAttemptTask(this.failoverTaskId, this.taskId); @@ -344,7 +344,7 @@ class ConnectionAttemptTask { } throw new AwsWrapperError(Messages.get("ClusterAwareReaderFailoverHandler.selectedTaskChosen", this.newHost.host)); } catch (error) { - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.NOT_AVAILABLE); throw error; } finally { await this.performFinalCleanup(); diff --git a/common/lib/plugins/failover/writer_failover_handler.ts b/common/lib/plugins/failover/writer_failover_handler.ts index e14a73fd3..dbc2320d6 100644 --- a/common/lib/plugins/failover/writer_failover_handler.ts +++ b/common/lib/plugins/failover/writer_failover_handler.ts @@ -37,12 +37,14 @@ export interface WriterFailoverHandler { function isCurrentHostWriter(topology: HostInfo[], originalWriterHost: HostInfo): boolean { const latestWriter = getWriter(topology); - const latestWriterAllAliases = latestWriter?.allAliases; - const currentAliases = originalWriterHost.allAliases; - if (currentAliases && latestWriterAllAliases) { - return [...currentAliases].filter((alias) => latestWriterAllAliases.has(alias)).length > 0; + if (!latestWriter || !originalWriterHost) { + return false; } - return false; + + return ( + (!!latestWriter.hostId && latestWriter.hostId === originalWriterHost.hostId) || + (!!latestWriter.hostAndPort && latestWriter.hostAndPort.toLowerCase() === originalWriterHost.hostAndPort.toLowerCase()) + ); } export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler { @@ -273,7 +275,7 @@ class ReconnectToWriterHandlerTask { } success = isCurrentHostWriter(latestTopology, this.originalWriterHost); - this.pluginService.setAvailability(this.originalWriterHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.originalWriterHost, HostAvailability.AVAILABLE); return new WriterFailoverResult( success, false, @@ -454,13 +456,13 @@ class WaitForNewWriterHandlerTask { let targetClient = null; try { targetClient = await this.pluginService.forceConnect(writerCandidate, props); - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.AVAILABLE); await this.callCloseClient(this.currentReaderTargetClient); await this.callCloseClient(this.currentClient); this.currentClient = targetClient; return true; } catch (error) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); await this.pluginService.abortTargetClient(targetClient); return false; } diff --git a/common/lib/plugins/failover2/failover2_plugin.ts b/common/lib/plugins/failover2/failover2_plugin.ts index f9e04fe6c..85288c657 100644 --- a/common/lib/plugins/failover2/failover2_plugin.ts +++ b/common/lib/plugins/failover2/failover2_plugin.ts @@ -150,7 +150,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin { throw error; } - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(hostInfo, HostAvailability.NOT_AVAILABLE); try { // Unable to directly connect, attempt failover. @@ -209,7 +209,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin { await this.invalidateCurrentClient(); const currentHostInfo: HostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } await this.failover(); this._lastError = error; @@ -247,7 +247,6 @@ export class Failover2Plugin extends AbstractConnectionPlugin { const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); const failoverEndTimeMs = Date.now() + this.failoverTimeoutSettingMs; try { @@ -265,7 +264,6 @@ export class Failover2Plugin extends AbstractConnectionPlugin { this.failoverReaderSuccessCounter.inc(); await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.pluginService.forceRefreshHostList(); } catch (error) { this.failoverReaderFailedCounter.inc(); diff --git a/common/lib/plugins/limitless/limitless_connection_context.ts b/common/lib/plugins/limitless/limitless_connection_context.ts index ccc244c9b..5891c2da6 100644 --- a/common/lib/plugins/limitless/limitless_connection_context.ts +++ b/common/lib/plugins/limitless/limitless_connection_context.ts @@ -17,6 +17,7 @@ import { HostInfo } from "../../host_info"; import { ClientWrapper } from "../../client_wrapper"; import { ConnectionPlugin } from "../../connection_plugin"; +import Map from "@arrows/array/src/map"; export class LimitlessConnectionContext { private readonly hostInfo: HostInfo; @@ -25,6 +26,7 @@ export class LimitlessConnectionContext { private readonly connectFunc: () => Promise; private routers: HostInfo[] | null; private plugin: ConnectionPlugin; + private connectionHostInfo: HostInfo | null; constructor( hostInfo: HostInfo, @@ -40,6 +42,7 @@ export class LimitlessConnectionContext { this.connectFunc = connectFunc; this.routers = routers; this.plugin = plugin; + this.connectionHostInfo = null; } public getHostInfo(): HostInfo { @@ -58,6 +61,14 @@ export class LimitlessConnectionContext { return this.connectFunc; } + getConnectionHostInfo(): HostInfo | null { + return this.connectionHostInfo; + } + + setConnectionHostInfo(value: HostInfo | null) { + this.connectionHostInfo = value; + } + public getRouters(): HostInfo[] | null { return this.routers; } diff --git a/common/lib/plugins/limitless/limitless_connection_plugin.ts b/common/lib/plugins/limitless/limitless_connection_plugin.ts index 775c6cbd6..768674ae3 100644 --- a/common/lib/plugins/limitless/limitless_connection_plugin.ts +++ b/common/lib/plugins/limitless/limitless_connection_plugin.ts @@ -65,6 +65,9 @@ export class LimitlessConnectionPlugin extends AbstractConnectionPlugin { await this.limitlessRouterService.establishConnection(context); if (context.getConnection() != null) { + if (context.getConnectionHostInfo() != null) { + this.pluginService.setRoutedHostInfo(context.getConnectionHostInfo()); + } return context.getConnection(); } throw new AwsWrapperError(Messages.get("LimitlessConnectionPlugin.failedToConnectToHost", hostInfo.host)); diff --git a/common/lib/plugins/limitless/limitless_router_service.ts b/common/lib/plugins/limitless/limitless_router_service.ts index b33569c2f..46850838d 100644 --- a/common/lib/plugins/limitless/limitless_router_service.ts +++ b/common/lib/plugins/limitless/limitless_router_service.ts @@ -118,6 +118,7 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { try { context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnectionHostInfo(selectedHostSpec); } catch (e) { logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); @@ -161,12 +162,12 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } } - let selectedHostSpec: HostInfo | undefined = undefined; + let selectedHostInfo: HostInfo | undefined = undefined; try { // Select healthiest router for best chance of connection over load-balancing with round-robin - selectedHostSpec = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); - logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostSpec ? selectedHostSpec.host : "undefined")); - if (!selectedHostSpec) { + selectedHostInfo = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); + logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostInfo ? selectedHostInfo.host : "undefined")); + if (!selectedHostInfo) { continue; } } catch (e) { @@ -179,13 +180,14 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } try { - context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnection(await this.pluginService.connect(selectedHostInfo, context.getProperties(), context.getPlugin())); if (context.getConnection()) { + context.setConnectionHostInfo(selectedHostInfo); return; } } catch (error) { - selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); - logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); + selectedHostInfo.setAvailability(HostAvailability.NOT_AVAILABLE); + logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostInfo.host)); } } throw new AwsWrapperError(Messages.get("LimitlessRouterServiceImpl.maxRetriesExceeded")); diff --git a/common/lib/utils/full_services_container.ts b/common/lib/utils/full_services_container.ts index 59902f882..c1be9827e 100644 --- a/common/lib/utils/full_services_container.ts +++ b/common/lib/utils/full_services_container.ts @@ -23,6 +23,7 @@ import { StorageService } from "./storage/storage_service"; import { MonitorService } from "./monitoring/monitor_service"; import { EventPublisher } from "./events/event"; import { ImportantEventService } from "./important_event_service"; +import { HostIdCacheService } from "./host_id_cache_service"; /** * Container for services used throughout the wrapper. @@ -37,6 +38,7 @@ export interface FullServicesContainer { hostListProviderService: HostListProviderService; pluginService: PluginService; importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; } export class FullServicesContainerImpl implements FullServicesContainer { @@ -49,6 +51,7 @@ export class FullServicesContainerImpl implements FullServicesContainer { hostListProviderService!: HostListProviderService; pluginService!: PluginService; importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; constructor( storageService: StorageService, diff --git a/common/lib/utils/host_id_cache_service.ts b/common/lib/utils/host_id_cache_service.ts new file mode 100644 index 000000000..9ac7e32c1 --- /dev/null +++ b/common/lib/utils/host_id_cache_service.ts @@ -0,0 +1,137 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../client_wrapper"; +import { HostInfo } from "../host_info"; +import { PluginService } from "../plugin_service"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; +import { RdsUtils } from "./rds_utils"; +import { RdsUrlType } from "./rds_url_type"; +import { AwsWrapperError } from "./errors"; +import { Messages } from "./messages"; + +/** + * Tuple of (instanceId, instanceName) identifying a connection's underlying instance, as + * returned by a topology-aware dialect. Either value may be null when identification fails. + */ +export type InstanceIdAndName = [instanceId: string | null, instanceName: string | null]; + +/** + * Identifies the underlying instance a connection is established to. + * + * For static endpoints (IP addresses or custom domain names) the identification result is cached, + * keyed by the connection's host name, to avoid repeatedly querying the database. + */ +export interface HostIdCacheService { + /** + * Identify the connected host, using the cache where possible. + * + * @param targetClient the connection to be identified. + * @param connectionHostInfo the {@link HostInfo} the connection was established with. + * @param pluginService the plugin service instance. + * @returns the identified {@link HostInfo}, or null if it cannot be determined. + */ + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise; +} + +function isTopologyAwareDialect(dialect: DatabaseDialect): dialect is DatabaseDialect & TopologyAwareDatabaseDialect { + return typeof (dialect as Partial).getInstanceId === "function"; +} + +export class HostIdCacheServiceImpl implements HostIdCacheService { + static readonly PROP_ENABLED = "AWS_NODEJS_HOST_CACHE_ENABLED"; + static readonly PROP_REGEXP = "AWS_NODEJS_HOST_CACHE_REGEXP"; + + private static readonly cache = new Map(); + private static readonly isEnabled = (process.env[HostIdCacheServiceImpl.PROP_ENABLED] ?? "true").toLowerCase() === "true"; + private static readonly hostRegexp = new RegExp(process.env[HostIdCacheServiceImpl.PROP_REGEXP] ?? ".*"); + private static readonly rdsHelper = new RdsUtils(); + + async identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise { + if (!targetClient || !connectionHostInfo || !pluginService) { + return null; + } + + const urlType: RdsUrlType = HostIdCacheServiceImpl.rdsHelper.identifyRdsType(connectionHostInfo.host); + switch (urlType) { + case RdsUrlType.RDS_INSTANCE: + return connectionHostInfo; + case RdsUrlType.IP_ADDRESS: + case RdsUrlType.OTHER: + // It might be a custom domain name. Cache the identification keyed by host name when allowed. + if (HostIdCacheServiceImpl.isEnabled && HostIdCacheServiceImpl.hostRegexp.test(connectionHostInfo.host)) { + return this.getCachedHostInfo(targetClient, connectionHostInfo, pluginService); + } + return pluginService.identifyConnection(targetClient); + default: + // Other hosts are dynamic and may change at any time, so they can't be cached. + return pluginService.identifyConnection(targetClient); + } + } + + protected async getCachedHostInfo( + targetClient: ClientWrapper, + connectionHostInfo: HostInfo, + pluginService: PluginService + ): Promise { + const host = connectionHostInfo.host; + + let instanceIdAndName = HostIdCacheServiceImpl.cache.get(host); + if (!instanceIdAndName) { + instanceIdAndName = await this.queryInstanceIdAndName(targetClient, pluginService); + HostIdCacheServiceImpl.cache.set(host, instanceIdAndName); + } + + const [instanceId, instanceName] = instanceIdAndName; + if (!instanceId && !instanceName) { + // We've already tried to identify the connection, but got nothing. + return null; + } + + let topology = pluginService.getAllHosts(); + if (!topology || topology.length === 0) { + const provider = pluginService.getHostListProvider(); + topology = provider ? await provider.forceRefresh() : null; + if (!topology || topology.length === 0) { + return null; + } + } + + return topology.find((candidate) => instanceId === candidate.hostId || instanceName === candidate.host) ?? null; + } + + private async queryInstanceIdAndName(targetClient: ClientWrapper, pluginService: PluginService): Promise { + const dialect = pluginService.getDialect(); + if (!isTopologyAwareDialect(dialect)) { + return [null, null]; + } + + try { + const [instanceId, instanceName] = await dialect.getInstanceId(targetClient); + return [instanceId ?? null, instanceName ?? null]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("HostIdCacheService.errorIdentifyConnection"), error); + } + } + + /** + * Clears the static host identification cache. Intended for test cleanup. + */ + static clearCache(): void { + HostIdCacheServiceImpl.cache.clear(); + } +} diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 32d9b9df7..94e619b9b 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -59,6 +59,7 @@ const MESSAGES: Record = { "RdsHostListProvider.noClusterId": "No clusterId found. Please ensure clusterId parameter is set to a non-empty string.", "ConnectionStringHostListProvider.parsedListEmpty": "Can't parse connection string: '%s'.", "ConnectionStringHostListProvider.errorIdentifyConnection": "An error occurred while obtaining the connection's host ID.", + "HostIdCacheService.errorIdentifyConnection": "An error occurred while identifying the connection's host ID.", "ExecuteTimePlugin.executeTime": "Executed method '%s' in %s milliseconds.", "ConnectTimePlugin.connectTime": "Connected to '%s' in %s milliseconds.", "ClusterAwareWriterFailoverHandler.failoverCalledWithInvalidTopology": "Failover was called with an invalid (null or empty) topology.", diff --git a/common/lib/utils/service_utils.ts b/common/lib/utils/service_utils.ts index b7bda542f..35c98b382 100644 --- a/common/lib/utils/service_utils.ts +++ b/common/lib/utils/service_utils.ts @@ -31,6 +31,7 @@ import { TelemetryFactory } from "./telemetry/telemetry_factory"; import { EventPublisher } from "./events/event"; import { PartialPluginService } from "../partial_plugin_service"; import { ConnectionUrlParser } from "./connection_url_parser"; +import { HostIdCacheServiceImpl } from "./host_id_cache_service"; export class ServiceUtils { private static readonly _instance: ServiceUtils = new ServiceUtils(); @@ -70,6 +71,7 @@ export class ServiceUtils { servicesContainer.pluginService = pluginService; servicesContainer.pluginManager = pluginManager; servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); return servicesContainer; } @@ -104,6 +106,7 @@ export class ServiceUtils { servicesContainer.pluginService = pluginService; servicesContainer.pluginManager = pluginManager; servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); return servicesContainer; } diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index 45b9bed7d..2048848c5 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -28,6 +28,7 @@ import { AwsWrapperError, ConnectionProvider, FailoverSuccessError, + HostInfo, InternalPooledConnectionProvider, TransactionIsolationLevel, UndefinedClientError, @@ -86,7 +87,9 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } diff --git a/pg/lib/client.ts b/pg/lib/client.ts index b8c55464a..bb0b3d9fd 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -215,7 +215,10 @@ class BaseAwsPgClient extends AwsClient implements PGClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } From 0ac222c1d3cddbb4b8ebb62e0fb3135d5d9bf2dc Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:42:05 +0000 Subject: [PATCH 2/3] chore: refactor efm plugins (#662) --- common/lib/connection_plugin_chain_builder.ts | 4 +- .../plugins/efm/base/connection_context.ts | 147 ++++++++ common/lib/plugins/efm/base/host_monitor.ts | 230 +++++++++++++ .../plugins/efm/base/host_monitor_service.ts | 117 +++++++ common/lib/plugins/efm/monitor.ts | 290 ---------------- .../plugins/efm/monitor_connection_context.ts | 155 --------- common/lib/plugins/efm/monitor_service.ts | 174 ---------- .../host_monitoring_connection_plugin.ts | 102 +++--- .../host_monitoring_plugin_factory.ts | 17 +- .../v2/host_monitoring2_connection_plugin.ts | 70 ++++ .../v2}/host_monitoring2_plugin_factory.ts | 17 +- .../host_monitoring2_connection_plugin.ts | 163 --------- common/lib/plugins/efm2/monitor.ts | 319 ------------------ .../efm2/monitor_connection_context.ts | 59 ---- common/lib/plugins/efm2/monitor_service.ts | 157 --------- .../profile/driver_configuration_profiles.ts | 2 +- common/lib/utils/monitoring/monitor.ts | 2 +- index.ts | 4 +- .../container/tests/performance.test.ts | 2 +- tests/unit/host_monitoring_plugin.test.ts | 91 ++--- tests/unit/monitor_connection_context.test.ts | 160 +++++---- tests/unit/monitor_impl.test.ts | 73 ++-- tests/unit/monitor_service_impl.test.ts | 247 ++++++-------- 23 files changed, 894 insertions(+), 1708 deletions(-) create mode 100644 common/lib/plugins/efm/base/connection_context.ts create mode 100644 common/lib/plugins/efm/base/host_monitor.ts create mode 100644 common/lib/plugins/efm/base/host_monitor_service.ts delete mode 100644 common/lib/plugins/efm/monitor.ts delete mode 100644 common/lib/plugins/efm/monitor_connection_context.ts delete mode 100644 common/lib/plugins/efm/monitor_service.ts rename common/lib/plugins/efm/{ => v1}/host_monitoring_connection_plugin.ts (64%) rename common/lib/plugins/efm/{ => v1}/host_monitoring_plugin_factory.ts (72%) create mode 100644 common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts rename common/lib/plugins/{efm2 => efm/v2}/host_monitoring2_plugin_factory.ts (72%) delete mode 100644 common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts delete mode 100644 common/lib/plugins/efm2/monitor.ts delete mode 100644 common/lib/plugins/efm2/monitor_connection_context.ts delete mode 100644 common/lib/plugins/efm2/monitor_service.ts diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 38ec0a4f1..1f4fd2777 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -31,7 +31,7 @@ import { StaleDnsPluginFactory } from "./plugins/stale_dns/stale_dns_plugin_fact import { FederatedAuthPluginFactory } from "./plugins/federated_auth/federated_auth_plugin_factory"; import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { OktaAuthPluginFactory } from "./plugins/federated_auth/okta_auth_plugin_factory"; -import { HostMonitoringPluginFactory } from "./plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "./plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "./plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "./plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; import { ConnectionProviderManager } from "./connection_provider_manager"; @@ -41,7 +41,7 @@ import { LimitlessConnectionPluginFactory } from "./plugins/limitless/limitless_ import { FastestResponseStrategyPluginFactory } from "./plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory"; import { CustomEndpointPluginFactory } from "./plugins/custom_endpoint/custom_endpoint_plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { HostMonitoring2PluginFactory } from "./plugins/efm2/host_monitoring2_plugin_factory"; +import { HostMonitoring2PluginFactory } from "./plugins/efm/v2/host_monitoring2_plugin_factory"; import { BlueGreenPluginFactory } from "./plugins/bluegreen/blue_green_plugin_factory"; import { GlobalDbFailoverPluginFactory } from "./plugins/gdb_failover/global_db_failover_plugin_factory"; import { FullServicesContainer } from "./utils/full_services_container"; diff --git a/common/lib/plugins/efm/base/connection_context.ts b/common/lib/plugins/efm/base/connection_context.ts new file mode 100644 index 000000000..bfcaffd56 --- /dev/null +++ b/common/lib/plugins/efm/base/connection_context.ts @@ -0,0 +1,147 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../../../client_wrapper"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; + +/** + * Monitoring context for each connection. This contains each connection's criteria for whether a + * host should be considered unhealthy. The context is shared between the main task and the + * monitor task. + */ +export interface ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + isActiveContext(): boolean; + isHostUnhealthy(): boolean; + setInactive(): void; + abortConnection(): Promise; + updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise; +} + +export class ConnectionContextImpl implements ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + private readonly failureDetectionTimeMillis: number; + private readonly connectionToAbortRef: WeakRef; + private readonly abortedConnectionsCounter: TelemetryCounter; + private readonly startMonitorTimeNano: number; + + private _activeContext: boolean = true; + private _hostUnhealthy: boolean = false; + private invalidHostStartTimeNano: number = 0; + private failureCount: number = 0; + + constructor( + connectionToAbort: ClientWrapper, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number, + abortedConnectionsCounter: TelemetryCounter + ) { + this.connectionToAbortRef = new WeakRef(connectionToAbort); + this.failureDetectionTimeMillis = failureDetectionTimeMillis; + this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; + this.failureDetectionCount = failureDetectionCount; + this.abortedConnectionsCounter = abortedConnectionsCounter; + this.startMonitorTimeNano = getCurrentTimeNano(); + this.expectedActiveMonitoringStartTimeNano = this.startMonitorTimeNano + this.failureDetectionTimeMillis * 1_000_000; + } + + isActiveContext(): boolean { + return this._activeContext; + } + + isHostUnhealthy(): boolean { + return this._hostUnhealthy; + } + + setInactive(): void { + this._activeContext = false; + } + + async abortConnection(): Promise { + const connectionToAbort = this.connectionToAbortRef.deref(); + if (connectionToAbort == null || !this._activeContext) { + return; + } + + try { + await connectionToAbort.abort(); + this.abortedConnectionsCounter.inc(); + } catch (error: any) { + // ignore + logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); + } + } + + /** + * Update whether the connection is still valid if the total elapsed time has passed the + * grace period. + */ + async updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise { + if (!this._activeContext) { + return; + } + + const totalElapsedTimeNano = statusCheckEndTimeNano - this.startMonitorTimeNano; + + if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1_000_000) { + await this.setConnectionValid(hostName, isValid, statusCheckStartTimeNano, statusCheckEndTimeNano); + } + } + + private async setConnectionValid( + hostName: string, + connectionValid: boolean, + statusCheckStartNano: number, + statusCheckEndNano: number + ): Promise { + if (!connectionValid) { + this.failureCount++; + + if (this.invalidHostStartTimeNano === 0) { + this.invalidHostStartTimeNano = statusCheckStartNano; + } + + const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; + const maxInvalidHostDurationNano = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount) * 1_000_000; + + if (invalidHostDurationNano >= maxInvalidHostDurationNano) { + logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); + this._hostUnhealthy = true; + await this.abortConnection(); + return; + } + + logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); + return; + } + + this.failureCount = 0; + this.invalidHostStartTimeNano = 0; + this._hostUnhealthy = false; + + logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); + } +} diff --git a/common/lib/plugins/efm/base/host_monitor.ts b/common/lib/plugins/efm/base/host_monitor.ts new file mode 100644 index 000000000..9cebcd58e --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor.ts @@ -0,0 +1,230 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext } from "./connection_context"; +import { HostInfo } from "../../../host_info"; +import { PluginService } from "../../../plugin_service"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; +import { getTimeInNanos } from "../../../utils/utils"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; +import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; +import { AbstractMonitor, MonitorState } from "../../../utils/monitoring/monitor"; + +export interface HostMonitor { + startMonitoring(context: ConnectionContext): void; + + stopMonitoring(context: ConnectionContext): void; + + clearContexts(): void; + + isStopped(): boolean; + + run(): Promise; + + releaseResources(): Promise; +} + +type ConnectionStatus = [isValid: boolean, elapsedTimeNano: number]; + +export class HostMonitorImpl extends AbstractMonitor implements HostMonitor { + private static readonly SLEEP_WHEN_INACTIVE_MILLIS = 100; + private static readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS = 3000; + private static readonly MONITOR_TERMINATION_TIMEOUT_SEC = 30; + + private readonly pluginService: PluginService; + private readonly telemetryFactory: TelemetryFactory; + private readonly hostInvalidCounter: TelemetryCounter; + private readonly properties: Map; + private readonly hostInfo: HostInfo; + private readonly monitorDisposalTimeMillis: number; + private contexts: ConnectionContext[] = []; + + private contextLastUsedTimestampNano: number; + private monitoringClient: ClientWrapper | null = null; + private delayTimeoutId: ReturnType | undefined; + private sleepTimeoutId: ReturnType | undefined; + + constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { + super(HostMonitorImpl.MONITOR_TERMINATION_TIMEOUT_SEC); + this.pluginService = pluginService; + this.telemetryFactory = this.pluginService.getTelemetryFactory(); + this.hostInfo = hostInfo; + this.properties = properties; + this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; + this.hostInvalidCounter = this.telemetryFactory.createCounter(`efm.nodeUnhealthy.count.${hostInfo.hostId || hostInfo.host}`); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + } + + startMonitoring(context: ConnectionContext): void { + if (this._stop) { + logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); + } + + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + this.contexts.push(context); + } + + stopMonitoring(context: ConnectionContext): void { + if (!context) { + logger.warn(Messages.get("MonitorImpl.contextNullWarning")); + return; + } + + context.setInactive(); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + } + + clearContexts(): void { + this.contexts.length = 0; + } + + isStopped(): boolean { + return this._stop || this.state === MonitorState.STOPPED; + } + + canDispose(): boolean { + return this.contexts.length === 0; + } + + async monitor(): Promise { + logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); + + try { + while (!this._stop) { + try { + this.lastActivityTimestampNanos = getTimeInNanos(); + + const activeContexts = this.contexts.filter((ctx) => ctx.isActiveContext()); + + if (activeContexts.length > 0) { + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + + const statusCheckStartTimeNano = getCurrentTimeNano(); + const [isValid, elapsedTimeNano] = await this.checkConnectionStatus(); + + let delayMillis = -1; + + for (const context of activeContexts) { + if (!context.isActiveContext()) { + continue; + } + + await context.updateConnectionStatus(this.hostInfo.url, statusCheckStartTimeNano, statusCheckStartTimeNano + elapsedTimeNano, isValid); + + if ( + context.isActiveContext() && + !context.isHostUnhealthy() && + statusCheckStartTimeNano >= context.expectedActiveMonitoringStartTimeNano + ) { + if (delayMillis === -1 || delayMillis > context.failureDetectionIntervalMillis) { + delayMillis = context.failureDetectionIntervalMillis; + } + } + } + + this.contexts = this.contexts.filter((ctx) => ctx.isActiveContext() && !ctx.isHostUnhealthy()); + + if (delayMillis === -1) { + delayMillis = HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS; + } else { + delayMillis -= Math.round(elapsedTimeNano / 1_000_000); + if (delayMillis <= 0) { + delayMillis = HostMonitorImpl.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; + } + } + + await new Promise((resolve) => { + this.delayTimeoutId = setTimeout(resolve, delayMillis); + }); + } else { + if (getCurrentTimeNano() - this.contextLastUsedTimestampNano >= this.monitorDisposalTimeMillis * 1_000_000) { + break; + } + await new Promise((resolve) => { + this.sleepTimeoutId = setTimeout(resolve, HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS); + }); + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); + } + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); + } + + logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); + } + + async close(): Promise { + this.contexts.length = 0; + await this.closeMonitoringClient(); + } + + async releaseResources(): Promise { + clearTimeout(this.delayTimeoutId); + clearTimeout(this.sleepTimeoutId); + await this.stop(); + } + + protected async checkConnectionStatus(): Promise { + const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); + connectContext.setAttribute("url", this.hostInfo.host); + return await connectContext.start(async () => { + const startNanos = getCurrentTimeNano(); + try { + if (this.monitoringClient != null && (await this.pluginService.isClientValid(this.monitoringClient))) { + return [true, getCurrentTimeNano() - startNanos]; + } + + await this.closeMonitoringClient(); + + const monitoringConnProperties = new Map(this.properties); + for (const key of monitoringConnProperties.keys()) { + if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { + continue; + } + monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); + monitoringConnProperties.delete(key); + } + + this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); + return [true, getCurrentTimeNano() - startNanos]; + } catch (error: any) { + this.hostInvalidCounter.inc(); + await this.closeMonitoringClient(); + return [false, getCurrentTimeNano() - startNanos]; + } + }); + } + + private async closeMonitoringClient(): Promise { + if (this.monitoringClient) { + try { + await this.pluginService.abortTargetClient(this.monitoringClient); + } catch { + // ignore + } + this.monitoringClient = null; + } + } +} diff --git a/common/lib/plugins/efm/base/host_monitor_service.ts b/common/lib/plugins/efm/base/host_monitor_service.ts new file mode 100644 index 000000000..1c976a179 --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor_service.ts @@ -0,0 +1,117 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext, ConnectionContextImpl } from "./connection_context"; +import { HostMonitor, HostMonitorImpl } from "./host_monitor"; +import { HostInfo } from "../../../host_info"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { MonitorService } from "../../../utils/monitoring/monitor_service"; +import { MonitorInitializer } from "../../../utils/monitoring/monitor"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export interface HostMonitorService { + startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise; + + stopMonitoring(context: ConnectionContext): void; + + releaseResources(): Promise; +} + +export class HostMonitorServiceImpl implements HostMonitorService { + private static readonly MONITOR_DISPOSAL_TIME_NS = BigInt(10 * 60 * 1_000_000_000); // 10 minutes + private static readonly INACTIVE_TIMEOUT_NS = BigInt(3 * 60 * 1_000_000_000); // 3 minutes + + private readonly servicesContainer: FullServicesContainer; + private readonly coreMonitorService: MonitorService; + private readonly abortedConnectionsCounter: TelemetryCounter; + + constructor(servicesContainer: FullServicesContainer) { + this.servicesContainer = servicesContainer; + this.coreMonitorService = servicesContainer.monitorService; + const telemetryFactory = servicesContainer.telemetryFactory; + this.abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); + + this.coreMonitorService.registerMonitorTypeIfAbsent( + HostMonitorImpl, + HostMonitorServiceImpl.MONITOR_DISPOSAL_TIME_NS, + HostMonitorServiceImpl.INACTIVE_TIMEOUT_NS, + new Set(), + undefined + ); + } + + async startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise { + const monitorKey = hostInfo.hostId || hostInfo.host; + + const monitor = await this.getMonitor(monitorKey, hostInfo, properties); + + if (!monitor) { + throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); + } + + const context = new ConnectionContextImpl( + connectionToAbort, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount, + this.abortedConnectionsCounter + ); + + monitor.startMonitoring(context); + return context; + } + + stopMonitoring(context: ConnectionContext): void { + context.setInactive(); + } + + private async getMonitor(monitorKey: string, hostInfo: HostInfo, properties: Map): Promise { + const monitorDisposalTimeMillis = WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties); + + const initializer: MonitorInitializer = { + createMonitor: (_servicesContainer) => + new HostMonitorImpl(this.servicesContainer.pluginService, hostInfo, properties, monitorDisposalTimeMillis) + }; + + return await this.coreMonitorService.runIfAbsent(HostMonitorImpl, monitorKey, this.servicesContainer, properties, initializer); + } + + async releaseResources(): Promise { + await this.coreMonitorService.stopAndRemoveMonitors(HostMonitorImpl); + } + + static async clearMonitors(monitorService: MonitorService): Promise { + await monitorService.stopAndRemoveMonitors(HostMonitorImpl); + } +} diff --git a/common/lib/plugins/efm/monitor.ts b/common/lib/plugins/efm/monitor.ts deleted file mode 100644 index 3094427f6..000000000 --- a/common/lib/plugins/efm/monitor.ts +++ /dev/null @@ -1,290 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - stopMonitoring(context: MonitorConnectionContext): void; - - clearContexts(): void; - - isStopped(): boolean; - - run(): Promise; - - releaseResources(): Promise; - - endMonitoringClient(): Promise; -} - -class ConnectionStatus { - isValid: boolean; - elapsedTimeNano: number; - - constructor(isValid: boolean, elapsedTimeNano: number) { - this.isValid = isValid; - this.elapsedTimeNano = elapsedTimeNano; - } -} - -export class MonitorImpl implements Monitor { - private readonly SLEEP_WHEN_INACTIVE_MILLIS: number = 100; - private readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS: number = 3000; - - private readonly activeContexts: MonitorConnectionContext[] = []; - private readonly newContexts: MonitorConnectionContext[] = []; - - private readonly pluginService: PluginService; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private readonly monitorDisposalTimeMillis: number; - private readonly telemetryFactory: TelemetryFactory; - private readonly instanceInvalidCounter: TelemetryCounter; - private contextLastUsedTimestampNanos: number; - private started = false; - private stopped: boolean = false; - private cancelled: boolean = false; - private monitoringClient: ClientWrapper | null = null; - private delayMillisTimeoutId: any; - private sleepWhenInactiveTimeoutId: any; - - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.properties = properties; - this.hostInfo = hostInfo; - this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - const instanceId = this.hostInfo.hostId ?? this.hostInfo.host; - this.instanceInvalidCounter = this.telemetryFactory.createCounter(`efm.hostUnhealthy.count.${instanceId}`); - } - - startRun() { - this.run(); - this.started = true; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.stopped) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const currentTimeNanos: number = getCurrentTimeNano(); - context.startMonitorTimeNano = currentTimeNanos; - this.contextLastUsedTimestampNanos = currentTimeNanos; - this.newContexts.push(context); - if (!this.started) { - this.startRun(); - } - } - - stopMonitoring(context: MonitorConnectionContext): void { - if (context == null) { - logger.warn(Messages.get("MonitorImpl.contextNullWarning")); - return; - } - - context.isActiveContext = false; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.cancelled) { - try { - let newMonitorContext: MonitorConnectionContext | undefined; - let firstAddedNewMonitorContext: MonitorConnectionContext | null = null; - const currentTimeNano: number = getCurrentTimeNano(); - while ((newMonitorContext = this.newContexts?.shift()) != null) { - if (firstAddedNewMonitorContext === newMonitorContext) { - this.newContexts.push(newMonitorContext); - - break; - } - - if (newMonitorContext.isActiveContext) { - if (newMonitorContext.expectedActiveMonitoringStartTimeNano > currentTimeNano) { - this.newContexts.push(newMonitorContext); - - firstAddedNewMonitorContext = firstAddedNewMonitorContext ?? newMonitorContext; - } else { - this.activeContexts.push(newMonitorContext); - } - } - } - - if (this.activeContexts.length > 0) { - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - this.contextLastUsedTimestampNanos = statusCheckStartTimeNanos; - - const status: ConnectionStatus = await this.checkConnectionStatus(); - let delayMillis: number = -1; - - let monitorContext: MonitorConnectionContext | undefined; - let firstAddedMonitorContext: MonitorConnectionContext | null = null; - - while ((monitorContext = this.activeContexts?.shift()) != null) { - // If context is already invalid, just skip it. - if (!monitorContext.isActiveContext) { - continue; - } - - if (firstAddedMonitorContext == monitorContext) { - // This context is already processed by this loop. - // Add it to the array and exit this loop. - - this.activeContexts?.push(monitorContext); - break; - } - - // Otherwise, process this context. - await monitorContext.updateConnectionStatus( - this.hostInfo.url, - statusCheckStartTimeNanos, - statusCheckStartTimeNanos + status.elapsedTimeNano, - status.isValid - ); - - if (monitorContext.isActiveContext && !monitorContext.isHostUnhealthy) { - this.activeContexts?.push(monitorContext); - - if (firstAddedMonitorContext == null) { - firstAddedMonitorContext = monitorContext; - } - - if (delayMillis === -1 || delayMillis > monitorContext.failureDetectionIntervalMillis) { - delayMillis = monitorContext.failureDetectionIntervalMillis; - } - } - } - - if (delayMillis === -1) { - // No active contexts. - delayMillis = this.SLEEP_WHEN_INACTIVE_MILLIS; - } else { - delayMillis -= Math.round(status.elapsedTimeNano / 1_000_000); - // Check for minimum delay between host health check; - if (delayMillis <= 0) { - delayMillis = this.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; - } - } - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, delayMillis); - }); - } else { - if (getCurrentTimeNano() - this.contextLastUsedTimestampNanos >= this.monitorDisposalTimeMillis * 1_000_000) { - break; - } - await new Promise((resolve) => { - this.sleepWhenInactiveTimeoutId = setTimeout(resolve, this.SLEEP_WHEN_INACTIVE_MILLIS); - }); - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - this.stopped = true; - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - return await connectContext.start(async () => { - const startNanos = getCurrentTimeNano(); - try { - const clientIsValid = this.monitoringClient && (await this.pluginService.isClientValid(this.monitoringClient)); - - if (this.monitoringClient !== null && clientIsValid) { - return Promise.resolve(new ConnectionStatus(clientIsValid, getCurrentTimeNano() - startNanos)); - } - - await this.endMonitoringClient(); - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return Promise.resolve(new ConnectionStatus(true, getCurrentTimeNano() - startNanos)); - } catch (error: any) { - this.instanceInvalidCounter.inc(); - await this.endMonitoringClient(); - return Promise.resolve(new ConnectionStatus(false, getCurrentTimeNano() - startNanos)); - } - }); - } - - clearContexts(): void { - this.activeContexts.length = 0; - this.newContexts.length = 0; - } - - isStopped(): boolean { - return this.stopped || this.cancelled; - } - - async releaseResources() { - this.cancelled = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenInactiveTimeoutId); - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient() { - if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - } -} diff --git a/common/lib/plugins/efm/monitor_connection_context.ts b/common/lib/plugins/efm/monitor_connection_context.ts deleted file mode 100644 index a68893228..000000000 --- a/common/lib/plugins/efm/monitor_connection_context.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { Monitor } from "./monitor"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { sleep } from "../../utils/utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { PluginService } from "../../plugin_service"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; - -export class MonitorConnectionContext { - readonly failureDetectionIntervalMillis: number; - private readonly failureDetectionTimeMillis: number; - private readonly failureDetectionCount: number; - private readonly telemetryAbortedConnectionCounter: TelemetryCounter; - readonly clientToAbort: ClientWrapper; - readonly monitor: Monitor; - readonly pluginService: PluginService; - - isActiveContext: boolean = true; - isHostUnhealthy: boolean = false; - startMonitorTimeNano: number = 0; - expectedActiveMonitoringStartTimeNano: number = 0; - private invalidHostStartTimeNano: number = 0; - private abortedConnectionCounter: number = 0; - failureCount: number = 0; - id: string = uniqueId("_monitorContext"); - - constructor( - monitor: Monitor, - clientToAbort: ClientWrapper, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - pluginService: PluginService, - telemetryAbortedConnectionCounter: TelemetryCounter - ) { - this.monitor = monitor; - this.clientToAbort = clientToAbort; - this.failureDetectionTimeMillis = failureDetectionTimeMillis; - this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; - this.failureDetectionCount = failureDetectionCount; - this.pluginService = pluginService; - this.telemetryAbortedConnectionCounter = telemetryAbortedConnectionCounter; - } - - resetInvalidHostStartTimeNano(): void { - this.invalidHostStartTimeNano = 0; - } - - isInvalidHostStartTimeDefined(): boolean { - return this.invalidHostStartTimeNano > 0; - } - - async abortConnection(): Promise { - if (this.clientToAbort == null || !this.isActiveContext) { - return Promise.resolve(); - } - - try { - await this.pluginService.abortTargetClient(this.clientToAbort); - this.telemetryAbortedConnectionCounter.inc(); - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - this.abortedConnectionCounter++; - } - - /** - * Update whether the connection is still valid if the total elapsed time has passed the grace period. - * @param hostName A host name for logging purposes. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @param isValid Whether the connection is valid. - */ - async updateConnectionStatus(hostName: string, statusCheckStartNano: number, statusCheckEndNano: number, isValid: boolean): Promise { - if (!this.isActiveContext) { - return; - } - - const totalElapsedTimeNano: number = statusCheckEndNano - this.startMonitorTimeNano; - if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1000000) { - await this.setConnectionValid(hostName, isValid, statusCheckStartNano, statusCheckEndNano); - } - } - - /** - * Set whether the connection to the server is still valid based on the monitoring settings set in the {@link AwsClient}. - * - *

These monitoring settings include: - * - *

    - *
  • {@code failureDetectionInterval} - *
  • {@code failureDetectionTime} - *
  • {@code failureDetectionCount} - *
- * - * @param hostName A host name for logging purposes. - * @param connectionValid Boolean indicating whether the server is still responsive. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @protected - */ - async setConnectionValid(hostName: string, connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number) { - if (!connectionValid) { - this.failureCount++; - - if (!this.isInvalidHostStartTimeDefined()) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano: number = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationMillis: number = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount); - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationMillis * 1_000_000) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); - this.isHostUnhealthy = true; - await this.abortConnection(); - return; - } - - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); - return; - } - - this.failureCount = 0; - this.resetInvalidHostStartTimeNano(); - this.isHostUnhealthy = false; - - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); - } - - async trackHealthStatus() { - while (!this.isHostUnhealthy && this.isActiveContext) { - await sleep(100); - } - - throw new AwsWrapperError("trackHealthStatus stopped"); - } -} diff --git a/common/lib/plugins/efm/monitor_service.ts b/common/lib/plugins/efm/monitor_service.ts deleted file mode 100644 index c81639f52..000000000 --- a/common/lib/plugins/efm/monitor_service.ts +++ /dev/null @@ -1,174 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError, IllegalArgumentError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; -import { ClientWrapper } from "../../client_wrapper"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - stopMonitoring(context: MonitorConnectionContext): Promise; - - stopMonitoringForAllConnections(hostKeys: Set): void; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - undefined, - async (monitor: Monitor) => { - await monitor.releaseResources(); - }, - "efm/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private cachedMonitorHostKeys: Set | undefined; - private cachedMonitorRef: WeakRef | undefined; - monitorSupplier = (pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) => - new MonitorImpl(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - if (hostKeys.size === 0) { - throw new IllegalArgumentError(Messages.get("MonitorService.emptyAliasSet", hostInfo.host)); - } - - let monitor: Monitor | null = this.cachedMonitorRef?.deref() ?? null; - - if (!monitor || (monitor && monitor.isStopped()) || this.cachedMonitorHostKeys?.size === 0 || this.cachedMonitorHostKeys !== hostKeys) { - monitor = await this.getMonitor(hostKeys, hostInfo, properties); - if (monitor) { - this.cachedMonitorRef = new WeakRef(monitor); - this.cachedMonitorHostKeys = hostKeys; - } - } - - const telemetryFactory = this.pluginService.getTelemetryFactory(); - const abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); - - if (monitor) { - const context = new MonitorConnectionContext( - monitor, - clientToAbort, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.pluginService, - abortedConnectionsCounter - ); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext) { - context.monitor.stopMonitoring(context); - await context.monitor.endMonitoringClient(); - } - - stopMonitoringForAllConnections(hostKeys: Set) { - let monitor: Monitor; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - if (monitor) { - monitor.clearContexts(); - return; - } - } - } - - async getMonitor(hostKeys: Set, hostInfo: HostInfo, properties: Map): Promise { - let monitor: Monitor; - let anyHostKey: string; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - anyHostKey = hostKey; - if (monitor) { - break; - } - } - - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - if (anyHostKey && (!monitor || (monitor && monitor.isStopped()))) { - monitor = MonitorServiceImpl.monitors.computeIfAbsent( - anyHostKey, - () => this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)), - cacheExpirationNanos - ); - - if (monitor && monitor.isStopped()) { - await monitor.releaseResources(); - MonitorServiceImpl.monitors.remove(anyHostKey); - monitor = this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)); - } - } - - if (monitor) { - this.populateMonitorMap(hostKeys, monitor, cacheExpirationNanos); - return monitor; - } - - return null; - } - - private populateMonitorMap(hostKeys: Set, monitor: Monitor, cacheExpirationNanos: bigint) { - for (const hostKey of hostKeys) { - MonitorServiceImpl.monitors.putIfAbsent(hostKey, monitor, cacheExpirationNanos); - } - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - this.cachedMonitorHostKeys = undefined; - this.cachedMonitorRef = undefined; - } - - // Used for performance testing. - static async clearMonitors() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts similarity index 64% rename from common/lib/plugins/efm/host_monitoring_connection_plugin.ts rename to common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts index 59ad986fb..3c7548d24 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts @@ -14,37 +14,42 @@ limitations under the License. */ -import { HostInfo, AwsWrapperError, UnavailableHostError, HostAvailability } from "../../"; -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; +import { HostInfo } from "../../../host_info"; +import { AwsWrapperError, UnavailableHostError } from "../../../utils/errors"; +import { PluginService } from "../../../plugin_service"; +import { HostChangeOptions } from "../../../host_change_options"; +import { OldConnectionSuggestionAction } from "../../../old_connection_suggestion_action"; +import { RdsUtils } from "../../../utils/rds_utils"; +import { AbstractConnectionPlugin } from "../../../abstract_connection_plugin"; +import { RdsUrlType } from "../../../utils/rds_url_type"; +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { HostListProvider } from "../../../host_list_provider/host_list_provider"; +import { CanReleaseResources } from "../../../can_release_resources"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { ClientWrapper } from "../../../client_wrapper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; +import { sleep } from "../../../utils/utils"; export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { + private static readonly UNHEALTHY_STATE = Symbol("unhealthy"); + id: string = uniqueId("_efmPlugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; + protected readonly properties: Map; + protected readonly pluginService: PluginService; + protected readonly rdsUtils: RdsUtils; + protected readonly monitorService: HostMonitorService; private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { super(); - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); + this.rdsUtils = new RdsUtils(); + this.monitorService = monitorService ?? new HostMonitorServiceImpl(servicesContainer); } getSubscribedMethods(): Set { @@ -78,20 +83,19 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return methodFunc(); } - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties) as number; - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties) as number; - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties) as number; + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); let result: T; - let monitorContext: MonitorConnectionContext | null = null; + let context: ConnectionContext | null = null; try { logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - monitorContext = await this.monitorService.startMonitoring( + context = await this.monitorService.startMonitoring( this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo.allAliases, monitoringHostInfo, this.properties, failureDetectionTimeMillis, @@ -99,19 +103,21 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp failureDetectionCount ); - result = await Promise.race([monitorContext.trackHealthStatus(), methodFunc()]) - .then((result: any) => { - return result; - }) - .catch((error: any) => { - throw error; + const methodPromise = methodFunc(); + const raceResult = await Promise.race([this.waitForUnhealthy(context), methodPromise]); + if (raceResult === HostMonitoringConnectionPlugin.UNHEALTHY_STATE) { + methodPromise.catch(() => { + // Attach a no-op rejection handler to prevent it from throwing an unhandled promise rejection error when waitForUnhealthy times out first. }); + throw new AwsWrapperError("Host monitoring detected unhealthy host"); + } + result = raceResult as T; } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext); + if (context != null) { + this.monitorService.stopMonitoring(context); logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - if (monitorContext.isHostUnhealthy) { + if (context.isHostUnhealthy()) { const monitoringHostInfo = await this.getMonitoringHostInfo(); const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; @@ -135,6 +141,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return result; } + private async waitForUnhealthy(context: ConnectionContext): Promise { + while (!context.isHostUnhealthy() && context.isActiveContext()) { + await sleep(100); + } + return HostMonitoringConnectionPlugin.UNHEALTHY_STATE; + } + private throwUnableToIdentifyConnection(host: HostInfo | null): never { const provider: HostListProvider | null = this.pluginService.getHostListProvider(); throw new AwsWrapperError( @@ -167,6 +180,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); this.throwUnableToIdentifyConnection(host); } + // Update identified HostInfo for the current connection await this.pluginService.setCurrentClient(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); } @@ -180,15 +194,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp } async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.WENT_DOWN) || changes.has(HostChangeOptions.HOST_DELETED)) { - const aliases = (await this.getMonitoringHostInfo())?.allAliases; - if (aliases && aliases.size > 0) { - this.monitorService.stopMonitoringForAllConnections(aliases); - } + if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { + this.monitoringHostInfo = null; } - - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; return OldConnectionSuggestionAction.NO_OPINION; } diff --git a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts similarity index 72% rename from common/lib/plugins/efm/host_monitoring_plugin_factory.ts rename to common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts index d6b430954..49f3d6284 100644 --- a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts @@ -14,12 +14,11 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; -import { FullServicesContainer } from "../../utils/full_services_container"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoringPluginFactory extends ConnectionPluginFactory { private static hostMonitoringPlugin: any; @@ -29,11 +28,7 @@ export class HostMonitoringPluginFactory extends ConnectionPluginFactory { if (!HostMonitoringPluginFactory.hostMonitoringPlugin) { HostMonitoringPluginFactory.hostMonitoringPlugin = await import("./host_monitoring_connection_plugin"); } - return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin( - servicesContainer.pluginService, - properties, - new RdsUtils() - ); + return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts new file mode 100644 index 000000000..3ca1194dd --- /dev/null +++ b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts @@ -0,0 +1,70 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { HostMonitoringConnectionPlugin } from "../v1/host_monitoring_connection_plugin"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export class HostMonitoring2ConnectionPlugin extends HostMonitoringConnectionPlugin { + id: string = uniqueId("_efm2Plugin"); + + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { + super(servicesContainer, properties, monitorService ?? new HostMonitorServiceImpl(servicesContainer)); + } + + async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { + const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); + + if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { + return methodFunc(); + } + + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); + + let result: T; + let context: ConnectionContext | null = null; + + try { + logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); + const monitoringHostInfo = await this.getMonitoringHostInfo(); + + context = await this.monitorService.startMonitoring( + this.pluginService.getCurrentClient().targetClient, + monitoringHostInfo, + this.properties, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount + ); + + result = await methodFunc(); + } finally { + if (context != null) { + this.monitorService.stopMonitoring(context); + logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); + } + } + + return result; + } +} diff --git a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts similarity index 72% rename from common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts rename to common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts index 0ac0319f4..e21e4c8be 100644 --- a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts +++ b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts @@ -14,12 +14,11 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; -import { FullServicesContainer } from "../../utils/full_services_container"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { private static hostMonitoring2Plugin: any; @@ -29,11 +28,7 @@ export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { if (!HostMonitoring2PluginFactory.hostMonitoring2Plugin) { HostMonitoring2PluginFactory.hostMonitoring2Plugin = await import("./host_monitoring2_connection_plugin"); } - return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin( - servicesContainer.pluginService, - properties, - new RdsUtils() - ); + return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts deleted file mode 100644 index 036b6dcd3..000000000 --- a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { HostInfo } from "../../host_info"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { AwsWrapperError } from "../../"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; - -export class HostMonitoring2ConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - id: string = uniqueId("_efm2Plugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; - private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { - super(); - this.pluginService = pluginService; - this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); - } - - getSubscribedMethods(): Set { - return new Set(["*"]); - } - - async connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - const targetClient = await connectFunc(); - if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); - if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); - } - } - return targetClient; - } - - async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { - const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); - - if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { - return methodFunc(); - } - - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); - - let result: T; - let monitorContext: MonitorConnectionContext | null = null; - - try { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); - const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - - monitorContext = await this.monitorService.startMonitoring( - this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo, - this.properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount - ); - - result = await methodFunc(); - } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext, this.pluginService.getCurrentClient().targetClient); - - logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - } - } - - return result; - } - - private throwUnableToIdentifyConnection(host: HostInfo | null): never { - const provider: HostListProvider | null = this.pluginService.getHostListProvider(); - throw new AwsWrapperError( - Messages.get( - "HostMonitoringConnectionPlugin.unableToIdentifyConnection", - host !== null ? host.host : "unknown host", - provider !== null ? provider.getHostProviderType() : "unknown provider" - ) - ); - } - - async getMonitoringHostInfo(): Promise { - if (this.monitoringHostInfo) { - return this.monitoringHostInfo; - } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); - if (this.monitoringHostInfo === null) { - this.throwUnableToIdentifyConnection(null); - } - const rdsUrlType: RdsUrlType = this.rdsUtils.identifyRdsType(this.monitoringHostInfo.url); - - try { - if (rdsUrlType.isRdsCluster) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); - if (this.monitoringHostInfo == null) { - const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); - this.throwUnableToIdentifyConnection(host); - } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); - } - } catch (error: any) { - if (!(error instanceof AwsWrapperError)) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.errorIdentifyingConnection", error.message)); - } - throw error; - } - - return this.monitoringHostInfo; - } - - async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; - } - return OldConnectionSuggestionAction.NO_OPINION; - } - - async releaseResources(): Promise { - await this.monitorService.releaseResources(); - } -} diff --git a/common/lib/plugins/efm2/monitor.ts b/common/lib/plugins/efm2/monitor.ts deleted file mode 100644 index f1d72a61a..000000000 --- a/common/lib/plugins/efm2/monitor.ts +++ /dev/null @@ -1,319 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { HostAvailability } from "../../host_availability/host_availability"; -import { MapUtils } from "../../utils/map_utils"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - run(): Promise; - - canDispose(): boolean; - - endMonitoringClient(): Promise; - - releaseResources(): Promise; -} - -export class MonitorImpl implements Monitor { - private static readonly TASK_SLEEP_MILLIS: number = 100; - private activeContexts: WeakRef[] = []; - static newContexts: Map>> = new Map>>(); - private readonly pluginService: PluginService; - private readonly telemetryFactory: TelemetryFactory; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private stopped: boolean = false; - - private monitoringClient: ClientWrapper | null = null; - - private readonly failureDetectionTimeNano: number; - private readonly failureDetectionIntervalNanos: number; - private readonly failureDetectionCount: number; - - private invalidHostStartTimeNano: number; - private failureCount: number; - private hostUnhealthy: boolean = false; - private readonly abortedConnectionsCounter: TelemetryCounter; - private delayMillisTimeoutId: any; - private sleepWhenHostHealthyTimeoutId: any; - - constructor( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.hostInfo = hostInfo; - this.properties = properties; - this.failureDetectionTimeNano = failureDetectionTimeMillis * 1000000; - this.failureDetectionIntervalNanos = failureDetectionIntervalMillis * 1000000; - this.failureDetectionCount = failureDetectionCount; - this.abortedConnectionsCounter = abortedConnectionsCounter; - - const hostId: string = this.hostInfo.hostId ?? this.hostInfo.host; - - this.telemetryFactory.createGauge(`efm2.newContexts.size.${hostId}`, () => MonitorImpl.newContexts.size === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.activeContexts.size.${hostId}`, () => this.activeContexts.length === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.hostHealthy.${hostId}`, () => (this.hostUnhealthy ? 0 : 1)); - - Promise.race([this.newContextRun(), this.run()]).finally(() => { - this.stopped = true; - }); - } - - canDispose(): boolean { - return this.activeContexts.length === 0 && MonitorImpl.newContexts.size === 0; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.isStopped()) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const startMonitorTimeNano = getCurrentTimeNano() + this.failureDetectionTimeNano; - const connectionQueue = MapUtils.computeIfAbsent( - MonitorImpl.newContexts, - startMonitorTimeNano, - () => new Array>() - ); - connectionQueue.push(new WeakRef(context)); - } - - async newContextRun(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoringTaskNewContext", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - const currentTimeNanos = getCurrentTimeNano(); - // Get entries with key (that is a time in nanos) less than current time. - const processedKeys: number[] = new Array(); - for (const [key, val] of MonitorImpl.newContexts.entries()) { - if (key < currentTimeNanos) { - const queue: Array> = val; - - processedKeys.push(key); - // Each value of found entry is a queue of monitoring contexts awaiting active monitoring. - // Add all contexts to an active monitoring contexts queue. - // Ignore disposed contexts. - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = queue?.shift()) != null) { - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - if (monitorContext && monitorContext.isActive()) { - this.activeContexts.push(monitorContextRef); - } - } - } - } - processedKeys.forEach((key) => { - MonitorImpl.newContexts.delete(key); - }); - await sleep(1000); - } - return; - } catch (err) { - // do nothing, exit task - } - logger.debug(Messages.get("MonitorImpl.stopMonitoringTaskNewContext", this.hostInfo.host)); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - try { - if (this.activeContexts.length === 0 && !this.hostUnhealthy) { - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, MonitorImpl.TASK_SLEEP_MILLIS); - }); - continue; - } - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - const isValid = await this.checkConnectionStatus(); - const statusCheckEndTimeNanos: number = getCurrentTimeNano(); - - await this.updateHostHealthStatus(isValid, statusCheckStartTimeNanos, statusCheckEndTimeNanos); - - if (this.hostUnhealthy) { - this.pluginService.setAvailability(this.hostInfo.aliases, HostAvailability.NOT_AVAILABLE); - } - const tmpActiveContexts: WeakRef[] = []; - - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = this.activeContexts?.shift()) != null) { - if (this.isStopped()) { - break; - } - - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - - if (!monitorContext) { - continue; - } - - if (this.hostUnhealthy) { - // Kill connection - monitorContext.setHostUnhealthy(true); - const clientToAbort = monitorContext.getClient(); - - monitorContext.setInactive(); - if (clientToAbort != null) { - await this.endMonitoringClient(clientToAbort); - - this.abortedConnectionsCounter.inc(); - } - } else if (monitorContext && monitorContext.isActive()) { - tmpActiveContexts.push(monitorContextRef); - } - } - - // activeContexts is empty now and tmpActiveContexts contains all yet active contexts - // Add active contexts back to the queue. - this.activeContexts.push(...tmpActiveContexts); - - const delayMillis = (this.failureDetectionIntervalNanos - (statusCheckEndTimeNanos - statusCheckStartTimeNanos)) / 1000000; - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout( - resolve, - delayMillis < MonitorImpl.TASK_SLEEP_MILLIS ? MonitorImpl.TASK_SLEEP_MILLIS : delayMillis - ); - }); - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - try { - if (!(await this.pluginService.isClientValid(this.monitoringClient))) { - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return true; - } - return true; - } catch (error: any) { - return false; - } - } - - isStopped(): boolean { - return this.stopped; - } - - updateHostHealthStatus(connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number): Promise { - if (!connectionValid) { - this.failureCount++; - - if (this.invalidHostStartTimeNano === 0) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationNano = this.failureDetectionIntervalNanos * Math.max(0, this.failureDetectionCount - 1); - - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationNano) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", this.hostInfo.host)); - this.hostUnhealthy = true; - return Promise.resolve(); - } - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", this.hostInfo.host)); - return Promise.resolve(); - } - - if (this.failureCount > 0) { - // Host is back alive - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", this.hostInfo.host)); - } - this.failureCount = 0; - this.invalidHostStartTimeNano = 0; - this.hostUnhealthy = false; - } - - async releaseResources() { - this.stopped = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenHostHealthyTimeoutId); - this.activeContexts = null; - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient(clientToAbort?: ClientWrapper) { - try { - if (clientToAbort) { - await this.pluginService.abortTargetClient(clientToAbort); - } else if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - this.stopped = true; - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } -} diff --git a/common/lib/plugins/efm2/monitor_connection_context.ts b/common/lib/plugins/efm2/monitor_connection_context.ts deleted file mode 100644 index 156b41963..000000000 --- a/common/lib/plugins/efm2/monitor_connection_context.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { uniqueId } from "../../../logutils"; -import { ClientWrapper } from "../../client_wrapper"; - -/** - * Monitoring context for each connection. This contains each connection's criteria for whether a - * server should be considered unhealthy. The context is shared between the main task and the monitor task. - */ -export class MonitorConnectionContext { - private clientToAbortRef: WeakRef | undefined; - isHostUnhealthy: boolean = false; - id: string = uniqueId("_monitorContext"); - - /** - * Constructor. - * - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - constructor(clientToAbort: ClientWrapper) { - if (clientToAbort) { - this.clientToAbortRef = new WeakRef(clientToAbort); - } - } - - setHostUnhealthy(hostUnhealthy: boolean) { - this.isHostUnhealthy = hostUnhealthy; - } - - shouldAbort(): boolean { - return this.isHostUnhealthy && this.clientToAbortRef != null; - } - - setInactive(): void { - this.clientToAbortRef = null; - } - - getClient(): ClientWrapper | null { - return this.clientToAbortRef?.deref() ?? null; - } - - isActive() { - return !!this.clientToAbortRef?.deref(); - } -} diff --git a/common/lib/plugins/efm2/monitor_service.ts b/common/lib/plugins/efm2/monitor_service.ts deleted file mode 100644 index 40a788178..000000000 --- a/common/lib/plugins/efm2/monitor_service.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { ClientWrapper } from "../../client_wrapper"; -import { logger } from "../../../logutils"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - /** - * Stop monitoring for a connection represented by the given {@link MonitorConnectionContext}. - * Removes the context from the {@link MonitorImpl}. - * - * @param context The {@link MonitorConnectionContext} representing a connection. - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - (monitor: Monitor) => monitor.canDispose(), - async (monitor: Monitor) => { - { - try { - await monitor.releaseResources(); - } catch (error) { - // ignore - } - } - }, - "efm2/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private telemetryFactory: TelemetryFactory; - private readonly abortedConnectionsCounter: TelemetryCounter; - monitorSupplier = ( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) => - new MonitorImpl( - pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - abortedConnectionsCounter - ); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - this.telemetryFactory = pluginService.getTelemetryFactory(); - this.abortedConnectionsCounter = this.telemetryFactory.createCounter("efm2.connections.aborted"); - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitor = await this.getMonitor(hostInfo, properties, failureDetectionTimeMillis, failureDetectionIntervalMillis, failureDetectionCount); - - if (monitor) { - const context = new MonitorConnectionContext(clientToAbort); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise { - context.setInactive(); - if (context.shouldAbort()) { - try { - await clientToAbort.abort(); - this.abortedConnectionsCounter.inc(); - } catch (error) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } - } - - async getMonitor( - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitorKey: string = `${failureDetectionTimeMillis.toString()} ${failureDetectionIntervalMillis.toString()} ${failureDetectionCount.toString()} ${hostInfo.host}`; - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - return MonitorServiceImpl.monitors.computeIfAbsent( - monitorKey, - () => - this.monitorSupplier( - this.pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.abortedConnectionsCounter - ), - cacheExpirationNanos - ); - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/profile/driver_configuration_profiles.ts b/common/lib/profile/driver_configuration_profiles.ts index 958d5dead..a9e2ffb62 100644 --- a/common/lib/profile/driver_configuration_profiles.ts +++ b/common/lib/profile/driver_configuration_profiles.ts @@ -17,7 +17,7 @@ import { ConfigurationProfile } from "./configuration_profile"; import { ConfigurationProfilePresetCodes } from "./configuration_profile_codes"; import { WrapperProperties } from "../wrapper_property"; -import { HostMonitoringPluginFactory } from "../plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "../plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "../plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "../plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting/read_write_splitting_plugin_factory"; diff --git a/common/lib/utils/monitoring/monitor.ts b/common/lib/utils/monitoring/monitor.ts index 81dadea43..5bf93ffcd 100644 --- a/common/lib/utils/monitoring/monitor.ts +++ b/common/lib/utils/monitoring/monitor.ts @@ -78,7 +78,7 @@ export abstract class AbstractMonitor implements Monitor { this.monitorPromise = this.run(); } - protected async run(): Promise { + async run(): Promise { try { this.state = MonitorState.RUNNING; this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); diff --git a/index.ts b/index.ts index 6292539cd..74a840bed 100644 --- a/index.ts +++ b/index.ts @@ -30,8 +30,8 @@ export { DefaultPlugin } from "./common/lib/plugins/default_plugin"; export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; export { FailoverPlugin } from "./common/lib/plugins/failover/failover_plugin"; export { Failover2Plugin } from "./common/lib/plugins/failover2/failover2_plugin"; -export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/host_monitoring_connection_plugin"; -export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm2/host_monitoring2_connection_plugin"; +export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; +export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm/v2/host_monitoring2_connection_plugin"; export { DeveloperConnectionPlugin } from "./common/lib/plugins/dev/developer_connection_plugin"; export { BlueGreenPlugin } from "./common/lib/plugins/bluegreen/blue_green_plugin"; export { AuroraConnectionTrackerPlugin } from "./common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin"; diff --git a/tests/integration/container/tests/performance.test.ts b/tests/integration/container/tests/performance.test.ts index f15d1268e..43d10ac1a 100644 --- a/tests/integration/container/tests/performance.test.ts +++ b/tests/integration/container/tests/performance.test.ts @@ -25,7 +25,7 @@ import { WrapperProperties, PluginManager } from "../../../../index"; import { features } from "./config"; import { PerfStat } from "./utils/perf_stat"; import { PerfTestUtility } from "./utils/perf_util"; -import { MonitorServiceImpl } from "../../../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl as MonitorServiceImpl } from "../../../../common/lib/plugins/efm/base/host_monitor_service"; const itIf = features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && diff --git a/tests/unit/host_monitoring_plugin.test.ts b/tests/unit/host_monitoring_plugin.test.ts index 02b70dd81..88c7c012c 100644 --- a/tests/unit/host_monitoring_plugin.test.ts +++ b/tests/unit/host_monitoring_plugin.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,15 +15,12 @@ */ import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/host_monitoring_connection_plugin"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; -import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; +import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; import { anything, instance, mock, reset, verify, when } from "ts-mockito"; -import { RdsUtils } from "../../common/lib/utils/rds_utils"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { AwsWrapperError, HostAvailability, HostInfo } from "../../common/lib"; +import { ConnectionContext, ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { AwsWrapperError, HostInfo } from "../../common/lib"; import { RdsUrlType } from "../../common/lib/utils/rds_url_type"; import { HostChangeOptions } from "../../common/lib/host_change_options"; import { OldConnectionSuggestionAction } from "../../common/lib/old_connection_suggestion_action"; @@ -31,6 +28,8 @@ import { Messages } from "../../common/lib/utils/messages"; import { AwsPGClient } from "../../pg/lib"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; const FAILURE_DETECTION_TIME = 10; const FAILURE_DETECTION_INTERVAL = 100; @@ -38,12 +37,10 @@ const FAILURE_DETECTION_COUNT = 5; const MONITOR_METHOD_NAME = "query"; const NO_MONITOR_METHOD_NAME = "end"; -const mockDialect: DatabaseDialect = mock(PgDatabaseDialect); -const mockMonitorService: MonitorServiceImpl = mock(MonitorServiceImpl); -const mockMonitorConnectionContext: MonitorConnectionContext = mock(MonitorConnectionContext); +const mockMonitorService: HostMonitorServiceImpl = mock(HostMonitorServiceImpl); +const mockConnectionContext: ConnectionContextImpl = mock(ConnectionContextImpl); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockClient: AwsPGClient = mock(AwsPGClient); -const mockRdsUtils = mock(RdsUtils); const mockHostInfo1 = mock(HostInfo); const mockHostInfo2 = mock(HostInfo); @@ -59,18 +56,19 @@ function incrementQueryCounter() { const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); function initDefaultMockReturns() { - when(mockDialect.getHostAliasQuery()).thenReturn("any"); - when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( - instance(mockMonitorConnectionContext) + when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( + instance(mockConnectionContext) ); + when(mockConnectionContext.isActiveContext()).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(false); when(mockHostInfo1.host).thenReturn("host"); when(mockHostInfo1.port).thenReturn(1234); when(mockHostInfo2.host).thenReturn("host"); when(mockHostInfo2.port).thenReturn(1234); when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); + when(mockPluginService.getRoutedHostInfo()).thenReturn(instance(mockHostInfo1)); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockClient.targetClient).thenReturn(mockClientWrapper); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_INSTANCE); properties.set(WrapperProperties.FAILURE_DETECTION_ENABLED.name, true); properties.set(WrapperProperties.FAILURE_DETECTION_TIME_MS.name, FAILURE_DETECTION_TIME); @@ -79,13 +77,18 @@ function initDefaultMockReturns() { } function initializePlugin() { - plugin = new HostMonitoringConnectionPlugin(instance(mockPluginService), properties, instance(mockRdsUtils), instance(mockMonitorService)); + const servicesContainer = { + pluginService: instance(mockPluginService), + telemetryFactory: new NullTelemetryFactory() + } as unknown as FullServicesContainer; + + plugin = new HostMonitoringConnectionPlugin(servicesContainer, properties, instance(mockMonitorService)); } describe("host monitoring plugin test", () => { beforeEach(() => { reset(mockMonitorService); - reset(mockMonitorConnectionContext); + reset(mockConnectionContext); reset(mockPluginService); reset(mockClient); @@ -99,7 +102,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -108,7 +111,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(NO_MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -117,7 +120,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).once(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).once(); verify(mockMonitorService.stopMonitoring(anything())).once(); expect(queryCounter).toBe(1); }); @@ -126,55 +129,25 @@ describe("host monitoring plugin test", () => { initializePlugin(); const expectedError = new AwsWrapperError("Error thrown during isClientValid"); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); when(mockPluginService.isClientValid(mockClientWrapper)).thenThrow(expectedError); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); }); - it("execute cleanup when abort connection throws error", async () => { + it("execute cleanup when connection is invalid throws unavailable host error", async () => { initializePlugin(); when(mockPluginService.isClientValid(mockClientWrapper)).thenResolve(false); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); const expectedError = new AwsWrapperError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", "host")); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); - verify(mockPluginService.setAvailability(anything(), HostAvailability.NOT_AVAILABLE)).once(); }); - it("test connect", async () => { + it("notify connection changed resets monitoring host info", async () => { initializePlugin(); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_WRITER_CLUSTER); - await plugin.connect(instance(mockHostInfo1), properties, true, () => { - return Promise.resolve(mockClientWrapper); - }); - verify(mockPluginService.fillAliases(anything(), anything())).once(); + const result = await plugin.notifyConnectionChanged(new Set([HostChangeOptions.HOSTNAME])); + expect(result).toBe(OldConnectionSuggestionAction.NO_OPINION); }); - - it.each([[HostChangeOptions.WENT_DOWN], [HostChangeOptions.HOST_DELETED]])( - "notify connection changed when node went down", - async (options: HostChangeOptions) => { - initializePlugin(); - - await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - - const aliases1 = new Set(["alias1", "alias2"]); - const aliases2 = new Set(["alias3", "alias4"]); - - when(mockHostInfo1.allAliases).thenReturn(aliases1); - when(mockHostInfo2.allAliases).thenReturn(aliases2); - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); - - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NodeKeys should contain {"alias1", "alias2"}. - verify(mockMonitorService.stopMonitoringForAllConnections(aliases1)).once(); - - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo2)); - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NotifyConnectionChanged should reset the monitoringHostSpec. - // NodeKeys should contain {"alias3", "alias4"} - verify(mockMonitorService.stopMonitoringForAllConnections(aliases2)).once(); - } - ); }); diff --git a/tests/unit/monitor_connection_context.test.ts b/tests/unit/monitor_connection_context.test.ts index 47e5eab01..3f11b3de3 100644 --- a/tests/unit/monitor_connection_context.test.ts +++ b/tests/unit/monitor_connection_context.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,113 +14,131 @@ limitations under the License. */ -import { instance, mock, spy, verify } from "ts-mockito"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; +import { mock } from "ts-mockito"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; +import { HostInfo } from "../../common/lib"; +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { getCurrentTimeNano } from "../../common/lib/utils/utils"; -const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); -const mockMonitor = mock(MonitorImpl); -const mockTargetClient = { - end() { - throw new Error("close"); - } -}; -const mockClientWrapper = mock(MySQLClientWrapper); +const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; const VALIDATION_INTERVAL_MILLIS = 50; -let context: MonitorConnectionContext; +let context: ConnectionContextImpl; -describe("monitor connection context test", () => { +describe("connection context test", () => { beforeEach(() => { - context = new MonitorConnectionContext( - instance(mockMonitor), - null, + context = new ConnectionContextImpl( + mockClientWrapper, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT, - instance(mockPluginService), new NullTelemetryFactory().createCounter("counter") ); }); it("isHostUnhealthy with valid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", true, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(0); + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + // Timestamps must exceed the grace period relative to context's startMonitorTimeNano (hrtime-based) + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, true); + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy with invalid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", false, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(1); + it("isHostUnhealthy with invalid connection - single failure", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); + // Single failure doesn't exceed threshold + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy exceeds failure detection count", async () => { - let currentTimeNano = Date.now(); - context.failureCount = 0; - context.resetInvalidHostStartTimeNano(); + it("isHostUnhealthy exceeds failure detection threshold", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - for (let i = 0; i < 5; i++) { - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; + // Use timestamps far enough in the future to exceed the grace period (failureDetectionTimeMillis). + // The context's startMonitorTimeNano is hrtime-based (~now), so we simulate checks starting after the grace period. + const futureStartNano = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - if (i >= FAILURE_DETECTION_COUNT - 1) { - expect(context.isHostUnhealthy).toBe(true); - } else { - expect(context.isHostUnhealthy).toBe(false); - } + // maxInvalidHostDuration = failureDetectionIntervalMillis * failureDetectionCount = 100 * 3 = 300ms + // Each check spans FAILURE_DETECTION_INTERVAL_MILLIS (100ms), so after 3 checks the duration exceeds threshold. + for (let i = 0; i < FAILURE_DETECTION_COUNT + 2; i++) { + const checkStart = futureStartNano + i * FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + const checkEnd = checkStart + FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); - currentTimeNano += VALIDATION_INTERVAL_MILLIS * 1_000_000; + if (ctx.isHostUnhealthy()) { + break; + } } - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - expect(context.isHostUnhealthy).toBe(true); + expect(ctx.isHostUnhealthy()).toBe(true); }); - it.each([[true], [false]])("updateConnectionStatus", async (isValid: boolean) => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - FAILURE_DETECTION_TIME_MILLIS * 1_000_000; + it("updateConnectionStatus skips within grace period", async () => { + // Use a context with a long grace period + const ctx = new ConnectionContextImpl( + mockClientWrapper, + 30000, // 30s grace + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, isValid); + // Call with a time that's within the grace period (elapsed < failureDetectionTime) + const now = getCurrentTimeNano(); + await ctx.updateConnectionStatus("test-node", now, now + 1_000_000, false); - verify(contextSpy.setConnectionValid("test-node", isValid, statusCheckStartTime, currentTimeNano)).once(); + // Should not have detected unhealthy since we're within grace period + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("updateConnectionStatus - inactive context", async () => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - 1000; - context.isActiveContext = false; + it("updateConnectionStatus skips for inactive context", async () => { + context.setInactive(); + expect(context.isActiveContext()).toBe(false); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, true); + const now = getCurrentTimeNano(); + await context.updateConnectionStatus("test-node", now, now + 100_000_000, false); - verify(contextSpy.setConnectionValid("test-node", true, statusCheckStartTime, currentTimeNano)).never(); + // Should remain not-unhealthy since the context is inactive + expect(context.isHostUnhealthy()).toBe(false); }); - it("abort client ignores error", async () => { - context = new MonitorConnectionContext( - instance(mockMonitor), - mockClientWrapper, - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("counter") - ); + it("setInactive marks context inactive", () => { + expect(context.isActiveContext()).toBe(true); + context.setInactive(); + expect(context.isActiveContext()).toBe(false); + }); + it("abortConnection does nothing for inactive context", async () => { + context.setInactive(); + // Should not throw await context.abortConnection(); }); }); diff --git a/tests/unit/monitor_impl.test.ts b/tests/unit/monitor_impl.test.ts index 8ba1116cc..2ba18316f 100644 --- a/tests/unit/monitor_impl.test.ts +++ b/tests/unit/monitor_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,61 +14,56 @@ limitations under the License. */ -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { anything, instance, mock, reset, when } from "ts-mockito"; import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo } from "../../common/lib"; -import { AwsClient } from "../../common/lib/aws_client"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; import { sleep } from "../../common/lib/utils/utils"; import { ClientWrapper } from "../../common/lib/client_wrapper"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; -class MonitorImplTest extends MonitorImpl { +class MonitorImplTest extends HostMonitorImpl { constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); } - override startRun() { - // do nothing + override checkConnectionStatus() { + // Exposes the protected checkConnectionStatus method for testing purposes. + return super.checkConnectionStatus(); } } const mockPluginService = mock(PluginServiceImpl); const mockHostInfo = mock(HostInfo); -const mockClient = mock(AwsClient); const mockClientWrapper: ClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const SHORT_INTERVAL_MILLIS = 30; const properties = new Map(); -let monitor: MonitorImpl; -let monitorSpy: MonitorImpl; +let monitor: MonitorImplTest; describe("monitor impl test", () => { beforeEach(() => { reset(mockPluginService); - when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockPluginService.forceConnect(anything(), anything())).thenResolve(mockClientWrapper); when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); monitor = new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); - monitorSpy = spy(monitor); }); - afterEach(() => { - monitor.releaseResources(); + afterEach(async () => { + await monitor.releaseResources(); }); it("is client healthy with no existing client", async () => { - const status = await monitor.checkConnectionStatus(); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); - verify(mockPluginService.forceConnect(anything(), anything())).once(); - expect(status.isValid).toBe(true); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + expect(isValid).toBe(true); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("is client healthy with existing client", async () => { @@ -77,13 +72,11 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status1 = await monitor.checkConnectionStatus(); - expect(status1.isValid).toBe(true); + const [isValid1] = await monitor.checkConnectionStatus(); + expect(isValid1).toBe(true); - const status2 = await monitor.checkConnectionStatus(); - expect(status2.isValid).toBe(true); - - verify(mockPluginService.isClientValid(mockClientWrapper)).twice(); + const [isValid2] = await monitor.checkConnectionStatus(); + expect(isValid2).toBe(true); }); it("is client healthy with error", async () => { @@ -92,31 +85,23 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status = await monitor.checkConnectionStatus(); - expect(status.isValid).toBe(false); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); + expect(isValid).toBe(false); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("run without context", async () => { - // Should end by itself. + // Should end by itself (monitorDisposalTimeMillis = 0). await monitor.run(); - verify(monitorSpy.checkConnectionStatus()).never(); }); it("run with context", async () => { - const monitorContextInstance = new MonitorConnectionContext( - monitor, - mockClientWrapper, - 30000, - 5000, - 3, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("name") - ); - monitor.startMonitoring(monitorContextInstance); - // Should end by itself. + when(mockPluginService.isClientValid(anything())).thenResolve(true); + + const context = new ConnectionContextImpl(mockClientWrapper, 30000, 5000, 3, new NullTelemetryFactory().createCounter("counter")); + monitor.startMonitoring(context); monitor.run(); await sleep(SHORT_INTERVAL_MILLIS); - monitor.stopMonitoring(monitorContextInstance); + monitor.stopMonitoring(context); }); }); diff --git a/tests/unit/monitor_service_impl.test.ts b/tests/unit/monitor_service_impl.test.ts index 676d8c2bd..4093196c3 100644 --- a/tests/unit/monitor_service_impl.test.ts +++ b/tests/unit/monitor_service_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,203 +14,168 @@ limitations under the License. */ -import { anything, capture, instance, mock, reset, verify, when } from "ts-mockito"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; -import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; +import { instance, mock, reset, when } from "ts-mockito"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo, HostInfoBuilder } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; - -class MonitorImplTest extends MonitorImpl { - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - } - - override startRun() { - // do nothing - } -} - -const mockPluginService = mock(PluginServiceImpl); -const mockMonitorA = mock(MonitorImpl); -const mockMonitorB = mock(MonitorImpl); -const mockHostInfo = mock(HostInfo); -const mockClientWrapper = mock(MySQLClientWrapper); +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { MonitorServiceImpl } from "../../common/lib/utils/monitoring/monitor_service"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { BatchingEventPublisher } from "../../common/lib/utils/events/batching_event_publisher"; +import { WrapperProperties } from "../../common/lib/wrapper_property"; const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; -const NODE_KEYS = new Set(["any"]); -const properties = new Map(); -let monitorService: MonitorServiceImpl; +const mockPluginService = mock(PluginServiceImpl); + +const properties: Map = new Map(); +properties.set(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.name, 600000); + +let monitorService: HostMonitorServiceImpl; +let coreMonitorService: MonitorServiceImpl; +let servicesContainer: FullServicesContainer; +let eventPublisher: BatchingEventPublisher; describe("monitor service impl test", () => { beforeEach(() => { reset(mockPluginService); - reset(mockMonitorA); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - monitorService = new MonitorServiceImpl(instance(mockPluginService)); - monitorService.monitorSupplier = () => new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); + const telemetryFactory = new NullTelemetryFactory(); + when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); + when(mockPluginService.isClientValid(undefined)).thenResolve(false); + + eventPublisher = new BatchingEventPublisher(60_000); + coreMonitorService = new MonitorServiceImpl(eventPublisher); + + servicesContainer = { + pluginService: instance(mockPluginService), + monitorService: coreMonitorService, + telemetryFactory: telemetryFactory + } as unknown as FullServicesContainer; + + monitorService = new HostMonitorServiceImpl(servicesContainer); }); afterEach(async () => { await monitorService.releaseResources(); + await coreMonitorService.releaseResources(); + eventPublisher.releaseResources(); }); - it("start monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring creates context", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - await monitorService.startMonitoring( + const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - }); - it("start monitoring called multiple times", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); - - const runs = 5; - - for (let i = 0; i < runs; i++) { - await monitorService.startMonitoring( - mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT - ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - } + expect(context).toBeDefined(); + expect(context).not.toBeNull(); + expect(context.isActiveContext()).toBe(true); + expect(context.isHostUnhealthy()).toBe(false); }); - it("start and stop monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("stop monitoring sets context inactive", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); - verify(mockMonitorA.stopMonitoring(anything())).once(); + monitorService.stopMonitoring(context); + expect(context.isActiveContext()).toBe(false); }); - it("stop monitoring called twice", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring reuses monitor for same host", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const context = await monitorService.startMonitoring( + await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); + const monitor1 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); - await monitorService.stopMonitoring(context); - verify(mockMonitorA.stopMonitoring(anything())).twice(); - }); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - it("stop monitoring for all connections with invalid node keys", async () => { - monitorService.stopMonitoringForAllConnections(new Set()); - monitorService.stopMonitoringForAllConnections(new Set(["foo"])); + const monitor2 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor2).toBe(monitor1); }); - it("stop monitoring for all connections", async () => { - const keysA = new Set(["monitorA"]); - const keysB = new Set(["monitorB"]); + it("start monitoring creates separate monitors for different hosts", async () => { + const hostInfoA = new HostInfoBuilder({ host: "host-a", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const hostInfoB = new HostInfoBuilder({ host: "host-b", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - monitorService.monitorSupplier = () => instance(mockMonitorA); - await monitorService.getMonitor( - keysA, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoA, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.monitorSupplier = () => instance(mockMonitorB); - await monitorService.getMonitor( - keysB, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoB, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.stopMonitoringForAllConnections(keysA); - verify(mockMonitorA.clearContexts()).once(); - - monitorService.stopMonitoringForAllConnections(keysB); - verify(mockMonitorB.clearContexts()).once(); + const monitorA = coreMonitorService.get(HostMonitorImpl, hostInfoA.host); + const monitorB = coreMonitorService.get(HostMonitorImpl, hostInfoB.host); + expect(monitorA).not.toBe(monitorB); }); - it("getMonitor called with multiple hosts in keys", async () => { - const keysA = new Set(["host1.domain", "host2.domain"]); - const keysB = new Set(["host2.domain"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - }); - - it("getMonitor called with different host keys", async () => { - const keys = new Set(["hostNEW.domain"]); - - const monitorOne = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); + it("release resources clears monitors", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const monitorOneDupe = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - - const monitorTwo = await monitorService.getMonitor(NODE_KEYS, instance(mockHostInfo), properties); - expect(monitorTwo).not.toBeNull(); - expect(monitorTwo).not.toBe(monitorOne); - }); - - it("getMonitor called with same keys in different sets", async () => { - const keysA = new Set(["hostA"]); - const keysB = new Set(["hostA", "hostB"]); - const keysC = new Set(["hostB"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - const monitorOneDupeTwo = await monitorService.getMonitor(keysC, instance(mockHostInfo), properties); - expect(monitorOneDupeTwo).toBe(monitorOne); - }); + await monitorService.releaseResources(); - it("startMonitoring with no host keys", async () => { - const keys: Set = new Set(); - expect(await monitorService.getMonitor(keys, instance(mockHostInfo), properties)).toBeNull(); + const monitor = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor).toBeNull(); }); }); From f96278da8a999bcb0aefef951afa88114cdd5894 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:54:48 -0700 Subject: [PATCH 3/3] chore: remove aliases --- .github/workflows/integration_tests.yml | 151 +++--------------- .../rds_host_list_provider.ts | 17 +- common/lib/partial_plugin_service.ts | 9 +- common/lib/plugin_service.ts | 13 +- ...rora_initial_connection_strategy_plugin.ts | 8 +- .../opened_connection_tracker.ts | 5 +- .../tracked_connection_list.ts | 110 +++++++++++++ .../v1/host_monitoring_connection_plugin.ts | 6 +- .../lib/plugins/failover/failover_plugin.ts | 2 +- .../limitless/limitless_connection_context.ts | 1 - common/lib/utils/messages.ts | 2 - mysql/lib/client.ts | 3 +- pg/lib/client.ts | 3 +- tests/unit/aurora_connection_tracker.test.ts | 3 +- tests/unit/failover2_plugin.test.ts | 5 - tests/unit/failover_plugin.test.ts | 7 - tests/unit/host_monitoring_plugin.test.ts | 3 + tests/unit/reader_failover_handler.test.ts | 2 +- tests/unit/writer_failover_handler.test.ts | 17 +- 19 files changed, 180 insertions(+), 187 deletions(-) create mode 100644 common/lib/plugins/connection_tracker/tracked_connection_list.ts diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 2218d9eac..501d3f12c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,157 +5,50 @@ on: push: branches: - main - - chore/remove-aliases paths-ignore: - - '**/*.md' - - '**/*.jpg' - - '**/README.txt' - - '**/LICENSE.txt' - - 'docs/**' - - 'ISSUE_TEMPLATE/**' - - '**/remove-old-artifacts.yml' - pull_request_target: - branches: - - main - paths-ignore: - - '**/*.md' - - '**/*.jpg' - - '**/README.txt' - - '**/LICENSE.txt' - - 'docs/**' - - 'ISSUE_TEMPLATE/**' - - '**/remove-old-artifacts.yml' + - "**/*.md" + - "**/*.jpg" + - "**/README.txt" + - "**/LICENSE.txt" + - "docs/**" + - "ISSUE_TEMPLATE/**" + - "**/remove-old-artifacts.yml" permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout jobs: - approve: - # Auto-approve for non-fork scenarios - if: > - github.event_name == 'push' || - github.event_name == 'workflow_dispatch' || - github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - steps: - - run: echo "Approved — not a fork PR" - approve-fork: - # Require manual approval for fork PRs - if: > - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest - environment: integration-tests - steps: - - run: echo "Fork PR approved by maintainer" - run-integration-tests-default: - name: Run Integration Tests (Default) - needs: [ approve, approve-fork ] - if: | - always() && - (needs.approve.result == 'success' || needs.approve-fork.result == 'success') && - !(needs.approve.result == 'failure' || needs.approve-fork.result == 'failure') - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - dbEngine: ["aurora-mysql", "aurora-postgres"] - - steps: - - name: 'Clone repository' - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: "Set up JDK 8" - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: - distribution: "corretto" - java-version: 8 - - name: Set up Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: "22.x" - - name: Install dependencies - run: npm install --no-save - - - name: Configure AWS Credentials - id: creds - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} - role-session-name: nodejs_int_default_tests - role-duration-seconds: 21600 - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - output-credentials: true - - - name: Run Integration Tests - run: | - ./gradlew --no-parallel --no-daemon test-${{ matrix.dbEngine }} --info - env: - RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - AURORA_MYSQL_DB_ENGINE_VERSION: default - AURORA_PG_DB_ENGINE_VERSION: default - - - name: "Get Github Action IP" - if: always() - id: ip - uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - - - name: "Remove Github Action IP" - if: always() - run: | - aws ec2 revoke-security-group-ingress \ - --group-name default \ - --protocol -1 \ - --port -1 \ - --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ - 2>&1 > /dev/null; - - - name: Archive results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: integration-report-default-${{ matrix.dbEngine }} - path: ./tests/integration/container/reports - retention-days: 5 - - run-integration-tests-latest: - name: Run Integration Tests (Latest) + run-integration-tests: + name: Run Integration Tests runs-on: ubuntu-latest - needs: run-integration-tests-default - if: | - always() && - needs.run-integration-tests-default.result == 'success' strategy: fail-fast: false matrix: - dbEngine: ["aurora-mysql", "aurora-postgres" ] + versions: [ "default", "latest" ] + dbEngine: ["aurora-mysql", "aurora-postgres", "multi-az-mysql", "multi-az-postgres" ] steps: - name: Clone repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v4 - name: "Set up JDK 8" - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + uses: actions/setup-java@v3 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: actions/setup-node@v4 with: - node-version: "22.x" + node-version: "20.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} - role-session-name: nodejs_int_latest_tests + role-session-name: nodejs_int_default_tests role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} output-credentials: true @@ -168,13 +61,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - AURORA_MYSQL_DB_ENGINE_VERSION: latest - AURORA_PG_DB_ENGINE_VERSION: latest + AURORA_MYSQL_DB_ENGINE_VERSION: ${{ matrix.dbEngine }} + AURORA_PG_DB_ENGINE_VERSION: ${{ matrix.versions }} - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 + uses: haythem/public-ip@v1.3 - name: "Remove Github Action IP" if: always() @@ -188,8 +81,8 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@v4 with: - name: integration-report-latest-${{ matrix.dbEngine }} + name: integration-report-default-${{ matrix.dbEngine }}-${{ matrix.versions}} path: ./tests/integration/container/reports retention-days: 5 diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index 4185d7db7..53f5d5955 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -185,30 +185,17 @@ export class RdsHostListProvider implements DynamicHostListProvider { } let topology = await this.refresh(); - let isForcedRefresh = false; - if (!topology) { topology = await this.forceRefresh(); - isForcedRefresh = true; } if (!topology) { return null; } + const instanceId = instanceIds[0]; const instanceName = instanceIds[1]; - let matches = topology.filter((host) => host.hostId === instanceName); - const foundHost = matches.length === 0 ? null : matches[0]; - - if (!foundHost && !isForcedRefresh) { - topology = await this.forceRefresh(); - if (!topology) { - return null; - } - } - - matches = topology.filter((host) => host.hostId === instanceName); - return matches.length === 0 ? null : matches[0]; + return topology.find((host) => instanceId === host.hostId || instanceName === host.host) ?? null; } async refresh(): Promise { diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts index 4203b2309..be99ac069 100644 --- a/common/lib/partial_plugin_service.ts +++ b/common/lib/partial_plugin_service.ts @@ -420,7 +420,14 @@ export class PartialPluginService implements PluginService, HostListProviderServ // constructor should already be updated and verified. } - async identifyConnection(targetClient: ClientWrapper): Promise { + identifyConnection(targetClient: ClientWrapper): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo | null): Promise; + async identifyConnection(targetClient: ClientWrapper, connectionHostInfo?: HostInfo | null): Promise { + const hostIdCacheService = this.servicesContainer.hostIdCacheService; + if (hostIdCacheService && connectionHostInfo) { + return hostIdCacheService.identifyConnection(targetClient, connectionHostInfo, this); + } + const provider = this.getHostListProvider(); if (!provider) { return Promise.reject(new AwsWrapperError(Messages.get("PluginService.errorIdentifyConnection"))); diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 1b80affbb..f2f0af523 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -498,6 +498,10 @@ export class PluginServiceImpl implements PluginService, HostListProviderService changes.set(host.url, hostChanges); } } + + if (changes.size > 0) { + this.servicesContainer.pluginManager?.notifyHostListChanged(changes); + } } updateConfigWithProperties(props: Map) { @@ -512,7 +516,14 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.routedHostInfo = routedHostInfo; } - identifyConnection(targetClient: ClientWrapper): Promise { + identifyConnection(targetClient: ClientWrapper): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo | null): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo?: HostInfo | null): Promise { + const hostIdCacheService = this.servicesContainer.hostIdCacheService; + if (hostIdCacheService && connectionHostInfo) { + return hostIdCacheService.identifyConnection(targetClient, connectionHostInfo, this); + } + const provider: HostListProvider | null = this.getHostListProvider(); if (!provider) { return Promise.reject(); diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts index cdabc7d1a..82688be6b 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts @@ -206,9 +206,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic // and return the current (writer) client. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); return readerCandidateClient; } await this.pluginService.abortTargetClient(readerCandidateClient); @@ -216,9 +214,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu continue; } // Reader connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); return readerCandidateClient; } catch (error: any) { await this.pluginService.abortTargetClient(readerCandidateClient); diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 6efa08391..764827aa0 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -133,8 +133,11 @@ export class OpenedConnectionTracker { } pruneNullConnections(): void { - for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + for (const [key, connectionList] of OpenedConnectionTracker.openedConnections.entries()) { connectionList.removeIf((ref) => !ref.deref()); + if (connectionList.isEmpty()) { + OpenedConnectionTracker.openedConnections.delete(key); + } } } diff --git a/common/lib/plugins/connection_tracker/tracked_connection_list.ts b/common/lib/plugins/connection_tracker/tracked_connection_list.ts new file mode 100644 index 000000000..083aadff1 --- /dev/null +++ b/common/lib/plugins/connection_tracker/tracked_connection_list.ts @@ -0,0 +1,110 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../../client_wrapper"; + +/** + * A handle to a single connection tracked within a {@link TrackedConnectionList}. + * Holding onto a host allows the tracked connection to be removed later without + * having to search the entire list. + */ +export class TrackedConnection { + private readonly list: TrackedConnectionList; + private readonly ref: WeakRef; + + constructor(list: TrackedConnectionList, ref: WeakRef) { + this.list = list; + this.ref = ref; + } + + /** + * Stop tracking the connection this host refers to. + */ + remove(): void { + this.list.removeRef(this.ref); + } +} + +/** + * Tracks opened connections for a single host. Connections are held via weak + * references so that clients which are no longer referenced elsewhere can be + * garbage collected without leaking through this list. + */ +export class TrackedConnectionList { + private readonly connections: WeakRef[] = []; + + /** + * Track a new connection and return a handle that can be used to stop tracking it. + */ + add(client: ClientWrapper): TrackedConnection { + const ref = new WeakRef(client); + this.connections.push(ref); + return new TrackedConnection(this, ref); + } + + /** + * Remove a specific weak reference from the list, if present. + */ + removeRef(ref: WeakRef): void { + const index = this.connections.indexOf(ref); + if (index !== -1) { + this.connections.splice(index, 1); + } + } + + /** + * Remove every tracked reference for which the predicate returns true. + */ + removeIf(predicate: (ref: WeakRef) => boolean): void { + for (let i = this.connections.length - 1; i >= 0; i--) { + if (predicate(this.connections[i])) { + this.connections.splice(i, 1); + } + } + } + + /** + * Whether there are no tracked references remaining. Note that a reference + * whose client has been garbage collected still counts until it is pruned. + */ + isEmpty(): boolean { + return this.connections.length === 0; + } + + /** + * Return the live clients that are still tracked, skipping any whose weak + * reference has been cleared. + */ + getConnections(): ClientWrapper[] { + const connections: ClientWrapper[] = []; + for (const ref of this.connections) { + const client = ref.deref(); + if (client) { + connections.push(client); + } + } + return connections; + } + + /** + * Return all live clients and clear the list. + */ + drainAll(): ClientWrapper[] { + const connections = this.getConnections(); + this.connections.length = 0; + return connections; + } +} diff --git a/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts index 3c7548d24..5ae1831ad 100644 --- a/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts @@ -22,6 +22,7 @@ import { OldConnectionSuggestionAction } from "../../../old_connection_suggestio import { RdsUtils } from "../../../utils/rds_utils"; import { AbstractConnectionPlugin } from "../../../abstract_connection_plugin"; import { RdsUrlType } from "../../../utils/rds_url_type"; +import { HostAvailability } from "../../../host_availability/host_availability"; import { WrapperProperties } from "../../../wrapper_property"; import { ConnectionContext } from "../base/connection_context"; import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; @@ -119,6 +120,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (context.isHostUnhealthy()) { const monitoringHostInfo = await this.getMonitoringHostInfo(); + this.pluginService.setAvailability(monitoringHostInfo, HostAvailability.NOT_AVAILABLE); const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; if (targetClient) { @@ -167,10 +169,10 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (this.monitoringHostInfo == null) { this.throwUnableToIdentifyConnection(null); } - const rdsUrlType: RdsUrlType = this.rdsUtils.identifyRdsType(this.monitoringHostInfo.url); + const rdsUrlType: RdsUrlType = this.rdsUtils.identifyRdsType(this.monitoringHostInfo.host); try { - if (rdsUrlType.isRdsCluster) { + if (rdsUrlType !== RdsUrlType.RDS_INSTANCE) { logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); this.monitoringHostInfo = await this.pluginService.identifyConnection( this.pluginService.getCurrentClient().targetClient!, diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index 73014d2dc..2b5d83727 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -327,7 +327,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); await this.updateTopology(true); - this.failoverWriterSuccessCounter.inc(); + this.failoverReaderSuccessCounter.inc(); } catch (error: any) { this.failoverReaderFailedCounter.inc(); throw error; diff --git a/common/lib/plugins/limitless/limitless_connection_context.ts b/common/lib/plugins/limitless/limitless_connection_context.ts index 5891c2da6..2c176098c 100644 --- a/common/lib/plugins/limitless/limitless_connection_context.ts +++ b/common/lib/plugins/limitless/limitless_connection_context.ts @@ -17,7 +17,6 @@ import { HostInfo } from "../../host_info"; import { ClientWrapper } from "../../client_wrapper"; import { ConnectionPlugin } from "../../connection_plugin"; -import Map from "@arrows/array/src/map"; export class LimitlessConnectionContext { private readonly hostInfo: HostInfo; diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 94e619b9b..e7aebe3d4 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -193,7 +193,6 @@ const MESSAGES: Record = { "MonitorImpl.startMonitoringTaskNewContext": "Start monitoring task for checking new contexts for '%s'", "MonitorImpl.stopMonitoringTaskNewContext": "Stop monitoring task for checking new contexts for '%s'", "MonitorService.startMonitoringNullMonitor": "Start monitoring called but could not find monitor for host: '%s'.", - "MonitorService.emptyAliasSet": "Empty alias set passed for '%s'. Set should not be empty.", "MonitorService.monitorClassMismatch": "The monitor stored at '%s' did not have the expected type. The expected type was '%s', but the monitor '%s' had a type of '%s'.", "MonitorService.monitorStuck": "Monitor '%s' has not been updated within the inactive timeout of %s milliseconds. The monitor will be stopped.", @@ -210,7 +209,6 @@ const MESSAGES: Record = { "PluginService.hostListEmpty": "Current host list is empty.", "PluginService.releaseResources": "Releasing resources.", "PluginService.failedToRetrieveHostPort": "Could not retrieve Host:Port for connection.", - "PluginService.nonEmptyAliases": "fillAliases called when HostInfo already contains the following aliases: '%s'.", "PluginService.forceMonitoringRefreshTimeout": "A timeout error occurred after waiting '%s' ms for refreshed topology.", "PluginService.requiredBlockingHostListProvider": "The detected host list provider is not a BlockingHostListProvider. A BlockingHostListProvider is required to force refresh the host list. Detected host list provider: '%s'.", diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index 2048848c5..b3ab7cbdb 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -87,7 +87,8 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { // Ignore } } - const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + const connectedHostInfo: HostInfo = + this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo() ?? result.hostInfo; await this.pluginService.setCurrentClient(result, connectedHostInfo); this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); diff --git a/pg/lib/client.ts b/pg/lib/client.ts index bb0b3d9fd..956b2bbd8 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -216,7 +216,8 @@ class BaseAwsPgClient extends AwsClient implements PGClient { } } - const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + const connectedHostInfo: HostInfo = + this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo() ?? result.hostInfo; await this.pluginService.setCurrentClient(result, connectedHostInfo); this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); diff --git a/tests/unit/aurora_connection_tracker.test.ts b/tests/unit/aurora_connection_tracker.test.ts index d6aad0791..bfbd76d77 100644 --- a/tests/unit/aurora_connection_tracker.test.ts +++ b/tests/unit/aurora_connection_tracker.test.ts @@ -74,8 +74,7 @@ describe("aurora connection tracker tests", () => { const client = await plugin.connect(hostInfo, props, isInitialConnection, mockConnectFunc); expect(client).toBe(mockClientWrapper); verify(mockTracker.populateOpenedConnectionQueue(hostInfo, mockClientWrapper)).called(); - const aliases = hostInfo.aliases; - expect(aliases.size).toBe(0); + verify(mockPluginService.setTrackedConnectionHost(anything())).called(); }); it("test invalidate opened connections when writer host not changed", async () => { diff --git a/tests/unit/failover2_plugin.test.ts b/tests/unit/failover2_plugin.test.ts index 8db14ad22..3eef605d0 100644 --- a/tests/unit/failover2_plugin.test.ts +++ b/tests/unit/failover2_plugin.test.ts @@ -114,7 +114,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").withRole(HostRole.READER).build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockHostInfo.getRawAvailability()).thenReturn(HostAvailability.AVAILABLE); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockPluginService.getHostInfoByStrategy(HostRole.READER, anything(), anything())).thenReturn(mockHostInfo); @@ -139,7 +138,6 @@ describe("reader failover handler", () => { const hosts = [hostInfo]; const test = new AwsWrapperError("test"); - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockHostInfo.getRawAvailability()).thenReturn(HostAvailability.AVAILABLE); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockPluginService.getAllHosts()).thenReturn(hosts); @@ -166,7 +164,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockPluginService.forceMonitoringRefresh(true, anything())).thenResolve(false); @@ -190,7 +187,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockPluginService.getAllHosts()).thenReturn(hosts); when(mockPluginService.forceMonitoringRefresh(true, anything())).thenResolve(true); @@ -219,7 +215,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").withRole(HostRole.WRITER).build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockPluginService.getAllHosts()).thenReturn(hosts); when(mockPluginService.forceMonitoringRefresh(true, anything())).thenResolve(true); diff --git a/tests/unit/failover_plugin.test.ts b/tests/unit/failover_plugin.test.ts index 42e510251..6f801e31a 100644 --- a/tests/unit/failover_plugin.test.ts +++ b/tests/unit/failover_plugin.test.ts @@ -128,10 +128,8 @@ describe("reader failover handler", () => { await plugin.notifyHostListChanged(changes); when(mockHostInfo.url).thenReturn("cluster-url/"); - when(mockHostInfo.allAliases).thenReturn(new Set(["instance"])); verify(mockPluginService.getCurrentHostInfo()).once(); - verify(mockHostInfo.allAliases).never(); }); it("test update topology", async () => { @@ -206,7 +204,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockHostInfo.getRawAvailability()).thenReturn(HostAvailability.AVAILABLE); when(mockPluginService.getHosts()).thenReturn(hosts); when(await mockPluginService.getHostRole(anything())).thenReturn(HostRole.WRITER); @@ -241,7 +238,6 @@ describe("reader failover handler", () => { const hosts = [hostInfo]; const test = new AwsWrapperError("test"); - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockHostInfo.getRawAvailability()).thenReturn(HostAvailability.AVAILABLE); when(mockPluginService.getHosts()).thenReturn(hosts); when(mockReaderResult.error).thenReturn(test); @@ -265,7 +261,6 @@ describe("reader failover handler", () => { const hosts = [hostInfo]; const test = new AwsWrapperError("test"); - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getAllHosts()).thenReturn(hosts); when(mockWriterResult.error).thenReturn(test); when(mockWriterFailoverHandler.failover(anything())).thenResolve(instance(mockWriterResult)); @@ -286,7 +281,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getAllHosts()).thenReturn(hosts); when(mockWriterResult.isConnected).thenReturn(false); when(mockWriterFailoverHandler.failover(anything())).thenResolve(instance(mockWriterResult)); @@ -316,7 +310,6 @@ describe("reader failover handler", () => { const hostInfo = builder.withHost("hostA").build(); const hosts = [hostInfo]; - when(mockHostInfo.allAliases).thenReturn(new Set(["alias1", "alias2"])); when(mockPluginService.getAllHosts()).thenReturn(hosts); when(mockWriterResult.isConnected).thenReturn(false); when(mockWriterResult.topology).thenReturn(hosts); diff --git a/tests/unit/host_monitoring_plugin.test.ts b/tests/unit/host_monitoring_plugin.test.ts index 88c7c012c..332ef8771 100644 --- a/tests/unit/host_monitoring_plugin.test.ts +++ b/tests/unit/host_monitoring_plugin.test.ts @@ -69,6 +69,9 @@ function initDefaultMockReturns() { when(mockPluginService.getRoutedHostInfo()).thenReturn(instance(mockHostInfo1)); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockClient.targetClient).thenReturn(mockClientWrapper); + // The monitoring host ("host") is a non-instance endpoint, so getMonitoringHostInfo identifies the + // underlying instance. Return the same host so identification succeeds in these tests. + when(mockPluginService.identifyConnection(mockClientWrapper, instance(mockHostInfo1))).thenResolve(instance(mockHostInfo1)); properties.set(WrapperProperties.FAILURE_DETECTION_ENABLED.name, true); properties.set(WrapperProperties.FAILURE_DETECTION_TIME_MS.name, FAILURE_DETECTION_TIME); diff --git a/tests/unit/reader_failover_handler.test.ts b/tests/unit/reader_failover_handler.test.ts index 412bdc5ac..c840c91ce 100644 --- a/tests/unit/reader_failover_handler.test.ts +++ b/tests/unit/reader_failover_handler.test.ts @@ -221,7 +221,7 @@ describe("reader failover handler", () => { expect(result.client).toStrictEqual(mockClientWrapper); verify(mockPluginService.setAvailability(anything(), HostAvailability.NOT_AVAILABLE)).never(); - verify(mockPluginService.setAvailability(fastHost.allAliases, HostAvailability.AVAILABLE)).atMost(2); + verify(mockPluginService.setAvailability(fastHost, HostAvailability.AVAILABLE)).atMost(2); clearTimeout(timeoutId); }, 30000); diff --git a/tests/unit/writer_failover_handler.test.ts b/tests/unit/writer_failover_handler.test.ts index 6daf39869..a6ac6d0af 100644 --- a/tests/unit/writer_failover_handler.test.ts +++ b/tests/unit/writer_failover_handler.test.ts @@ -85,11 +85,6 @@ describe("writer failover handler", () => { const mockHostListProviderService = mock(); beforeEach(() => { - writer.addAlias("writer-host"); - newWriterHost.addAlias("new-writer-host"); - readerA.addAlias("reader-a-host"); - readerB.addAlias("reader-b-host"); - when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); // Mock ServiceUtils.createMinimalServiceContainerFrom to return a container @@ -141,7 +136,7 @@ describe("writer failover handler", () => { expect(result.isNewHost).toBe(false); expect(result.client).toBe(mockClientWrapper); - verify(mockPluginService.setAvailability(writer.allAliases, HostAvailability.AVAILABLE)).called(); + verify(mockPluginService.setAvailability(writer, HostAvailability.AVAILABLE)).called(); }); it("test reconnect to writer - slow reader A", async () => { @@ -174,7 +169,7 @@ describe("writer failover handler", () => { expect(result.isNewHost).toBe(false); expect(result.client).toBe(mockClientWrapper); - verify(mockPluginService.setAvailability(writer.allAliases, HostAvailability.AVAILABLE)).called(); + verify(mockPluginService.setAvailability(writer, HostAvailability.AVAILABLE)).called(); clearTimeout(timeoutId); }, 10000); @@ -209,7 +204,7 @@ describe("writer failover handler", () => { expect(result.isNewHost).toBe(false); expect(result.client).toBe(mockClientWrapper); - verify(mockPluginService.setAvailability(writer.allAliases, HostAvailability.AVAILABLE)).called(); + verify(mockPluginService.setAvailability(writer, HostAvailability.AVAILABLE)).called(); clearTimeout(timeoutId); }, 10000); @@ -247,7 +242,7 @@ describe("writer failover handler", () => { expect(result.topology.length).toBe(3); expect(result.topology[0].host).toBe("new-writer-host"); - verify(mockPluginService.setAvailability(newWriterHost.allAliases, HostAvailability.AVAILABLE)).once(); + verify(mockPluginService.setAvailability(newWriterHost, HostAvailability.AVAILABLE)).once(); clearTimeout(timeoutId); }, 10000); @@ -285,7 +280,7 @@ describe("writer failover handler", () => { expect(result.topology[0].host).toBe("new-writer-host"); verify(mockPluginService.forceRefreshHostList()).atLeast(1); - verify(mockPluginService.setAvailability(newWriterHost.allAliases, HostAvailability.AVAILABLE)).once(); + verify(mockPluginService.setAvailability(newWriterHost, HostAvailability.AVAILABLE)).once(); clearTimeout(timeoutId); }, 10000); @@ -359,6 +354,6 @@ describe("writer failover handler", () => { expect(result.isConnected).toBe(false); expect(result.isNewHost).toBe(false); - verify(mockPluginService.setAvailability(newWriterHost.allAliases, HostAvailability.NOT_AVAILABLE)).atLeast(1); + verify(mockPluginService.setAvailability(newWriterHost, HostAvailability.NOT_AVAILABLE)).atLeast(1); }, 10000); });